CernVM-FS  2.10.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
mountpoint.cc
Go to the documentation of this file.
1 
4 #include "cvmfs_config.h"
5 #include "mountpoint.h"
6 
7 #include <errno.h>
8 #include <fcntl.h>
9 #include <inttypes.h>
10 #include <stdint.h>
11 #include <unistd.h>
12 
13 #include <algorithm>
14 #include <cassert>
15 #include <climits>
16 #include <cstring>
17 #include <vector>
18 
19 #include "duplex_fuse.h" // NOLINT
20 
21 #ifndef CVMFS_LIBCVMFS
22 #ifdef FUSE_CAP_EXPORT_SUPPORT
23 #define CVMFS_NFS_SUPPORT
24 #else
25 #warning "No NFS support, Fuse too old"
26 #endif
27 #endif
28 
29 #include "authz/authz_curl.h"
30 #include "authz/authz_fetch.h"
31 #include "authz/authz_session.h"
32 #include "backoff.h"
33 #include "cache.h"
34 #include "cache_extern.h"
35 #include "cache_posix.h"
36 #include "cache_ram.h"
37 #include "cache_tiered.h"
38 #include "catalog.h"
39 #include "catalog_mgr_client.h"
40 #include "clientctx.h"
41 #include "crypto/signature.h"
42 #include "download.h"
43 #include "duplex_sqlite3.h"
44 #include "fetch.h"
45 #include "file_chunk.h"
46 #include "globals.h"
47 #include "glue_buffer.h"
48 #include "google/protobuf/stubs/common.h"
49 #include "history.h"
50 #include "history_sqlite.h"
51 #include "lru_md.h"
52 #include "manifest.h"
53 #include "manifest_fetch.h"
54 #include "nfs_maps.h"
55 #ifdef CVMFS_NFS_SUPPORT
56 #include "nfs_maps_leveldb.h"
57 #include "nfs_maps_sqlite.h"
58 #endif
59 #include "options.h"
60 #include "quota_posix.h"
62 #include "sqlitemem.h"
63 #include "sqlitevfs.h"
64 #include "statistics.h"
65 #include "tracer.h"
66 #include "util/concurrency.h"
67 #include "util/logging.h"
68 #include "util/platform.h"
69 #include "util/pointer.h"
70 #include "util/posix.h"
71 #include "util/string.h"
72 #include "util/uuid.h"
73 #include "wpad.h"
74 
75 using namespace std; // NOLINT
76 
77 
78 bool FileSystem::g_alive = false;
79 const char *FileSystem::kDefaultCacheBase = "/var/lib/cvmfs";
80 const char *FileSystem::kDefaultCacheMgrInstance = "default";
81 
82 
83 FileSystem::IoErrorInfo::IoErrorInfo() : counter_(NULL), timestamp_last_(0) { }
84 
86  counter_->Set(0);
87  timestamp_last_ = 0;
88 }
89 
91  perf::Inc(counter_);
92  timestamp_last_ = time(NULL);
93 }
94 
96 
97 int64_t FileSystem::IoErrorInfo::count() { return counter_->Get(); }
98 
99 time_t FileSystem::IoErrorInfo::timestamp_last() { return timestamp_last_; }
100 
101 
102 
107 bool FileSystem::CheckInstanceName(const std::string &instance) {
108  if (instance.length() > 24)
109  return false;
110  sanitizer::CacheInstanceSanitizer instance_sanitizer;
111  if (!instance_sanitizer.IsValid(instance)) {
112  boot_error_ = "invalid instance name (" + instance + "), " +
113  "only characters a-z, A-Z, 0-9, _ are allowed";
115  return false;
116  }
117  return true;
118 }
119 
120 
126 {
127  if (settings.is_alien && settings.is_shared) {
128  boot_error_ = "Failure: shared local disk cache and alien cache mutually "
129  "exclusive. Please turn off shared local disk cache.";
131  return false;
132  }
133  if (settings.is_alien && settings.is_managed) {
134  boot_error_ = "Failure: quota management and alien cache mutually "
135  "exclusive. Please turn off quota limit.";
137  return false;
138  }
139 
140  if (type_ == kFsLibrary) {
141  if (settings.is_shared || settings.is_managed) {
142  boot_error_ = "Failure: libcvmfs supports only unmanaged exclusive cache "
143  "or alien cache.";
145  return false;
146  }
147  }
148 
149  if (settings.cache_base_defined && settings.cache_dir_defined) {
150  boot_error_ =
151  "'CVMFS_CACHE_BASE' and 'CVMFS_CACHE_DIR' are mutually exclusive";
153  return false;
154  }
155 
156  return true;
157 }
158 
159 
166  file_system(new FileSystem(fs_info));
167 
168  file_system->SetupLogging();
169  LogCvmfs(kLogCvmfs, kLogDebug, "Options:\n%s",
170  file_system->options_mgr()->Dump().c_str());
171 
172  file_system->CreateStatistics();
173  file_system->SetupSqlite();
174  if (!file_system->DetermineNfsMode())
175  return file_system.Release();
176  if (!file_system->SetupWorkspace())
177  return file_system.Release();
178 
179  // Redirect SQlite temp directory to workspace (global variable)
180  unsigned length_tempdir = file_system->workspace_.length() + 1;
181  sqlite3_temp_directory = static_cast<char *>(sqlite3_malloc(length_tempdir));
182  snprintf(sqlite3_temp_directory,
183  length_tempdir,
184  "%s",
185  file_system->workspace_.c_str());
186 
187  if (!file_system->TriageCacheMgr())
188  return file_system.Release();
189  file_system->SetupUuid();
190  if (!file_system->SetupNfsMaps())
191  return file_system.Release();
192  bool retval = sqlite::RegisterVfsRdOnly(
193  file_system->cache_mgr_,
194  file_system->statistics_,
196  assert(retval);
197  file_system->has_custom_sqlitevfs_ = true;
198 
200 
201  file_system->boot_status_ = loader::kFailOk;
202  return file_system.Release();
203 }
204 
205 
208 
209  // Register the ShortString's static counters
210  statistics_->Register("pathstring.n_instances", "Number of instances");
211  statistics_->Register("pathstring.n_overflows", "Number of overflows");
212  statistics_->Register("namestring.n_instances", "Number of instances");
213  statistics_->Register("namestring.n_overflows", "Number of overflows");
214  statistics_->Register("linkstring.n_instances", "Number of instances");
215  statistics_->Register("linkstring.n_overflows", "Number of overflows");
216 
217  // Callback counters
218  n_fs_open_ = statistics_->Register("cvmfs.n_fs_open",
219  "Overall number of file open operations");
220  n_fs_dir_open_ = statistics_->Register("cvmfs.n_fs_dir_open",
221  "Overall number of directory open operations");
222  n_fs_lookup_ = statistics_->Register("cvmfs.n_fs_lookup",
223  "Number of lookups");
224  n_fs_lookup_negative_ = statistics_->Register("cvmfs.n_fs_lookup_negative",
225  "Number of negative lookups");
226  n_fs_stat_ = statistics_->Register("cvmfs.n_fs_stat", "Number of stats");
227  n_fs_stat_stale_ = statistics_->Register("cvmfs.n_fs_stat_stale",
228  "Number of stats for stale (open, meanwhile changed) regular files");
229  n_fs_read_ = statistics_->Register("cvmfs.n_fs_read", "Number of files read");
230  n_fs_readlink_ = statistics_->Register("cvmfs.n_fs_readlink",
231  "Number of links read");
232  n_fs_forget_ = statistics_->Register("cvmfs.n_fs_forget",
233  "Number of inode forgets");
234  n_fs_inode_replace_ = statistics_->Register("cvmfs.n_fs_inode_replace",
235  "Number of stale inodes that got replaced by an up-to-date version");
236  no_open_files_ = statistics_->Register("cvmfs.no_open_files",
237  "Number of currently opened files");
238  no_open_dirs_ = statistics_->Register("cvmfs.no_open_dirs",
239  "Number of currently opened directories");
240  io_error_info_.SetCounter(statistics_->Register("cvmfs.n_io_error",
241  "Number of I/O errors"));
242 
243  string optarg;
244  if (options_mgr_->GetValue("CVMFS_INSTRUMENT_FUSE", &optarg) &&
245  options_mgr_->IsOn(optarg))
246  {
248  }
249 
250  hist_fs_lookup_ = new Log2Histogram(30);
251  hist_fs_forget_ = new Log2Histogram(30);
258  hist_fs_open_ = new Log2Histogram(30);
259  hist_fs_read_ = new Log2Histogram(30);
261 }
262 
263 
269  const string &instance
270 ) {
271  string optarg;
273 
274  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SHARED", instance),
275  &optarg)
276  && options_mgr_->IsOn(optarg))
277  {
278  settings.is_shared = true;
279  }
280  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SERVER_MODE", instance),
281  &optarg)
282  && options_mgr_->IsOn(optarg))
283  {
284  settings.avoid_rename = true;
285  }
286 
287  if (type_ == kFsFuse)
288  settings.quota_limit = kDefaultQuotaLimit;
289  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_QUOTA_LIMIT", instance),
290  &optarg))
291  {
292  settings.quota_limit = String2Int64(optarg) * 1024 * 1024;
293  }
294  if (settings.quota_limit > 0)
295  settings.is_managed = true;
296 
297  settings.cache_path = kDefaultCacheBase;
298  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_BASE", instance),
299  &optarg))
300  {
301  settings.cache_path = MakeCanonicalPath(optarg);
302  settings.cache_base_defined = true;
303  }
304  if (settings.is_shared) {
305  settings.cache_path += "/shared";
306  } else {
307  settings.cache_path += "/" + name_;
308  }
309 
310  // CheckCacheMode makes sure that CVMFS_CACHE_DIR and CVMFS_CACHE_BASE are
311  // not set at the same time.
312  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_DIR", instance),
313  &optarg))
314  {
315  settings.cache_dir_defined = true;
316  settings.cache_path = optarg;
317  }
318  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_ALIEN", instance),
319  &optarg))
320  {
321  settings.is_alien = true;
322  settings.cache_path = optarg;
323  }
324  // We already changed the cwd to the workspace
325  // Which is only done if using FUSE
326  if ((type_ == kFsFuse) && (settings.cache_path == workspace_fullpath_)) {
327  settings.cache_path = ".";
328  }
329 
330  // The cache workspace usually is the cache directory, unless explicitly
331  // set otherwise
332  settings.workspace = settings.cache_path;
333  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_WORKSPACE", instance),
334  &optarg) ||
335  options_mgr_->GetValue("CVMFS_WORKSPACE", &optarg))
336  {
337  // Used for the shared quota manager
338  settings.workspace = optarg;
339  }
340 
341  return settings;
342 }
343 
344 
346  string optarg;
347 
348  if (options_mgr_->GetValue("CVMFS_NFS_SOURCE", &optarg) &&
349  options_mgr_->IsOn(optarg))
350  {
351  nfs_mode_ |= kNfsMaps;
352  if (options_mgr_->GetValue("CVMFS_NFS_SHARED", &optarg)) {
354  nfs_maps_dir_ = optarg;
355  }
356  }
357 
358  if ((type_ == kFsLibrary) && (nfs_mode_ != kNfsNone)) {
359  boot_error_ = "Failure: libcvmfs does not support NFS export.";
361  return false;
362  }
363  return true;
364 }
365 
366 
368  : name_(fs_info.name)
369  , exe_path_(fs_info.exe_path)
370  , type_(fs_info.type)
371  , options_mgr_(fs_info.options_mgr)
372  , wait_workspace_(fs_info.wait_workspace)
373  , foreground_(fs_info.foreground)
374  , n_fs_open_(NULL)
375  , n_fs_dir_open_(NULL)
376  , n_fs_lookup_(NULL)
377  , n_fs_lookup_negative_(NULL)
378  , n_fs_stat_(NULL)
379  , n_fs_stat_stale_(NULL)
380  , n_fs_read_(NULL)
381  , n_fs_readlink_(NULL)
382  , n_fs_forget_(NULL)
383  , n_fs_inode_replace_(NULL)
384  , no_open_files_(NULL)
385  , no_open_dirs_(NULL)
386  , statistics_(NULL)
387  , fd_workspace_lock_(-1)
388  , found_previous_crash_(false)
390  , cache_mgr_(NULL)
391  , uuid_cache_(NULL)
392  , nfs_maps_(NULL)
393  , has_custom_sqlitevfs_(false)
394 {
395  assert(!g_alive);
396  g_alive = true;
397  g_uid = geteuid();
398  g_gid = getegid();
399 
400  string optarg;
401  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SERVER_MODE",
403  &optarg)
404  && options_mgr_->IsOn(optarg))
405  {
406  g_raw_symlinks = true;
407  }
408 }
409 
410 
413 
416 
417  delete uuid_cache_;
418  delete nfs_maps_;
419  delete cache_mgr_;
420 
421  if (sqlite3_temp_directory) {
422  sqlite3_free(sqlite3_temp_directory);
423  sqlite3_temp_directory = NULL;
424  }
425 
426  if (!path_crash_guard_.empty())
427  unlink(path_crash_guard_.c_str());
428  if (!path_workspace_lock_.empty())
429  unlink(path_workspace_lock_.c_str());
430  if (fd_workspace_lock_ >= 0)
432 
433  sqlite3_shutdown();
435 
436  delete hist_fs_lookup_;
437  delete hist_fs_forget_multi_;
438  delete hist_fs_forget_;
439  delete hist_fs_getattr_;
440  delete hist_fs_readlink_;
441  delete hist_fs_opendir_;
442  delete hist_fs_releasedir_;
443  delete hist_fs_readdir_;
444  delete hist_fs_open_;
445  delete hist_fs_read_;
446  delete hist_fs_release_;
447  delete statistics_;
448 
449  SetLogSyslogPrefix("");
450  SetLogMicroSyslog("");
451  SetLogDebugFile("");
452  google::protobuf::ShutdownProtobufLibrary();
453  g_alive = false;
454 }
455 
456 
458  path_workspace_lock_ = workspace_ + "/lock." + name_;
460  if (fd_workspace_lock_ >= 0)
461  return true;
462 
463  if (fd_workspace_lock_ == -1) {
464  boot_error_ = "could not acquire workspace lock (" +
465  StringifyInt(errno) + ")";
467  return false;
468  }
469 
470  assert(fd_workspace_lock_ == -2);
471 
472  if (!wait_workspace_) {
474  return false;
475  }
476 
478  if (fd_workspace_lock_ < 0) {
479  boot_error_ = "could not acquire workspace lock (" +
480  StringifyInt(errno) + ")";
482  return false;
483  }
484  return true;
485 }
486 
487 
489  void *user_data __attribute__((unused)),
490  int sqlite_extended_error,
491  const char *message)
492 {
493  int log_dest = kLogDebug;
494  int sqlite_error = sqlite_extended_error & 0xFF;
495  switch (sqlite_error) {
496  case SQLITE_INTERNAL:
497  case SQLITE_PERM:
498  case SQLITE_NOMEM:
499  case SQLITE_IOERR:
500  case SQLITE_CORRUPT:
501  case SQLITE_FULL:
502  case SQLITE_CANTOPEN:
503  case SQLITE_MISUSE:
504  case SQLITE_FORMAT:
505  case SQLITE_NOTADB:
506  log_dest |= kLogSyslogErr;
507  break;
508  case SQLITE_WARNING:
509  case SQLITE_NOTICE:
510  default:
511  break;
512  }
513  LogCvmfs(kLogCvmfs, log_dest, "SQlite3: %s (%d)",
514  message, sqlite_extended_error);
515 }
516 
517 
524  const string &generic_parameter,
525  const string &instance)
526 {
527  assert(HasPrefix(generic_parameter, "CVMFS_CACHE_", false));
528 
529  if (instance == kDefaultCacheMgrInstance) {
530  // Compatibility parameter names
531  if ((generic_parameter == "CVMFS_CACHE_SHARED") &&
532  !options_mgr_->IsDefined(generic_parameter))
533  {
534  return "CVMFS_SHARED_CACHE";
535  }
536  if ((generic_parameter == "CVMFS_CACHE_ALIEN") &&
537  !options_mgr_->IsDefined(generic_parameter))
538  {
539  return "CVMFS_ALIEN_CACHE";
540  }
541  if ((generic_parameter == "CVMFS_CACHE_SERVER_MODE") &&
542  !options_mgr_->IsDefined(generic_parameter))
543  {
544  return "CVMFS_SERVER_CACHE_MODE";
545  }
546  if ((generic_parameter == "CVMFS_CACHE_QUOTA_LIMIT") &&
547  !options_mgr_->IsDefined(generic_parameter))
548  {
549  return "CVMFS_QUOTA_LIMIT";
550  }
551  return generic_parameter;
552  }
553 
554  return "CVMFS_CACHE_" + instance + "_" + generic_parameter.substr(12);
555 }
556 
557 
560 }
561 
562 
566 CacheManager *FileSystem::SetupCacheMgr(const string &instance) {
567  if (constructed_instances_.find(instance) != constructed_instances_.end()) {
568  boot_error_ = "circular cache definition: " + instance;
570  return NULL;
571  }
572  constructed_instances_.insert(instance);
573 
574  LogCvmfs(kLogCvmfs, kLogDebug, "setting up cache manager instance %s",
575  instance.c_str());
576  string instance_type;
577  if (instance == kDefaultCacheMgrInstance) {
578  instance_type = "posix";
579  } else {
580  options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_TYPE", instance),
581  &instance_type);
582  }
583  if (instance_type == "posix") {
584  return SetupPosixCacheMgr(instance);
585  } else if (instance_type == "ram") {
586  return SetupRamCacheMgr(instance);
587  } else if (instance_type == "tiered") {
588  return SetupTieredCacheMgr(instance);
589  } else if (instance_type == "external") {
590  return SetupExternalCacheMgr(instance);
591  } else {
592  boot_error_ = "invalid cache manager type for '" + instance + "':" +
593  instance_type;
595  return NULL;
596  }
597 }
598 
599 
601  string optarg;
602  unsigned nfiles = kDefaultNfiles;
603  if (options_mgr_->GetValue("CVMFS_NFILES", &optarg))
604  nfiles = String2Uint64(optarg);
605  vector<string> cmd_line;
606  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_CMDLINE", instance),
607  &optarg))
608  {
609  cmd_line = SplitString(optarg, ',');
610  }
611 
612  if (!options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_LOCATOR", instance),
613  &optarg))
614  {
615  boot_error_ = MkCacheParm("CVMFS_CACHE_LOCATOR", instance) + " missing";
617  return NULL;
618  }
619 
621  ExternalCacheManager::CreatePlugin(optarg, cmd_line));
622  if (!plugin_handle->IsValid()) {
623  boot_error_ = plugin_handle->error_msg();
625  return NULL;
626  }
628  plugin_handle->fd_connection(), nfiles, name_ + ":" + instance);
629  if (cache_mgr == NULL) {
630  boot_error_ = "failed to create external cache manager for " + instance;
632  return NULL;
633  }
634  cache_mgr->AcquireQuotaManager(ExternalQuotaManager::Create(cache_mgr));
635  return cache_mgr;
636 }
637 
638 
641  if (!CheckPosixCacheSettings(settings))
642  return NULL;
643 
645  settings.cache_path,
646  settings.is_alien,
649  if (!cache_mgr.IsValid()) {
650  boot_error_ = "Failed to setup posix cache '" + instance + "' in " +
651  settings.cache_path + ": " + strerror(errno);
653  return NULL;
654  }
655 
656  // Sentinel file for future use
657  // Might be a read-only cache
658  const bool ignore_failure = settings.is_alien;
659  CreateFile(settings.cache_path + "/.cvmfscache", 0600, ignore_failure);
660 
661  if (settings.is_managed) {
662  if (!SetupPosixQuotaMgr(settings, cache_mgr.weak_ref()))
663  return NULL;
664  }
665  return cache_mgr.Release();
666 }
667 
668 
669 CacheManager *FileSystem::SetupRamCacheMgr(const string &instance) {
670  string optarg;
671  unsigned nfiles = kDefaultNfiles;
672  if (options_mgr_->GetValue("CVMFS_NFILES", &optarg)) {
673  nfiles = String2Uint64(optarg);
674  }
675  uint64_t sz_cache_bytes;
676  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SIZE", instance),
677  &optarg))
678  {
679  if (HasSuffix(optarg, "%", false)) {
680  sz_cache_bytes = platform_memsize() * String2Uint64(optarg)/100;
681  } else {
682  sz_cache_bytes = String2Uint64(optarg) * 1024 * 1024;
683  }
684  } else {
685  sz_cache_bytes = platform_memsize() >> 5; // ~3%
686  }
688  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_MALLOC", instance),
689  &optarg))
690  {
691  if (optarg == "libc") {
693  } else if (optarg == "heap") {
695  } else {
696  boot_error_ = "Failure: unknown malloc " +
697  MkCacheParm("CVMFS_CACHE_MALLOC", instance) + "=" + optarg;
699  return NULL;
700  }
701  }
702  sz_cache_bytes = RoundUp8(std::max(static_cast<uint64_t>(40 * 1024 * 1024),
703  sz_cache_bytes));
705  sz_cache_bytes,
706  nfiles,
707  alloc,
708  perf::StatisticsTemplate("cache." + instance, statistics_));
709  if (cache_mgr == NULL) {
710  boot_error_ = "failed to create ram cache manager for " + instance;
712  return NULL;
713  }
714  cache_mgr->AcquireQuotaManager(new NoopQuotaManager());
715  return cache_mgr;
716 }
717 
718 
720  string optarg;
721  if (!options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_UPPER", instance),
722  &optarg))
723  {
724  boot_error_ = MkCacheParm("CVMFS_CACHE_UPPER", instance) + " missing";
726  return NULL;
727  }
728  UniquePtr<CacheManager> upper(SetupCacheMgr(optarg));
729  if (!upper.IsValid())
730  return NULL;
731 
732  if (!options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_LOWER", instance),
733  &optarg))
734  {
735  boot_error_ = MkCacheParm("CVMFS_CACHE_LOWER", instance) + " missing";
737  return NULL;
738  }
739  UniquePtr<CacheManager> lower(SetupCacheMgr(optarg));
740  if (!lower.IsValid())
741  return NULL;
742 
743  CacheManager *tiered =
744  TieredCacheManager::Create(upper.Release(), lower.Release());
745  if (tiered == NULL) {
746  boot_error_ = "Failed to setup tiered cache manager " + instance;
748  return NULL;
749  }
750  if (options_mgr_->GetValue(
751  MkCacheParm("CVMFS_CACHE_LOWER_READONLY", instance), &optarg) &&
752  options_mgr_->IsOn(optarg))
753  {
754  static_cast<TieredCacheManager*>(tiered)->SetLowerReadOnly();
755  }
756  return tiered;
757 }
758 
759 
761  path_crash_guard_ = workspace_ + "/running." + name_;
762  platform_stat64 info;
763  int retval = platform_stat(path_crash_guard_.c_str(), &info);
764  if (retval == 0) {
765  found_previous_crash_ = true;
767  "looks like cvmfs has been crashed previously");
768  }
769  retval = open(path_crash_guard_.c_str(), O_RDONLY | O_CREAT, 0600);
770  if (retval < 0) {
771  boot_error_ = "could not open running sentinel (" +
772  StringifyInt(errno) + ")";
774  return false;
775  }
776  close(retval);
777  return true;
778 }
779 
780 
782  if (type_ == kFsFuse) {
783  // Try to jump to workspace / cache directory. This tests, if it is
784  // accessible and it brings speed later on.
785  int retval = chdir(workspace_.c_str());
786  if (retval != 0) {
787  boot_error_ = "workspace " + workspace_ + " is unavailable";
789  return false;
790  }
791  workspace_ = ".";
792  return true;
793  }
794 
795  // Note: as of version 2.4 support for CVMFS_CWD_CACHE is dropped due to
796  // disproportionate large complexity to configure correctly. This affects
797  // only libcvmfs, mostly the legacy part.
798  // string optarg;
799  // if (options_mgr_->GetValue("CVMFS_CWD_CACHE", &optarg) &&
800  // options_mgr_->IsOn(optarg))
801  // {
802  // ...
803  // }
804  return true;
805 }
806 
807 
809  string optarg;
810  if (options_mgr_->GetValue("CVMFS_SYSLOG_LEVEL", &optarg))
812  if (options_mgr_->GetValue("CVMFS_SYSLOG_FACILITY", &optarg))
814  if (options_mgr_->GetValue("CVMFS_USYSLOG", &optarg))
815  SetLogMicroSyslog(optarg);
816  if (options_mgr_->GetValue("CVMFS_DEBUGLOG", &optarg))
817  SetLogDebugFile(optarg);
818  if (options_mgr_->GetValue("CVMFS_SYSLOG_PREFIX", &optarg)) {
819  SetLogSyslogPrefix(optarg);
820  } else {
822  }
823 }
824 
825 
827 #ifdef CVMFS_NFS_SUPPORT
828  if (!IsHaNfsSource())
830 
831  string no_nfs_sentinel;
832  if (cache_mgr_->id() == kPosixCacheManager) {
833  PosixCacheManager *posix_cache_mgr =
834  reinterpret_cast<PosixCacheManager *>(cache_mgr_);
835  no_nfs_sentinel = posix_cache_mgr->cache_path() + "/no_nfs_maps." + name_;
836  if (!IsNfsSource()) {
837  // Might be a read-only cache
838  const bool ignore_failure = posix_cache_mgr->alien_cache();
839  CreateFile(no_nfs_sentinel, 0600, ignore_failure);
840  return true;
841  }
842  } else {
843  if (IsNfsSource()) {
844  boot_error_ = "NFS source only works with POSIX cache manager.";
846  return false;
847  }
848  return true;
849  }
850 
852  assert(IsNfsSource());
853  if (!no_nfs_sentinel.empty() && FileExists(no_nfs_sentinel)) {
854  boot_error_ = "Cache was used without NFS maps before. "
855  "It has to be wiped out.";
857  return false;
858  }
859 
860  // nfs maps need to be protected by workspace lock
861  PosixCacheManager *posix_cache_mgr =
862  reinterpret_cast<PosixCacheManager *>(cache_mgr_);
863  if (posix_cache_mgr->cache_path() != workspace_) {
864  boot_error_ = "Cache directory and workspace must be identical for "
865  "NFS export";
867  return false;
868  }
869 
870  string inode_cache_dir = nfs_maps_dir_ + "/nfs_maps." + name_;
871  if (!MkdirDeep(inode_cache_dir, 0700)) {
872  boot_error_ = "Failed to initialize NFS maps";
874  return false;
875  }
876 
877  // TODO(jblomer): make this a manager class
878  if (IsHaNfsSource()) {
880  inode_cache_dir,
883  statistics_);
884  } else {
886  inode_cache_dir,
889  statistics_);
890  }
891 
892  if (nfs_maps_ == NULL) {
893  boot_error_ = "Failed to initialize NFS maps";
895  return false;
896  }
897 
898  string optarg;
899  if (options_mgr_->GetValue("CVMFS_NFS_INTERLEAVED_INODES", &optarg)) {
900  vector<string> tokens = SplitString(optarg, '%');
901  if (tokens.size() != 2) {
902  boot_error_ =
903  "invalid format for CVMFS_NFS_INTERLEAVED_INODES: " + optarg;
905  return false;
906  }
908  String2Uint64(tokens[0]));
909  }
910 
911  return true;
912 
913 #else
914  return true;
915 #endif
916 }
917 
918 
921  CacheManager *cache_mgr
922 ) {
923  assert(settings.quota_limit >= 0);
924  int64_t quota_threshold = settings.quota_limit / 2;
925  string cache_workspace = settings.cache_path;
926  if (settings.cache_path != settings.workspace) {
928  "using workspace %s to protect cache database in %s",
929  settings.workspace.c_str(), settings.cache_path.c_str());
930  cache_workspace += ":" + settings.workspace;
931  }
932  PosixQuotaManager *quota_mgr;
933 
934  if (settings.is_shared) {
936  exe_path_,
937  cache_workspace,
938  settings.quota_limit,
939  quota_threshold,
940  foreground_);
941  if (quota_mgr == NULL) {
942  boot_error_ = "Failed to initialize shared lru cache";
944  return false;
945  }
946  } else {
947  quota_mgr = PosixQuotaManager::Create(
948  cache_workspace,
949  settings.quota_limit,
950  quota_threshold,
952  if (quota_mgr == NULL) {
953  boot_error_ = "Failed to initialize lru cache";
955  return false;
956  }
957  }
958 
959  if (quota_mgr->GetSize() > quota_mgr->GetCapacity()) {
961  "cache is already beyond quota size "
962  "(size: %" PRId64 ", capacity: %" PRId64 "), cleaning up",
963  quota_mgr->GetSize(), quota_mgr->GetCapacity());
964  if (!quota_mgr->Cleanup(quota_threshold)) {
965  delete quota_mgr;
966  boot_error_ = "Failed to clean up cache";
968  return false;
969  }
970  }
971 
972  int retval = cache_mgr->AcquireQuotaManager(quota_mgr);
973  assert(retval);
975  "CernVM-FS: quota initialized, current size %luMB",
976  quota_mgr->GetSize() / (1024 * 1024));
977  return true;
978 }
979 
980 
982  // Make sure SQlite starts clean after initialization
983  sqlite3_shutdown();
984 
985  int retval;
986  retval = sqlite3_config(SQLITE_CONFIG_LOG, FileSystem::LogSqliteError, NULL);
987  assert(retval == SQLITE_OK);
988  retval = sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
989  assert(retval == SQLITE_OK);
991 
992  // Disable SQlite3 file locking
993  retval = sqlite3_vfs_register(sqlite3_vfs_find("unix-none"), 1);
994  assert(retval == SQLITE_OK);
995 }
996 
997 
999  string optarg;
1000  // This is very similar to "determine cache dir". It's for backward
1001  // compatibility with classic cache configuration where there was no
1002  // distinction between workspace and cache.
1003  // Complicated cache configurations should explicitly set CVMFS_WORKSPACE.
1005  if (options_mgr_->GetValue("CVMFS_CACHE_BASE", &optarg))
1006  workspace_ = MakeCanonicalPath(optarg);
1007  if (options_mgr_->GetValue("CVMFS_SHARED_CACHE", &optarg) &&
1008  options_mgr_->IsOn(optarg))
1009  {
1010  workspace_ += "/shared";
1011  } else {
1012  workspace_ += "/" + name_;
1013  }
1014  if (options_mgr_->GetValue("CVMFS_CACHE_DIR", &optarg)) {
1015  if (options_mgr_->IsDefined("CVMFS_CACHE_BASE")) {
1016  boot_error_ =
1017  "'CVMFS_CACHE_BASE' and 'CVMFS_CACHE_DIR' are mutually exclusive";
1019  return false;
1020  }
1021  workspace_ = optarg;
1022  }
1023  if (options_mgr_->GetValue("CVMFS_WORKSPACE", &optarg))
1024  workspace_ = optarg;
1026 
1027  // If workspace and alien cache are the same directory, we need to open
1028  // permission now to 0770 to avoid a race when fixing it later
1029  const int mode = 0770;
1030  if (!MkdirDeep(workspace_, mode, false)) {
1031  boot_error_ = "cannot create workspace directory " + workspace_;
1033  return false;
1034  }
1035 
1036  if (!LockWorkspace())
1037  return false;
1038  if (!SetupCwd())
1039  return false;
1040  if (!SetupCrashGuard())
1041  return false;
1042 
1043  return true;
1044 }
1045 
1046 
1049  if (uuid_cache_ == NULL) {
1051  "failed to load/store %s/uuid", workspace_.c_str());
1053  assert(uuid_cache_ != NULL);
1054  }
1055 }
1056 
1057 
1063  if ((cache_mgr_ != NULL) && (cache_mgr_->id() == kPosixCacheManager)) {
1064  PosixCacheManager *posix_cache_mgr =
1065  reinterpret_cast<PosixCacheManager *>(cache_mgr_);
1066  posix_cache_mgr->TearDown2ReadOnly();
1067  }
1068 
1069  unlink(path_crash_guard_.c_str());
1070  LogCvmfs(kLogCache, kLogSyslog, "switch to read-only cache mode");
1071  SetLogMicroSyslog("");
1072 }
1073 
1074 
1075 void FileSystem::RemapCatalogFd(int from, int to) {
1076  sqlite::RegisterFdMapping(from, to);
1077 }
1078 
1079 
1082  string instance;
1083  if (options_mgr_->GetValue("CVMFS_CACHE_PRIMARY", &instance) &&
1084  !instance.empty())
1085  {
1086  if (!CheckInstanceName(instance))
1087  return false;
1088  cache_mgr_instance_ = instance;
1089  }
1090 
1092  return cache_mgr_ != NULL;
1093 }
1094 
1095 
1096 //------------------------------------------------------------------------------
1097 
1098 
1099 const char *MountPoint::kDefaultAuthzSearchPath = "/usr/libexec/cvmfs/authz";
1100 const char *MountPoint::kDefaultBlacklist = "/etc/cvmfs/blacklist";
1101 
1103  blacklist_paths_.clear();
1104  string blacklist;
1105  if (!options_mgr_->GetValue("CVMFS_BLACKLIST", &blacklist))
1106  blacklist = kDefaultBlacklist;
1107  blacklist_paths_.push_back(blacklist);
1108 
1109  bool append = false;
1110  if (FileExists(blacklist)) {
1111  if (!signature_mgr_->LoadBlacklist(blacklist, append)) {
1112  boot_error_ = "failed to load blacklist " + blacklist;
1114  return false;
1115  }
1116  append = true;
1117  }
1118 
1119  string config_repository_path;
1120  if (options_mgr_->HasConfigRepository(fqrn_, &config_repository_path)) {
1121  blacklist = config_repository_path + "blacklist";
1122  blacklist_paths_.push_back(blacklist);
1123  if (FileExists(blacklist)) {
1124  if (!signature_mgr_->LoadBlacklist(blacklist, append)) {
1125  boot_error_ = "failed to load blacklist from config repository";
1127  return false;
1128  }
1129  }
1130  }
1131 
1132  return true;
1133 }
1134 
1135 
1141  bool result = true;
1142  bool append = false;
1143  for (unsigned i = 0; i < blacklist_paths_.size(); ++i) {
1144  string blacklist = blacklist_paths_[i];
1145  if (FileExists(blacklist)) {
1146  bool retval = signature_mgr_->LoadBlacklist(blacklist, append);
1147  // TODO(jblomer): this can leave us with a half-loaded blacklist
1148  if (!retval)
1149  return false;
1150  append = true;
1151  }
1152  }
1153  return result;
1154 }
1155 
1156 
1162  const string &fqrn,
1163  FileSystem *file_system,
1164  OptionsManager *options_mgr)
1165 {
1166  if (options_mgr == NULL)
1167  options_mgr = file_system->options_mgr();
1168  UniquePtr<MountPoint> mountpoint(new MountPoint(
1169  fqrn, file_system, options_mgr));
1170 
1171  // At this point, we have a repository name, the type (fuse or library) and
1172  // an options manager (which can be the same than the FileSystem's one).
1173 
1174  mountpoint->CreateStatistics();
1175  mountpoint->CreateAuthz();
1176  mountpoint->backoff_throttle_ = new BackoffThrottle();
1177 
1178  if (!mountpoint->CreateSignatureManager() || !mountpoint->CheckBlacklists())
1179  return mountpoint.Release();
1180  if (!mountpoint->CreateDownloadManagers())
1181  return mountpoint.Release();
1182  if (!mountpoint->CreateResolvConfWatcher()) {
1183  return mountpoint.Release();
1184  }
1185  mountpoint->CreateFetchers();
1186  if (!mountpoint->CreateCatalogManager())
1187  return mountpoint.Release();
1188  if (!mountpoint->CreateTracer())
1189  return mountpoint.Release();
1190 
1191  mountpoint->ReEvaluateAuthz();
1192  mountpoint->CreateTables();
1193  if (!mountpoint->SetupBehavior())
1194  return mountpoint.Release();
1195 
1196  mountpoint->boot_status_ = loader::kFailOk;
1197  return mountpoint.Release();
1198 }
1199 
1200 
1202  string optarg;
1203  string authz_helper;
1204  if (options_mgr_->GetValue("CVMFS_AUTHZ_HELPER", &optarg))
1205  authz_helper = optarg;
1206  string authz_search_path(kDefaultAuthzSearchPath);
1207  if (options_mgr_->GetValue("CVMFS_AUTHZ_SEARCH_PATH", &optarg))
1208  authz_search_path = optarg;
1209 
1211  fqrn_,
1212  authz_helper,
1213  authz_search_path,
1214  options_mgr_);
1215  assert(authz_fetcher_ != NULL);
1216 
1219  statistics_);
1220  assert(authz_session_mgr_ != NULL);
1221 
1223  assert(authz_attachment_ != NULL);
1224 }
1225 
1226 
1228  string optarg;
1229 
1231 
1233  if (!SetupOwnerMaps())
1234  return false;
1235  shash::Any root_hash;
1236  if (!DetermineRootHash(&root_hash))
1237  return false;
1238 
1239  bool retval;
1240  if (root_hash.IsNull()) {
1241  retval = catalog_mgr_->Init();
1242  } else {
1243  fixed_catalog_ = true;
1244  bool alt_root_path =
1245  options_mgr_->GetValue("CVMFS_ALT_ROOT_PATH", &optarg) &&
1246  options_mgr_->IsOn(optarg);
1247  retval = catalog_mgr_->InitFixed(root_hash, alt_root_path);
1248  }
1249  if (!retval) {
1250  boot_error_ = "Failed to initialize root file catalog";
1252  return false;
1253  }
1254 
1256  boot_error_ = "repository revision blacklisted";
1258  return false;
1259  }
1260 
1261  if (options_mgr_->GetValue("CVMFS_AUTO_UPDATE", &optarg) &&
1262  !options_mgr_->IsOn(optarg))
1263  {
1264  fixed_catalog_ = true;
1265  }
1266 
1267  if (options_mgr_->GetValue("CVMFS_CATALOG_WATERMARK", &optarg)) {
1269  } else {
1270  unsigned soft_limit;
1271  unsigned hard_limit;
1272  GetLimitNoFile(&soft_limit, &hard_limit);
1273  catalog_mgr_->SetCatalogWatermark(soft_limit / 4);
1274  }
1275 
1276  if (catalog_mgr_->volatile_flag()) {
1277  LogCvmfs(kLogCvmfs, kLogDebug, "content of repository flagged as VOLATILE");
1278  }
1279 
1280  return true;
1281 }
1282 
1283 
1285  string optarg;
1288  perf::StatisticsTemplate("download", statistics_));
1290 
1291  if (options_mgr_->GetValue("CVMFS_SERVER_URL", &optarg)) {
1292  download_mgr_->SetHostChain(optarg);
1293  }
1294 
1296  SetupHttpTuning();
1297 
1298  string forced_proxy_template;
1299  if (options_mgr_->GetValue("CVMFS_PROXY_TEMPLATE", &optarg))
1300  forced_proxy_template = optarg;
1302  forced_proxy_template);
1303 
1304  string proxies;
1305  if (options_mgr_->GetValue("CVMFS_HTTP_PROXY", &optarg))
1306  proxies = optarg;
1308  proxies,
1309  file_system_->workspace() + "/proxies" + GetUniqFileSuffix(),
1310  download_mgr_);
1311  if (proxies == "") {
1312  boot_error_ = "failed to discover HTTP proxy servers";
1314  return false;
1315  }
1316  string fallback_proxies;
1317  if (options_mgr_->GetValue("CVMFS_FALLBACK_PROXY", &optarg))
1318  fallback_proxies = optarg;
1319  download_mgr_->SetProxyChain(proxies, fallback_proxies,
1321 
1322  bool do_geosort = options_mgr_->GetValue("CVMFS_USE_GEOAPI", &optarg) &&
1323  options_mgr_->IsOn(optarg);
1324  if (do_geosort) {
1326  }
1327  if (options_mgr_->GetValue("CVMFS_MAX_SERVERS", &optarg)) {
1328  unsigned max_servers = String2Uint64(optarg);
1329  std::vector<std::string> host_chain;
1330  download_mgr_->GetHostInfo(&host_chain, NULL, NULL);
1331  if (max_servers > 0 && max_servers < host_chain.size()) {
1332  host_chain.resize(max_servers);
1333  download_mgr_->SetHostChain(host_chain);
1334  }
1335  }
1336 
1337  if (options_mgr_->GetValue("CVMFS_USE_SSL_SYSTEM_CA", &optarg) &&
1338  options_mgr_->IsOn(optarg)) {
1340  }
1341 
1342  if (options_mgr_->GetValue("CVMFS_PROXY_SHARD", &optarg) &&
1343  options_mgr_->IsOn(optarg)) {
1345  }
1346 
1347  return SetupExternalDownloadMgr(do_geosort);
1348 }
1349 
1351  std::string roaming_value;
1352  options_mgr_->GetValue("CVMFS_DNS_ROAMING", &roaming_value);
1353  if (options_mgr_->IsDefined("CVMFS_DNS_ROAMING") &&
1354  options_mgr_->IsOn(roaming_value)) {
1356  "DNS roaming is enabled for this repository.");
1357  // Create a file watcher to update the DNS settings of the download
1358  // managers when there are changes to /etc/resolv.conf
1360 
1361  if (resolv_conf_watcher_) {
1362  ResolvConfEventHandler *handler =
1364  resolv_conf_watcher_->RegisterHandler("/etc/resolv.conf", handler);
1365  }
1366  } else {
1368  "DNS roaming is disabled for this repository.");
1369  }
1370  return true;
1371 }
1372 
1374  fetcher_ = new cvmfs::Fetcher(
1376  download_mgr_,
1379 
1380  const bool is_external_data = true;
1385  perf::StatisticsTemplate("fetch-external", statistics_),
1386  is_external_data);
1387 }
1388 
1389 
1391  string optarg;
1393  signature_mgr_->Init();
1394 
1395  string public_keys;
1396  if (options_mgr_->GetValue("CVMFS_PUBLIC_KEY", &optarg)) {
1397  public_keys = optarg;
1398  } else if (options_mgr_->GetValue("CVMFS_KEYS_DIR", &optarg)) {
1399  // Collect .pub files from CVMFS_KEYS_DIR
1400  public_keys = JoinStrings(FindFilesBySuffix(optarg, ".pub"), ":");
1401  } else {
1402  public_keys =
1403  JoinStrings(FindFilesBySuffix("/etc/cvmfs/keys", ".pub"), ":");
1404  }
1405 
1406  if (!signature_mgr_->LoadPublicRsaKeys(public_keys)) {
1407  boot_error_ = "failed to load public key(s)";
1409  return false;
1410  }
1411  LogCvmfs(kLogCvmfs, kLogDebug, "CernVM-FS: using public key(s) %s",
1412  public_keys.c_str());
1413 
1414  if (options_mgr_->GetValue("CVMFS_TRUSTED_CERTS", &optarg)) {
1415  if (!signature_mgr_->LoadTrustedCaCrl(optarg)) {
1416  boot_error_ = "failed to load trusted certificates";
1418  return false;
1419  }
1420  }
1421 
1422  return true;
1423 }
1424 
1425 
1429  return;
1430 
1431  // TODO(jblomer): this should be registered by the tracker
1432  statistics_->Register("inode_tracker.n_insert",
1433  "overall number of accessed inodes");
1434  statistics_->Register("inode_tracker.n_remove",
1435  "overall number of evicted inodes");
1436  statistics_->Register("inode_tracker.no_reference",
1437  "currently active inodes");
1438  statistics_->Register("inode_tracker.n_hit_inode",
1439  "overall number of inode lookups");
1440  statistics_->Register("inode_tracker.n_hit_path",
1441  "overall number of successful path lookups");
1442  statistics_->Register("inode_tracker.n_miss_path",
1443  "overall number of unsuccessful path lookups");
1444 
1445  statistics_->Register("dentry_tracker.n_insert",
1446  "overall number of added negative cache entries");
1447  statistics_->Register("dentry_tracker.n_remove",
1448  "overall number of evicted negative cache entries");
1449  statistics_->Register("dentry_tracker.n_prune",
1450  "overall number of prune calls");
1451 
1452  statistics_->Register("page_cache_tracker.n_insert",
1453  "overall number of added page cache entries");
1454  statistics_->Register("page_cache_tracker.n_remove",
1455  "overall number of evicted page cache entries");
1456  statistics_->Register("page_cache_tracker.n_open_direct",
1457  "overall number of direct I/O open calls");
1458  statistics_->Register("page_cache_tracker.n_open_flush",
1459  "overall number of open calls where the file's page cache gets flushed");
1460  statistics_->Register("page_cache_tracker.n_open_cached",
1461  "overall number of open calls where the file's page cache is reused");
1462 }
1463 
1464 
1467  // Libcvmfs simplified tables
1470  return;
1471  }
1472 
1473  chunk_tables_ = new ChunkTables();
1474 
1475  string optarg;
1476  uint64_t mem_cache_size = kDefaultMemcacheSize;
1477  if (options_mgr_->GetValue("CVMFS_MEMCACHE_SIZE", &optarg))
1478  mem_cache_size = String2Uint64(optarg) * 1024 * 1024;
1479 
1480  const double memcache_unit_size =
1481  (static_cast<double>(kInodeCacheFactor) * lru::Md5PathCache::GetEntrySize())
1483  const unsigned memcache_num_units =
1484  mem_cache_size / static_cast<unsigned>(memcache_unit_size);
1485  // Number of cache entries must be a multiple of 64
1486  const unsigned mask_64 = ~((1 << 6) - 1);
1487  inode_cache_ = new lru::InodeCache(memcache_num_units & mask_64, statistics_);
1488  path_cache_ = new lru::PathCache(memcache_num_units & mask_64, statistics_);
1489  md5path_cache_ = new lru::Md5PathCache((memcache_num_units * 7) & mask_64,
1490  statistics_);
1491 
1495  if (file_system_->IsNfsSource())
1497 }
1498 
1508  string optarg;
1509  tracer_ = new Tracer();
1510  if (options_mgr_->GetValue("CVMFS_TRACEFILE", &optarg)) {
1512  boot_error_ = "tracer is only supported in the fuse module";
1514  return false;
1515  }
1516  string tracebuffer_file = optarg;
1517  uint64_t tracebuffer_size = kTracerBufferSize;
1518  uint64_t tracebuffer_threshold = kTracerFlushThreshold;
1519 
1520  if (options_mgr_->GetValue("CVMFS_TRACEBUFFER", &optarg)) {
1521  tracebuffer_size = String2Uint64(optarg);
1522  }
1523  if (options_mgr_->GetValue("CVMFS_TRACEBUFFER_THRESHOLD",
1524  &optarg)) {
1525  tracebuffer_threshold = String2Uint64(optarg);
1526  }
1527  assert(tracebuffer_size <= INT_MAX
1528  && tracebuffer_threshold <= INT_MAX);
1530  "Initialising tracer with buffer size %" PRIu64 " and threshold %" PRIu64,
1531  tracebuffer_size, tracebuffer_threshold);
1532  tracer_->Activate(tracebuffer_size, tracebuffer_threshold,
1533  tracebuffer_file);
1534  }
1535  return true;
1536 }
1537 
1538 
1540  string optarg;
1541  if (options_mgr_->GetValue("CVMFS_ROOT_HASH", &optarg)) {
1542  *root_hash = MkFromHexPtr(shash::HexPtr(optarg), shash::kSuffixCatalog);
1543  return true;
1544  }
1545 
1546  if (!options_mgr_->IsDefined("CVMFS_REPOSITORY_TAG") &&
1547  !options_mgr_->IsDefined("CVMFS_REPOSITORY_DATE"))
1548  {
1549  root_hash->SetNull();
1550  return true;
1551  }
1552 
1553  string history_path;
1554  if (!FetchHistory(&history_path))
1555  return false;
1556  UnlinkGuard history_file(history_path);
1558  history::SqliteHistory::Open(history_path));
1559  if (!tag_db.IsValid()) {
1561  "failed to open history database (%s)", history_path.c_str());
1562  boot_error_ = "failed to open history database";
1564  return false;
1565  }
1566 
1568  bool retval;
1569  if (!options_mgr_->GetValue("CVMFS_REPOSITORY_TAG", &repository_tag_)) {
1570  string repository_date;
1571  // options_mgr_->IsDefined("CVMFS_REPOSITORY_DATE") must be true
1572  options_mgr_->GetValue("CVMFS_REPOSITORY_DATE", &repository_date);
1573  time_t repository_utctime = IsoTimestamp2UtcTime(repository_date);
1574  if (repository_utctime == 0) {
1575  boot_error_ = "invalid timestamp in CVMFS_REPOSITORY_DATE: " +
1576  repository_date + ". Use YYYY-MM-DDTHH:MM:SSZ";
1578  return false;
1579  }
1580  retval = tag_db->GetByDate(repository_utctime, &tag);
1581  if (!retval) {
1582  boot_error_ = "no repository state as early as utc timestamp " +
1583  StringifyTime(repository_utctime, true);
1585  return false;
1586  }
1588  "time stamp %s UTC resolved to tag '%s'",
1589  StringifyTime(repository_utctime, true).c_str(),
1590  tag.name.c_str());
1591  repository_tag_ = tag.name;
1592  } else {
1593  retval = tag_db->GetByName(repository_tag_, &tag);
1594  if (!retval) {
1595  boot_error_ = "no such tag: " + repository_tag_;
1597  return false;
1598  }
1599  }
1600  LogCvmfs(kLogCvmfs, kLogDebug, "mounting tag %s", tag.name.c_str());
1601 
1602  *root_hash = tag.root_hash;
1603  return true;
1604 }
1605 
1606 
1607 bool MountPoint::FetchHistory(std::string *history_path) {
1608  manifest::Failures retval_mf;
1609  manifest::ManifestEnsemble ensemble;
1610  retval_mf = manifest::Fetch("", fqrn_, 0, NULL, signature_mgr_, download_mgr_,
1611  &ensemble);
1612  if (retval_mf != manifest::kFailOk) {
1613  boot_error_ = "Failed to fetch manifest";
1615  return false;
1616  }
1617  shash::Any history_hash = ensemble.manifest->history();
1618  if (history_hash.IsNull()) {
1619  boot_error_ = "No history";
1621  return false;
1622  }
1623 
1624  int fd = fetcher_->Fetch(
1625  history_hash,
1627  "tag database for " + fqrn_,
1630  if (fd < 0) {
1631  boot_error_ = "failed to download history: " + StringifyInt(-fd);
1633  return false;
1634  }
1635  // We have the custom sqlite vfs driver installed
1636  *history_path = "@" + StringifyInt(fd);
1637  return true;
1638 }
1639 
1640 
1642  unsigned max_ttl;
1643  {
1644  MutexLockGuard lock_guard(lock_max_ttl_);
1645  max_ttl = max_ttl_sec_;
1646  }
1647  const unsigned catalog_ttl_sec = catalog_mgr_->GetTTL();
1648 
1649  return max_ttl ? std::min(max_ttl, catalog_ttl_sec) : catalog_ttl_sec;
1650 }
1651 
1652 
1654  MutexLockGuard lock_guard(lock_max_ttl_);
1655  return max_ttl_sec_ / 60;
1656 }
1657 
1658 
1664  return "." + file_system_->name() + "-" + fqrn_;
1665 }
1666 
1667 
1669  const string &fqrn,
1670  FileSystem *file_system,
1671  OptionsManager *options_mgr)
1672  : fqrn_(fqrn)
1673  , uuid_(cvmfs::Uuid::Create(""))
1674  , file_system_(file_system)
1675  , options_mgr_(options_mgr)
1676  , statistics_(NULL)
1677  , authz_fetcher_(NULL)
1678  , authz_session_mgr_(NULL)
1679  , authz_attachment_(NULL)
1680  , backoff_throttle_(NULL)
1681  , signature_mgr_(NULL)
1682  , download_mgr_(NULL)
1683  , external_download_mgr_(NULL)
1684  , fetcher_(NULL)
1685  , external_fetcher_(NULL)
1686  , inode_annotation_(NULL)
1687  , catalog_mgr_(NULL)
1688  , chunk_tables_(NULL)
1689  , simple_chunk_tables_(NULL)
1690  , inode_cache_(NULL)
1691  , path_cache_(NULL)
1692  , md5path_cache_(NULL)
1693  , tracer_(NULL)
1694  , inode_tracker_(NULL)
1695  , dentry_tracker_(NULL)
1696  , page_cache_tracker_(NULL)
1697  , resolv_conf_watcher_(NULL)
1698  , max_ttl_sec_(kDefaultMaxTtlSec)
1699  , kcache_timeout_sec_(static_cast<double>(kDefaultKCacheTtlSec))
1700  , fixed_catalog_(false)
1701  , enforce_acls_(false)
1702  , has_membership_req_(false)
1703  , talk_socket_path_(std::string("./cvmfs_io.") + fqrn)
1704  , talk_socket_uid_(0)
1705  , talk_socket_gid_(0)
1706 {
1707  int retval = pthread_mutex_init(&lock_max_ttl_, NULL);
1708  assert(retval == 0);
1709 }
1710 
1711 
1713  pthread_mutex_destroy(&lock_max_ttl_);
1714 
1715  delete page_cache_tracker_;
1716  delete dentry_tracker_;
1717  delete inode_tracker_;
1718  delete tracer_;
1719  delete md5path_cache_;
1720  delete path_cache_;
1721  delete inode_cache_;
1722  delete simple_chunk_tables_;
1723  delete chunk_tables_;
1724 
1725  delete catalog_mgr_;
1726  delete inode_annotation_;
1727  delete external_fetcher_;
1728  delete fetcher_;
1729  if (external_download_mgr_ != NULL) {
1731  delete external_download_mgr_;
1732  }
1733  if (download_mgr_ != NULL) {
1734  download_mgr_->Fini();
1735  delete download_mgr_;
1736  }
1737  if (signature_mgr_ != NULL) {
1738  signature_mgr_->Fini();
1739  delete signature_mgr_;
1740  }
1741 
1742  if (resolv_conf_watcher_ != NULL) {
1744  delete resolv_conf_watcher_;
1745  }
1746 
1747  delete backoff_throttle_;
1748  delete authz_attachment_;
1749  delete authz_session_mgr_;
1750  delete authz_fetcher_;
1751  delete statistics_;
1752  delete uuid_;
1753 }
1754 
1755 
1757  string old_membership_req = membership_req_;
1759  if (old_membership_req != membership_req_) {
1762  }
1763 }
1764 
1765 
1766 void MountPoint::SetMaxTtlMn(unsigned value_minutes) {
1767  MutexLockGuard lock_guard(lock_max_ttl_);
1768  max_ttl_sec_ = value_minutes * 60;
1769 }
1770 
1771 
1773  string optarg;
1774 
1775  if (options_mgr_->GetValue("CVMFS_MAX_TTL", &optarg))
1776  SetMaxTtlMn(String2Uint64(optarg));
1777 
1778  if (options_mgr_->GetValue("CVMFS_KCACHE_TIMEOUT", &optarg)) {
1779  // Can be negative and should then be interpreted as 0.0
1781  std::max(0.0, static_cast<double>(String2Int64(optarg)));
1782  }
1783  LogCvmfs(kLogCvmfs, kLogDebug, "kernel caches expire after %d seconds",
1784  static_cast<int>(kcache_timeout_sec_));
1785 
1786  MagicXattrManager::EVisibility xattr_visibility =
1788  if (options_mgr_->GetValue("CVMFS_HIDE_MAGIC_XATTRS", &optarg)) {
1789  if (options_mgr_->IsOn(optarg))
1790  xattr_visibility = MagicXattrManager::kVisibilityNever;
1791  else if (options_mgr_->IsOff(optarg))
1792  xattr_visibility = MagicXattrManager::kVisibilityAlways;
1793  }
1794  if (options_mgr_->GetValue("CVMFS_MAGIC_XATTRS_VISIBILITY", &optarg)) {
1795  if (ToUpper(optarg) == "ROOTONLY") {
1796  xattr_visibility = MagicXattrManager::kVisibilityRootOnly;
1797  } else if (ToUpper(optarg) == "NEVER") {
1798  xattr_visibility = MagicXattrManager::kVisibilityNever;
1799  } else if (ToUpper(optarg) == "ALWAYS") {
1800  xattr_visibility = MagicXattrManager::kVisibilityAlways;
1801  } else {
1803  "unsupported setting: CVMFS_MAGIC_XATTRS_VISIBILITY=%s",
1804  optarg.c_str());
1805  }
1806  }
1807  magic_xattr_mgr_ = new MagicXattrManager(this, xattr_visibility);
1808 
1809 
1810  if (options_mgr_->GetValue("CVMFS_ENFORCE_ACLS", &optarg)
1811  && options_mgr_->IsOn(optarg))
1812  {
1813  enforce_acls_ = true;
1814  }
1815 
1816  if (options_mgr_->GetValue("CVMFS_TALK_SOCKET", &optarg)) {
1817  talk_socket_path_ = optarg;
1818  }
1819  if (options_mgr_->GetValue("CVMFS_TALK_OWNER", &optarg)) {
1820  bool retval = GetUidOf(optarg, &talk_socket_uid_, &talk_socket_gid_);
1821  if (!retval) {
1822  boot_error_ = "unknown owner of cvmfs_talk socket: " + optarg;
1824  return false;
1825  }
1826  }
1827 
1828  return true;
1829 }
1830 
1831 
1837  string optarg;
1838  unsigned dns_timeout_ms = download::DownloadManager::kDnsDefaultTimeoutMs;
1839  unsigned dns_retries = download::DownloadManager::kDnsDefaultRetries;
1840  if (options_mgr_->GetValue("CVMFS_DNS_TIMEOUT", &optarg))
1841  dns_timeout_ms = String2Uint64(optarg) * 1000;
1842  if (options_mgr_->GetValue("CVMFS_DNS_RETRIES", &optarg))
1843  dns_retries = String2Uint64(optarg);
1844  manager->SetDnsParameters(dns_retries, dns_timeout_ms);
1845 
1846  // Rest has to be after SetDnsParameters because SetDnsParameters might
1847  // construct a new resolver object
1848 
1849  unsigned dns_min_ttl = dns::Resolver::kDefaultMinTtl;
1850  unsigned dns_max_ttl = dns::Resolver::kDefaultMaxTtl;
1851  if (options_mgr_->GetValue("CVMFS_DNS_MIN_TTL", &optarg))
1852  dns_min_ttl = String2Uint64(optarg);
1853  if (options_mgr_->GetValue("CVMFS_DNS_MAX_TTL", &optarg))
1854  dns_max_ttl = String2Uint64(optarg);
1855  manager->SetDnsTtlLimits(dns_min_ttl, dns_max_ttl);
1856 
1857  if (options_mgr_->GetValue("CVMFS_DNS_SERVER", &optarg)) {
1858  download_mgr_->SetDnsServer(optarg);
1859  }
1860 
1861  if (options_mgr_->GetValue("CVMFS_IPFAMILY_PREFER", &optarg)) {
1862  switch (String2Int64(optarg)) {
1863  case 4:
1865  break;
1866  case 6:
1868  break;
1869  }
1870  }
1871  if (options_mgr_->GetValue("CVMFS_MAX_IPADDR_PER_PROXY", &optarg))
1872  manager->SetMaxIpaddrPerProxy(String2Uint64(optarg));
1873 }
1874 
1875 
1877  string optarg;
1879  download_mgr_->Clone(perf::StatisticsTemplate("download-external",
1880  statistics_));
1881 
1882  unsigned timeout;
1883  unsigned timeout_direct;
1884  download_mgr_->GetTimeout(&timeout, &timeout_direct);
1885  if (options_mgr_->GetValue("CVMFS_EXTERNAL_TIMEOUT", &optarg)) {
1886  timeout = String2Uint64(optarg);
1887  }
1888  if (options_mgr_->GetValue("CVMFS_EXTERNAL_TIMEOUT_DIRECT", &optarg)) {
1889  timeout_direct = String2Uint64(optarg);
1890  }
1891  external_download_mgr_->SetTimeout(timeout, timeout_direct);
1892 
1893  if (options_mgr_->GetValue("CVMFS_EXTERNAL_URL", &optarg)) {
1895  if (dogeosort) {
1896  std::vector<std::string> host_chain;
1897  external_download_mgr_->GetHostInfo(&host_chain, NULL, NULL);
1898  download_mgr_->GeoSortServers(&host_chain);
1899  external_download_mgr_->SetHostChain(host_chain);
1900  }
1901  } else {
1903  }
1904 
1905  if (options_mgr_->GetValue("CVMFS_EXTERNAL_MAX_SERVERS", &optarg)) {
1906  unsigned max_servers = String2Uint64(optarg);
1907  std::vector<std::string> host_chain;
1908  external_download_mgr_->GetHostInfo(&host_chain, NULL, NULL);
1909  if (max_servers > 0 && max_servers < host_chain.size()) {
1910  host_chain.resize(max_servers);
1911  external_download_mgr_->SetHostChain(host_chain);
1912  }
1913  }
1914 
1915  string proxies = "DIRECT";
1916  if (options_mgr_->GetValue("CVMFS_EXTERNAL_HTTP_PROXY", &optarg)) {
1918  optarg,
1919  file_system_->workspace() + "/proxies-external" + GetUniqFileSuffix(),
1921  if (proxies == "") {
1922  boot_error_ = "failed to discover external HTTP proxy servers";
1924  return false;
1925  }
1926  }
1927  string fallback_proxies;
1928  if (options_mgr_->GetValue("CVMFS_EXTERNAL_FALLBACK_PROXY", &optarg))
1929  fallback_proxies = optarg;
1931  proxies, fallback_proxies, download::DownloadManager::kSetProxyBoth);
1932 
1933  return true;
1934 }
1935 
1936 
1938  string optarg;
1939 
1940  // TODO(jblomer): avoid double default settings
1941 
1942  unsigned timeout = kDefaultTimeoutSec;
1943  unsigned timeout_direct = kDefaultTimeoutSec;
1944  if (options_mgr_->GetValue("CVMFS_TIMEOUT", &optarg))
1945  timeout = String2Uint64(optarg);
1946  if (options_mgr_->GetValue("CVMFS_TIMEOUT_DIRECT", &optarg))
1947  timeout_direct = String2Uint64(optarg);
1948  download_mgr_->SetTimeout(timeout, timeout_direct);
1949 
1950  unsigned max_retries = kDefaultRetries;
1951  unsigned backoff_init = kDefaultBackoffInitMs;
1952  unsigned backoff_max = kDefaultBackoffMaxMs;
1953  if (options_mgr_->GetValue("CVMFS_MAX_RETRIES", &optarg))
1954  max_retries = String2Uint64(optarg);
1955  if (options_mgr_->GetValue("CVMFS_BACKOFF_INIT", &optarg))
1956  backoff_init = String2Uint64(optarg) * 1000;
1957  if (options_mgr_->GetValue("CVMFS_BACKOFF_MAX", &optarg))
1958  backoff_max = String2Uint64(optarg) * 1000;
1959  download_mgr_->SetRetryParameters(max_retries, backoff_init, backoff_max);
1960 
1961  if (options_mgr_->GetValue("CVMFS_LOW_SPEED_LIMIT", &optarg))
1963  if (options_mgr_->GetValue("CVMFS_PROXY_RESET_AFTER", &optarg))
1965  if (options_mgr_->GetValue("CVMFS_HOST_RESET_AFTER", &optarg))
1967 
1968  if (options_mgr_->GetValue("CVMFS_FOLLOW_REDIRECTS", &optarg) &&
1969  options_mgr_->IsOn(optarg))
1970  {
1972  }
1973  if (options_mgr_->GetValue("CVMFS_SEND_INFO_HEADER", &optarg) &&
1974  options_mgr_->IsOn(optarg))
1975  {
1977  }
1978 }
1979 
1980 
1982  string optarg;
1983 
1984  if (file_system_->IsNfsSource()) {
1986  } else {
1988  }
1989  if (options_mgr_->GetValue("CVMFS_INITIAL_GENERATION", &optarg)) {
1991  }
1992 
1995  }
1996 }
1997 
1998 
2000  string optarg;
2001  catalog::OwnerMap uid_map;
2002  catalog::OwnerMap gid_map;
2003 
2004  if (options_mgr_->GetValue("CVMFS_UID_MAP", &optarg)) {
2005  if (!uid_map.Read(optarg)) {
2006  boot_error_ = "failed to parse uid map " + optarg;
2008  return false;
2009  }
2010  }
2011  if (options_mgr_->GetValue("CVMFS_GID_MAP", &optarg)) {
2012  if (!gid_map.Read(optarg)) {
2013  boot_error_ = "failed to parse gid map " + optarg;
2015  return false;
2016  }
2017  }
2018  catalog_mgr_->SetOwnerMaps(uid_map, gid_map);
2019 
2020  // TODO(jblomer): make local to catalog manager
2021  if (options_mgr_->GetValue("CVMFS_CLAIM_OWNERSHIP", &optarg) &&
2022  options_mgr_->IsOn(optarg))
2023  {
2024  g_claim_ownership = true;
2025  }
2026 
2027  return true;
2028 }
IoErrorInfo io_error_info_
Definition: mountpoint.h:321
OptionsManager * options_mgr()
Definition: mountpoint.h:226
#define LogCvmfs(source, mask,...)
Definition: logging.h:22
std::string exe_path()
Definition: mountpoint.h:197
static const unsigned kDnsDefaultTimeoutMs
Definition: download.h:398
bool GetVOMSAuthz(std::string *authz) const
void SetLogSyslogFacility(const int local_facility)
Definition: logging.cc:183
FileSystem * file_system_
Definition: mountpoint.h:548
Counter * Register(const std::string &name, const std::string &desc)
Definition: statistics.cc:139
void SetCounter(perf::Counter *c)
Definition: mountpoint.cc:95
void RegisterFdMapping(int from, int to)
Definition: sqlitevfs.cc:541
std::string nfs_maps_dir_
Definition: mountpoint.h:378
void SetLogSyslogLevel(const int level)
Definition: logging.cc:151
bool IsNull() const
Definition: hash.h:383
bool foreground_
Definition: mountpoint.h:307
void Activate(const int buffer_size, const int flush_threshold, const std::string &trace_file)
Definition: tracer.cc:25
download::DownloadManager * download_mgr_
Definition: repository.h:141
struct stat64 platform_stat64
Log2Histogram * hist_fs_forget_multi_
Definition: mountpoint.h:326
static FileWatcher * Create()
Definition: file_watcher.cc:44
void SetupInodeAnnotation()
Definition: mountpoint.cc:1981
bool SetupCwd()
Definition: mountpoint.cc:781
catalog::InodeAnnotation * inode_annotation_
Definition: mountpoint.h:564
static NfsMapsSqlite * Create(const std::string &db_dir, const uint64_t root_inode, const bool rebuild, perf::Statistics *statistics_)
void TearDown2ReadOnly()
Definition: cache_posix.cc:539
std::string workspace_fullpath_
Definition: mountpoint.h:348
bool DetermineNfsMode()
Definition: mountpoint.cc:345
static SqliteHistory * Open(const std::string &file_name)
void SetHostChain(const std::string &host_list)
static const unsigned kTracerFlushThreshold
Definition: mountpoint.h:515
virtual uint64_t GetSize()
Definition: quota_posix.cc:674
static Publisher * Create(const SettingsPublisher &settings)
uid_t talk_socket_uid_
Definition: mountpoint.h:592
void SetLowSpeedLimit(const unsigned low_speed_limit)
Definition: download.cc:1893
perf::Counter * n_fs_stat_stale_
Definition: mountpoint.h:314
Log2Histogram * hist_fs_opendir_
Definition: mountpoint.h:329
bool LockWorkspace()
Definition: mountpoint.cc:457
bool CreateTracer()
Definition: mountpoint.cc:1507
bool wait_workspace_
Definition: mountpoint.h:306
virtual bool Cleanup(const uint64_t leave_size)
Definition: quota_posix.cc:125
double kcache_timeout_sec_
Definition: mountpoint.h:581
signature::SignatureManager * signature_mgr_
Definition: mountpoint.h:559
void CreateFile(const std::string &path, const int mode, const bool ignore_failure)
Definition: posix.cc:275
perf::Counter * n_fs_lookup_
Definition: mountpoint.h:311
std::string name
Definition: history.h:89
AuthzSessionManager * authz_session_mgr_
Definition: mountpoint.h:556
static const unsigned kDefaultRetries
Definition: mountpoint.h:508
string JoinStrings(const vector< string > &strings, const string &joint)
Definition: string.cc:323
perf::Statistics * statistics_
Definition: mountpoint.h:322
download::DownloadManager * download_mgr_
Definition: mountpoint.h:560
static const unsigned kDefaultTimeoutSec
Definition: mountpoint.h:507
std::string exe_path_
Definition: mountpoint.h:300
bool CheckInstanceName(const std::string &instance)
Definition: mountpoint.cc:107
perf::Statistics * statistics_
Definition: mountpoint.h:554
Log2Histogram * hist_fs_readlink_
Definition: mountpoint.h:328
void SetProxyChain(const std::string &proxy_list, const std::string &fallback_proxy_list, const ProxySetModes set_mode)
Definition: download.cc:2413
static NfsMapsLeveldb * Create(const std::string &leveldb_dir, const uint64_t root_inode, const bool rebuild, perf::Statistics *statistics)
gid_t talk_socket_gid_
Definition: mountpoint.h:593
void SetupSqlite()
Definition: mountpoint.cc:981
bool IsOn(const std::string &param_value) const
Definition: options.cc:409
std::string GetUniqFileSuffix()
Definition: mountpoint.cc:1663
download::DownloadManager * external_download_mgr_
Definition: mountpoint.h:561
MagicXattrManager * magic_xattr_mgr_
Definition: mountpoint.h:575
CacheManager * SetupCacheMgr(const std::string &instance)
Definition: mountpoint.cc:566
const SettingsPublisher & settings() const
Definition: repository.h:316
perf::Statistics * statistics_
Definition: repository.h:139
void SetupHttpTuning()
Definition: mountpoint.cc:1937
static const char * kDefaultBlacklist
Definition: mountpoint.h:516
virtual bool AcquireQuotaManager(QuotaManager *quota_mgr)
Definition: cache_extern.cc:85
unsigned max_ttl_sec_
Definition: mountpoint.h:579
static const unsigned kDefaultMaxTtl
Definition: dns.h:182
bool CreateDownloadManagers()
Definition: mountpoint.cc:1284
std::string talk_socket_path_
Definition: mountpoint.h:591
perf::Counter * n_fs_stat_
Definition: mountpoint.h:313
perf::Counter * no_open_files_
Definition: mountpoint.h:319
assert((mem||(size==0))&&"Out Of Memory")
void SetupLogging()
Definition: mountpoint.cc:808
int fd_workspace_lock_
Definition: mountpoint.h:349
void CreateStatistics()
Definition: mountpoint.cc:1426
bool UnregisterVfsRdOnly()
Definition: sqlitevfs.cc:522
CacheManager * SetupRamCacheMgr(const std::string &instance)
Definition: mountpoint.cc:669
static void LogSqliteError(void *user_data __attribute__((unused)), int sqlite_extended_error, const char *message)
Definition: mountpoint.cc:488
static void CleanupInstance()
Definition: sqlitemem.cc:170
static const unsigned kNfsMaps
Definition: mountpoint.h:180
std::string workspace_
Definition: mountpoint.h:342
string StringifyTime(const time_t seconds, const bool utc)
Definition: string.cc:105
PosixCacheSettings DeterminePosixCacheSettings(const std::string &instance)
Definition: mountpoint.cc:268
void SetDnsServer(const std::string &address)
Definition: download.cc:1822
int platform_stat(const char *path, platform_stat64 *buf)
void AssignGlobalArenas()
Definition: sqlitemem.cc:137
bool has_membership_req_
Definition: mountpoint.h:589
virtual CacheManagerIds id()=0
void SetLogMicroSyslog(const std::string &filename)
Definition: logging.cc:272
bool IsOff(const std::string &param_value) const
Definition: options.cc:416
void ReEvaluateAuthz()
Definition: mountpoint.cc:1756
bool HasConfigRepository(const std::string &fqrn, std::string *config_path)
Definition: options.cc:256
std::set< std::string > constructed_instances_
Definition: mountpoint.h:377
bool DetermineRootHash(shash::Any *root_hash)
Definition: mountpoint.cc:1539
#define SetLogDebugFile(filename)
void SetTimeout(const unsigned seconds_proxy, const unsigned seconds_direct)
Definition: download.cc:1879
bool CreateCatalogManager()
Definition: mountpoint.cc:1227
struct cvmcache_object_info __attribute__
Definition: atomic.h:24
void RemapCatalogFd(int from, int to)
Definition: mountpoint.cc:1075
static const unsigned kNfsNone
Definition: mountpoint.h:176
std::string path_crash_guard_
Definition: mountpoint.h:355
IntegerMap< uint64_t > OwnerMap
Definition: catalog.h:41
catalog::WritableCatalogManager * catalog_mgr_
Definition: repository.h:384
bool SetupWorkspace()
Definition: mountpoint.cc:998
static const char * kDefaultAuthzSearchPath
Definition: mountpoint.h:499
Type type()
Definition: mountpoint.h:228
FileSystem(const FileSystemInfo &fs_info)
Definition: mountpoint.cc:367
static const unsigned kLibPathCacheSize
Definition: mountpoint.h:487
std::string boot_error_
Definition: mountpoint.h:86
OptionsManager * options_mgr_
Definition: mountpoint.h:552
Log2Histogram * hist_fs_getattr_
Definition: mountpoint.h:327
bool IsValid(const std::string &input) const
Definition: sanitizer.cc:114
Log2Histogram * hist_fs_releasedir_
Definition: mountpoint.h:330
Definition: tracer.h:35
bool FileExists(const std::string &path)
Definition: posix.cc:778
bool LoadTrustedCaCrl(const std::string &path_list)
Definition: signature.cc:588
void SetDnsTtlLimits(const unsigned min_seconds, const unsigned max_seconds)
Definition: download.cc:1858
std::string cache_path()
Definition: cache_posix.h:105
perf::Counter * n_fs_readlink_
Definition: mountpoint.h:316
void Init(const unsigned max_pool_handles, const perf::StatisticsTemplate &statistics)
Definition: download.cc:1606
uint64_t platform_memsize()
int64_t String2Int64(const string &value)
Definition: string.cc:222
void SetMaxTtlMn(unsigned value_minutes)
Definition: mountpoint.cc:1766
string ToUpper(const string &mixed_case)
Definition: string.cc:474
static const unsigned kDefaultBackoffInitMs
Definition: mountpoint.h:509
perf::Counter * n_fs_forget_
Definition: mountpoint.h:317
MountPoint(const std::string &fqrn, FileSystem *file_system, OptionsManager *options_mgr)
Definition: mountpoint.cc:1668
void GetLimitNoFile(unsigned *soft_limit, unsigned *hard_limit)
Definition: posix.cc:1457
Tracer * tracer_
Definition: mountpoint.h:571
void SetProxyGroupResetDelay(const unsigned seconds)
Definition: download.cc:2710
static CacheManager * Create(CacheManager *upper_cache, CacheManager *lower_cache)
static bool g_alive
Definition: mountpoint.h:236
static const unsigned kDefaultNumConnections
Definition: mountpoint.h:503
void TearDown2ReadOnly()
Definition: mountpoint.cc:1062
gid_t g_gid
Definition: globals.cc:14
uuid_t uuid
Definition: uuid.h:41
static const unsigned kNfsMapsHa
Definition: mountpoint.h:184
lru::Md5PathCache * md5path_cache_
Definition: mountpoint.h:570
std::string name_
Definition: mountpoint.h:299
vector< string > SplitString(const string &str, char delim)
Definition: string.cc:288
std::string Dump()
Definition: options.cc:454
bool RegisterVfsRdOnly(CacheManager *cache_mgr, perf::Statistics *statistics, const VfsOptions options)
Definition: sqlitevfs.cc:453
static AuthzSessionManager * Create(AuthzFetcher *authz_fetcher, perf::Statistics *statistics)
std::string membership_req_
Definition: mountpoint.h:588
glue::DentryTracker * dentry_tracker_
Definition: mountpoint.h:573
perf::Counter * n_fs_read_
Definition: mountpoint.h:315
signature::SignatureManager * signature_mgr_
Definition: repository.h:140
static const char * kDefaultCacheMgrInstance
Definition: mountpoint.h:240
bool HasSuffix(const std::string &str, const std::string &suffix, const bool ignore_case)
Definition: string.cc:279
virtual uint64_t GetCapacity()
Definition: quota_posix.cc:582
glue::PageCacheTracker * page_cache_tracker_
Definition: mountpoint.h:574
bool found_previous_crash_
Definition: mountpoint.h:361
const char kSuffixCatalog
Definition: hash.h:54
loader::Failures boot_status_
Definition: mountpoint.h:85
Failures Fetch(const std::string &base_url, const std::string &repository_name, const uint64_t minimum_timestamp, const shash::Any *base_catalog, signature::SignatureManager *signature_manager, download::DownloadManager *download_manager, ManifestEnsemble *ensemble)
bool TriageCacheMgr()
Definition: mountpoint.cc:1080
bool CheckPosixCacheSettings(const PosixCacheSettings &settings)
Definition: mountpoint.cc:124
perf::Counter * n_fs_inode_replace_
Definition: mountpoint.h:318
bool IsNfsSource()
Definition: mountpoint.h:189
ChunkTables * chunk_tables_
Definition: mountpoint.h:566
CacheManager * cache_mgr()
Definition: mountpoint.h:195
uid_t g_uid
Definition: globals.cc:13
void SetupDnsTuning(download::DownloadManager *manager)
Definition: mountpoint.cc:1836
cvmfs::Uuid * uuid_cache_
Definition: mountpoint.h:388
unsigned GetEffectiveTtlSec()
Definition: mountpoint.cc:1641
int TryLockFile(const std::string &path)
Definition: posix.cc:900
bool MkdirDeep(const std::string &path, const mode_t mode, bool verify_writable)
Definition: posix.cc:833
int LockFile(const std::string &path)
Definition: posix.cc:958
void SetIpPreference(const dns::IpPreference preference)
Definition: download.cc:1868
static const unsigned kDefaultBackoffMaxMs
Definition: mountpoint.h:510
lru::InodeCache * inode_cache_
Definition: mountpoint.h:568
CacheManager * SetupPosixCacheMgr(const std::string &instance)
Definition: mountpoint.cc:639
perf::Counter * n_fs_open_
Definition: mountpoint.h:309
static uint64_t RoundUp8(const uint64_t size)
Definition: smalloc.h:37
bool SetupPosixQuotaMgr(const PosixCacheSettings &settings, CacheManager *cache_mgr)
Definition: mountpoint.cc:919
static SqliteMemoryManager * GetInstance()
Definition: sqlitemem.h:137
static PosixQuotaManager * Create(const std::string &cache_workspace, const uint64_t limit, const uint64_t cleanup_threshold, const bool rebuild_database)
Definition: quota_posix.cc:230
unsigned nfs_mode_
Definition: mountpoint.h:382
void SetRetryParameters(const unsigned max_retries, const unsigned backoff_init_ms, const unsigned backoff_max_ms)
Definition: download.cc:2729
string StringifyInt(const int64_t value)
Definition: string.cc:78
Type type_
Definition: mountpoint.h:301
static const unsigned kInodeCacheFactor
Definition: mountpoint.h:491
void SetMaxIpaddrPerProxy(unsigned limit)
Definition: download.cc:2740
DownloadManager * Clone(const perf::StatisticsTemplate &statistics)
Definition: download.cc:2773
void Inc(class Counter *counter)
Definition: statistics.h:50
bool HasPrefix(const string &str, const string &prefix, const bool ignore_case)
Definition: string.cc:265
Log2Histogram * hist_fs_readdir_
Definition: mountpoint.h:331
bool GetValue(const std::string &key, std::string *value) const
Definition: options.cc:376
cvmfs::Uuid * uuid_
Definition: mountpoint.h:544
OptionsManager * options_mgr_
Definition: cvmfs.cc:146
bool LoadBlacklist(const std::string &path_blacklist, bool append)
Definition: signature.cc:544
BackoffThrottle * backoff_throttle_
Definition: mountpoint.h:558
AuthzAttachment * authz_attachment_
Definition: mountpoint.h:557
void SetProxyTemplates(const std::string &direct, const std::string &forced)
Definition: download.cc:2746
SimpleChunkTables * simple_chunk_tables_
Definition: mountpoint.h:567
bool InitFixed(const shash::Any &root_hash, bool alternative_path)
static const unsigned kDefaultNfiles
Definition: mountpoint.h:239
static FileSystem * Create(const FileSystemInfo &fs_info)
Definition: mountpoint.cc:164
bool has_custom_sqlitevfs_
Definition: mountpoint.h:399
bool ReloadBlacklists()
Definition: mountpoint.cc:1140
std::string name()
Definition: mountpoint.h:222
time_t IsoTimestamp2UtcTime(const std::string &iso8601)
Definition: string.cc:198
static Uuid * Create(const std::string &store_path)
Definition: uuid.cc:23
void CreateStatistics()
Definition: mountpoint.cc:206
pthread_mutex_t lock_max_ttl_
Definition: mountpoint.h:580
void CreateAuthz()
Definition: mountpoint.cc:1201
bool g_raw_symlinks
Definition: globals.cc:12
static const char * kDefaultCacheBase
Definition: mountpoint.h:237
void SetNull()
Definition: hash.h:400
string ResolveProxyDescription(const string &cvmfs_proxies, const std::string &path_fallback_cache, DownloadManager *download_manager)
Definition: wpad.cc:206
uint64_t String2Uint64(const string &value)
Definition: string.cc:228
static const unsigned kTracerBufferSize
Definition: mountpoint.h:514
void SetupUuid()
Definition: mountpoint.cc:1047
catalog::ClientCatalogManager * catalog_mgr_
Definition: mountpoint.h:565
bool CreateSignatureManager()
Definition: mountpoint.cc:1390
std::string fqrn_
Definition: mountpoint.h:543
unsigned GetMaxTtlMn()
Definition: mountpoint.cc:1653
std::string error_msg() const
Definition: cache_extern.h:45
bool GetUidOf(const std::string &username, uid_t *uid, gid_t *main_gid)
Definition: posix.cc:1331
bool SetupBehavior()
Definition: mountpoint.cc:1772
static ExternalCacheManager * Create(int fd_connection, unsigned max_open_fds, const std::string &ident)
AuthzFetcher * authz_fetcher_
Definition: mountpoint.h:555
bool FetchHistory(std::string *history_path)
Definition: mountpoint.cc:1607
perf::Statistics * statistics()
Definition: mountpoint.h:227
bool enforce_acls_
Definition: mountpoint.h:583
bool IsHaNfsSource()
Definition: mountpoint.h:190
void CreateTables()
Definition: mountpoint.cc:1465
shash::Any history() const
Definition: manifest.h:127
bool SetupCrashGuard()
Definition: mountpoint.cc:760
static const unsigned kDefaultQuotaLimit
Definition: mountpoint.h:238
static PosixQuotaManager * CreateShared(const std::string &exe_path, const std::string &cache_workspace, const uint64_t limit, const uint64_t cleanup_threshold, bool foreground)
Definition: quota_posix.cc:262
void ResetErrorCounters()
Definition: mountpoint.cc:558
Definition: mutex.h:42
Log2Histogram * hist_fs_lookup_
Definition: mountpoint.h:324
shash::Any root_hash
Definition: history.h:90
bool LoadPublicRsaKeys(const std::string &path_list)
Definition: signature.cc:313
void RegisterHandler(const std::string &file_path, EventHandler *handler)
Definition: file_watcher.cc:56
void GetTimeout(unsigned *seconds_proxy, unsigned *seconds_direct)
Definition: download.cc:1902
static const unsigned kDefaultMemcacheSize
Definition: mountpoint.h:495
FileSystem * file_system_
Definition: cvmfs.cc:121
std::string repository_tag_
Definition: mountpoint.h:584
static const unsigned kDefaultMinTtl
Definition: dns.h:177
bool SetupOwnerMaps()
Definition: mountpoint.cc:1999
void SetDnsParameters(const unsigned retries, const unsigned timeout_ms)
Definition: download.cc:1840
bool CreateResolvConfWatcher()
Definition: mountpoint.cc:1350
Any MkFromHexPtr(const HexPtr hex, const char suffix)
Definition: hash.cc:83
static ExternalQuotaManager * Create(ExternalCacheManager *cache_mgr)
Log2Histogram * hist_fs_read_
Definition: mountpoint.h:333
void SetCatalogWatermark(unsigned limit)
static bool g_is_enabled
Definition: algorithm.h:200
bool SetupNfsMaps()
Definition: mountpoint.cc:826
bool CheckBlacklists()
Definition: mountpoint.cc:1102
std::vector< std::string > blacklist_paths_
Definition: mountpoint.h:585
cvmfs::Fetcher * fetcher_
Definition: mountpoint.h:562
file_watcher::FileWatcher * resolv_conf_watcher_
Definition: mountpoint.h:577
static const unsigned kDnsDefaultRetries
Definition: download.h:397
bool GeoSortServers(std::vector< std::string > *servers, std::vector< uint64_t > *output_order=NULL)
Definition: download.cc:2132
bool SetupExternalDownloadMgr(bool dogeosort)
Definition: mountpoint.cc:1876
void SetOwnerMaps(const OwnerMap &uid_map, const OwnerMap &gid_map)
CacheManager * SetupExternalCacheMgr(const std::string &instance)
Definition: mountpoint.cc:600
static void CleanupInstance()
Definition: clientctx.cc:18
perf::Counter * n_fs_lookup_negative_
Definition: mountpoint.h:312
Log2Histogram * hist_fs_open_
Definition: mountpoint.h:332
virtual bool AcquireQuotaManager(QuotaManager *quota_mgr)=0
std::string cache_mgr_instance_
Definition: mountpoint.h:372
std::string path_workspace_lock_
Definition: mountpoint.h:350
virtual void IncGeneration(const uint64_t by)=0
Log2Histogram * hist_fs_release_
Definition: mountpoint.h:334
void SetLogSyslogPrefix(const std::string &prefix)
Definition: logging.cc:241
std::string MakeCanonicalPath(const std::string &path)
Definition: posix.cc:96
OptionsManager * options_mgr_
Definition: mountpoint.h:305
perf::Counter * no_open_dirs_
Definition: mountpoint.h:320
glue::InodeTracker * inode_tracker_
Definition: mountpoint.h:572
bool fixed_catalog_
Definition: mountpoint.h:582
static PluginHandle * CreatePlugin(const std::string &locator, const std::vector< std::string > &cmd_line)
bool IsDefined(const std::string &key)
Definition: options.cc:370
void GetHostInfo(std::vector< std::string > *host_chain, std::vector< int > *rtt, unsigned *current_host)
Definition: download.cc:1946
void set_membership(const std::string &m)
Definition: authz_curl.h:25
Statistics * Fork()
Definition: statistics.cc:47
CacheManager * SetupTieredCacheMgr(const std::string &instance)
Definition: mountpoint.cc:719
std::vector< std::string > FindFilesBySuffix(const std::string &dir, const std::string &suffix)
Definition: posix.cc:1111
std::string workspace()
Definition: mountpoint.h:230
bool g_claim_ownership
Definition: globals.cc:11
lru::PathCache * path_cache_
Definition: mountpoint.h:569
CacheManager * cache_mgr_
Definition: mountpoint.h:383
void SetCredentialsAttachment(CredentialsAttachment *ca)
Definition: download.cc:1806
static ClientCtx * GetInstance()
Definition: clientctx.cc:45
virtual bool AcquireQuotaManager(QuotaManager *quota_mgr)
Definition: cache_ram.cc:70
cvmfs::Uuid * uuid_cache()
Definition: mountpoint.h:229
cvmfs::Fetcher * external_fetcher_
Definition: mountpoint.h:563
perf::Counter * n_fs_dir_open_
Definition: mountpoint.h:310
static const uint64_t kSizeUnknown
Definition: cache.h:72
std::string MkCacheParm(const std::string &generic_parameter, const std::string &instance)
Definition: mountpoint.cc:523
void UnlockFile(const int filedes)
Definition: posix.cc:982
NfsMaps * nfs_maps_
Definition: mountpoint.h:394
void CreateFetchers()
Definition: mountpoint.cc:1373
void SetInodeAnnotation(InodeAnnotation *new_annotation)
int Fetch(const shash::Any &id, const uint64_t size, const std::string &name, const zlib::Algorithms compression_algorithm, const CacheManager::ObjectType object_type, const std::string &alt_url="", off_t range_offset=-1)
Definition: fetch.cc:82
virtual void SetInodeResidue(unsigned residue_class, unsigned remainder)
Definition: nfs_maps.h:31
void SetHostResetDelay(const unsigned seconds)
Definition: download.cc:2720
static MountPoint * Create(const std::string &fqrn, FileSystem *file_system, OptionsManager *options_mgr=NULL)
Definition: mountpoint.cc:1161
Log2Histogram * hist_fs_forget_
Definition: mountpoint.h:325
static PosixCacheManager * Create(const std::string &cache_path, const bool alien_cache, const RenameWorkarounds rename_workaround=kRenameNormal)
Definition: cache_posix.cc:230