CernVM-FS  2.13.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
mountpoint.cc
Go to the documentation of this file.
1 
5 #include "mountpoint.h"
6 
7 #include <errno.h>
8 #include <fcntl.h>
9 #include <inttypes.h>
10 #include <stdint.h>
11 #include <unistd.h>
12 
13 #include <algorithm>
14 #include <cassert>
15 #include <climits>
16 #include <cstring>
17 #include <vector>
18 
19 #include "duplex_fuse.h" // NOLINT
20 
21 #ifndef CVMFS_LIBCVMFS
22 #ifdef FUSE_CAP_EXPORT_SUPPORT
23 #define CVMFS_NFS_SUPPORT
24 #else
25 #warning "No NFS support, Fuse too old"
26 #endif
27 #endif
28 
29 #include "authz/authz_curl.h"
30 #include "authz/authz_fetch.h"
31 #include "authz/authz_session.h"
32 #include "backoff.h"
33 #include "cache.h"
34 #include "cache_extern.h"
35 #include "cache_posix.h"
36 #include "cache_ram.h"
37 #include "cache_stream.h"
38 #include "cache_tiered.h"
39 #include "catalog.h"
40 #include "catalog_mgr_client.h"
41 #include "clientctx.h"
42 #include "crypto/signature.h"
43 #include "duplex_sqlite3.h"
44 #include "fetch.h"
45 #include "file_chunk.h"
46 #include "globals.h"
47 #include "glue_buffer.h"
48 #include "google/protobuf/stubs/common.h"
49 #include "history.h"
50 #include "history_sqlite.h"
51 #include "lru_md.h"
52 #include "manifest.h"
53 #include "manifest_fetch.h"
54 #include "network/download.h"
55 #include "nfs_maps.h"
56 #ifdef CVMFS_NFS_SUPPORT
57 #include "nfs_maps_leveldb.h"
58 #include "nfs_maps_sqlite.h"
59 #endif
60 #include "options.h"
61 #include "quota_posix.h"
63 #include "sqlitemem.h"
64 #include "sqlitevfs.h"
65 #include "statistics.h"
66 #include "telemetry_aggregator.h"
67 #include "tracer.h"
68 #include "util/concurrency.h"
69 #include "util/logging.h"
70 #include "util/platform.h"
71 #include "util/pointer.h"
72 #include "util/posix.h"
73 #include "util/string.h"
74 #include "util/uuid.h"
75 #include "wpad.h"
76 
77 using namespace std; // NOLINT
78 
79 
80 bool FileSystem::g_alive = false;
81 const char *FileSystem::kDefaultCacheBase = "/var/lib/cvmfs";
82 const char *FileSystem::kDefaultCacheMgrInstance = "default";
83 
84 
85 FileSystem::IoErrorInfo::IoErrorInfo() : counter_(NULL), timestamp_last_(0) { }
86 
88  counter_->Set(0);
89  timestamp_last_ = 0;
90 }
91 
93  perf::Inc(counter_);
94  timestamp_last_ = time(NULL);
95 }
96 
98 
99 int64_t FileSystem::IoErrorInfo::count() { return counter_->Get(); }
100 
101 time_t FileSystem::IoErrorInfo::timestamp_last() { return timestamp_last_; }
102 
103 
104 
109 bool FileSystem::CheckInstanceName(const std::string &instance) {
110  if (instance.length() > 24)
111  return false;
112  sanitizer::CacheInstanceSanitizer instance_sanitizer;
113  if (!instance_sanitizer.IsValid(instance)) {
114  boot_error_ = "invalid instance name (" + instance + "), " +
115  "only characters a-z, A-Z, 0-9, _ are allowed";
117  return false;
118  }
119  return true;
120 }
121 
122 
128 {
129  if (settings.is_alien && settings.is_shared) {
130  boot_error_ = "Failure: shared local disk cache and alien cache mutually "
131  "exclusive. Please turn off shared local disk cache.";
133  return false;
134  }
135  if (settings.is_alien && settings.is_managed) {
136  boot_error_ = "Failure: quota management and alien cache mutually "
137  "exclusive. Please turn off quota limit.";
139  return false;
140  }
141 
142  if (type_ == kFsLibrary) {
143  if (settings.is_shared || settings.is_managed) {
144  boot_error_ = "Failure: libcvmfs supports only unmanaged exclusive cache "
145  "or alien cache.";
147  return false;
148  }
149  }
150 
151  if (settings.cache_base_defined && settings.cache_dir_defined) {
152  boot_error_ =
153  "'CVMFS_CACHE_BASE' and 'CVMFS_CACHE_DIR' are mutually exclusive";
155  return false;
156  }
157 
158  return true;
159 }
160 
161 
168  file_system(new FileSystem(fs_info));
169 
170  file_system->SetupGlobalEnvironmentParams();
171 
172  file_system->SetupLogging();
173  LogCvmfs(kLogCvmfs, kLogDebug, "Options:\n%s",
174  file_system->options_mgr()->Dump().c_str());
175 
176  file_system->CreateStatistics();
177  file_system->SetupSqlite();
178  if (!file_system->DetermineNfsMode())
179  return file_system.Release();
180  if (!file_system->SetupWorkspace())
181  return file_system.Release();
182 
183  // Redirect SQlite temp directory to workspace (global variable)
184  unsigned length_tempdir = file_system->workspace_.length() + 1;
185  sqlite3_temp_directory = static_cast<char *>(sqlite3_malloc(length_tempdir));
186  snprintf(sqlite3_temp_directory,
187  length_tempdir,
188  "%s",
189  file_system->workspace_.c_str());
190 
191  if (!file_system->TriageCacheMgr())
192  return file_system.Release();
193  file_system->SetupUuid();
194  if (!file_system->SetupNfsMaps())
195  return file_system.Release();
196  bool retval = sqlite::RegisterVfsRdOnly(
197  file_system->cache_mgr_,
198  file_system->statistics_,
200  assert(retval);
201  file_system->has_custom_sqlitevfs_ = true;
202 
204 
205  file_system->boot_status_ = loader::kFailOk;
206  return file_system.Release();
207 }
208 
209 
212 
213  // Register the ShortString's static counters
214  statistics_->Register("pathstring.n_instances", "Number of instances");
215  statistics_->Register("pathstring.n_overflows", "Number of overflows");
216  statistics_->Register("namestring.n_instances", "Number of instances");
217  statistics_->Register("namestring.n_overflows", "Number of overflows");
218  statistics_->Register("linkstring.n_instances", "Number of instances");
219  statistics_->Register("linkstring.n_overflows", "Number of overflows");
220 
221  // Callback counters
222  n_fs_open_ = statistics_->Register("cvmfs.n_fs_open",
223  "Overall number of file open operations");
224  n_fs_dir_open_ = statistics_->Register("cvmfs.n_fs_dir_open",
225  "Overall number of directory open operations");
226  n_fs_lookup_ = statistics_->Register("cvmfs.n_fs_lookup",
227  "Number of lookups");
228  n_fs_lookup_negative_ = statistics_->Register("cvmfs.n_fs_lookup_negative",
229  "Number of negative lookups");
230  n_fs_stat_ = statistics_->Register("cvmfs.n_fs_stat", "Number of stats");
231  n_fs_stat_stale_ = statistics_->Register("cvmfs.n_fs_stat_stale",
232  "Number of stats for stale (open, meanwhile changed) regular files");
233  n_fs_statfs_ = statistics_->Register("cvmfs.n_fs_statfs",
234  "Overall number of statsfs calls");
235  n_fs_statfs_cached_ = statistics_->Register("cvmfs.n_fs_statfs_cached",
236  "Number of statsfs calls that accessed the cached statfs info");
237  n_fs_read_ = statistics_->Register("cvmfs.n_fs_read", "Number of files read");
238  n_fs_readlink_ = statistics_->Register("cvmfs.n_fs_readlink",
239  "Number of links read");
240  n_fs_forget_ = statistics_->Register("cvmfs.n_fs_forget",
241  "Number of inode forgets");
242  n_fs_inode_replace_ = statistics_->Register("cvmfs.n_fs_inode_replace",
243  "Number of stale inodes that got replaced by an up-to-date version");
244  no_open_files_ = statistics_->Register("cvmfs.no_open_files",
245  "Number of currently opened files");
246  no_open_dirs_ = statistics_->Register("cvmfs.no_open_dirs",
247  "Number of currently opened directories");
248  io_error_info_.SetCounter(statistics_->Register("cvmfs.n_io_error",
249  "Number of I/O errors"));
250  n_eio_total_ = statistics_->Register("eio.total",
251  "EIO returned to calling process. Sum of individual eio counters");
252  n_eio_01_ = statistics_->Register("eio.01",
253  "EIO returned by cvmfs.cc:cvmfs_lookup() - lookup failed");
254  n_eio_02_ = statistics_->Register("eio.02",
255  "EIO returned by cvmfs.cc:ReplyNegative() "
256  "- CVMFS-specific metadata not found");
257  n_eio_03_ = statistics_->Register("eio.03",
258  "EIO returned by cvmfs.cc:cvmfs_opendir() - failed to open directory ");
259  n_eio_04_ = statistics_->Register("eio.04",
260  "EIO returned by cvmfs.cc:cvmfs_open() "
261  "- chunked file does not have any chunks");
262  n_eio_05_ = statistics_->Register("eio.05",
263  "EIO returned by cvmfs.cc:cvmfs_read() - failed to fetch chunk");
264  n_eio_06_ = statistics_->Register("eio.06",
265  "EIO returned by cvmfs.cc:cvmfs_open() - failed to open file");
266  n_eio_07_ = statistics_->Register("eio.07",
267  "EIO returned by cvmfs.cc:cvmfs_read() - failed to read chunk");
268  n_eio_08_ = statistics_->Register("eio.08",
269  "EIO returned by cvmfs.cc:cvmfs_read() - failed to read file");
270  n_emfile_ = statistics_->Register("eio.emfile",
271  "EMFILE returned by cvmfs.cc:cvmfs_read(): too many open files");
272 
273  string optarg;
274  if (options_mgr_->GetValue("CVMFS_INSTRUMENT_FUSE", &optarg) &&
275  options_mgr_->IsOn(optarg))
276  {
278  }
279 
280  hist_fs_lookup_ = new Log2Histogram(30);
281  hist_fs_forget_ = new Log2Histogram(30);
288  hist_fs_open_ = new Log2Histogram(30);
289  hist_fs_read_ = new Log2Histogram(30);
291 }
292 
293 
299  const string &instance
300 ) {
301  string optarg;
303 
304  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_REFCOUNT", instance),
305  &optarg)
306  && options_mgr_->IsOff(optarg))
307  {
308  settings.do_refcount = false;
309  }
310 
311  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SHARED", instance),
312  &optarg)
313  && options_mgr_->IsOn(optarg))
314  {
315  settings.is_shared = true;
316  }
317  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SERVER_MODE", instance),
318  &optarg)
319  && options_mgr_->IsOn(optarg))
320  {
321  settings.avoid_rename = true;
322  }
323 
324  if (type_ == kFsFuse)
325  settings.quota_limit = kDefaultQuotaLimit;
326  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_QUOTA_LIMIT", instance),
327  &optarg))
328  {
329  settings.quota_limit = String2Int64(optarg) * 1024 * 1024;
330  }
331  if (settings.quota_limit > 0)
332  settings.is_managed = true;
333 
334  settings.cache_path = kDefaultCacheBase;
335  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_BASE", instance),
336  &optarg))
337  {
338  settings.cache_path = MakeCanonicalPath(optarg);
339  settings.cache_base_defined = true;
340  }
341  if (settings.is_shared) {
342  settings.cache_path += "/shared";
343  } else {
344  settings.cache_path += "/" + name_;
345  }
346 
347  // CheckCacheMode makes sure that CVMFS_CACHE_DIR and CVMFS_CACHE_BASE are
348  // not set at the same time.
349  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_DIR", instance),
350  &optarg))
351  {
352  settings.cache_dir_defined = true;
353  settings.cache_path = optarg;
354  }
355  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_ALIEN", instance),
356  &optarg))
357  {
358  settings.is_alien = true;
359  settings.cache_path = optarg;
360  }
361  // We already changed the cwd to the workspace
362  // Which is only done if using FUSE
363  if ((type_ == kFsFuse) && (settings.cache_path == workspace_fullpath_)) {
364  settings.cache_path = ".";
365  }
366 
367  // The cache workspace usually is the cache directory, unless explicitly
368  // set otherwise
369  settings.workspace = settings.cache_path;
370  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_WORKSPACE", instance),
371  &optarg) ||
372  options_mgr_->GetValue("CVMFS_WORKSPACE", &optarg))
373  {
374  // Used for the shared quota manager
375  settings.workspace = optarg;
376  }
377 
378  return settings;
379 }
380 
381 
383  string optarg;
384 
385  if (options_mgr_->GetValue("CVMFS_NFS_SOURCE", &optarg) &&
386  options_mgr_->IsOn(optarg))
387  {
388  nfs_mode_ |= kNfsMaps;
389  if (options_mgr_->GetValue("CVMFS_NFS_SHARED", &optarg)) {
391  nfs_maps_dir_ = optarg;
392  }
393  }
394 
395  if ((type_ == kFsLibrary) && (nfs_mode_ != kNfsNone)) {
396  boot_error_ = "Failure: libcvmfs does not support NFS export.";
398  return false;
399  }
400  return true;
401 }
402 
403 
405  : name_(fs_info.name)
406  , exe_path_(fs_info.exe_path)
407  , type_(fs_info.type)
408  , options_mgr_(fs_info.options_mgr)
409  , wait_workspace_(fs_info.wait_workspace)
410  , foreground_(fs_info.foreground)
411  , n_fs_open_(NULL)
412  , n_fs_dir_open_(NULL)
413  , n_fs_lookup_(NULL)
414  , n_fs_lookup_negative_(NULL)
415  , n_fs_stat_(NULL)
416  , n_fs_stat_stale_(NULL)
417  , n_fs_statfs_(NULL)
418  , n_fs_statfs_cached_(NULL)
419  , n_fs_read_(NULL)
420  , n_fs_readlink_(NULL)
421  , n_fs_forget_(NULL)
422  , n_fs_inode_replace_(NULL)
423  , no_open_files_(NULL)
424  , no_open_dirs_(NULL)
425  , n_eio_total_(NULL)
426  , n_eio_01_(NULL)
427  , n_eio_02_(NULL)
428  , n_eio_03_(NULL)
429  , n_eio_04_(NULL)
430  , n_eio_05_(NULL)
431  , n_eio_06_(NULL)
432  , n_eio_07_(NULL)
433  , n_eio_08_(NULL)
434  , n_emfile_(NULL)
435  , statistics_(NULL)
436  , fd_workspace_lock_(-1)
437  , found_previous_crash_(false)
439  , cache_mgr_(NULL)
440  , uuid_cache_(NULL)
441  , nfs_maps_(NULL)
442  , has_custom_sqlitevfs_(false)
443 {
444  assert(!g_alive);
445  g_alive = true;
446  g_uid = geteuid();
447  g_gid = getegid();
448 
449  string optarg;
450  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SERVER_MODE",
452  &optarg)
453  && options_mgr_->IsOn(optarg))
454  {
455  g_raw_symlinks = true;
456  }
457 }
458 
459 
462 
465 
466  delete uuid_cache_;
467  delete nfs_maps_;
468  delete cache_mgr_;
469 
470  if (sqlite3_temp_directory) {
471  sqlite3_free(sqlite3_temp_directory);
472  sqlite3_temp_directory = NULL;
473  }
474 
475  if (!path_crash_guard_.empty())
476  unlink(path_crash_guard_.c_str());
477  if (!path_workspace_lock_.empty())
478  unlink(path_workspace_lock_.c_str());
479  if (fd_workspace_lock_ >= 0)
481 
482  sqlite3_shutdown();
484 
485  delete hist_fs_lookup_;
486  delete hist_fs_forget_multi_;
487  delete hist_fs_forget_;
488  delete hist_fs_getattr_;
489  delete hist_fs_readlink_;
490  delete hist_fs_opendir_;
491  delete hist_fs_releasedir_;
492  delete hist_fs_readdir_;
493  delete hist_fs_open_;
494  delete hist_fs_read_;
495  delete hist_fs_release_;
496  delete statistics_;
497 
498  SetLogSyslogPrefix("");
499  SetLogMicroSyslog("");
500  SetLogDebugFile("");
501  google::protobuf::ShutdownProtobufLibrary();
502  g_alive = false;
503 }
504 
505 
507  path_workspace_lock_ = workspace_ + "/lock." + name_;
509  if (fd_workspace_lock_ >= 0)
510  return true;
511 
512  if (fd_workspace_lock_ == -1) {
513  boot_error_ = "could not acquire workspace lock (" +
514  StringifyInt(errno) + ")";
516  return false;
517  }
518 
519  assert(fd_workspace_lock_ == -2);
520 
521  if (!wait_workspace_) {
523  return false;
524  }
525 
527  if (fd_workspace_lock_ < 0) {
528  boot_error_ = "could not acquire workspace lock (" +
529  StringifyInt(errno) + ")";
531  return false;
532  }
533  return true;
534 }
535 
536 
538  void *user_data __attribute__((unused)),
539  int sqlite_extended_error,
540  const char *message)
541 {
542  int log_dest = kLogDebug;
543  int sqlite_error = sqlite_extended_error & 0xFF;
544  switch (sqlite_error) {
545  case SQLITE_INTERNAL:
546  case SQLITE_PERM:
547  case SQLITE_NOMEM:
548  case SQLITE_IOERR:
549  case SQLITE_CORRUPT:
550  case SQLITE_FULL:
551  case SQLITE_CANTOPEN:
552  case SQLITE_MISUSE:
553  case SQLITE_FORMAT:
554  case SQLITE_NOTADB:
555  log_dest |= kLogSyslogErr;
556  break;
557  case SQLITE_WARNING:
558  case SQLITE_NOTICE:
559  default:
560  break;
561  }
562  LogCvmfs(kLogCvmfs, log_dest, "SQlite3: %s (%d)",
563  message, sqlite_extended_error);
564 }
565 
566 
573  const string &generic_parameter,
574  const string &instance)
575 {
576  assert(HasPrefix(generic_parameter, "CVMFS_CACHE_", false));
577 
578  if (instance == kDefaultCacheMgrInstance) {
579  // Compatibility parameter names
580  if ((generic_parameter == "CVMFS_CACHE_SHARED") &&
581  !options_mgr_->IsDefined(generic_parameter))
582  {
583  return "CVMFS_SHARED_CACHE";
584  }
585  if ((generic_parameter == "CVMFS_CACHE_ALIEN") &&
586  !options_mgr_->IsDefined(generic_parameter))
587  {
588  return "CVMFS_ALIEN_CACHE";
589  }
590  if ((generic_parameter == "CVMFS_CACHE_SERVER_MODE") &&
591  !options_mgr_->IsDefined(generic_parameter))
592  {
593  return "CVMFS_SERVER_CACHE_MODE";
594  }
595  if ((generic_parameter == "CVMFS_CACHE_QUOTA_LIMIT") &&
596  !options_mgr_->IsDefined(generic_parameter))
597  {
598  return "CVMFS_QUOTA_LIMIT";
599  }
600  return generic_parameter;
601  }
602 
603  return "CVMFS_CACHE_" + instance + "_" + generic_parameter.substr(12);
604 }
605 
606 
609  n_eio_total_->Set(0);
610  n_eio_01_->Set(0);
611  n_eio_02_->Set(0);
612  n_eio_03_->Set(0);
613  n_eio_04_->Set(0);
614  n_eio_05_->Set(0);
615  n_eio_06_->Set(0);
616  n_eio_07_->Set(0);
617  n_eio_08_->Set(0);
618  n_emfile_->Set(0);
619 }
620 
621 
625 CacheManager *FileSystem::SetupCacheMgr(const string &instance) {
626  if (constructed_instances_.find(instance) != constructed_instances_.end()) {
627  boot_error_ = "circular cache definition: " + instance;
629  return NULL;
630  }
631  constructed_instances_.insert(instance);
632 
633  LogCvmfs(kLogCvmfs, kLogDebug, "setting up cache manager instance %s",
634  instance.c_str());
635  string instance_type;
636  if (instance == kDefaultCacheMgrInstance) {
637  instance_type = "posix";
638  } else {
639  options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_TYPE", instance),
640  &instance_type);
641  }
642  if (instance_type == "posix") {
643  return SetupPosixCacheMgr(instance);
644  } else if (instance_type == "ram") {
645  return SetupRamCacheMgr(instance);
646  } else if (instance_type == "tiered") {
647  return SetupTieredCacheMgr(instance);
648  } else if (instance_type == "external") {
649  return SetupExternalCacheMgr(instance);
650  } else {
651  boot_error_ = "invalid cache manager type for '" + instance + "':" +
652  instance_type;
654  return NULL;
655  }
656 }
657 
658 
660  string optarg;
661  unsigned nfiles = kDefaultNfiles;
662  if (options_mgr_->GetValue("CVMFS_NFILES", &optarg))
663  nfiles = String2Uint64(optarg);
664  vector<string> cmd_line;
665  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_CMDLINE", instance),
666  &optarg))
667  {
668  cmd_line = SplitString(optarg, ',');
669  }
670 
671  if (!options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_LOCATOR", instance),
672  &optarg))
673  {
674  boot_error_ = MkCacheParm("CVMFS_CACHE_LOCATOR", instance) + " missing";
676  return NULL;
677  }
678 
680  ExternalCacheManager::CreatePlugin(optarg, cmd_line));
681  if (!plugin_handle->IsValid()) {
682  boot_error_ = plugin_handle->error_msg();
684  return NULL;
685  }
687  plugin_handle->fd_connection(), nfiles, name_ + ":" + instance);
688  if (cache_mgr == NULL) {
689  boot_error_ = "failed to create external cache manager for " + instance;
691  return NULL;
692  }
693  cache_mgr->AcquireQuotaManager(ExternalQuotaManager::Create(cache_mgr));
694  return cache_mgr;
695 }
696 
697 
700  if (!CheckPosixCacheSettings(settings))
701  return NULL;
703  settings.cache_path,
704  settings.is_alien,
707  settings.do_refcount));
708  if (!cache_mgr.IsValid()) {
709  boot_error_ = "Failed to setup posix cache '" + instance + "' in " +
710  settings.cache_path + ": " + strerror(errno);
712  return NULL;
713  }
714 
715  // Sentinel file for future use
716  // Might be a read-only cache
717  const bool ignore_failure = settings.is_alien;
718  CreateFile(settings.cache_path + "/.cvmfscache", 0600, ignore_failure);
719 
720  if (settings.is_managed) {
721  if (!SetupPosixQuotaMgr(settings, cache_mgr.weak_ref()))
722  return NULL;
723  }
724  return cache_mgr.Release();
725 }
726 
727 
728 CacheManager *FileSystem::SetupRamCacheMgr(const string &instance) {
729  string optarg;
730  unsigned nfiles = kDefaultNfiles;
731  if (options_mgr_->GetValue("CVMFS_NFILES", &optarg)) {
732  nfiles = String2Uint64(optarg);
733  }
734  uint64_t sz_cache_bytes;
735  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SIZE", instance),
736  &optarg))
737  {
738  if (HasSuffix(optarg, "%", false)) {
739  sz_cache_bytes = platform_memsize() * String2Uint64(optarg)/100;
740  } else {
741  sz_cache_bytes = String2Uint64(optarg) * 1024 * 1024;
742  }
743  } else {
744  sz_cache_bytes = platform_memsize() >> 5; // ~3%
745  }
747  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_MALLOC", instance),
748  &optarg))
749  {
750  if (optarg == "libc") {
752  } else if (optarg == "heap") {
754  } else {
755  boot_error_ = "Failure: unknown malloc " +
756  MkCacheParm("CVMFS_CACHE_MALLOC", instance) + "=" + optarg;
758  return NULL;
759  }
760  }
761  sz_cache_bytes = RoundUp8(std::max(static_cast<uint64_t>(40 * 1024 * 1024),
762  sz_cache_bytes));
764  sz_cache_bytes,
765  nfiles,
766  alloc,
767  perf::StatisticsTemplate("cache." + instance, statistics_));
768  if (cache_mgr == NULL) {
769  boot_error_ = "failed to create ram cache manager for " + instance;
771  return NULL;
772  }
773  cache_mgr->AcquireQuotaManager(new NoopQuotaManager());
774  return cache_mgr;
775 }
776 
777 
779  string optarg;
780  if (!options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_UPPER", instance),
781  &optarg))
782  {
783  boot_error_ = MkCacheParm("CVMFS_CACHE_UPPER", instance) + " missing";
785  return NULL;
786  }
787  UniquePtr<CacheManager> upper(SetupCacheMgr(optarg));
788  if (!upper.IsValid())
789  return NULL;
790 
791  if (!options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_LOWER", instance),
792  &optarg))
793  {
794  boot_error_ = MkCacheParm("CVMFS_CACHE_LOWER", instance) + " missing";
796  return NULL;
797  }
798  UniquePtr<CacheManager> lower(SetupCacheMgr(optarg));
799  if (!lower.IsValid())
800  return NULL;
801 
802  CacheManager *tiered =
803  TieredCacheManager::Create(upper.Release(), lower.Release());
804  if (tiered == NULL) {
805  boot_error_ = "Failed to setup tiered cache manager " + instance;
807  return NULL;
808  }
809  if (options_mgr_->GetValue(
810  MkCacheParm("CVMFS_CACHE_LOWER_READONLY", instance), &optarg) &&
811  options_mgr_->IsOn(optarg))
812  {
813  static_cast<TieredCacheManager*>(tiered)->SetLowerReadOnly();
814  }
815  return tiered;
816 }
817 
818 
820  path_crash_guard_ = workspace_ + "/running." + name_;
821  platform_stat64 info;
822  int retval = platform_stat(path_crash_guard_.c_str(), &info);
823  if (retval == 0) {
824  found_previous_crash_ = true;
826  "looks like cvmfs has been crashed previously");
827  }
828  retval = open(path_crash_guard_.c_str(), O_RDONLY | O_CREAT, 0600);
829  if (retval < 0) {
830  boot_error_ = "could not open running sentinel (" +
831  StringifyInt(errno) + ")";
833  return false;
834  }
835  close(retval);
836  return true;
837 }
838 
839 
841  if (type_ == kFsFuse) {
842  // Try to jump to workspace / cache directory. This tests, if it is
843  // accessible and it brings speed later on.
844  int retval = chdir(workspace_.c_str());
845  if (retval != 0) {
846  boot_error_ = "workspace " + workspace_ + " is unavailable";
848  return false;
849  }
850  workspace_ = ".";
851  return true;
852  }
853 
854  // Note: as of version 2.4 support for CVMFS_CWD_CACHE is dropped due to
855  // disproportionate large complexity to configure correctly. This affects
856  // only libcvmfs, mostly the legacy part.
857  // string optarg;
858  // if (options_mgr_->GetValue("CVMFS_CWD_CACHE", &optarg) &&
859  // options_mgr_->IsOn(optarg))
860  // {
861  // ...
862  // }
863  return true;
864 }
865 
866 
871  setenv("CVMFS_ARCH", GetArch().c_str(), 1 /* overwrite */);
872 }
873 
874 
876  const OptionsManager &options_mgr, const std::string &prefix)
877 {
879 
880  string optarg;
881  if (options_mgr.GetValue("CVMFS_SYSLOG_LEVEL", &optarg))
883  if (options_mgr.GetValue("CVMFS_SYSLOG_FACILITY", &optarg))
885  if (options_mgr.GetValue("CVMFS_USYSLOG", &optarg))
886  SetLogMicroSyslog(optarg);
887  if (options_mgr.GetValue("CVMFS_DEBUGLOG", &optarg))
888  SetLogDebugFile(optarg);
889  if (options_mgr.GetValue("CVMFS_SYSLOG_PREFIX", &optarg)) {
890  SetLogSyslogPrefix(optarg);
891  } else {
892  SetLogSyslogPrefix(prefix);
893  }
894 }
895 
896 
899 }
900 
901 
903 #ifdef CVMFS_NFS_SUPPORT
904  if (!IsHaNfsSource())
906 
907  string no_nfs_sentinel;
908  if (cache_mgr_->id() == kPosixCacheManager) {
909  PosixCacheManager *posix_cache_mgr =
910  reinterpret_cast<PosixCacheManager *>(cache_mgr_);
911  no_nfs_sentinel = posix_cache_mgr->cache_path() + "/no_nfs_maps." + name_;
912  if (!IsNfsSource()) {
913  // Might be a read-only cache
914  const bool ignore_failure = posix_cache_mgr->alien_cache();
915  CreateFile(no_nfs_sentinel, 0600, ignore_failure);
916  return true;
917  }
918  } else {
919  if (IsNfsSource()) {
920  boot_error_ = "NFS source only works with POSIX cache manager.";
922  return false;
923  }
924  return true;
925  }
926 
928  assert(IsNfsSource());
929  if (!no_nfs_sentinel.empty() && FileExists(no_nfs_sentinel)) {
930  boot_error_ = "Cache was used without NFS maps before. "
931  "It has to be wiped out.";
933  return false;
934  }
935 
936  // nfs maps need to be protected by workspace lock
937  PosixCacheManager *posix_cache_mgr =
938  reinterpret_cast<PosixCacheManager *>(cache_mgr_);
939  if (posix_cache_mgr->cache_path() != workspace_) {
940  boot_error_ = "Cache directory and workspace must be identical for "
941  "NFS export";
943  return false;
944  }
945 
946  string inode_cache_dir = nfs_maps_dir_ + "/nfs_maps." + name_;
947  if (!MkdirDeep(inode_cache_dir, 0700)) {
948  boot_error_ = "Failed to initialize NFS maps";
950  return false;
951  }
952 
953  // TODO(jblomer): make this a manager class
954  if (IsHaNfsSource()) {
956  inode_cache_dir,
959  statistics_);
960  } else {
962  inode_cache_dir,
965  statistics_);
966  }
967 
968  if (nfs_maps_ == NULL) {
969  boot_error_ = "Failed to initialize NFS maps";
971  return false;
972  }
973 
974  string optarg;
975  if (options_mgr_->GetValue("CVMFS_NFS_INTERLEAVED_INODES", &optarg)) {
976  vector<string> tokens = SplitString(optarg, '%');
977  if (tokens.size() != 2) {
978  boot_error_ =
979  "invalid format for CVMFS_NFS_INTERLEAVED_INODES: " + optarg;
981  return false;
982  }
984  String2Uint64(tokens[0]));
985  }
986 
987  return true;
988 
989 #else
990  return true;
991 #endif
992 }
993 
994 
997  CacheManager *cache_mgr
998 ) {
999  assert(settings.quota_limit >= 0);
1000  int64_t quota_threshold = settings.quota_limit / 2;
1001  string cache_workspace = settings.cache_path;
1002  if (settings.cache_path != settings.workspace) {
1004  "using workspace %s to protect cache database in %s",
1005  settings.workspace.c_str(), settings.cache_path.c_str());
1006  cache_workspace += ":" + settings.workspace;
1007  }
1008  PosixQuotaManager *quota_mgr;
1009 
1010  if (settings.is_shared) {
1011  quota_mgr = PosixQuotaManager::CreateShared(
1012  exe_path_,
1013  cache_workspace,
1014  settings.quota_limit,
1015  quota_threshold,
1016  foreground_);
1017  if (quota_mgr == NULL) {
1018  boot_error_ = "Failed to initialize shared lru cache";
1020  return false;
1021  }
1022  } else {
1023  quota_mgr = PosixQuotaManager::Create(
1024  cache_workspace,
1025  settings.quota_limit,
1026  quota_threshold,
1028  if (quota_mgr == NULL) {
1029  boot_error_ = "Failed to initialize lru cache";
1031  return false;
1032  }
1033  }
1034 
1035  if (quota_mgr->GetSize() > quota_mgr->GetCapacity()) {
1037  "cache is already beyond quota size "
1038  "(size: %" PRId64 ", capacity: %" PRId64 "), cleaning up",
1039  quota_mgr->GetSize(), quota_mgr->GetCapacity());
1040  if (!quota_mgr->Cleanup(quota_threshold)) {
1041  delete quota_mgr;
1042  boot_error_ = "Failed to clean up cache";
1044  return false;
1045  }
1046  }
1047 
1048  int retval = cache_mgr->AcquireQuotaManager(quota_mgr);
1049  assert(retval);
1051  "CernVM-FS: quota initialized, current size %luMB",
1052  quota_mgr->GetSize() / (1024 * 1024));
1053  return true;
1054 }
1055 
1056 
1058  // Make sure SQlite starts clean after initialization
1059  sqlite3_shutdown();
1060 
1061  int retval;
1062  retval = sqlite3_config(SQLITE_CONFIG_LOG, FileSystem::LogSqliteError, NULL);
1063  assert(retval == SQLITE_OK);
1064  retval = sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
1065  assert(retval == SQLITE_OK);
1067 
1068  // Disable SQlite3 file locking
1069  retval = sqlite3_vfs_register(sqlite3_vfs_find("unix-none"), 1);
1070  assert(retval == SQLITE_OK);
1071 }
1072 
1073 
1075  string optarg;
1076  // This is very similar to "determine cache dir". It's for backward
1077  // compatibility with classic cache configuration where there was no
1078  // distinction between workspace and cache.
1079  // Complicated cache configurations should explicitly set CVMFS_WORKSPACE.
1081  if (options_mgr_->GetValue("CVMFS_CACHE_BASE", &optarg))
1082  workspace_ = MakeCanonicalPath(optarg);
1083  if (options_mgr_->GetValue("CVMFS_SHARED_CACHE", &optarg) &&
1084  options_mgr_->IsOn(optarg))
1085  {
1086  workspace_ += "/shared";
1087  } else {
1088  workspace_ += "/" + name_;
1089  }
1090  if (options_mgr_->GetValue("CVMFS_CACHE_DIR", &optarg)) {
1091  if (options_mgr_->IsDefined("CVMFS_CACHE_BASE")) {
1092  boot_error_ =
1093  "'CVMFS_CACHE_BASE' and 'CVMFS_CACHE_DIR' are mutually exclusive";
1095  return false;
1096  }
1097  workspace_ = optarg;
1098  }
1099  if (options_mgr_->GetValue("CVMFS_WORKSPACE", &optarg))
1100  workspace_ = optarg;
1102 
1103  // If workspace and alien cache are the same directory, we need to open
1104  // permission now to 0770 to avoid a race when fixing it later
1105  const int mode = 0770;
1106  if (!MkdirDeep(workspace_, mode, false)) {
1107  boot_error_ = "cannot create workspace directory " + workspace_;
1109  return false;
1110  }
1111 
1112  if (!LockWorkspace())
1113  return false;
1114  if (!SetupCwd())
1115  return false;
1116  if (!SetupCrashGuard())
1117  return false;
1118 
1119  return true;
1120 }
1121 
1122 
1125  if (uuid_cache_ == NULL) {
1127  "failed to load/store %s/uuid", workspace_.c_str());
1129  assert(uuid_cache_ != NULL);
1130  }
1131 }
1132 
1133 
1139  if ((cache_mgr_ != NULL) &&
1140  (cache_mgr_->id() == kPosixCacheManager)) {
1141  PosixCacheManager *posix_cache_mgr =
1142  reinterpret_cast<PosixCacheManager *>(cache_mgr_);
1143  posix_cache_mgr->TearDown2ReadOnly();
1144  }
1145 
1146  unlink(path_crash_guard_.c_str());
1147  LogCvmfs(kLogCache, kLogSyslog, "switch to read-only cache mode");
1148  SetLogMicroSyslog("");
1149 }
1150 
1151 
1152 void FileSystem::RemapCatalogFd(int from, int to) {
1153  sqlite::RegisterFdMapping(from, to);
1154 }
1155 
1157  cache_mgr_ = new_cache_mgr;
1158  sqlite::ReplaceCacheManager(new_cache_mgr);
1159 }
1160 
1161 
1164  string instance;
1165  if (options_mgr_->GetValue("CVMFS_CACHE_PRIMARY", &instance) &&
1166  !instance.empty())
1167  {
1168  if (!CheckInstanceName(instance))
1169  return false;
1170  cache_mgr_instance_ = instance;
1171  }
1172 
1174  if (cache_mgr_ == NULL)
1175  return false;
1176 
1177  std::string optarg;
1178  if (options_mgr_->GetValue("CVMFS_STREAMING_CACHE", &optarg) &&
1179  options_mgr_->IsOn(optarg))
1180  {
1181  unsigned nfiles = kDefaultNfiles;
1182  if (options_mgr_->GetValue("CVMFS_NFILES", &optarg))
1183  nfiles = String2Uint64(optarg);
1184  size_t buffer_size = StreamingCacheManager::kDefaultBufferSize;
1185  if (options_mgr_->GetValue("CVMFS_STREAMING_CACHE_BUFFER_SIZE", &optarg))
1186  buffer_size = String2Uint64(optarg);
1187  cache_mgr_ = new StreamingCacheManager(nfiles, cache_mgr_, NULL, NULL,
1188  buffer_size, statistics_);
1189  }
1190 
1191  return true;
1192 }
1193 
1194 
1195 //------------------------------------------------------------------------------
1196 
1197 
1198 const char *MountPoint::kDefaultAuthzSearchPath = "/usr/libexec/cvmfs/authz";
1199 const char *MountPoint::kDefaultBlacklist = "/etc/cvmfs/blacklist";
1200 
1202  blacklist_paths_.clear();
1203  string blacklist;
1204  if (!options_mgr_->GetValue("CVMFS_BLACKLIST", &blacklist))
1205  blacklist = kDefaultBlacklist;
1206  blacklist_paths_.push_back(blacklist);
1207 
1208  bool append = false;
1209  if (FileExists(blacklist)) {
1210  if (!signature_mgr_->LoadBlacklist(blacklist, append)) {
1211  boot_error_ = "failed to load blacklist " + blacklist;
1213  return false;
1214  }
1215  append = true;
1216  }
1217 
1218  string config_repository_path;
1219  if (options_mgr_->HasConfigRepository(fqrn_, &config_repository_path)) {
1220  blacklist = config_repository_path + "blacklist";
1221  blacklist_paths_.push_back(blacklist);
1222  if (FileExists(blacklist)) {
1223  if (!signature_mgr_->LoadBlacklist(blacklist, append)) {
1224  boot_error_ = "failed to load blacklist from config repository";
1226  return false;
1227  }
1228  }
1229  }
1230 
1231  return true;
1232 }
1233 
1234 
1240  bool result = true;
1241  bool append = false;
1242  for (unsigned i = 0; i < blacklist_paths_.size(); ++i) {
1243  string blacklist = blacklist_paths_[i];
1244  if (FileExists(blacklist)) {
1245  bool retval = signature_mgr_->LoadBlacklist(blacklist, append);
1246  // TODO(jblomer): this can leave us with a half-loaded blacklist
1247  if (!retval)
1248  return false;
1249  append = true;
1250  }
1251  }
1252  return result;
1253 }
1254 
1264  cache_symlinks_ = false;
1265 }
1266 
1272  fuse_expire_entry_ = true;
1273 }
1274 
1275 
1281  const string &fqrn,
1282  FileSystem *file_system,
1283  OptionsManager *options_mgr)
1284 {
1285  if (options_mgr == NULL)
1286  options_mgr = file_system->options_mgr();
1287  UniquePtr<MountPoint> mountpoint(new MountPoint(
1288  fqrn, file_system, options_mgr));
1289 
1290  // At this point, we have a repository name, the type (fuse or library) and
1291  // an options manager (which can be the same than the FileSystem's one).
1292 
1293  mountpoint->CreateStatistics();
1294  mountpoint->CreateAuthz();
1295  mountpoint->backoff_throttle_ = new BackoffThrottle();
1296 
1297  if (!mountpoint->CreateSignatureManager() || !mountpoint->CheckBlacklists())
1298  return mountpoint.Release();
1299  if (!mountpoint->CreateDownloadManagers())
1300  return mountpoint.Release();
1301  if (file_system->cache_mgr()->id() == kStreamingCacheManager) {
1302  StreamingCacheManager *streaming_cachemgr =
1303  dynamic_cast<StreamingCacheManager *>(file_system->cache_mgr());
1304  streaming_cachemgr->SetRegularDownloadManager(mountpoint->download_mgr());
1305  streaming_cachemgr->SetExternalDownloadManager(
1306  mountpoint->external_download_mgr());
1307  }
1308  if (!mountpoint->CreateResolvConfWatcher()) {
1309  return mountpoint.Release();
1310  }
1311  mountpoint->CreateFetchers();
1312  if (!mountpoint->CreateCatalogManager())
1313  return mountpoint.Release();
1314  if (!mountpoint->CreateTracer())
1315  return mountpoint.Release();
1316 
1317  mountpoint->ReEvaluateAuthz();
1318  mountpoint->CreateTables();
1319  if (!mountpoint->SetupBehavior())
1320  return mountpoint.Release();
1321 
1322  mountpoint->boot_status_ = loader::kFailOk;
1323  return mountpoint.Release();
1324 }
1325 
1326 
1328  string optarg;
1329  string authz_helper;
1330  if (options_mgr_->GetValue("CVMFS_AUTHZ_HELPER", &optarg))
1331  authz_helper = optarg;
1332  string authz_search_path(kDefaultAuthzSearchPath);
1333  if (options_mgr_->GetValue("CVMFS_AUTHZ_SEARCH_PATH", &optarg))
1334  authz_search_path = optarg;
1335 
1337  fqrn_,
1338  authz_helper,
1339  authz_search_path,
1340  options_mgr_);
1341  assert(authz_fetcher_ != NULL);
1342 
1345  statistics_);
1346  assert(authz_session_mgr_ != NULL);
1347 
1349  assert(authz_attachment_ != NULL);
1350 }
1351 
1352 
1354  string optarg;
1355 
1357 
1359  if (!SetupOwnerMaps())
1360  return false;
1361  shash::Any root_hash;
1362  if (!DetermineRootHash(&root_hash))
1363  return false;
1364 
1365  bool retval;
1366  if (root_hash.IsNull()) {
1367  retval = catalog_mgr_->Init();
1368  } else {
1369  fixed_catalog_ = true;
1370  bool alt_root_path =
1371  options_mgr_->GetValue("CVMFS_ALT_ROOT_PATH", &optarg) &&
1372  options_mgr_->IsOn(optarg);
1373  retval = catalog_mgr_->InitFixed(root_hash, alt_root_path);
1374  }
1375  if (!retval) {
1376  boot_error_ = "Failed to initialize root file catalog";
1378  return false;
1379  }
1380 
1382  boot_error_ = "repository revision blacklisted";
1384  return false;
1385  }
1386 
1387  if (options_mgr_->GetValue("CVMFS_AUTO_UPDATE", &optarg) &&
1388  !options_mgr_->IsOn(optarg))
1389  {
1390  fixed_catalog_ = true;
1391  }
1392 
1393  if (options_mgr_->GetValue("CVMFS_CATALOG_WATERMARK", &optarg)) {
1395  } else {
1396  unsigned soft_limit;
1397  unsigned hard_limit;
1398  GetLimitNoFile(&soft_limit, &hard_limit);
1399  catalog_mgr_->SetCatalogWatermark(soft_limit / 4);
1400  }
1401 
1402  if (catalog_mgr_->volatile_flag()) {
1403  LogCvmfs(kLogCvmfs, kLogDebug, "content of repository flagged as VOLATILE");
1404  }
1405 
1406  return true;
1407 }
1408 
1409 
1411  string optarg;
1413  perf::StatisticsTemplate("download", statistics_));
1415 
1416  // must be set before proxy and host chains are being initialized
1417  // error output is handled in SetShardingPolicy
1418  if (options_mgr_->GetValue("CVMFS_PROXY_SHARDING_POLICY", &optarg)) {
1419  if (optarg.compare("EXTERNAL") == 0) {
1422  }
1423  }
1424 
1425  if (options_mgr_->GetValue("CVMFS_FAILOVER_INDEFINITELY", &optarg) &&
1426  options_mgr_->IsOn(optarg)) {
1428  }
1429 
1430  if (options_mgr_->GetValue("CVMFS_METALINK_URL", &optarg)) {
1431  download_mgr_->SetMetalinkChain(optarg);
1432  // host chain will be set later when the metalink server is contacted
1434  // metalink requires redirects
1436  } else if (options_mgr_->GetValue("CVMFS_SERVER_URL", &optarg)) {
1437  download_mgr_->SetHostChain(optarg);
1438  }
1439 
1440  if (options_mgr_->GetValue("_CVMFS_DEVEL_IGNORE_SIGNATURE_FAILURES", &optarg)
1441  && options_mgr_->IsOn(optarg)) {
1444  "Development option: Activate ignore signature failures during download. "
1445  "DO NOT USE IN PRODUCTION");
1446  }
1447 
1449  SetupHttpTuning();
1450 
1451  string forced_proxy_template;
1452  if (options_mgr_->GetValue("CVMFS_PROXY_TEMPLATE", &optarg))
1453  forced_proxy_template = optarg;
1455  forced_proxy_template);
1456 
1457  string proxies;
1458  if (options_mgr_->GetValue("CVMFS_HTTP_PROXY", &optarg))
1459  proxies = optarg;
1461  proxies,
1462  file_system_->workspace() + "/proxies" + GetUniqFileSuffix(),
1463  download_mgr_);
1464  if (proxies == "") {
1465  boot_error_ = "failed to discover HTTP proxy servers";
1467  return false;
1468  }
1469  string fallback_proxies;
1470  if (options_mgr_->GetValue("CVMFS_FALLBACK_PROXY", &optarg))
1471  fallback_proxies = optarg;
1472  download_mgr_->SetProxyChain(proxies, fallback_proxies,
1474 
1475  bool do_geosort = options_mgr_->GetValue("CVMFS_USE_GEOAPI", &optarg) &&
1476  options_mgr_->IsOn(optarg);
1477  if (do_geosort) {
1479  }
1480  if (options_mgr_->GetValue("CVMFS_MAX_SERVERS", &optarg)) {
1481  unsigned max_servers = String2Uint64(optarg);
1482  std::vector<std::string> host_chain;
1483  download_mgr_->GetHostInfo(&host_chain, NULL, NULL);
1484  if (max_servers > 0 && max_servers < host_chain.size()) {
1485  host_chain.resize(max_servers);
1486  download_mgr_->SetHostChain(host_chain);
1487  }
1488  }
1489 
1490  if (options_mgr_->GetValue("CVMFS_USE_SSL_SYSTEM_CA", &optarg) &&
1491  options_mgr_->IsOn(optarg)) {
1493  }
1494 
1495  if (options_mgr_->GetValue("CVMFS_PROXY_SHARD", &optarg) &&
1496  options_mgr_->IsOn(optarg)) {
1498  }
1499 
1500  // configure http tracing header
1501  if (options_mgr_->GetValue("CVMFS_HTTP_TRACING", &optarg) &&
1502  options_mgr_->IsOn(optarg)) {
1504  if (options_mgr_->GetValue("CVMFS_HTTP_TRACING_HEADERS", &optarg)) {
1505  if (optarg.size() > 1000) {
1507  "CVMFS_HTTP_TRACING_HEADERS too large ( max 1000 chars, given %ld )",
1508  optarg.size());
1509  } else {
1510  std::vector<std::string> tokens = SplitString(optarg, '|');
1511  sanitizer::AlphaNumSanitizer sanitizer;
1512 
1513  for (unsigned int i = 0; i < tokens.size(); i++) {
1514  std::string token = Trim(tokens[i]);
1515 
1516  std::vector<std::string> key_val = SplitString(token, ':');
1517 
1518  if (key_val.size() != 2) {
1520  "Http tracing header: Skipping current token part of "
1521  "CVMFS_HTTP_TRACING_HEADERS! Invalid "
1522  "<key:value> pair. Token: %s", token.c_str());
1523  continue;
1524  }
1525 
1526  std::string prefix = "X-CVMFS-";
1527  std::string key = Trim(key_val[0]);
1528 
1529  if (!sanitizer.IsValid(key)) {
1531  "Http tracing header: Skipping current token part of "
1532  "CVMFS_HTTP_TRACING_HEADERS! Invalid key. Only alphanumeric keys "
1533  "are allowed (a-z, A-Z, 0-9). Token: %s", token.c_str());
1534  continue;
1535  }
1536 
1537  std::string final_token = prefix + key + ": " + Trim(key_val[1]);
1538 
1539  download_mgr_->AddHTTPTracingHeader(final_token);
1540  }
1541  }
1542  }
1543  }
1544 
1545  return SetupExternalDownloadMgr(do_geosort);
1546 }
1547 
1549  std::string roaming_value;
1550  options_mgr_->GetValue("CVMFS_DNS_ROAMING", &roaming_value);
1551  if (options_mgr_->IsDefined("CVMFS_DNS_ROAMING") &&
1552  options_mgr_->IsOn(roaming_value)) {
1554  "DNS roaming is enabled for this repository.");
1555  // Create a file watcher to update the DNS settings of the download
1556  // managers when there are changes to /etc/resolv.conf
1558 
1559  if (resolv_conf_watcher_) {
1560  ResolvConfEventHandler *handler =
1562  resolv_conf_watcher_->RegisterHandler("/etc/resolv.conf", handler);
1563  }
1564  } else {
1566  "DNS roaming is disabled for this repository.");
1567  }
1568  return true;
1569 }
1570 
1572  fetcher_ = new cvmfs::Fetcher(
1574  download_mgr_,
1577 
1582  perf::StatisticsTemplate("fetch-external", statistics_));
1583 }
1584 
1585 
1587  string optarg;
1589  signature_mgr_->Init();
1590 
1591  string public_keys;
1592  if (options_mgr_->GetValue("CVMFS_PUBLIC_KEY", &optarg)) {
1593  public_keys = optarg;
1594  } else if (options_mgr_->GetValue("CVMFS_KEYS_DIR", &optarg)) {
1595  // Collect .pub files from CVMFS_KEYS_DIR
1596  public_keys = JoinStrings(FindFilesBySuffix(optarg, ".pub"), ":");
1597  } else {
1598  public_keys =
1599  JoinStrings(FindFilesBySuffix("/etc/cvmfs/keys", ".pub"), ":");
1600  }
1601 
1602  if (!signature_mgr_->LoadPublicRsaKeys(public_keys)) {
1603  boot_error_ = "failed to load public key(s)";
1605  return false;
1606  }
1607 
1608  if (public_keys.size() > 0) {
1609  LogCvmfs(kLogCvmfs, kLogDebug, "CernVM-FS: using public key(s) %s",
1610  public_keys.c_str());
1611  } else {
1612  LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogWarn, "no public key loaded");
1613  }
1614 
1615  return true;
1616 }
1617 
1618 
1622  return;
1623 
1624  // TODO(jblomer): this should be registered by the tracker
1625  statistics_->Register("inode_tracker.n_insert",
1626  "overall number of accessed inodes");
1627  statistics_->Register("inode_tracker.n_remove",
1628  "overall number of evicted inodes");
1629  statistics_->Register("inode_tracker.no_reference",
1630  "currently active inodes");
1631  statistics_->Register("inode_tracker.n_hit_inode",
1632  "overall number of inode lookups");
1633  statistics_->Register("inode_tracker.n_hit_path",
1634  "overall number of successful path lookups");
1635  statistics_->Register("inode_tracker.n_miss_path",
1636  "overall number of unsuccessful path lookups");
1637 
1638  statistics_->Register("dentry_tracker.n_insert",
1639  "overall number of added negative cache entries");
1640  statistics_->Register("dentry_tracker.n_remove",
1641  "overall number of evicted negative cache entries");
1642  statistics_->Register("dentry_tracker.n_prune",
1643  "overall number of prune calls");
1644 
1645  statistics_->Register("page_cache_tracker.n_insert",
1646  "overall number of added page cache entries");
1647  statistics_->Register("page_cache_tracker.n_remove",
1648  "overall number of evicted page cache entries");
1649  statistics_->Register("page_cache_tracker.n_open_direct",
1650  "overall number of direct I/O open calls");
1651  statistics_->Register("page_cache_tracker.n_open_flush",
1652  "overall number of open calls where the file's page cache gets flushed");
1653  statistics_->Register("page_cache_tracker.n_open_cached",
1654  "overall number of open calls where the file's page cache is reused");
1655 }
1656 
1657 
1660  // Libcvmfs simplified tables
1663  return;
1664  }
1665 
1666  chunk_tables_ = new ChunkTables();
1667 
1668  string optarg;
1669  uint64_t mem_cache_size = kDefaultMemcacheSize;
1670  if (options_mgr_->GetValue("CVMFS_MEMCACHE_SIZE", &optarg))
1671  mem_cache_size = String2Uint64(optarg) * 1024 * 1024;
1672 
1673  const double memcache_unit_size =
1674  (static_cast<double>(kInodeCacheFactor) * lru::Md5PathCache::GetEntrySize())
1676  const unsigned memcache_num_units =
1677  mem_cache_size / static_cast<unsigned>(memcache_unit_size);
1678  // Number of cache entries must be a multiple of 64
1679  const unsigned mask_64 = ~((1 << 6) - 1);
1680  inode_cache_ = new lru::InodeCache(memcache_num_units & mask_64, statistics_);
1681  path_cache_ = new lru::PathCache(memcache_num_units & mask_64, statistics_);
1682  md5path_cache_ = new lru::Md5PathCache((memcache_num_units * 7) & mask_64,
1683  statistics_);
1684 
1688  if (file_system_->IsNfsSource())
1690 }
1691 
1701  string optarg;
1702  tracer_ = new Tracer();
1703  if (options_mgr_->GetValue("CVMFS_TRACEFILE", &optarg)) {
1705  boot_error_ = "tracer is only supported in the fuse module";
1707  return false;
1708  }
1709  string tracebuffer_file = optarg;
1710  uint64_t tracebuffer_size = kTracerBufferSize;
1711  uint64_t tracebuffer_threshold = kTracerFlushThreshold;
1712 
1713  if (options_mgr_->GetValue("CVMFS_TRACEBUFFER", &optarg)) {
1714  tracebuffer_size = String2Uint64(optarg);
1715  }
1716  if (options_mgr_->GetValue("CVMFS_TRACEBUFFER_THRESHOLD",
1717  &optarg)) {
1718  tracebuffer_threshold = String2Uint64(optarg);
1719  }
1720  assert(tracebuffer_size <= INT_MAX
1721  && tracebuffer_threshold <= INT_MAX);
1723  "Initialising tracer with buffer size %" PRIu64 " and threshold %" PRIu64,
1724  tracebuffer_size, tracebuffer_threshold);
1725  tracer_->Activate(tracebuffer_size, tracebuffer_threshold,
1726  tracebuffer_file);
1727  }
1728  return true;
1729 }
1730 
1731 
1733  string optarg;
1734  if (options_mgr_->GetValue("CVMFS_ROOT_HASH", &optarg)) {
1735  *root_hash = MkFromHexPtr(shash::HexPtr(optarg), shash::kSuffixCatalog);
1736  return true;
1737  }
1738 
1739  if (!options_mgr_->IsDefined("CVMFS_REPOSITORY_TAG") &&
1740  !options_mgr_->IsDefined("CVMFS_REPOSITORY_DATE"))
1741  {
1742  root_hash->SetNull();
1743  return true;
1744  }
1745 
1746  string history_path;
1747  if (!FetchHistory(&history_path))
1748  return false;
1749  UnlinkGuard history_file(history_path);
1751  history::SqliteHistory::Open(history_path));
1752  if (!tag_db.IsValid()) {
1754  "failed to open history database (%s)", history_path.c_str());
1755  boot_error_ = "failed to open history database";
1757  return false;
1758  }
1759 
1761  bool retval;
1762  if (!options_mgr_->GetValue("CVMFS_REPOSITORY_TAG", &repository_tag_)) {
1763  string repository_date;
1764  // options_mgr_->IsDefined("CVMFS_REPOSITORY_DATE") must be true
1765  options_mgr_->GetValue("CVMFS_REPOSITORY_DATE", &repository_date);
1766  time_t repository_utctime = IsoTimestamp2UtcTime(repository_date);
1767  if (repository_utctime == 0) {
1768  boot_error_ = "invalid timestamp in CVMFS_REPOSITORY_DATE: " +
1769  repository_date + ". Use YYYY-MM-DDTHH:MM:SSZ";
1771  return false;
1772  }
1773  retval = tag_db->GetByDate(repository_utctime, &tag);
1774  if (!retval) {
1775  boot_error_ = "no repository state as early as utc timestamp " +
1776  StringifyTime(repository_utctime, true);
1778  return false;
1779  }
1781  "time stamp %s UTC resolved to tag '%s'",
1782  StringifyTime(repository_utctime, true).c_str(),
1783  tag.name.c_str());
1784  repository_tag_ = tag.name;
1785  } else {
1786  retval = tag_db->GetByName(repository_tag_, &tag);
1787  if (!retval) {
1788  boot_error_ = "no such tag: " + repository_tag_;
1790  return false;
1791  }
1792  }
1793  LogCvmfs(kLogCvmfs, kLogDebug, "mounting tag %s", tag.name.c_str());
1794 
1795  *root_hash = tag.root_hash;
1796  return true;
1797 }
1798 
1799 
1800 bool MountPoint::FetchHistory(std::string *history_path) {
1801  manifest::Failures retval_mf;
1802  manifest::ManifestEnsemble ensemble;
1803  retval_mf = manifest::Fetch("", fqrn_, 0, NULL, signature_mgr_, download_mgr_,
1804  &ensemble);
1805  if (retval_mf != manifest::kFailOk) {
1806  boot_error_ = "Failed to fetch manifest";
1808  return false;
1809  }
1810  shash::Any history_hash = ensemble.manifest->history();
1811  if (history_hash.IsNull()) {
1812  boot_error_ = "No history";
1814  return false;
1815  }
1816 
1817  CacheManager::Label label;
1819  label.path = fqrn_;
1820  int fd = fetcher_->Fetch(CacheManager::LabeledObject(history_hash, label));
1821  if (fd < 0) {
1822  boot_error_ = "failed to download history: " + StringifyInt(-fd);
1824  return false;
1825  }
1826  // We have the custom sqlite vfs driver installed
1827  *history_path = "@" + StringifyInt(fd);
1828  return true;
1829 }
1830 
1831 
1833  unsigned max_ttl;
1834  {
1835  MutexLockGuard lock_guard(lock_max_ttl_);
1836  max_ttl = max_ttl_sec_;
1837  }
1838  const unsigned catalog_ttl_sec = catalog_mgr_->GetTTL();
1839 
1840  return max_ttl ? std::min(max_ttl, catalog_ttl_sec) : catalog_ttl_sec;
1841 }
1842 
1843 
1845  MutexLockGuard lock_guard(lock_max_ttl_);
1846  return max_ttl_sec_ / 60;
1847 }
1848 
1849 
1855  return "." + file_system_->name() + "-" + fqrn_;
1856 }
1857 
1858 
1860  const string &fqrn,
1861  FileSystem *file_system,
1862  OptionsManager *options_mgr)
1863  : fqrn_(fqrn)
1864  , uuid_(cvmfs::Uuid::Create(""))
1865  , file_system_(file_system)
1866  , options_mgr_(options_mgr)
1867  , statistics_(NULL)
1868  , telemetry_aggr_(NULL)
1869  , authz_fetcher_(NULL)
1870  , authz_session_mgr_(NULL)
1871  , authz_attachment_(NULL)
1872  , backoff_throttle_(NULL)
1873  , signature_mgr_(NULL)
1874  , download_mgr_(NULL)
1875  , external_download_mgr_(NULL)
1876  , fetcher_(NULL)
1877  , external_fetcher_(NULL)
1878  , inode_annotation_(NULL)
1879  , catalog_mgr_(NULL)
1880  , chunk_tables_(NULL)
1881  , simple_chunk_tables_(NULL)
1882  , inode_cache_(NULL)
1883  , path_cache_(NULL)
1884  , md5path_cache_(NULL)
1885  , tracer_(NULL)
1886  , inode_tracker_(NULL)
1887  , dentry_tracker_(NULL)
1888  , page_cache_tracker_(NULL)
1889  , statfs_cache_(NULL)
1890  , resolv_conf_watcher_(NULL)
1891  , max_ttl_sec_(kDefaultMaxTtlSec)
1892  , kcache_timeout_sec_(static_cast<double>(kDefaultKCacheTtlSec))
1893  , fixed_catalog_(false)
1894  , enforce_acls_(false)
1895  , cache_symlinks_(false)
1896  , fuse_expire_entry_(false)
1897  , has_membership_req_(false)
1898  , talk_socket_path_(std::string("./cvmfs_io.") + fqrn)
1899  , talk_socket_uid_(0)
1900  , talk_socket_gid_(0)
1901 {
1902  int retval = pthread_mutex_init(&lock_max_ttl_, NULL);
1903  assert(retval == 0);
1904 }
1905 
1906 
1908  pthread_mutex_destroy(&lock_max_ttl_);
1909 
1910  delete page_cache_tracker_;
1911  delete dentry_tracker_;
1912  delete inode_tracker_;
1913  delete tracer_;
1914  delete md5path_cache_;
1915  delete path_cache_;
1916  delete inode_cache_;
1917  delete simple_chunk_tables_;
1918  delete chunk_tables_;
1919 
1920  delete catalog_mgr_;
1921  delete inode_annotation_;
1922  delete external_fetcher_;
1923  delete fetcher_;
1924 
1925  delete external_download_mgr_;
1926  delete download_mgr_;
1927 
1928  if (signature_mgr_ != NULL) {
1929  signature_mgr_->Fini();
1930  delete signature_mgr_;
1931  }
1932 
1933  if (resolv_conf_watcher_ != NULL) {
1935  delete resolv_conf_watcher_;
1936  }
1937 
1938  delete backoff_throttle_;
1939  delete authz_attachment_;
1940  delete authz_session_mgr_;
1941  delete authz_fetcher_;
1942  delete telemetry_aggr_;
1943  delete statistics_;
1944  delete uuid_;
1945 
1946  delete statfs_cache_;
1947 }
1948 
1949 
1951  string old_membership_req = membership_req_;
1953  if (old_membership_req != membership_req_) {
1956  }
1957 }
1958 
1959 
1960 void MountPoint::SetMaxTtlMn(unsigned value_minutes) {
1961  MutexLockGuard lock_guard(lock_max_ttl_);
1962  max_ttl_sec_ = value_minutes * 60;
1963 }
1964 
1966  string optarg;
1967 
1968  if (options_mgr_->GetValue("CVMFS_MAX_TTL", &optarg))
1969  SetMaxTtlMn(String2Uint64(optarg));
1970 
1971  if (options_mgr_->GetValue("CVMFS_KCACHE_TIMEOUT", &optarg)) {
1972  // Can be negative and should then be interpreted as 0.0
1974  std::max(0.0, static_cast<double>(String2Int64(optarg)));
1975  }
1976  LogCvmfs(kLogCvmfs, kLogDebug, "kernel caches expire after %d seconds",
1977  static_cast<int>(kcache_timeout_sec_));
1978 
1979  uint64_t statfs_time_cache_valid = 0;
1980  if (options_mgr_->GetValue("CVMFS_STATFS_CACHE_TIMEOUT", &optarg)) {
1981  statfs_time_cache_valid = static_cast<uint64_t>(String2Uint64(optarg));
1982  }
1983  LogCvmfs(kLogCvmfs, kLogDebug, "statfs cache expires after %d seconds",
1984  static_cast<int>(statfs_time_cache_valid));
1985  statfs_cache_ = new StatfsCache(statfs_time_cache_valid);
1986 
1987  MagicXattrManager::EVisibility xattr_visibility =
1989  if (options_mgr_->GetValue("CVMFS_HIDE_MAGIC_XATTRS", &optarg)) {
1990  if (options_mgr_->IsOn(optarg))
1991  xattr_visibility = MagicXattrManager::kVisibilityNever;
1992  else if (options_mgr_->IsOff(optarg))
1993  xattr_visibility = MagicXattrManager::kVisibilityAlways;
1994  }
1995  if (options_mgr_->GetValue("CVMFS_MAGIC_XATTRS_VISIBILITY", &optarg)) {
1996  if (ToUpper(optarg) == "ROOTONLY") {
1997  xattr_visibility = MagicXattrManager::kVisibilityRootOnly;
1998  } else if (ToUpper(optarg) == "NEVER") {
1999  xattr_visibility = MagicXattrManager::kVisibilityNever;
2000  } else if (ToUpper(optarg) == "ALWAYS") {
2001  xattr_visibility = MagicXattrManager::kVisibilityAlways;
2002  } else {
2004  "unsupported setting: CVMFS_MAGIC_XATTRS_VISIBILITY=%s",
2005  optarg.c_str());
2006  }
2007  }
2008 
2009  std::set<gid_t> protected_xattr_gids;
2010  if (options_mgr_->GetValue("CVMFS_XATTR_PRIVILEGED_GIDS", &optarg)) {
2011  std::vector<string> tmp = SplitString(optarg, ',');
2012 
2013  for (size_t i = 0; i < tmp.size(); i++) {
2014  std::string trimmed = Trim(tmp[i]);
2016  "Privileged gid for xattr added: %s", trimmed.c_str());
2017  protected_xattr_gids.insert(static_cast<gid_t>(String2Uint64(trimmed)));
2018  }
2019  }
2020  std::set<std::string> protected_xattrs;
2021  if (options_mgr_->GetValue("CVMFS_XATTR_PROTECTED_XATTRS", &optarg)) {
2022  std::vector<string> tmp = SplitString(optarg, ',');
2023 
2024  for (size_t i = 0; i < tmp.size(); i++) {
2025  std::string trimmed = Trim(tmp[i]);
2027  "Protected xattr added: %s", trimmed.c_str());
2028  protected_xattrs.insert(trimmed);
2029  }
2030 
2031  // root has always access to xattr
2032  if (protected_xattr_gids.count(0) < 1) {
2033  protected_xattr_gids.insert(0);
2035  "Automatically added root to have access to protected xattrs.");
2036  }
2037  }
2038  magic_xattr_mgr_ = new MagicXattrManager(this, xattr_visibility,
2039  protected_xattrs, protected_xattr_gids);
2040 
2041 
2042  if (options_mgr_->GetValue("CVMFS_ENFORCE_ACLS", &optarg)
2043  && options_mgr_->IsOn(optarg))
2044  {
2045  enforce_acls_ = true;
2046  }
2047 
2048  if (options_mgr_->GetValue("CVMFS_CACHE_SYMLINKS", &optarg)
2049  && options_mgr_->IsOn(optarg))
2050  {
2051  cache_symlinks_ = true;
2052  }
2053 
2054 
2055 
2056  if (options_mgr_->GetValue("CVMFS_TALK_SOCKET", &optarg)) {
2057  talk_socket_path_ = optarg;
2058  }
2059  if (options_mgr_->GetValue("CVMFS_TALK_OWNER", &optarg)) {
2060  bool retval = GetUidOf(optarg, &talk_socket_uid_, &talk_socket_gid_);
2061  if (!retval) {
2062  boot_error_ = "unknown owner of cvmfs_talk socket: " + optarg;
2064  return false;
2065  }
2066  }
2067 
2068  // this can be later be changed to switch through different
2069  // telemetryAggregators
2070  if (options_mgr_->GetValue("CVMFS_TELEMETRY_SEND", &optarg)
2071  && options_mgr_->IsOn(optarg)) {
2072  int telemetry_send_rate_sec = kDefaultTelemetrySendRateSec;
2073  if (options_mgr_->GetValue("CVMFS_TELEMETRY_RATE", &optarg)) {
2074  telemetry_send_rate_sec = static_cast<int>(String2Uint64(optarg));
2075 
2076  // minimum send rate: 5sec
2077  if (telemetry_send_rate_sec < kMinimumTelemetrySendRateSec) {
2078  telemetry_send_rate_sec = kMinimumTelemetrySendRateSec;
2079  }
2080 
2082  telemetry_send_rate_sec,
2083  options_mgr_,
2084  this,
2085  fqrn_,
2088  "Enable telemetry to report every %d seconds",
2089  telemetry_send_rate_sec);
2090  }
2091  }
2092 
2093  return true;
2094 }
2095 
2096 
2102  string optarg;
2103  unsigned dns_timeout_ms = download::DownloadManager::kDnsDefaultTimeoutMs;
2104  unsigned dns_retries = download::DownloadManager::kDnsDefaultRetries;
2105  if (options_mgr_->GetValue("CVMFS_DNS_TIMEOUT", &optarg))
2106  dns_timeout_ms = String2Uint64(optarg) * 1000;
2107  if (options_mgr_->GetValue("CVMFS_DNS_RETRIES", &optarg))
2108  dns_retries = String2Uint64(optarg);
2109  manager->SetDnsParameters(dns_retries, dns_timeout_ms);
2110 
2111  // Rest has to be after SetDnsParameters because SetDnsParameters might
2112  // construct a new resolver object
2113 
2114  unsigned dns_min_ttl = dns::Resolver::kDefaultMinTtl;
2115  unsigned dns_max_ttl = dns::Resolver::kDefaultMaxTtl;
2116  if (options_mgr_->GetValue("CVMFS_DNS_MIN_TTL", &optarg))
2117  dns_min_ttl = String2Uint64(optarg);
2118  if (options_mgr_->GetValue("CVMFS_DNS_MAX_TTL", &optarg))
2119  dns_max_ttl = String2Uint64(optarg);
2120  manager->SetDnsTtlLimits(dns_min_ttl, dns_max_ttl);
2121 
2122  if (options_mgr_->GetValue("CVMFS_DNS_SERVER", &optarg)) {
2123  download_mgr_->SetDnsServer(optarg);
2124  }
2125 
2126  if (options_mgr_->GetValue("CVMFS_IPFAMILY_PREFER", &optarg)) {
2127  switch (String2Int64(optarg)) {
2128  case 4:
2130  break;
2131  case 6:
2133  break;
2134  }
2135  }
2136  if (options_mgr_->GetValue("CVMFS_MAX_IPADDR_PER_PROXY", &optarg))
2137  manager->SetMaxIpaddrPerProxy(String2Uint64(optarg));
2138 }
2139 
2140 
2142  string optarg;
2144  download_mgr_->Clone(perf::StatisticsTemplate("download-external",
2145  statistics_), "external");
2146 
2147  unsigned timeout;
2148  unsigned timeout_direct;
2149  download_mgr_->GetTimeout(&timeout, &timeout_direct);
2150  if (options_mgr_->GetValue("CVMFS_EXTERNAL_TIMEOUT", &optarg)) {
2151  timeout = String2Uint64(optarg);
2152  }
2153  if (options_mgr_->GetValue("CVMFS_EXTERNAL_TIMEOUT_DIRECT", &optarg)) {
2154  timeout_direct = String2Uint64(optarg);
2155  }
2156  external_download_mgr_->SetTimeout(timeout, timeout_direct);
2157 
2158  if (options_mgr_->GetValue("CVMFS_EXTERNAL_METALINK", &optarg)) {
2160  // host chain will be set later when the metalink server is contacted
2162  // metalink requires redirects
2164  } else if (options_mgr_->GetValue("CVMFS_EXTERNAL_URL", &optarg)) {
2166  if (dogeosort) {
2167  std::vector<std::string> host_chain;
2168  external_download_mgr_->GetHostInfo(&host_chain, NULL, NULL);
2169  download_mgr_->GeoSortServers(&host_chain);
2170  external_download_mgr_->SetHostChain(host_chain);
2171  }
2172  } else {
2174  }
2175 
2176  if (options_mgr_->GetValue("CVMFS_EXTERNAL_MAX_SERVERS", &optarg)) {
2177  unsigned max_servers = String2Uint64(optarg);
2178  std::vector<std::string> host_chain;
2179  external_download_mgr_->GetHostInfo(&host_chain, NULL, NULL);
2180  if (max_servers > 0 && max_servers < host_chain.size()) {
2181  host_chain.resize(max_servers);
2182  external_download_mgr_->SetHostChain(host_chain);
2183  }
2184  }
2185 
2186  string proxies = "DIRECT";
2187  if (options_mgr_->GetValue("CVMFS_EXTERNAL_HTTP_PROXY", &optarg)) {
2189  optarg,
2190  file_system_->workspace() + "/proxies-external" + GetUniqFileSuffix(),
2192  if (proxies == "") {
2193  boot_error_ = "failed to discover external HTTP proxy servers";
2195  return false;
2196  }
2197  }
2198  string fallback_proxies;
2199  if (options_mgr_->GetValue("CVMFS_EXTERNAL_FALLBACK_PROXY", &optarg))
2200  fallback_proxies = optarg;
2202  proxies, fallback_proxies, download::DownloadManager::kSetProxyBoth);
2203 
2204  return true;
2205 }
2206 
2207 
2209  string optarg;
2210 
2211  // TODO(jblomer): avoid double default settings
2212 
2213  unsigned timeout = kDefaultTimeoutSec;
2214  unsigned timeout_direct = kDefaultTimeoutSec;
2215  if (options_mgr_->GetValue("CVMFS_TIMEOUT", &optarg))
2216  timeout = String2Uint64(optarg);
2217  if (options_mgr_->GetValue("CVMFS_TIMEOUT_DIRECT", &optarg))
2218  timeout_direct = String2Uint64(optarg);
2219  download_mgr_->SetTimeout(timeout, timeout_direct);
2220 
2221  unsigned max_retries = kDefaultRetries;
2222  unsigned backoff_init = kDefaultBackoffInitMs;
2223  unsigned backoff_max = kDefaultBackoffMaxMs;
2224  if (options_mgr_->GetValue("CVMFS_MAX_RETRIES", &optarg))
2225  max_retries = String2Uint64(optarg);
2226  if (options_mgr_->GetValue("CVMFS_BACKOFF_INIT", &optarg))
2227  backoff_init = String2Uint64(optarg) * 1000;
2228  if (options_mgr_->GetValue("CVMFS_BACKOFF_MAX", &optarg))
2229  backoff_max = String2Uint64(optarg) * 1000;
2230  download_mgr_->SetRetryParameters(max_retries, backoff_init, backoff_max);
2231 
2232  if (options_mgr_->GetValue("CVMFS_LOW_SPEED_LIMIT", &optarg))
2234  if (options_mgr_->GetValue("CVMFS_PROXY_RESET_AFTER", &optarg)) {
2236  // Use the proxy reset delay as the default for the metalink reset delay
2238  }
2239  if (options_mgr_->GetValue("CVMFS_METALINK_RESET_AFTER", &optarg))
2241  if (options_mgr_->GetValue("CVMFS_HOST_RESET_AFTER", &optarg))
2243 
2244  if (options_mgr_->GetValue("CVMFS_FOLLOW_REDIRECTS", &optarg) &&
2245  options_mgr_->IsOn(optarg))
2246  {
2248  }
2249  if (options_mgr_->GetValue("CVMFS_SEND_INFO_HEADER", &optarg) &&
2250  options_mgr_->IsOn(optarg))
2251  {
2253  }
2254 }
2255 
2256 
2258  string optarg;
2259 
2260  if (file_system_->IsNfsSource()) {
2262  } else {
2264  }
2265  if (options_mgr_->GetValue("CVMFS_INITIAL_GENERATION", &optarg)) {
2267  }
2268 
2271  }
2272 }
2273 
2274 
2276  string optarg;
2277  catalog::OwnerMap uid_map;
2278  catalog::OwnerMap gid_map;
2279 
2280  if (options_mgr_->GetValue("CVMFS_UID_MAP", &optarg)) {
2281  if (!uid_map.Read(optarg)) {
2282  boot_error_ = "failed to parse uid map " + optarg;
2284  return false;
2285  }
2286  }
2287  if (options_mgr_->GetValue("CVMFS_GID_MAP", &optarg)) {
2288  if (!gid_map.Read(optarg)) {
2289  boot_error_ = "failed to parse gid map " + optarg;
2291  return false;
2292  }
2293  }
2294  catalog_mgr_->SetOwnerMaps(uid_map, gid_map);
2295 
2296  // TODO(jblomer): make local to catalog manager
2297  if (options_mgr_->GetValue("CVMFS_CLAIM_OWNERSHIP", &optarg) &&
2298  options_mgr_->IsOn(optarg))
2299  {
2300  g_claim_ownership = true;
2301  }
2302  if (options_mgr_->GetValue("CVMFS_WORLD_READABLE", &optarg) &&
2303  options_mgr_->IsOn(optarg))
2304  {
2305  g_world_readable = true;
2306  }
2307 
2308 
2309 
2310  return true;
2311 }
perf::Counter * n_eio_07_
Definition: mountpoint.h:354
IoErrorInfo io_error_info_
Definition: mountpoint.h:357
OptionsManager * options_mgr()
Definition: mountpoint.h:248
std::string exe_path()
Definition: mountpoint.h:207
static const unsigned kDnsDefaultTimeoutMs
Definition: download.h:177
bool GetVOMSAuthz(std::string *authz) const
void SetLogSyslogFacility(const int local_facility)
Definition: logging.cc:183
FileSystem * file_system_
Definition: mountpoint.h:629
Counter * Register(const std::string &name, const std::string &desc)
Definition: statistics.cc:160
void SetCounter(perf::Counter *c)
Definition: mountpoint.cc:97
void RegisterFdMapping(int from, int to)
Definition: sqlitevfs.cc:541
std::string nfs_maps_dir_
Definition: mountpoint.h:414
void SetLogSyslogLevel(const int level)
Definition: logging.cc:151
bool IsNull() const
Definition: hash.h:383
bool foreground_
Definition: mountpoint.h:331
void Activate(const int buffer_size, const int flush_threshold, const std::string &trace_file)
Definition: tracer.cc:25
download::DownloadManager * download_mgr_
Definition: repository.h:141
struct stat64 platform_stat64
perf::Counter * n_eio_08_
Definition: mountpoint.h:355
perf::Counter * n_emfile_
Definition: mountpoint.h:356
Log2Histogram * hist_fs_forget_multi_
Definition: mountpoint.h:362
perf::Counter * n_fs_statfs_cached_
Definition: mountpoint.h:340
static FileWatcher * Create()
Definition: file_watcher.cc:44
void SetupInodeAnnotation()
Definition: mountpoint.cc:2257
bool SetupCwd()
Definition: mountpoint.cc:840
catalog::InodeAnnotation * inode_annotation_
Definition: mountpoint.h:646
static NfsMapsSqlite * Create(const std::string &db_dir, const uint64_t root_inode, const bool rebuild, perf::Statistics *statistics_)
void TearDown2ReadOnly()
Definition: cache_posix.cc:613
std::string workspace_fullpath_
Definition: mountpoint.h:384
bool DetermineNfsMode()
Definition: mountpoint.cc:382
static SqliteHistory * Open(const std::string &file_name)
void SetHostChain(const std::string &host_list)
static const unsigned kTracerFlushThreshold
Definition: mountpoint.h:591
virtual uint64_t GetSize()
Definition: quota_posix.cc:777
static Publisher * Create(const SettingsPublisher &settings)
uid_t talk_socket_uid_
Definition: mountpoint.h:677
void SetMetalinkChain(const std::string &metalink_list)
void SetLowSpeedLimit(const unsigned low_speed_limit)
Definition: download.cc:2201
perf::Counter * n_fs_stat_stale_
Definition: mountpoint.h:338
Log2Histogram * hist_fs_opendir_
Definition: mountpoint.h:365
bool LockWorkspace()
Definition: mountpoint.cc:506
bool CreateTracer()
Definition: mountpoint.cc:1700
bool wait_workspace_
Definition: mountpoint.h:330
virtual bool Cleanup(const uint64_t leave_size)
Definition: quota_posix.cc:125
double kcache_timeout_sec_
Definition: mountpoint.h:664
signature::SignatureManager * signature_mgr_
Definition: mountpoint.h:641
void CreateFile(const std::string &path, const int mode, const bool ignore_failure)
Definition: posix.cc:280
perf::Counter * n_fs_lookup_
Definition: mountpoint.h:335
std::string name
Definition: history.h:88
AuthzSessionManager * authz_session_mgr_
Definition: mountpoint.h:638
static const unsigned kDefaultRetries
Definition: mountpoint.h:584
string Trim(const string &raw, bool trim_newline)
Definition: string.cc:462
string JoinStrings(const vector< string > &strings, const string &joint)
Definition: string.cc:359
perf::Statistics * statistics_
Definition: mountpoint.h:358
download::DownloadManager * download_mgr_
Definition: mountpoint.h:642
static const unsigned kDefaultTimeoutSec
Definition: mountpoint.h:583
std::string exe_path_
Definition: mountpoint.h:324
bool CheckInstanceName(const std::string &instance)
Definition: mountpoint.cc:109
perf::Statistics * statistics_
Definition: mountpoint.h:635
Log2Histogram * hist_fs_readlink_
Definition: mountpoint.h:364
static void SetupLoggingStandalone(const OptionsManager &options_mgr, const std::string &prefix)
Definition: mountpoint.cc:875
void SetProxyChain(const std::string &proxy_list, const std::string &fallback_proxy_list, const ProxySetModes set_mode)
Definition: download.cc:2812
std::string fqrn() const
Definition: mountpoint.h:514
static NfsMapsLeveldb * Create(const std::string &leveldb_dir, const uint64_t root_inode, const bool rebuild, perf::Statistics *statistics)
gid_t talk_socket_gid_
Definition: mountpoint.h:678
void SetupSqlite()
Definition: mountpoint.cc:1057
perf::Counter * n_eio_total_
Definition: mountpoint.h:347
bool IsOn(const std::string &param_value) const
Definition: options.cc:409
std::string GetUniqFileSuffix()
Definition: mountpoint.cc:1854
StatfsCache * statfs_cache_
Definition: mountpoint.h:658
download::DownloadManager * external_download_mgr_
Definition: mountpoint.h:643
MagicXattrManager * magic_xattr_mgr_
Definition: mountpoint.h:657
bool fuse_expire_entry_
Definition: mountpoint.h:668
static const size_t kDefaultBufferSize
Definition: cache_stream.h:33
CacheManager * SetupCacheMgr(const std::string &instance)
Definition: mountpoint.cc:625
const SettingsPublisher & settings() const
Definition: repository.h:317
perf::Statistics * statistics_
Definition: repository.h:139
void SetupHttpTuning()
Definition: mountpoint.cc:2208
static const char * kDefaultBlacklist
Definition: mountpoint.h:592
virtual bool AcquireQuotaManager(QuotaManager *quota_mgr)
Definition: cache_extern.cc:85
unsigned max_ttl_sec_
Definition: mountpoint.h:662
static const unsigned kDefaultMaxTtl
Definition: dns.h:182
bool CreateDownloadManagers()
Definition: mountpoint.cc:1410
std::string talk_socket_path_
Definition: mountpoint.h:676
perf::Counter * n_fs_stat_
Definition: mountpoint.h:337
perf::Counter * n_eio_05_
Definition: mountpoint.h:352
perf::Counter * no_open_files_
Definition: mountpoint.h:345
assert((mem||(size==0))&&"Out Of Memory")
void ReplaceCacheManager(CacheManager *new_cache_mgr)
Definition: sqlitevfs.cc:546
static const int kLabelHistory
Definition: cache.h:87
void SetupLogging()
Definition: mountpoint.cc:897
static void SetupGlobalEnvironmentParams()
Definition: mountpoint.cc:870
int fd_workspace_lock_
Definition: mountpoint.h:385
void CreateStatistics()
Definition: mountpoint.cc:1619
bool UnregisterVfsRdOnly()
Definition: sqlitevfs.cc:522
CacheManager * SetupRamCacheMgr(const std::string &instance)
Definition: mountpoint.cc:728
bool cache_symlinks_
Definition: mountpoint.h:667
static void LogSqliteError(void *user_data __attribute__((unused)), int sqlite_extended_error, const char *message)
Definition: mountpoint.cc:537
static void CleanupInstance()
Definition: sqlitemem.cc:170
void EnableFuseExpireEntry()
Definition: mountpoint.cc:1271
static const unsigned kNfsMaps
Definition: mountpoint.h:182
std::string workspace_
Definition: mountpoint.h:378
string StringifyTime(const time_t seconds, const bool utc)
Definition: string.cc:105
PosixCacheSettings DeterminePosixCacheSettings(const std::string &instance)
Definition: mountpoint.cc:298
void SetDnsServer(const std::string &address)
Definition: download.cc:2129
int platform_stat(const char *path, platform_stat64 *buf)
void AssignGlobalArenas()
Definition: sqlitemem.cc:137
DownloadManager * Clone(const perf::StatisticsTemplate &statistics, const std::string &cloned_name)
Definition: download.cc:3225
bool has_membership_req_
Definition: mountpoint.h:674
virtual CacheManagerIds id()=0
void SetLogMicroSyslog(const std::string &filename)
Definition: logging.cc:272
bool IsOff(const std::string &param_value) const
Definition: options.cc:416
perf::Counter * n_eio_01_
Definition: mountpoint.h:348
void ReEvaluateAuthz()
Definition: mountpoint.cc:1950
bool HasConfigRepository(const std::string &fqrn, std::string *config_path)
Definition: options.cc:256
std::set< std::string > constructed_instances_
Definition: mountpoint.h:413
bool DetermineRootHash(shash::Any *root_hash)
Definition: mountpoint.cc:1732
#define SetLogDebugFile(filename)
void SetTimeout(const unsigned seconds_proxy, const unsigned seconds_direct)
Definition: download.cc:2187
bool CreateCatalogManager()
Definition: mountpoint.cc:1353
struct cvmcache_object_info __attribute__
Definition: atomic.h:24
void RemapCatalogFd(int from, int to)
Definition: mountpoint.cc:1152
static const unsigned kNfsNone
Definition: mountpoint.h:178
std::string path_crash_guard_
Definition: mountpoint.h:391
IntegerMap< uint64_t > OwnerMap
Definition: catalog.h:41
catalog::WritableCatalogManager * catalog_mgr_
Definition: repository.h:385
bool SetupWorkspace()
Definition: mountpoint.cc:1074
static const char * kDefaultAuthzSearchPath
Definition: mountpoint.h:575
Type type()
Definition: mountpoint.h:250
FileSystem(const FileSystemInfo &fs_info)
Definition: mountpoint.cc:404
static const unsigned kLibPathCacheSize
Definition: mountpoint.h:563
void SetFqrn(const std::string &fqrn)
Definition: download.h:248
std::string boot_error_
Definition: mountpoint.h:88
OptionsManager * options_mgr_
Definition: mountpoint.h:633
Log2Histogram * hist_fs_getattr_
Definition: mountpoint.h:363
bool IsValid(const std::string &input) const
Definition: sanitizer.cc:114
Log2Histogram * hist_fs_releasedir_
Definition: mountpoint.h:366
Definition: tracer.h:35
bool FileExists(const std::string &path)
Definition: posix.cc:802
void ReplaceCacheManager(CacheManager *new_cache_mgr)
Definition: mountpoint.cc:1156
void DisableCacheSymlinks()
Definition: mountpoint.cc:1263
void SetDnsTtlLimits(const unsigned min_seconds, const unsigned max_seconds)
Definition: download.cc:2166
std::string cache_path()
Definition: cache_posix.h:107
perf::Counter * n_fs_readlink_
Definition: mountpoint.h:342
uint64_t platform_memsize()
int64_t String2Int64(const string &value)
Definition: string.cc:240
void SetMaxTtlMn(unsigned value_minutes)
Definition: mountpoint.cc:1960
string ToUpper(const string &mixed_case)
Definition: string.cc:510
static const unsigned kDefaultBackoffInitMs
Definition: mountpoint.h:585
perf::Counter * n_fs_forget_
Definition: mountpoint.h:343
MountPoint(const std::string &fqrn, FileSystem *file_system, OptionsManager *options_mgr)
Definition: mountpoint.cc:1859
void GetLimitNoFile(unsigned *soft_limit, unsigned *hard_limit)
Definition: posix.cc:1491
Tracer * tracer_
Definition: mountpoint.h:653
void SetProxyGroupResetDelay(const unsigned seconds)
Definition: download.cc:3126
static CacheManager * Create(CacheManager *upper_cache, CacheManager *lower_cache)
static bool g_alive
Definition: mountpoint.h:258
static const unsigned kDefaultNumConnections
Definition: mountpoint.h:579
void Set(const int64_t val)
Definition: statistics.h:33
void TearDown2ReadOnly()
Definition: mountpoint.cc:1138
gid_t g_gid
Definition: globals.cc:14
uuid_t uuid
Definition: uuid.h:41
static const unsigned kNfsMapsHa
Definition: mountpoint.h:186
lru::Md5PathCache * md5path_cache_
Definition: mountpoint.h:652
std::string name_
Definition: mountpoint.h:323
vector< string > SplitString(const string &str, char delim)
Definition: string.cc:308
static TelemetryAggregator * Create(Statistics *statistics, int send_rate, OptionsManager *options_mgr, MountPoint *mount_point, const std::string &fqrn, const TelemetrySelector type)
std::string Dump()
Definition: options.cc:454
static PosixCacheManager * Create(const std::string &cache_path, const bool alien_cache, const RenameWorkarounds rename_workaround=kRenameNormal, const bool do_refcount=true)
Definition: cache_posix.cc:272
bool RegisterVfsRdOnly(CacheManager *cache_mgr, perf::Statistics *statistics, const VfsOptions options)
Definition: sqlitevfs.cc:453
static AuthzSessionManager * Create(AuthzFetcher *authz_fetcher, perf::Statistics *statistics)
std::string membership_req_
Definition: mountpoint.h:673
glue::DentryTracker * dentry_tracker_
Definition: mountpoint.h:655
int Fetch(const CacheManager::LabeledObject &object, const std::string &alt_url="")
Definition: fetch.cc:81
perf::Counter * n_fs_read_
Definition: mountpoint.h:341
signature::SignatureManager * signature_mgr_
Definition: repository.h:140
perf::Counter * n_eio_06_
Definition: mountpoint.h:353
static const char * kDefaultCacheMgrInstance
Definition: mountpoint.h:262
bool HasSuffix(const std::string &str, const std::string &suffix, const bool ignore_case)
Definition: string.cc:299
virtual uint64_t GetCapacity()
Definition: quota_posix.cc:657
glue::PageCacheTracker * page_cache_tracker_
Definition: mountpoint.h:656
bool SetShardingPolicy(const ShardingPolicySelector type)
Definition: download.cc:3206
bool found_previous_crash_
Definition: mountpoint.h:397
const char kSuffixCatalog
Definition: hash.h:54
loader::Failures boot_status_
Definition: mountpoint.h:87
Failures Fetch(const std::string &base_url, const std::string &repository_name, const uint64_t minimum_timestamp, const shash::Any *base_catalog, signature::SignatureManager *signature_manager, download::DownloadManager *download_manager, ManifestEnsemble *ensemble)
bool TriageCacheMgr()
Definition: mountpoint.cc:1162
bool CheckPosixCacheSettings(const PosixCacheSettings &settings)
Definition: mountpoint.cc:126
perf::TelemetryAggregator * telemetry_aggr_
Definition: mountpoint.h:636
perf::Counter * n_fs_inode_replace_
Definition: mountpoint.h:344
bool IsNfsSource()
Definition: mountpoint.h:195
ChunkTables * chunk_tables_
Definition: mountpoint.h:648
CacheManager * cache_mgr()
Definition: mountpoint.h:205
uid_t g_uid
Definition: globals.cc:13
void SetupDnsTuning(download::DownloadManager *manager)
Definition: mountpoint.cc:2101
cvmfs::Uuid * uuid_cache_
Definition: mountpoint.h:424
unsigned GetEffectiveTtlSec()
Definition: mountpoint.cc:1832
int TryLockFile(const std::string &path)
Definition: posix.cc:924
bool MkdirDeep(const std::string &path, const mode_t mode, bool verify_writable)
Definition: posix.cc:857
int LockFile(const std::string &path)
Definition: posix.cc:982
void SetIpPreference(const dns::IpPreference preference)
Definition: download.cc:2176
static const unsigned kDefaultBackoffMaxMs
Definition: mountpoint.h:586
lru::InodeCache * inode_cache_
Definition: mountpoint.h:650
CacheManager * SetupPosixCacheMgr(const std::string &instance)
Definition: mountpoint.cc:698
download::DownloadManager * download_mgr()
Definition: mountpoint.h:505
perf::Counter * n_fs_open_
Definition: mountpoint.h:333
static uint64_t RoundUp8(const uint64_t size)
Definition: smalloc.h:37
bool SetupPosixQuotaMgr(const PosixCacheSettings &settings, CacheManager *cache_mgr)
Definition: mountpoint.cc:995
static SqliteMemoryManager * GetInstance()
Definition: sqlitemem.h:137
static PosixQuotaManager * Create(const std::string &cache_workspace, const uint64_t limit, const uint64_t cleanup_threshold, const bool rebuild_database)
Definition: quota_posix.cc:232
unsigned nfs_mode_
Definition: mountpoint.h:418
void SetRetryParameters(const unsigned max_retries, const unsigned backoff_init_ms, const unsigned backoff_max_ms)
Definition: download.cc:3154
string StringifyInt(const int64_t value)
Definition: string.cc:78
Type type_
Definition: mountpoint.h:325
static const unsigned kInodeCacheFactor
Definition: mountpoint.h:567
void SetMaxIpaddrPerProxy(unsigned limit)
Definition: download.cc:3165
void EnableIgnoreSignatureFailures()
Definition: download.cc:3190
std::string GetArch()
Definition: posix.cc:1457
void Inc(class Counter *counter)
Definition: statistics.h:50
bool HasPrefix(const string &str, const string &prefix, const bool ignore_case)
Definition: string.cc:285
perf::Counter * n_eio_04_
Definition: mountpoint.h:351
Log2Histogram * hist_fs_readdir_
Definition: mountpoint.h:367
bool GetValue(const std::string &key, std::string *value) const
Definition: options.cc:376
cvmfs::Uuid * uuid_
Definition: mountpoint.h:625
void SetExternalDownloadManager(download::DownloadManager *download_mgr)
Definition: cache_stream.h:61
OptionsManager * options_mgr_
Definition: cvmfs.cc:153
bool LoadBlacklist(const std::string &path_blacklist, bool append)
Definition: signature.cc:547
BackoffThrottle * backoff_throttle_
Definition: mountpoint.h:640
AuthzAttachment * authz_attachment_
Definition: mountpoint.h:639
void SetProxyTemplates(const std::string &direct, const std::string &forced)
Definition: download.cc:3171
SimpleChunkTables * simple_chunk_tables_
Definition: mountpoint.h:649
bool InitFixed(const shash::Any &root_hash, bool alternative_path)
perf::Counter * n_eio_03_
Definition: mountpoint.h:350
static const unsigned kDefaultNfiles
Definition: mountpoint.h:261
static FileSystem * Create(const FileSystemInfo &fs_info)
Definition: mountpoint.cc:166
bool has_custom_sqlitevfs_
Definition: mountpoint.h:435
perf::Counter * n_fs_statfs_
Definition: mountpoint.h:339
bool ReloadBlacklists()
Definition: mountpoint.cc:1239
std::string name()
Definition: mountpoint.h:234
time_t IsoTimestamp2UtcTime(const std::string &iso8601)
Definition: string.cc:216
static Uuid * Create(const std::string &store_path)
Definition: uuid.cc:23
void CreateStatistics()
Definition: mountpoint.cc:210
pthread_mutex_t lock_max_ttl_
Definition: mountpoint.h:663
void CreateAuthz()
Definition: mountpoint.cc:1327
bool g_raw_symlinks
Definition: globals.cc:12
bool g_world_readable
Definition: globals.cc:15
static const char * kDefaultCacheBase
Definition: mountpoint.h:259
void SetNull()
Definition: hash.h:400
string ResolveProxyDescription(const string &cvmfs_proxies, const std::string &path_fallback_cache, DownloadManager *download_manager)
Definition: wpad.cc:207
uint64_t String2Uint64(const string &value)
Definition: string.cc:246
static const unsigned kTracerBufferSize
Definition: mountpoint.h:590
void SetupUuid()
Definition: mountpoint.cc:1123
perf::Counter * n_eio_02_
Definition: mountpoint.h:349
catalog::ClientCatalogManager * catalog_mgr_
Definition: mountpoint.h:647
bool CreateSignatureManager()
Definition: mountpoint.cc:1586
std::string fqrn_
Definition: mountpoint.h:624
unsigned GetMaxTtlMn()
Definition: mountpoint.cc:1844
std::string error_msg() const
Definition: cache_extern.h:45
bool GetUidOf(const std::string &username, uid_t *uid, gid_t *main_gid)
Definition: posix.cc:1355
bool SetupBehavior()
Definition: mountpoint.cc:1965
static ExternalCacheManager * Create(int fd_connection, unsigned max_open_fds, const std::string &ident)
AuthzFetcher * authz_fetcher_
Definition: mountpoint.h:637
bool FetchHistory(std::string *history_path)
Definition: mountpoint.cc:1800
perf::Statistics * statistics()
Definition: mountpoint.h:249
bool enforce_acls_
Definition: mountpoint.h:666
bool IsHaNfsSource()
Definition: mountpoint.h:196
void CreateTables()
Definition: mountpoint.cc:1658
shash::Any history() const
Definition: manifest.h:135
bool SetupCrashGuard()
Definition: mountpoint.cc:819
static const unsigned kDefaultQuotaLimit
Definition: mountpoint.h:260
static PosixQuotaManager * CreateShared(const std::string &exe_path, const std::string &cache_workspace, const uint64_t limit, const uint64_t cleanup_threshold, bool foreground)
Definition: quota_posix.cc:264
void ResetErrorCounters()
Definition: mountpoint.cc:607
Definition: mutex.h:42
Log2Histogram * hist_fs_lookup_
Definition: mountpoint.h:360
shash::Any root_hash
Definition: history.h:89
bool LoadPublicRsaKeys(const std::string &path_list)
Definition: signature.cc:305
void SetRegularDownloadManager(download::DownloadManager *download_mgr)
Definition: cache_stream.h:58
void RegisterHandler(const std::string &file_path, EventHandler *handler)
Definition: file_watcher.cc:56
void GetTimeout(unsigned *seconds_proxy, unsigned *seconds_direct)
Definition: download.cc:2210
static const unsigned kDefaultMemcacheSize
Definition: mountpoint.h:571
FileSystem * file_system_
Definition: cvmfs.cc:128
std::string repository_tag_
Definition: mountpoint.h:669
static const unsigned kDefaultMinTtl
Definition: dns.h:177
bool SetupOwnerMaps()
Definition: mountpoint.cc:2275
void SetDnsParameters(const unsigned retries, const unsigned timeout_ms)
Definition: download.cc:2148
bool CreateResolvConfWatcher()
Definition: mountpoint.cc:1548
Any MkFromHexPtr(const HexPtr hex, const char suffix)
Definition: hash.cc:83
static ExternalQuotaManager * Create(ExternalCacheManager *cache_mgr)
Log2Histogram * hist_fs_read_
Definition: mountpoint.h:369
void SetCatalogWatermark(unsigned limit)
static bool g_is_enabled
Definition: algorithm.h:201
bool SetupNfsMaps()
Definition: mountpoint.cc:902
std::string path
Definition: cache.h:133
bool CheckBlacklists()
Definition: mountpoint.cc:1201
void SetMetalinkResetDelay(const unsigned seconds)
Definition: download.cc:3136
std::vector< std::string > blacklist_paths_
Definition: mountpoint.h:670
cvmfs::Fetcher * fetcher_
Definition: mountpoint.h:644
file_watcher::FileWatcher * resolv_conf_watcher_
Definition: mountpoint.h:660
static const unsigned kDnsDefaultRetries
Definition: download.h:176
bool GeoSortServers(std::vector< std::string > *servers, std::vector< uint64_t > *output_order=NULL)
Definition: download.cc:2525
bool SetupExternalDownloadMgr(bool dogeosort)
Definition: mountpoint.cc:2141
void SetOwnerMaps(const OwnerMap &uid_map, const OwnerMap &gid_map)
CacheManager * SetupExternalCacheMgr(const std::string &instance)
Definition: mountpoint.cc:659
static void CleanupInstance()
Definition: clientctx.cc:18
perf::Counter * n_fs_lookup_negative_
Definition: mountpoint.h:336
Log2Histogram * hist_fs_open_
Definition: mountpoint.h:368
virtual bool AcquireQuotaManager(QuotaManager *quota_mgr)=0
std::string cache_mgr_instance_
Definition: mountpoint.h:408
std::string path_workspace_lock_
Definition: mountpoint.h:386
virtual void IncGeneration(const uint64_t by)=0
Log2Histogram * hist_fs_release_
Definition: mountpoint.h:370
void SetLogSyslogPrefix(const std::string &prefix)
Definition: logging.cc:241
std::string MakeCanonicalPath(const std::string &path)
Definition: posix.cc:98
static const int kDefaultTelemetrySendRateSec
Definition: mountpoint.h:596
OptionsManager * options_mgr_
Definition: mountpoint.h:329
perf::Counter * no_open_dirs_
Definition: mountpoint.h:346
glue::InodeTracker * inode_tracker_
Definition: mountpoint.h:654
bool fixed_catalog_
Definition: mountpoint.h:665
static PluginHandle * CreatePlugin(const std::string &locator, const std::vector< std::string > &cmd_line)
bool IsDefined(const std::string &key)
Definition: options.cc:370
static const int kMinimumTelemetrySendRateSec
Definition: mountpoint.h:597
void GetHostInfo(std::vector< std::string > *host_chain, std::vector< int > *rtt, unsigned *current_host)
Definition: download.cc:2293
void set_membership(const std::string &m)
Definition: authz_curl.h:25
Statistics * Fork()
Definition: statistics.cc:47
CacheManager * SetupTieredCacheMgr(const std::string &instance)
Definition: mountpoint.cc:778
std::vector< std::string > FindFilesBySuffix(const std::string &dir, const std::string &suffix)
Definition: posix.cc:1135
std::string workspace()
Definition: mountpoint.h:252
bool g_claim_ownership
Definition: globals.cc:11
lru::PathCache * path_cache_
Definition: mountpoint.h:651
CacheManager * cache_mgr_
Definition: mountpoint.h:419
void AddHTTPTracingHeader(const std::string &header)
Definition: download.cc:3198
void SetCredentialsAttachment(CredentialsAttachment *ca)
Definition: download.cc:2113
static ClientCtx * GetInstance()
Definition: clientctx.cc:45
virtual bool AcquireQuotaManager(QuotaManager *quota_mgr)
Definition: cache_ram.cc:73
cvmfs::Uuid * uuid_cache()
Definition: mountpoint.h:251
cvmfs::Fetcher * external_fetcher_
Definition: mountpoint.h:645
perf::Counter * n_fs_dir_open_
Definition: mountpoint.h:334
std::string MkCacheParm(const std::string &generic_parameter, const std::string &instance)
Definition: mountpoint.cc:572
void UnlockFile(const int filedes)
Definition: posix.cc:1006
NfsMaps * nfs_maps_
Definition: mountpoint.h:430
download::DownloadManager * external_download_mgr()
Definition: mountpoint.h:506
void CreateFetchers()
Definition: mountpoint.cc:1571
void SetInodeAnnotation(InodeAnnotation *new_annotation)
virtual void SetInodeResidue(unsigned residue_class, unsigned remainder)
Definition: nfs_maps.h:31
void SetHostResetDelay(const unsigned seconds)
Definition: download.cc:3145
static MountPoint * Create(const std::string &fqrn, FileSystem *file_system, OptionsManager *options_mgr=NULL)
Definition: mountpoint.cc:1280
Log2Histogram * hist_fs_forget_
Definition: mountpoint.h:361
CVMFS_EXPORT void LogCvmfs(const LogSource source, const int mask, const char *format,...)
Definition: logging.cc:528