CernVM-FS  2.11.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
mountpoint.cc
Go to the documentation of this file.
1 
4 #include "cvmfs_config.h"
5 #include "mountpoint.h"
6 
7 #include <errno.h>
8 #include <fcntl.h>
9 #include <inttypes.h>
10 #include <stdint.h>
11 #include <unistd.h>
12 
13 #include <algorithm>
14 #include <cassert>
15 #include <climits>
16 #include <cstring>
17 #include <vector>
18 
19 #include "duplex_fuse.h" // NOLINT
20 
21 #ifndef CVMFS_LIBCVMFS
22 #ifdef FUSE_CAP_EXPORT_SUPPORT
23 #define CVMFS_NFS_SUPPORT
24 #else
25 #warning "No NFS support, Fuse too old"
26 #endif
27 #endif
28 
29 #include "authz/authz_curl.h"
30 #include "authz/authz_fetch.h"
31 #include "authz/authz_session.h"
32 #include "backoff.h"
33 #include "cache.h"
34 #include "cache_extern.h"
35 #include "cache_posix.h"
36 #include "cache_ram.h"
37 #include "cache_tiered.h"
38 #include "catalog.h"
39 #include "catalog_mgr_client.h"
40 #include "clientctx.h"
41 #include "crypto/signature.h"
42 #include "duplex_sqlite3.h"
43 #include "fetch.h"
44 #include "file_chunk.h"
45 #include "globals.h"
46 #include "glue_buffer.h"
47 #include "google/protobuf/stubs/common.h"
48 #include "history.h"
49 #include "history_sqlite.h"
50 #include "lru_md.h"
51 #include "manifest.h"
52 #include "manifest_fetch.h"
53 #include "network/download.h"
54 #include "nfs_maps.h"
55 #ifdef CVMFS_NFS_SUPPORT
56 #include "nfs_maps_leveldb.h"
57 #include "nfs_maps_sqlite.h"
58 #endif
59 #include "options.h"
60 #include "quota_posix.h"
62 #include "sqlitemem.h"
63 #include "sqlitevfs.h"
64 #include "statistics.h"
65 #include "telemetry_aggregator.h"
66 #include "tracer.h"
67 #include "util/concurrency.h"
68 #include "util/logging.h"
69 #include "util/platform.h"
70 #include "util/pointer.h"
71 #include "util/posix.h"
72 #include "util/string.h"
73 #include "util/uuid.h"
74 #include "wpad.h"
75 
76 using namespace std; // NOLINT
77 
78 
79 bool FileSystem::g_alive = false;
80 const char *FileSystem::kDefaultCacheBase = "/var/lib/cvmfs";
81 const char *FileSystem::kDefaultCacheMgrInstance = "default";
82 
83 
84 FileSystem::IoErrorInfo::IoErrorInfo() : counter_(NULL), timestamp_last_(0) { }
85 
87  counter_->Set(0);
88  timestamp_last_ = 0;
89 }
90 
92  perf::Inc(counter_);
93  timestamp_last_ = time(NULL);
94 }
95 
97 
98 int64_t FileSystem::IoErrorInfo::count() { return counter_->Get(); }
99 
100 time_t FileSystem::IoErrorInfo::timestamp_last() { return timestamp_last_; }
101 
102 
103 
108 bool FileSystem::CheckInstanceName(const std::string &instance) {
109  if (instance.length() > 24)
110  return false;
111  sanitizer::CacheInstanceSanitizer instance_sanitizer;
112  if (!instance_sanitizer.IsValid(instance)) {
113  boot_error_ = "invalid instance name (" + instance + "), " +
114  "only characters a-z, A-Z, 0-9, _ are allowed";
116  return false;
117  }
118  return true;
119 }
120 
121 
127 {
128  if (settings.is_alien && settings.is_shared) {
129  boot_error_ = "Failure: shared local disk cache and alien cache mutually "
130  "exclusive. Please turn off shared local disk cache.";
132  return false;
133  }
134  if (settings.is_alien && settings.is_managed) {
135  boot_error_ = "Failure: quota management and alien cache mutually "
136  "exclusive. Please turn off quota limit.";
138  return false;
139  }
140 
141  if (type_ == kFsLibrary) {
142  if (settings.is_shared || settings.is_managed) {
143  boot_error_ = "Failure: libcvmfs supports only unmanaged exclusive cache "
144  "or alien cache.";
146  return false;
147  }
148  }
149 
150  if (settings.cache_base_defined && settings.cache_dir_defined) {
151  boot_error_ =
152  "'CVMFS_CACHE_BASE' and 'CVMFS_CACHE_DIR' are mutually exclusive";
154  return false;
155  }
156 
157  return true;
158 }
159 
160 
167  file_system(new FileSystem(fs_info));
168 
169  file_system->SetupGlobalEnvironmentParams();
170 
171  file_system->SetupLogging();
172  LogCvmfs(kLogCvmfs, kLogDebug, "Options:\n%s",
173  file_system->options_mgr()->Dump().c_str());
174 
175  file_system->CreateStatistics();
176  file_system->SetupSqlite();
177  if (!file_system->DetermineNfsMode())
178  return file_system.Release();
179  if (!file_system->SetupWorkspace())
180  return file_system.Release();
181 
182  // Redirect SQlite temp directory to workspace (global variable)
183  unsigned length_tempdir = file_system->workspace_.length() + 1;
184  sqlite3_temp_directory = static_cast<char *>(sqlite3_malloc(length_tempdir));
185  snprintf(sqlite3_temp_directory,
186  length_tempdir,
187  "%s",
188  file_system->workspace_.c_str());
189 
190  if (!file_system->TriageCacheMgr())
191  return file_system.Release();
192  file_system->SetupUuid();
193  if (!file_system->SetupNfsMaps())
194  return file_system.Release();
195  bool retval = sqlite::RegisterVfsRdOnly(
196  file_system->cache_mgr_,
197  file_system->statistics_,
199  assert(retval);
200  file_system->has_custom_sqlitevfs_ = true;
201 
203 
204  file_system->boot_status_ = loader::kFailOk;
205  return file_system.Release();
206 }
207 
208 
211 
212  // Register the ShortString's static counters
213  statistics_->Register("pathstring.n_instances", "Number of instances");
214  statistics_->Register("pathstring.n_overflows", "Number of overflows");
215  statistics_->Register("namestring.n_instances", "Number of instances");
216  statistics_->Register("namestring.n_overflows", "Number of overflows");
217  statistics_->Register("linkstring.n_instances", "Number of instances");
218  statistics_->Register("linkstring.n_overflows", "Number of overflows");
219 
220  // Callback counters
221  n_fs_open_ = statistics_->Register("cvmfs.n_fs_open",
222  "Overall number of file open operations");
223  n_fs_dir_open_ = statistics_->Register("cvmfs.n_fs_dir_open",
224  "Overall number of directory open operations");
225  n_fs_lookup_ = statistics_->Register("cvmfs.n_fs_lookup",
226  "Number of lookups");
227  n_fs_lookup_negative_ = statistics_->Register("cvmfs.n_fs_lookup_negative",
228  "Number of negative lookups");
229  n_fs_stat_ = statistics_->Register("cvmfs.n_fs_stat", "Number of stats");
230  n_fs_stat_stale_ = statistics_->Register("cvmfs.n_fs_stat_stale",
231  "Number of stats for stale (open, meanwhile changed) regular files");
232  n_fs_statfs_ = statistics_->Register("cvmfs.n_fs_statfs",
233  "Overall number of statsfs calls");
234  n_fs_statfs_cached_ = statistics_->Register("cvmfs.n_fs_statfs_cached",
235  "Number of statsfs calls that accessed the cached statfs info");
236  n_fs_read_ = statistics_->Register("cvmfs.n_fs_read", "Number of files read");
237  n_fs_readlink_ = statistics_->Register("cvmfs.n_fs_readlink",
238  "Number of links read");
239  n_fs_forget_ = statistics_->Register("cvmfs.n_fs_forget",
240  "Number of inode forgets");
241  n_fs_inode_replace_ = statistics_->Register("cvmfs.n_fs_inode_replace",
242  "Number of stale inodes that got replaced by an up-to-date version");
243  no_open_files_ = statistics_->Register("cvmfs.no_open_files",
244  "Number of currently opened files");
245  no_open_dirs_ = statistics_->Register("cvmfs.no_open_dirs",
246  "Number of currently opened directories");
247  io_error_info_.SetCounter(statistics_->Register("cvmfs.n_io_error",
248  "Number of I/O errors"));
249  n_eio_total_ = statistics_->Register("eio.total",
250  "EIO returned to calling process. Sum of individual eio counters");
251  n_eio_01_ = statistics_->Register("eio.01",
252  "EIO returned to calling process. cvmfs.cc:cvmfs_lookup()");
253  n_eio_02_ = statistics_->Register("eio.02",
254  "EIO returned to calling process. cvmfs.cc:ReplyNegative()");
255  n_eio_03_ = statistics_->Register("eio.03",
256  "EIO returned to calling process. cvmfs.cc:cvmfs_opendir()");
257  n_eio_04_ = statistics_->Register("eio.04",
258  "EIO returned to calling process. cvmfs.cc:cvmfs_open()");
259  n_eio_05_ = statistics_->Register("eio.05",
260  "EIO returned to calling process. cvmfs.cc:cvmfs_read()");
261  n_eio_06_ = statistics_->Register("eio.06",
262  "EIO returned to calling process. cvmfs.cc:cvmfs_open()");
263  n_eio_07_ = statistics_->Register("eio.07",
264  "EIO returned to calling process. cvmfs.cc:cvmfs_read()");
265  n_eio_08_ = statistics_->Register("eio.08",
266  "EIO returned to calling process. cvmfs.cc:cvmfs_read()");
267 
268  string optarg;
269  if (options_mgr_->GetValue("CVMFS_INSTRUMENT_FUSE", &optarg) &&
270  options_mgr_->IsOn(optarg))
271  {
273  }
274 
275  hist_fs_lookup_ = new Log2Histogram(30);
276  hist_fs_forget_ = new Log2Histogram(30);
283  hist_fs_open_ = new Log2Histogram(30);
284  hist_fs_read_ = new Log2Histogram(30);
286 }
287 
288 
294  const string &instance
295 ) {
296  string optarg;
298 
299  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SHARED", instance),
300  &optarg)
301  && options_mgr_->IsOn(optarg))
302  {
303  settings.is_shared = true;
304  }
305  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SERVER_MODE", instance),
306  &optarg)
307  && options_mgr_->IsOn(optarg))
308  {
309  settings.avoid_rename = true;
310  }
311 
312  if (type_ == kFsFuse)
313  settings.quota_limit = kDefaultQuotaLimit;
314  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_QUOTA_LIMIT", instance),
315  &optarg))
316  {
317  settings.quota_limit = String2Int64(optarg) * 1024 * 1024;
318  }
319  if (settings.quota_limit > 0)
320  settings.is_managed = true;
321 
322  settings.cache_path = kDefaultCacheBase;
323  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_BASE", instance),
324  &optarg))
325  {
326  settings.cache_path = MakeCanonicalPath(optarg);
327  settings.cache_base_defined = true;
328  }
329  if (settings.is_shared) {
330  settings.cache_path += "/shared";
331  } else {
332  settings.cache_path += "/" + name_;
333  }
334 
335  // CheckCacheMode makes sure that CVMFS_CACHE_DIR and CVMFS_CACHE_BASE are
336  // not set at the same time.
337  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_DIR", instance),
338  &optarg))
339  {
340  settings.cache_dir_defined = true;
341  settings.cache_path = optarg;
342  }
343  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_ALIEN", instance),
344  &optarg))
345  {
346  settings.is_alien = true;
347  settings.cache_path = optarg;
348  }
349  // We already changed the cwd to the workspace
350  // Which is only done if using FUSE
351  if ((type_ == kFsFuse) && (settings.cache_path == workspace_fullpath_)) {
352  settings.cache_path = ".";
353  }
354 
355  // The cache workspace usually is the cache directory, unless explicitly
356  // set otherwise
357  settings.workspace = settings.cache_path;
358  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_WORKSPACE", instance),
359  &optarg) ||
360  options_mgr_->GetValue("CVMFS_WORKSPACE", &optarg))
361  {
362  // Used for the shared quota manager
363  settings.workspace = optarg;
364  }
365 
366  return settings;
367 }
368 
369 
371  string optarg;
372 
373  if (options_mgr_->GetValue("CVMFS_NFS_SOURCE", &optarg) &&
374  options_mgr_->IsOn(optarg))
375  {
376  nfs_mode_ |= kNfsMaps;
377  if (options_mgr_->GetValue("CVMFS_NFS_SHARED", &optarg)) {
379  nfs_maps_dir_ = optarg;
380  }
381  }
382 
383  if ((type_ == kFsLibrary) && (nfs_mode_ != kNfsNone)) {
384  boot_error_ = "Failure: libcvmfs does not support NFS export.";
386  return false;
387  }
388  return true;
389 }
390 
391 
393  : name_(fs_info.name)
394  , exe_path_(fs_info.exe_path)
395  , type_(fs_info.type)
396  , options_mgr_(fs_info.options_mgr)
397  , wait_workspace_(fs_info.wait_workspace)
398  , foreground_(fs_info.foreground)
399  , n_fs_open_(NULL)
400  , n_fs_dir_open_(NULL)
401  , n_fs_lookup_(NULL)
402  , n_fs_lookup_negative_(NULL)
403  , n_fs_stat_(NULL)
404  , n_fs_stat_stale_(NULL)
405  , n_fs_statfs_(NULL)
406  , n_fs_statfs_cached_(NULL)
407  , n_fs_read_(NULL)
408  , n_fs_readlink_(NULL)
409  , n_fs_forget_(NULL)
410  , n_fs_inode_replace_(NULL)
411  , no_open_files_(NULL)
412  , no_open_dirs_(NULL)
413  , n_eio_total_(NULL)
414  , n_eio_01_(NULL)
415  , n_eio_02_(NULL)
416  , n_eio_03_(NULL)
417  , n_eio_04_(NULL)
418  , n_eio_05_(NULL)
419  , n_eio_06_(NULL)
420  , n_eio_07_(NULL)
421  , n_eio_08_(NULL)
422  , statistics_(NULL)
423  , fd_workspace_lock_(-1)
424  , found_previous_crash_(false)
426  , cache_mgr_(NULL)
427  , uuid_cache_(NULL)
428  , nfs_maps_(NULL)
429  , has_custom_sqlitevfs_(false)
430 {
431  assert(!g_alive);
432  g_alive = true;
433  g_uid = geteuid();
434  g_gid = getegid();
435 
436  string optarg;
437  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SERVER_MODE",
439  &optarg)
440  && options_mgr_->IsOn(optarg))
441  {
442  g_raw_symlinks = true;
443  }
444 }
445 
446 
449 
452 
453  delete uuid_cache_;
454  delete nfs_maps_;
455  delete cache_mgr_;
456 
457  if (sqlite3_temp_directory) {
458  sqlite3_free(sqlite3_temp_directory);
459  sqlite3_temp_directory = NULL;
460  }
461 
462  if (!path_crash_guard_.empty())
463  unlink(path_crash_guard_.c_str());
464  if (!path_workspace_lock_.empty())
465  unlink(path_workspace_lock_.c_str());
466  if (fd_workspace_lock_ >= 0)
468 
469  sqlite3_shutdown();
471 
472  delete hist_fs_lookup_;
473  delete hist_fs_forget_multi_;
474  delete hist_fs_forget_;
475  delete hist_fs_getattr_;
476  delete hist_fs_readlink_;
477  delete hist_fs_opendir_;
478  delete hist_fs_releasedir_;
479  delete hist_fs_readdir_;
480  delete hist_fs_open_;
481  delete hist_fs_read_;
482  delete hist_fs_release_;
483  delete statistics_;
484 
485  SetLogSyslogPrefix("");
486  SetLogMicroSyslog("");
487  SetLogDebugFile("");
488  google::protobuf::ShutdownProtobufLibrary();
489  g_alive = false;
490 }
491 
492 
494  path_workspace_lock_ = workspace_ + "/lock." + name_;
496  if (fd_workspace_lock_ >= 0)
497  return true;
498 
499  if (fd_workspace_lock_ == -1) {
500  boot_error_ = "could not acquire workspace lock (" +
501  StringifyInt(errno) + ")";
503  return false;
504  }
505 
506  assert(fd_workspace_lock_ == -2);
507 
508  if (!wait_workspace_) {
510  return false;
511  }
512 
514  if (fd_workspace_lock_ < 0) {
515  boot_error_ = "could not acquire workspace lock (" +
516  StringifyInt(errno) + ")";
518  return false;
519  }
520  return true;
521 }
522 
523 
525  void *user_data __attribute__((unused)),
526  int sqlite_extended_error,
527  const char *message)
528 {
529  int log_dest = kLogDebug;
530  int sqlite_error = sqlite_extended_error & 0xFF;
531  switch (sqlite_error) {
532  case SQLITE_INTERNAL:
533  case SQLITE_PERM:
534  case SQLITE_NOMEM:
535  case SQLITE_IOERR:
536  case SQLITE_CORRUPT:
537  case SQLITE_FULL:
538  case SQLITE_CANTOPEN:
539  case SQLITE_MISUSE:
540  case SQLITE_FORMAT:
541  case SQLITE_NOTADB:
542  log_dest |= kLogSyslogErr;
543  break;
544  case SQLITE_WARNING:
545  case SQLITE_NOTICE:
546  default:
547  break;
548  }
549  LogCvmfs(kLogCvmfs, log_dest, "SQlite3: %s (%d)",
550  message, sqlite_extended_error);
551 }
552 
553 
560  const string &generic_parameter,
561  const string &instance)
562 {
563  assert(HasPrefix(generic_parameter, "CVMFS_CACHE_", false));
564 
565  if (instance == kDefaultCacheMgrInstance) {
566  // Compatibility parameter names
567  if ((generic_parameter == "CVMFS_CACHE_SHARED") &&
568  !options_mgr_->IsDefined(generic_parameter))
569  {
570  return "CVMFS_SHARED_CACHE";
571  }
572  if ((generic_parameter == "CVMFS_CACHE_ALIEN") &&
573  !options_mgr_->IsDefined(generic_parameter))
574  {
575  return "CVMFS_ALIEN_CACHE";
576  }
577  if ((generic_parameter == "CVMFS_CACHE_SERVER_MODE") &&
578  !options_mgr_->IsDefined(generic_parameter))
579  {
580  return "CVMFS_SERVER_CACHE_MODE";
581  }
582  if ((generic_parameter == "CVMFS_CACHE_QUOTA_LIMIT") &&
583  !options_mgr_->IsDefined(generic_parameter))
584  {
585  return "CVMFS_QUOTA_LIMIT";
586  }
587  return generic_parameter;
588  }
589 
590  return "CVMFS_CACHE_" + instance + "_" + generic_parameter.substr(12);
591 }
592 
593 
596  n_eio_total_->Set(0);
597  n_eio_01_->Set(0);
598  n_eio_02_->Set(0);
599  n_eio_03_->Set(0);
600  n_eio_04_->Set(0);
601  n_eio_05_->Set(0);
602  n_eio_06_->Set(0);
603  n_eio_07_->Set(0);
604  n_eio_08_->Set(0);
605 }
606 
607 
611 CacheManager *FileSystem::SetupCacheMgr(const string &instance) {
612  if (constructed_instances_.find(instance) != constructed_instances_.end()) {
613  boot_error_ = "circular cache definition: " + instance;
615  return NULL;
616  }
617  constructed_instances_.insert(instance);
618 
619  LogCvmfs(kLogCvmfs, kLogDebug, "setting up cache manager instance %s",
620  instance.c_str());
621  string instance_type;
622  if (instance == kDefaultCacheMgrInstance) {
623  instance_type = "posix";
624  } else {
625  options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_TYPE", instance),
626  &instance_type);
627  }
628  if (instance_type == "posix") {
629  return SetupPosixCacheMgr(instance);
630  } else if (instance_type == "ram") {
631  return SetupRamCacheMgr(instance);
632  } else if (instance_type == "tiered") {
633  return SetupTieredCacheMgr(instance);
634  } else if (instance_type == "external") {
635  return SetupExternalCacheMgr(instance);
636  } else {
637  boot_error_ = "invalid cache manager type for '" + instance + "':" +
638  instance_type;
640  return NULL;
641  }
642 }
643 
644 
646  string optarg;
647  unsigned nfiles = kDefaultNfiles;
648  if (options_mgr_->GetValue("CVMFS_NFILES", &optarg))
649  nfiles = String2Uint64(optarg);
650  vector<string> cmd_line;
651  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_CMDLINE", instance),
652  &optarg))
653  {
654  cmd_line = SplitString(optarg, ',');
655  }
656 
657  if (!options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_LOCATOR", instance),
658  &optarg))
659  {
660  boot_error_ = MkCacheParm("CVMFS_CACHE_LOCATOR", instance) + " missing";
662  return NULL;
663  }
664 
666  ExternalCacheManager::CreatePlugin(optarg, cmd_line));
667  if (!plugin_handle->IsValid()) {
668  boot_error_ = plugin_handle->error_msg();
670  return NULL;
671  }
673  plugin_handle->fd_connection(), nfiles, name_ + ":" + instance);
674  if (cache_mgr == NULL) {
675  boot_error_ = "failed to create external cache manager for " + instance;
677  return NULL;
678  }
679  cache_mgr->AcquireQuotaManager(ExternalQuotaManager::Create(cache_mgr));
680  return cache_mgr;
681 }
682 
683 
686  if (!CheckPosixCacheSettings(settings))
687  return NULL;
688 
690  settings.cache_path,
691  settings.is_alien,
694  if (!cache_mgr.IsValid()) {
695  boot_error_ = "Failed to setup posix cache '" + instance + "' in " +
696  settings.cache_path + ": " + strerror(errno);
698  return NULL;
699  }
700 
701  // Sentinel file for future use
702  // Might be a read-only cache
703  const bool ignore_failure = settings.is_alien;
704  CreateFile(settings.cache_path + "/.cvmfscache", 0600, ignore_failure);
705 
706  if (settings.is_managed) {
707  if (!SetupPosixQuotaMgr(settings, cache_mgr.weak_ref()))
708  return NULL;
709  }
710  return cache_mgr.Release();
711 }
712 
713 
714 CacheManager *FileSystem::SetupRamCacheMgr(const string &instance) {
715  string optarg;
716  unsigned nfiles = kDefaultNfiles;
717  if (options_mgr_->GetValue("CVMFS_NFILES", &optarg)) {
718  nfiles = String2Uint64(optarg);
719  }
720  uint64_t sz_cache_bytes;
721  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SIZE", instance),
722  &optarg))
723  {
724  if (HasSuffix(optarg, "%", false)) {
725  sz_cache_bytes = platform_memsize() * String2Uint64(optarg)/100;
726  } else {
727  sz_cache_bytes = String2Uint64(optarg) * 1024 * 1024;
728  }
729  } else {
730  sz_cache_bytes = platform_memsize() >> 5; // ~3%
731  }
733  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_MALLOC", instance),
734  &optarg))
735  {
736  if (optarg == "libc") {
738  } else if (optarg == "heap") {
740  } else {
741  boot_error_ = "Failure: unknown malloc " +
742  MkCacheParm("CVMFS_CACHE_MALLOC", instance) + "=" + optarg;
744  return NULL;
745  }
746  }
747  sz_cache_bytes = RoundUp8(std::max(static_cast<uint64_t>(40 * 1024 * 1024),
748  sz_cache_bytes));
750  sz_cache_bytes,
751  nfiles,
752  alloc,
753  perf::StatisticsTemplate("cache." + instance, statistics_));
754  if (cache_mgr == NULL) {
755  boot_error_ = "failed to create ram cache manager for " + instance;
757  return NULL;
758  }
759  cache_mgr->AcquireQuotaManager(new NoopQuotaManager());
760  return cache_mgr;
761 }
762 
763 
765  string optarg;
766  if (!options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_UPPER", instance),
767  &optarg))
768  {
769  boot_error_ = MkCacheParm("CVMFS_CACHE_UPPER", instance) + " missing";
771  return NULL;
772  }
773  UniquePtr<CacheManager> upper(SetupCacheMgr(optarg));
774  if (!upper.IsValid())
775  return NULL;
776 
777  if (!options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_LOWER", instance),
778  &optarg))
779  {
780  boot_error_ = MkCacheParm("CVMFS_CACHE_LOWER", instance) + " missing";
782  return NULL;
783  }
784  UniquePtr<CacheManager> lower(SetupCacheMgr(optarg));
785  if (!lower.IsValid())
786  return NULL;
787 
788  CacheManager *tiered =
789  TieredCacheManager::Create(upper.Release(), lower.Release());
790  if (tiered == NULL) {
791  boot_error_ = "Failed to setup tiered cache manager " + instance;
793  return NULL;
794  }
795  if (options_mgr_->GetValue(
796  MkCacheParm("CVMFS_CACHE_LOWER_READONLY", instance), &optarg) &&
797  options_mgr_->IsOn(optarg))
798  {
799  static_cast<TieredCacheManager*>(tiered)->SetLowerReadOnly();
800  }
801  return tiered;
802 }
803 
804 
806  path_crash_guard_ = workspace_ + "/running." + name_;
807  platform_stat64 info;
808  int retval = platform_stat(path_crash_guard_.c_str(), &info);
809  if (retval == 0) {
810  found_previous_crash_ = true;
812  "looks like cvmfs has been crashed previously");
813  }
814  retval = open(path_crash_guard_.c_str(), O_RDONLY | O_CREAT, 0600);
815  if (retval < 0) {
816  boot_error_ = "could not open running sentinel (" +
817  StringifyInt(errno) + ")";
819  return false;
820  }
821  close(retval);
822  return true;
823 }
824 
825 
827  if (type_ == kFsFuse) {
828  // Try to jump to workspace / cache directory. This tests, if it is
829  // accessible and it brings speed later on.
830  int retval = chdir(workspace_.c_str());
831  if (retval != 0) {
832  boot_error_ = "workspace " + workspace_ + " is unavailable";
834  return false;
835  }
836  workspace_ = ".";
837  return true;
838  }
839 
840  // Note: as of version 2.4 support for CVMFS_CWD_CACHE is dropped due to
841  // disproportionate large complexity to configure correctly. This affects
842  // only libcvmfs, mostly the legacy part.
843  // string optarg;
844  // if (options_mgr_->GetValue("CVMFS_CWD_CACHE", &optarg) &&
845  // options_mgr_->IsOn(optarg))
846  // {
847  // ...
848  // }
849  return true;
850 }
851 
852 
857  setenv("CVMFS_ARCH", GetArch().c_str(), 1 /* overwrite */);
858 }
859 
860 
862  const OptionsManager &options_mgr, const std::string &prefix)
863 {
865 
866  string optarg;
867  if (options_mgr.GetValue("CVMFS_SYSLOG_LEVEL", &optarg))
869  if (options_mgr.GetValue("CVMFS_SYSLOG_FACILITY", &optarg))
871  if (options_mgr.GetValue("CVMFS_USYSLOG", &optarg))
872  SetLogMicroSyslog(optarg);
873  if (options_mgr.GetValue("CVMFS_DEBUGLOG", &optarg))
874  SetLogDebugFile(optarg);
875  if (options_mgr.GetValue("CVMFS_SYSLOG_PREFIX", &optarg)) {
876  SetLogSyslogPrefix(optarg);
877  } else {
878  SetLogSyslogPrefix(prefix);
879  }
880 }
881 
882 
885 }
886 
887 
889 #ifdef CVMFS_NFS_SUPPORT
890  if (!IsHaNfsSource())
892 
893  string no_nfs_sentinel;
894  if (cache_mgr_->id() == kPosixCacheManager) {
895  PosixCacheManager *posix_cache_mgr =
896  reinterpret_cast<PosixCacheManager *>(cache_mgr_);
897  no_nfs_sentinel = posix_cache_mgr->cache_path() + "/no_nfs_maps." + name_;
898  if (!IsNfsSource()) {
899  // Might be a read-only cache
900  const bool ignore_failure = posix_cache_mgr->alien_cache();
901  CreateFile(no_nfs_sentinel, 0600, ignore_failure);
902  return true;
903  }
904  } else {
905  if (IsNfsSource()) {
906  boot_error_ = "NFS source only works with POSIX cache manager.";
908  return false;
909  }
910  return true;
911  }
912 
914  assert(IsNfsSource());
915  if (!no_nfs_sentinel.empty() && FileExists(no_nfs_sentinel)) {
916  boot_error_ = "Cache was used without NFS maps before. "
917  "It has to be wiped out.";
919  return false;
920  }
921 
922  // nfs maps need to be protected by workspace lock
923  PosixCacheManager *posix_cache_mgr =
924  reinterpret_cast<PosixCacheManager *>(cache_mgr_);
925  if (posix_cache_mgr->cache_path() != workspace_) {
926  boot_error_ = "Cache directory and workspace must be identical for "
927  "NFS export";
929  return false;
930  }
931 
932  string inode_cache_dir = nfs_maps_dir_ + "/nfs_maps." + name_;
933  if (!MkdirDeep(inode_cache_dir, 0700)) {
934  boot_error_ = "Failed to initialize NFS maps";
936  return false;
937  }
938 
939  // TODO(jblomer): make this a manager class
940  if (IsHaNfsSource()) {
942  inode_cache_dir,
945  statistics_);
946  } else {
948  inode_cache_dir,
951  statistics_);
952  }
953 
954  if (nfs_maps_ == NULL) {
955  boot_error_ = "Failed to initialize NFS maps";
957  return false;
958  }
959 
960  string optarg;
961  if (options_mgr_->GetValue("CVMFS_NFS_INTERLEAVED_INODES", &optarg)) {
962  vector<string> tokens = SplitString(optarg, '%');
963  if (tokens.size() != 2) {
964  boot_error_ =
965  "invalid format for CVMFS_NFS_INTERLEAVED_INODES: " + optarg;
967  return false;
968  }
970  String2Uint64(tokens[0]));
971  }
972 
973  return true;
974 
975 #else
976  return true;
977 #endif
978 }
979 
980 
983  CacheManager *cache_mgr
984 ) {
985  assert(settings.quota_limit >= 0);
986  int64_t quota_threshold = settings.quota_limit / 2;
987  string cache_workspace = settings.cache_path;
988  if (settings.cache_path != settings.workspace) {
990  "using workspace %s to protect cache database in %s",
991  settings.workspace.c_str(), settings.cache_path.c_str());
992  cache_workspace += ":" + settings.workspace;
993  }
994  PosixQuotaManager *quota_mgr;
995 
996  if (settings.is_shared) {
998  exe_path_,
999  cache_workspace,
1000  settings.quota_limit,
1001  quota_threshold,
1002  foreground_);
1003  if (quota_mgr == NULL) {
1004  boot_error_ = "Failed to initialize shared lru cache";
1006  return false;
1007  }
1008  } else {
1009  quota_mgr = PosixQuotaManager::Create(
1010  cache_workspace,
1011  settings.quota_limit,
1012  quota_threshold,
1014  if (quota_mgr == NULL) {
1015  boot_error_ = "Failed to initialize lru cache";
1017  return false;
1018  }
1019  }
1020 
1021  if (quota_mgr->GetSize() > quota_mgr->GetCapacity()) {
1023  "cache is already beyond quota size "
1024  "(size: %" PRId64 ", capacity: %" PRId64 "), cleaning up",
1025  quota_mgr->GetSize(), quota_mgr->GetCapacity());
1026  if (!quota_mgr->Cleanup(quota_threshold)) {
1027  delete quota_mgr;
1028  boot_error_ = "Failed to clean up cache";
1030  return false;
1031  }
1032  }
1033 
1034  int retval = cache_mgr->AcquireQuotaManager(quota_mgr);
1035  assert(retval);
1037  "CernVM-FS: quota initialized, current size %luMB",
1038  quota_mgr->GetSize() / (1024 * 1024));
1039  return true;
1040 }
1041 
1042 
1044  // Make sure SQlite starts clean after initialization
1045  sqlite3_shutdown();
1046 
1047  int retval;
1048  retval = sqlite3_config(SQLITE_CONFIG_LOG, FileSystem::LogSqliteError, NULL);
1049  assert(retval == SQLITE_OK);
1050  retval = sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
1051  assert(retval == SQLITE_OK);
1053 
1054  // Disable SQlite3 file locking
1055  retval = sqlite3_vfs_register(sqlite3_vfs_find("unix-none"), 1);
1056  assert(retval == SQLITE_OK);
1057 }
1058 
1059 
1061  string optarg;
1062  // This is very similar to "determine cache dir". It's for backward
1063  // compatibility with classic cache configuration where there was no
1064  // distinction between workspace and cache.
1065  // Complicated cache configurations should explicitly set CVMFS_WORKSPACE.
1067  if (options_mgr_->GetValue("CVMFS_CACHE_BASE", &optarg))
1068  workspace_ = MakeCanonicalPath(optarg);
1069  if (options_mgr_->GetValue("CVMFS_SHARED_CACHE", &optarg) &&
1070  options_mgr_->IsOn(optarg))
1071  {
1072  workspace_ += "/shared";
1073  } else {
1074  workspace_ += "/" + name_;
1075  }
1076  if (options_mgr_->GetValue("CVMFS_CACHE_DIR", &optarg)) {
1077  if (options_mgr_->IsDefined("CVMFS_CACHE_BASE")) {
1078  boot_error_ =
1079  "'CVMFS_CACHE_BASE' and 'CVMFS_CACHE_DIR' are mutually exclusive";
1081  return false;
1082  }
1083  workspace_ = optarg;
1084  }
1085  if (options_mgr_->GetValue("CVMFS_WORKSPACE", &optarg))
1086  workspace_ = optarg;
1088 
1089  // If workspace and alien cache are the same directory, we need to open
1090  // permission now to 0770 to avoid a race when fixing it later
1091  const int mode = 0770;
1092  if (!MkdirDeep(workspace_, mode, false)) {
1093  boot_error_ = "cannot create workspace directory " + workspace_;
1095  return false;
1096  }
1097 
1098  if (!LockWorkspace())
1099  return false;
1100  if (!SetupCwd())
1101  return false;
1102  if (!SetupCrashGuard())
1103  return false;
1104 
1105  return true;
1106 }
1107 
1108 
1111  if (uuid_cache_ == NULL) {
1113  "failed to load/store %s/uuid", workspace_.c_str());
1115  assert(uuid_cache_ != NULL);
1116  }
1117 }
1118 
1119 
1125  if ((cache_mgr_ != NULL) && (cache_mgr_->id() == kPosixCacheManager)) {
1126  PosixCacheManager *posix_cache_mgr =
1127  reinterpret_cast<PosixCacheManager *>(cache_mgr_);
1128  posix_cache_mgr->TearDown2ReadOnly();
1129  }
1130 
1131  unlink(path_crash_guard_.c_str());
1132  LogCvmfs(kLogCache, kLogSyslog, "switch to read-only cache mode");
1133  SetLogMicroSyslog("");
1134 }
1135 
1136 
1137 void FileSystem::RemapCatalogFd(int from, int to) {
1138  sqlite::RegisterFdMapping(from, to);
1139 }
1140 
1141 
1144  string instance;
1145  if (options_mgr_->GetValue("CVMFS_CACHE_PRIMARY", &instance) &&
1146  !instance.empty())
1147  {
1148  if (!CheckInstanceName(instance))
1149  return false;
1150  cache_mgr_instance_ = instance;
1151  }
1152 
1154  return cache_mgr_ != NULL;
1155 }
1156 
1157 
1158 //------------------------------------------------------------------------------
1159 
1160 
1161 const char *MountPoint::kDefaultAuthzSearchPath = "/usr/libexec/cvmfs/authz";
1162 const char *MountPoint::kDefaultBlacklist = "/etc/cvmfs/blacklist";
1163 
1165  blacklist_paths_.clear();
1166  string blacklist;
1167  if (!options_mgr_->GetValue("CVMFS_BLACKLIST", &blacklist))
1168  blacklist = kDefaultBlacklist;
1169  blacklist_paths_.push_back(blacklist);
1170 
1171  bool append = false;
1172  if (FileExists(blacklist)) {
1173  if (!signature_mgr_->LoadBlacklist(blacklist, append)) {
1174  boot_error_ = "failed to load blacklist " + blacklist;
1176  return false;
1177  }
1178  append = true;
1179  }
1180 
1181  string config_repository_path;
1182  if (options_mgr_->HasConfigRepository(fqrn_, &config_repository_path)) {
1183  blacklist = config_repository_path + "blacklist";
1184  blacklist_paths_.push_back(blacklist);
1185  if (FileExists(blacklist)) {
1186  if (!signature_mgr_->LoadBlacklist(blacklist, append)) {
1187  boot_error_ = "failed to load blacklist from config repository";
1189  return false;
1190  }
1191  }
1192  }
1193 
1194  return true;
1195 }
1196 
1197 
1203  bool result = true;
1204  bool append = false;
1205  for (unsigned i = 0; i < blacklist_paths_.size(); ++i) {
1206  string blacklist = blacklist_paths_[i];
1207  if (FileExists(blacklist)) {
1208  bool retval = signature_mgr_->LoadBlacklist(blacklist, append);
1209  // TODO(jblomer): this can leave us with a half-loaded blacklist
1210  if (!retval)
1211  return false;
1212  append = true;
1213  }
1214  }
1215  return result;
1216 }
1217 
1227  cache_symlinks_ = false;
1228 }
1229 
1235  fuse_expire_entry_ = true;
1236 }
1237 
1238 
1244  const string &fqrn,
1245  FileSystem *file_system,
1246  OptionsManager *options_mgr)
1247 {
1248  if (options_mgr == NULL)
1249  options_mgr = file_system->options_mgr();
1250  UniquePtr<MountPoint> mountpoint(new MountPoint(
1251  fqrn, file_system, options_mgr));
1252 
1253  // At this point, we have a repository name, the type (fuse or library) and
1254  // an options manager (which can be the same than the FileSystem's one).
1255 
1256  mountpoint->CreateStatistics();
1257  mountpoint->CreateAuthz();
1258  mountpoint->backoff_throttle_ = new BackoffThrottle();
1259 
1260  if (!mountpoint->CreateSignatureManager() || !mountpoint->CheckBlacklists())
1261  return mountpoint.Release();
1262  if (!mountpoint->CreateDownloadManagers())
1263  return mountpoint.Release();
1264  if (!mountpoint->CreateResolvConfWatcher()) {
1265  return mountpoint.Release();
1266  }
1267  mountpoint->CreateFetchers();
1268  if (!mountpoint->CreateCatalogManager())
1269  return mountpoint.Release();
1270  if (!mountpoint->CreateTracer())
1271  return mountpoint.Release();
1272 
1273  mountpoint->ReEvaluateAuthz();
1274  mountpoint->CreateTables();
1275  if (!mountpoint->SetupBehavior())
1276  return mountpoint.Release();
1277 
1278  mountpoint->boot_status_ = loader::kFailOk;
1279  return mountpoint.Release();
1280 }
1281 
1282 
1284  string optarg;
1285  string authz_helper;
1286  if (options_mgr_->GetValue("CVMFS_AUTHZ_HELPER", &optarg))
1287  authz_helper = optarg;
1288  string authz_search_path(kDefaultAuthzSearchPath);
1289  if (options_mgr_->GetValue("CVMFS_AUTHZ_SEARCH_PATH", &optarg))
1290  authz_search_path = optarg;
1291 
1293  fqrn_,
1294  authz_helper,
1295  authz_search_path,
1296  options_mgr_);
1297  assert(authz_fetcher_ != NULL);
1298 
1301  statistics_);
1302  assert(authz_session_mgr_ != NULL);
1303 
1305  assert(authz_attachment_ != NULL);
1306 }
1307 
1308 
1310  string optarg;
1311 
1313 
1315  if (!SetupOwnerMaps())
1316  return false;
1317  shash::Any root_hash;
1318  if (!DetermineRootHash(&root_hash))
1319  return false;
1320 
1321  bool retval;
1322  if (root_hash.IsNull()) {
1323  retval = catalog_mgr_->Init();
1324  } else {
1325  fixed_catalog_ = true;
1326  bool alt_root_path =
1327  options_mgr_->GetValue("CVMFS_ALT_ROOT_PATH", &optarg) &&
1328  options_mgr_->IsOn(optarg);
1329  retval = catalog_mgr_->InitFixed(root_hash, alt_root_path);
1330  }
1331  if (!retval) {
1332  boot_error_ = "Failed to initialize root file catalog";
1334  return false;
1335  }
1336 
1338  boot_error_ = "repository revision blacklisted";
1340  return false;
1341  }
1342 
1343  if (options_mgr_->GetValue("CVMFS_AUTO_UPDATE", &optarg) &&
1344  !options_mgr_->IsOn(optarg))
1345  {
1346  fixed_catalog_ = true;
1347  }
1348 
1349  if (options_mgr_->GetValue("CVMFS_CATALOG_WATERMARK", &optarg)) {
1351  } else {
1352  unsigned soft_limit;
1353  unsigned hard_limit;
1354  GetLimitNoFile(&soft_limit, &hard_limit);
1355  catalog_mgr_->SetCatalogWatermark(soft_limit / 4);
1356  }
1357 
1358  if (catalog_mgr_->volatile_flag()) {
1359  LogCvmfs(kLogCvmfs, kLogDebug, "content of repository flagged as VOLATILE");
1360  }
1361 
1362  return true;
1363 }
1364 
1365 
1367  string optarg;
1370  perf::StatisticsTemplate("download", statistics_));
1372 
1373  if (options_mgr_->GetValue("CVMFS_SERVER_URL", &optarg)) {
1374  download_mgr_->SetHostChain(optarg);
1375  }
1376 
1378  SetupHttpTuning();
1379 
1380  string forced_proxy_template;
1381  if (options_mgr_->GetValue("CVMFS_PROXY_TEMPLATE", &optarg))
1382  forced_proxy_template = optarg;
1384  forced_proxy_template);
1385 
1386  string proxies;
1387  if (options_mgr_->GetValue("CVMFS_HTTP_PROXY", &optarg))
1388  proxies = optarg;
1390  proxies,
1391  file_system_->workspace() + "/proxies" + GetUniqFileSuffix(),
1392  download_mgr_);
1393  if (proxies == "") {
1394  boot_error_ = "failed to discover HTTP proxy servers";
1396  return false;
1397  }
1398  string fallback_proxies;
1399  if (options_mgr_->GetValue("CVMFS_FALLBACK_PROXY", &optarg))
1400  fallback_proxies = optarg;
1401  download_mgr_->SetProxyChain(proxies, fallback_proxies,
1403 
1404  bool do_geosort = options_mgr_->GetValue("CVMFS_USE_GEOAPI", &optarg) &&
1405  options_mgr_->IsOn(optarg);
1406  if (do_geosort) {
1408  }
1409  if (options_mgr_->GetValue("CVMFS_MAX_SERVERS", &optarg)) {
1410  unsigned max_servers = String2Uint64(optarg);
1411  std::vector<std::string> host_chain;
1412  download_mgr_->GetHostInfo(&host_chain, NULL, NULL);
1413  if (max_servers > 0 && max_servers < host_chain.size()) {
1414  host_chain.resize(max_servers);
1415  download_mgr_->SetHostChain(host_chain);
1416  }
1417  }
1418 
1419  if (options_mgr_->GetValue("CVMFS_USE_SSL_SYSTEM_CA", &optarg) &&
1420  options_mgr_->IsOn(optarg)) {
1422  }
1423 
1424  if (options_mgr_->GetValue("CVMFS_PROXY_SHARD", &optarg) &&
1425  options_mgr_->IsOn(optarg)) {
1427  }
1428 
1429  return SetupExternalDownloadMgr(do_geosort);
1430 }
1431 
1433  std::string roaming_value;
1434  options_mgr_->GetValue("CVMFS_DNS_ROAMING", &roaming_value);
1435  if (options_mgr_->IsDefined("CVMFS_DNS_ROAMING") &&
1436  options_mgr_->IsOn(roaming_value)) {
1438  "DNS roaming is enabled for this repository.");
1439  // Create a file watcher to update the DNS settings of the download
1440  // managers when there are changes to /etc/resolv.conf
1442 
1443  if (resolv_conf_watcher_) {
1444  ResolvConfEventHandler *handler =
1446  resolv_conf_watcher_->RegisterHandler("/etc/resolv.conf", handler);
1447  }
1448  } else {
1450  "DNS roaming is disabled for this repository.");
1451  }
1452  return true;
1453 }
1454 
1456  fetcher_ = new cvmfs::Fetcher(
1458  download_mgr_,
1461 
1462  const bool is_external_data = true;
1467  perf::StatisticsTemplate("fetch-external", statistics_),
1468  is_external_data);
1469 }
1470 
1471 
1473  string optarg;
1475  signature_mgr_->Init();
1476 
1477  string public_keys;
1478  if (options_mgr_->GetValue("CVMFS_PUBLIC_KEY", &optarg)) {
1479  public_keys = optarg;
1480  } else if (options_mgr_->GetValue("CVMFS_KEYS_DIR", &optarg)) {
1481  // Collect .pub files from CVMFS_KEYS_DIR
1482  public_keys = JoinStrings(FindFilesBySuffix(optarg, ".pub"), ":");
1483  } else {
1484  public_keys =
1485  JoinStrings(FindFilesBySuffix("/etc/cvmfs/keys", ".pub"), ":");
1486  }
1487 
1488  if (!signature_mgr_->LoadPublicRsaKeys(public_keys)) {
1489  boot_error_ = "failed to load public key(s)";
1491  return false;
1492  }
1493  LogCvmfs(kLogCvmfs, kLogDebug, "CernVM-FS: using public key(s) %s",
1494  public_keys.c_str());
1495 
1496  if (options_mgr_->GetValue("CVMFS_TRUSTED_CERTS", &optarg)) {
1497  if (!signature_mgr_->LoadTrustedCaCrl(optarg)) {
1498  boot_error_ = "failed to load trusted certificates";
1500  return false;
1501  }
1502  }
1503 
1504  return true;
1505 }
1506 
1507 
1511  return;
1512 
1513  // TODO(jblomer): this should be registered by the tracker
1514  statistics_->Register("inode_tracker.n_insert",
1515  "overall number of accessed inodes");
1516  statistics_->Register("inode_tracker.n_remove",
1517  "overall number of evicted inodes");
1518  statistics_->Register("inode_tracker.no_reference",
1519  "currently active inodes");
1520  statistics_->Register("inode_tracker.n_hit_inode",
1521  "overall number of inode lookups");
1522  statistics_->Register("inode_tracker.n_hit_path",
1523  "overall number of successful path lookups");
1524  statistics_->Register("inode_tracker.n_miss_path",
1525  "overall number of unsuccessful path lookups");
1526 
1527  statistics_->Register("dentry_tracker.n_insert",
1528  "overall number of added negative cache entries");
1529  statistics_->Register("dentry_tracker.n_remove",
1530  "overall number of evicted negative cache entries");
1531  statistics_->Register("dentry_tracker.n_prune",
1532  "overall number of prune calls");
1533 
1534  statistics_->Register("page_cache_tracker.n_insert",
1535  "overall number of added page cache entries");
1536  statistics_->Register("page_cache_tracker.n_remove",
1537  "overall number of evicted page cache entries");
1538  statistics_->Register("page_cache_tracker.n_open_direct",
1539  "overall number of direct I/O open calls");
1540  statistics_->Register("page_cache_tracker.n_open_flush",
1541  "overall number of open calls where the file's page cache gets flushed");
1542  statistics_->Register("page_cache_tracker.n_open_cached",
1543  "overall number of open calls where the file's page cache is reused");
1544 }
1545 
1546 
1549  // Libcvmfs simplified tables
1552  return;
1553  }
1554 
1555  chunk_tables_ = new ChunkTables();
1556 
1557  string optarg;
1558  uint64_t mem_cache_size = kDefaultMemcacheSize;
1559  if (options_mgr_->GetValue("CVMFS_MEMCACHE_SIZE", &optarg))
1560  mem_cache_size = String2Uint64(optarg) * 1024 * 1024;
1561 
1562  const double memcache_unit_size =
1563  (static_cast<double>(kInodeCacheFactor) * lru::Md5PathCache::GetEntrySize())
1565  const unsigned memcache_num_units =
1566  mem_cache_size / static_cast<unsigned>(memcache_unit_size);
1567  // Number of cache entries must be a multiple of 64
1568  const unsigned mask_64 = ~((1 << 6) - 1);
1569  inode_cache_ = new lru::InodeCache(memcache_num_units & mask_64, statistics_);
1570  path_cache_ = new lru::PathCache(memcache_num_units & mask_64, statistics_);
1571  md5path_cache_ = new lru::Md5PathCache((memcache_num_units * 7) & mask_64,
1572  statistics_);
1573 
1577  if (file_system_->IsNfsSource())
1579 }
1580 
1590  string optarg;
1591  tracer_ = new Tracer();
1592  if (options_mgr_->GetValue("CVMFS_TRACEFILE", &optarg)) {
1594  boot_error_ = "tracer is only supported in the fuse module";
1596  return false;
1597  }
1598  string tracebuffer_file = optarg;
1599  uint64_t tracebuffer_size = kTracerBufferSize;
1600  uint64_t tracebuffer_threshold = kTracerFlushThreshold;
1601 
1602  if (options_mgr_->GetValue("CVMFS_TRACEBUFFER", &optarg)) {
1603  tracebuffer_size = String2Uint64(optarg);
1604  }
1605  if (options_mgr_->GetValue("CVMFS_TRACEBUFFER_THRESHOLD",
1606  &optarg)) {
1607  tracebuffer_threshold = String2Uint64(optarg);
1608  }
1609  assert(tracebuffer_size <= INT_MAX
1610  && tracebuffer_threshold <= INT_MAX);
1612  "Initialising tracer with buffer size %" PRIu64 " and threshold %" PRIu64,
1613  tracebuffer_size, tracebuffer_threshold);
1614  tracer_->Activate(tracebuffer_size, tracebuffer_threshold,
1615  tracebuffer_file);
1616  }
1617  return true;
1618 }
1619 
1620 
1622  string optarg;
1623  if (options_mgr_->GetValue("CVMFS_ROOT_HASH", &optarg)) {
1624  *root_hash = MkFromHexPtr(shash::HexPtr(optarg), shash::kSuffixCatalog);
1625  return true;
1626  }
1627 
1628  if (!options_mgr_->IsDefined("CVMFS_REPOSITORY_TAG") &&
1629  !options_mgr_->IsDefined("CVMFS_REPOSITORY_DATE"))
1630  {
1631  root_hash->SetNull();
1632  return true;
1633  }
1634 
1635  string history_path;
1636  if (!FetchHistory(&history_path))
1637  return false;
1638  UnlinkGuard history_file(history_path);
1640  history::SqliteHistory::Open(history_path));
1641  if (!tag_db.IsValid()) {
1643  "failed to open history database (%s)", history_path.c_str());
1644  boot_error_ = "failed to open history database";
1646  return false;
1647  }
1648 
1650  bool retval;
1651  if (!options_mgr_->GetValue("CVMFS_REPOSITORY_TAG", &repository_tag_)) {
1652  string repository_date;
1653  // options_mgr_->IsDefined("CVMFS_REPOSITORY_DATE") must be true
1654  options_mgr_->GetValue("CVMFS_REPOSITORY_DATE", &repository_date);
1655  time_t repository_utctime = IsoTimestamp2UtcTime(repository_date);
1656  if (repository_utctime == 0) {
1657  boot_error_ = "invalid timestamp in CVMFS_REPOSITORY_DATE: " +
1658  repository_date + ". Use YYYY-MM-DDTHH:MM:SSZ";
1660  return false;
1661  }
1662  retval = tag_db->GetByDate(repository_utctime, &tag);
1663  if (!retval) {
1664  boot_error_ = "no repository state as early as utc timestamp " +
1665  StringifyTime(repository_utctime, true);
1667  return false;
1668  }
1670  "time stamp %s UTC resolved to tag '%s'",
1671  StringifyTime(repository_utctime, true).c_str(),
1672  tag.name.c_str());
1673  repository_tag_ = tag.name;
1674  } else {
1675  retval = tag_db->GetByName(repository_tag_, &tag);
1676  if (!retval) {
1677  boot_error_ = "no such tag: " + repository_tag_;
1679  return false;
1680  }
1681  }
1682  LogCvmfs(kLogCvmfs, kLogDebug, "mounting tag %s", tag.name.c_str());
1683 
1684  *root_hash = tag.root_hash;
1685  return true;
1686 }
1687 
1688 
1689 bool MountPoint::FetchHistory(std::string *history_path) {
1690  manifest::Failures retval_mf;
1691  manifest::ManifestEnsemble ensemble;
1692  retval_mf = manifest::Fetch("", fqrn_, 0, NULL, signature_mgr_, download_mgr_,
1693  &ensemble);
1694  if (retval_mf != manifest::kFailOk) {
1695  boot_error_ = "Failed to fetch manifest";
1697  return false;
1698  }
1699  shash::Any history_hash = ensemble.manifest->history();
1700  if (history_hash.IsNull()) {
1701  boot_error_ = "No history";
1703  return false;
1704  }
1705 
1706  int fd = fetcher_->Fetch(
1707  history_hash,
1709  "tag database for " + fqrn_,
1712  if (fd < 0) {
1713  boot_error_ = "failed to download history: " + StringifyInt(-fd);
1715  return false;
1716  }
1717  // We have the custom sqlite vfs driver installed
1718  *history_path = "@" + StringifyInt(fd);
1719  return true;
1720 }
1721 
1722 
1724  unsigned max_ttl;
1725  {
1726  MutexLockGuard lock_guard(lock_max_ttl_);
1727  max_ttl = max_ttl_sec_;
1728  }
1729  const unsigned catalog_ttl_sec = catalog_mgr_->GetTTL();
1730 
1731  return max_ttl ? std::min(max_ttl, catalog_ttl_sec) : catalog_ttl_sec;
1732 }
1733 
1734 
1736  MutexLockGuard lock_guard(lock_max_ttl_);
1737  return max_ttl_sec_ / 60;
1738 }
1739 
1740 
1746  return "." + file_system_->name() + "-" + fqrn_;
1747 }
1748 
1749 
1751  const string &fqrn,
1752  FileSystem *file_system,
1753  OptionsManager *options_mgr)
1754  : fqrn_(fqrn)
1755  , uuid_(cvmfs::Uuid::Create(""))
1756  , file_system_(file_system)
1757  , options_mgr_(options_mgr)
1758  , statistics_(NULL)
1759  , telemetry_aggr_(NULL)
1760  , authz_fetcher_(NULL)
1761  , authz_session_mgr_(NULL)
1762  , authz_attachment_(NULL)
1763  , backoff_throttle_(NULL)
1764  , signature_mgr_(NULL)
1765  , download_mgr_(NULL)
1766  , external_download_mgr_(NULL)
1767  , fetcher_(NULL)
1768  , external_fetcher_(NULL)
1769  , inode_annotation_(NULL)
1770  , catalog_mgr_(NULL)
1771  , chunk_tables_(NULL)
1772  , simple_chunk_tables_(NULL)
1773  , inode_cache_(NULL)
1774  , path_cache_(NULL)
1775  , md5path_cache_(NULL)
1776  , tracer_(NULL)
1777  , inode_tracker_(NULL)
1778  , dentry_tracker_(NULL)
1779  , page_cache_tracker_(NULL)
1780  , statfs_cache_(NULL)
1781  , resolv_conf_watcher_(NULL)
1782  , max_ttl_sec_(kDefaultMaxTtlSec)
1783  , kcache_timeout_sec_(static_cast<double>(kDefaultKCacheTtlSec))
1784  , fixed_catalog_(false)
1785  , enforce_acls_(false)
1786  , cache_symlinks_(false)
1787  , fuse_expire_entry_(false)
1788  , has_membership_req_(false)
1789  , talk_socket_path_(std::string("./cvmfs_io.") + fqrn)
1790  , talk_socket_uid_(0)
1791  , talk_socket_gid_(0)
1792 {
1793  int retval = pthread_mutex_init(&lock_max_ttl_, NULL);
1794  assert(retval == 0);
1795 }
1796 
1797 
1799  pthread_mutex_destroy(&lock_max_ttl_);
1800 
1801  delete page_cache_tracker_;
1802  delete dentry_tracker_;
1803  delete inode_tracker_;
1804  delete tracer_;
1805  delete md5path_cache_;
1806  delete path_cache_;
1807  delete inode_cache_;
1808  delete simple_chunk_tables_;
1809  delete chunk_tables_;
1810 
1811  delete catalog_mgr_;
1812  delete inode_annotation_;
1813  delete external_fetcher_;
1814  delete fetcher_;
1815  if (external_download_mgr_ != NULL) {
1817  delete external_download_mgr_;
1818  }
1819  if (download_mgr_ != NULL) {
1820  download_mgr_->Fini();
1821  delete download_mgr_;
1822  }
1823  if (signature_mgr_ != NULL) {
1824  signature_mgr_->Fini();
1825  delete signature_mgr_;
1826  }
1827 
1828  if (resolv_conf_watcher_ != NULL) {
1830  delete resolv_conf_watcher_;
1831  }
1832 
1833  delete backoff_throttle_;
1834  delete authz_attachment_;
1835  delete authz_session_mgr_;
1836  delete authz_fetcher_;
1837  delete telemetry_aggr_;
1838  delete statistics_;
1839  delete uuid_;
1840 
1841  delete statfs_cache_;
1842 }
1843 
1844 
1846  string old_membership_req = membership_req_;
1848  if (old_membership_req != membership_req_) {
1851  }
1852 }
1853 
1854 
1855 void MountPoint::SetMaxTtlMn(unsigned value_minutes) {
1856  MutexLockGuard lock_guard(lock_max_ttl_);
1857  max_ttl_sec_ = value_minutes * 60;
1858 }
1859 
1861  string optarg;
1862 
1863  if (options_mgr_->GetValue("CVMFS_MAX_TTL", &optarg))
1864  SetMaxTtlMn(String2Uint64(optarg));
1865 
1866  if (options_mgr_->GetValue("CVMFS_KCACHE_TIMEOUT", &optarg)) {
1867  // Can be negative and should then be interpreted as 0.0
1869  std::max(0.0, static_cast<double>(String2Int64(optarg)));
1870  }
1871  LogCvmfs(kLogCvmfs, kLogDebug, "kernel caches expire after %d seconds",
1872  static_cast<int>(kcache_timeout_sec_));
1873 
1874  uint64_t statfs_time_cache_valid = 0;
1875  if (options_mgr_->GetValue("CVMFS_STATFS_CACHE_TIMEOUT", &optarg)) {
1876  statfs_time_cache_valid = static_cast<uint64_t>(String2Uint64(optarg));
1877  }
1878  LogCvmfs(kLogCvmfs, kLogDebug, "statfs cache expires after %d seconds",
1879  static_cast<int>(statfs_time_cache_valid));
1880  statfs_cache_ = new StatfsCache(statfs_time_cache_valid);
1881 
1882  MagicXattrManager::EVisibility xattr_visibility =
1884  if (options_mgr_->GetValue("CVMFS_HIDE_MAGIC_XATTRS", &optarg)) {
1885  if (options_mgr_->IsOn(optarg))
1886  xattr_visibility = MagicXattrManager::kVisibilityNever;
1887  else if (options_mgr_->IsOff(optarg))
1888  xattr_visibility = MagicXattrManager::kVisibilityAlways;
1889  }
1890  if (options_mgr_->GetValue("CVMFS_MAGIC_XATTRS_VISIBILITY", &optarg)) {
1891  if (ToUpper(optarg) == "ROOTONLY") {
1892  xattr_visibility = MagicXattrManager::kVisibilityRootOnly;
1893  } else if (ToUpper(optarg) == "NEVER") {
1894  xattr_visibility = MagicXattrManager::kVisibilityNever;
1895  } else if (ToUpper(optarg) == "ALWAYS") {
1896  xattr_visibility = MagicXattrManager::kVisibilityAlways;
1897  } else {
1899  "unsupported setting: CVMFS_MAGIC_XATTRS_VISIBILITY=%s",
1900  optarg.c_str());
1901  }
1902  }
1903 
1904  std::set<gid_t> protected_xattr_gids;
1905  if (options_mgr_->GetValue("CVMFS_XATTR_PRIVILEGED_GIDS", &optarg)) {
1906  std::vector<string> tmp = SplitString(optarg, ',');
1907 
1908  for (size_t i = 0; i < tmp.size(); i++) {
1909  std::string trimmed = Trim(tmp[i]);
1911  "Privileged gid for xattr added: %s", trimmed.c_str());
1912  protected_xattr_gids.insert(static_cast<gid_t>(String2Uint64(trimmed)));
1913  }
1914  }
1915  std::set<std::string> protected_xattrs;
1916  if (options_mgr_->GetValue("CVMFS_XATTR_PROTECTED_XATTRS", &optarg)) {
1917  std::vector<string> tmp = SplitString(optarg, ',');
1918 
1919  for (size_t i = 0; i < tmp.size(); i++) {
1920  std::string trimmed = Trim(tmp[i]);
1922  "Protected xattr added: %s", trimmed.c_str());
1923  protected_xattrs.insert(trimmed);
1924  }
1925 
1926  // root has always access to xattr
1927  if (protected_xattr_gids.count(0) < 1) {
1928  protected_xattr_gids.insert(0);
1930  "Automatically added root to have access to protected xattrs.");
1931  }
1932  }
1933  magic_xattr_mgr_ = new MagicXattrManager(this, xattr_visibility,
1934  protected_xattrs, protected_xattr_gids);
1935 
1936 
1937  if (options_mgr_->GetValue("CVMFS_ENFORCE_ACLS", &optarg)
1938  && options_mgr_->IsOn(optarg))
1939  {
1940  enforce_acls_ = true;
1941  }
1942 
1943  if (options_mgr_->GetValue("CVMFS_CACHE_SYMLINKS", &optarg)
1944  && options_mgr_->IsOn(optarg))
1945  {
1946  cache_symlinks_ = true;
1947  }
1948 
1949 
1950 
1951  if (options_mgr_->GetValue("CVMFS_TALK_SOCKET", &optarg)) {
1952  talk_socket_path_ = optarg;
1953  }
1954  if (options_mgr_->GetValue("CVMFS_TALK_OWNER", &optarg)) {
1955  bool retval = GetUidOf(optarg, &talk_socket_uid_, &talk_socket_gid_);
1956  if (!retval) {
1957  boot_error_ = "unknown owner of cvmfs_talk socket: " + optarg;
1959  return false;
1960  }
1961  }
1962 
1963  // this can be later be changed to switch through different
1964  // telemetryAggregators
1965  if (options_mgr_->GetValue("CVMFS_TELEMETRY_SEND", &optarg)
1966  && options_mgr_->IsOn(optarg)) {
1967  int telemetry_send_rate_sec = kDefaultTelemetrySendRateSec;
1968  if (options_mgr_->GetValue("CVMFS_TELEMETRY_RATE", &optarg)) {
1969  telemetry_send_rate_sec = static_cast<int>(String2Uint64(optarg));
1970 
1971  // minimum send rate: 5sec
1972  if (telemetry_send_rate_sec < kMinimumTelemetrySendRateSec) {
1973  telemetry_send_rate_sec = kMinimumTelemetrySendRateSec;
1974  }
1975 
1977  telemetry_send_rate_sec,
1978  options_mgr_,
1979  fqrn_,
1982  "Enable telemetry to report every %d seconds",
1983  telemetry_send_rate_sec);
1984  }
1985  }
1986 
1987  return true;
1988 }
1989 
1990 
1996  string optarg;
1997  unsigned dns_timeout_ms = download::DownloadManager::kDnsDefaultTimeoutMs;
1998  unsigned dns_retries = download::DownloadManager::kDnsDefaultRetries;
1999  if (options_mgr_->GetValue("CVMFS_DNS_TIMEOUT", &optarg))
2000  dns_timeout_ms = String2Uint64(optarg) * 1000;
2001  if (options_mgr_->GetValue("CVMFS_DNS_RETRIES", &optarg))
2002  dns_retries = String2Uint64(optarg);
2003  manager->SetDnsParameters(dns_retries, dns_timeout_ms);
2004 
2005  // Rest has to be after SetDnsParameters because SetDnsParameters might
2006  // construct a new resolver object
2007 
2008  unsigned dns_min_ttl = dns::Resolver::kDefaultMinTtl;
2009  unsigned dns_max_ttl = dns::Resolver::kDefaultMaxTtl;
2010  if (options_mgr_->GetValue("CVMFS_DNS_MIN_TTL", &optarg))
2011  dns_min_ttl = String2Uint64(optarg);
2012  if (options_mgr_->GetValue("CVMFS_DNS_MAX_TTL", &optarg))
2013  dns_max_ttl = String2Uint64(optarg);
2014  manager->SetDnsTtlLimits(dns_min_ttl, dns_max_ttl);
2015 
2016  if (options_mgr_->GetValue("CVMFS_DNS_SERVER", &optarg)) {
2017  download_mgr_->SetDnsServer(optarg);
2018  }
2019 
2020  if (options_mgr_->GetValue("CVMFS_IPFAMILY_PREFER", &optarg)) {
2021  switch (String2Int64(optarg)) {
2022  case 4:
2024  break;
2025  case 6:
2027  break;
2028  }
2029  }
2030  if (options_mgr_->GetValue("CVMFS_MAX_IPADDR_PER_PROXY", &optarg))
2031  manager->SetMaxIpaddrPerProxy(String2Uint64(optarg));
2032 }
2033 
2034 
2036  string optarg;
2038  download_mgr_->Clone(perf::StatisticsTemplate("download-external",
2039  statistics_));
2040 
2041  unsigned timeout;
2042  unsigned timeout_direct;
2043  download_mgr_->GetTimeout(&timeout, &timeout_direct);
2044  if (options_mgr_->GetValue("CVMFS_EXTERNAL_TIMEOUT", &optarg)) {
2045  timeout = String2Uint64(optarg);
2046  }
2047  if (options_mgr_->GetValue("CVMFS_EXTERNAL_TIMEOUT_DIRECT", &optarg)) {
2048  timeout_direct = String2Uint64(optarg);
2049  }
2050  external_download_mgr_->SetTimeout(timeout, timeout_direct);
2051 
2052  if (options_mgr_->GetValue("CVMFS_EXTERNAL_URL", &optarg)) {
2054  if (dogeosort) {
2055  std::vector<std::string> host_chain;
2056  external_download_mgr_->GetHostInfo(&host_chain, NULL, NULL);
2057  download_mgr_->GeoSortServers(&host_chain);
2058  external_download_mgr_->SetHostChain(host_chain);
2059  }
2060  } else {
2062  }
2063 
2064  if (options_mgr_->GetValue("CVMFS_EXTERNAL_MAX_SERVERS", &optarg)) {
2065  unsigned max_servers = String2Uint64(optarg);
2066  std::vector<std::string> host_chain;
2067  external_download_mgr_->GetHostInfo(&host_chain, NULL, NULL);
2068  if (max_servers > 0 && max_servers < host_chain.size()) {
2069  host_chain.resize(max_servers);
2070  external_download_mgr_->SetHostChain(host_chain);
2071  }
2072  }
2073 
2074  string proxies = "DIRECT";
2075  if (options_mgr_->GetValue("CVMFS_EXTERNAL_HTTP_PROXY", &optarg)) {
2077  optarg,
2078  file_system_->workspace() + "/proxies-external" + GetUniqFileSuffix(),
2080  if (proxies == "") {
2081  boot_error_ = "failed to discover external HTTP proxy servers";
2083  return false;
2084  }
2085  }
2086  string fallback_proxies;
2087  if (options_mgr_->GetValue("CVMFS_EXTERNAL_FALLBACK_PROXY", &optarg))
2088  fallback_proxies = optarg;
2090  proxies, fallback_proxies, download::DownloadManager::kSetProxyBoth);
2091 
2092  return true;
2093 }
2094 
2095 
2097  string optarg;
2098 
2099  // TODO(jblomer): avoid double default settings
2100 
2101  unsigned timeout = kDefaultTimeoutSec;
2102  unsigned timeout_direct = kDefaultTimeoutSec;
2103  if (options_mgr_->GetValue("CVMFS_TIMEOUT", &optarg))
2104  timeout = String2Uint64(optarg);
2105  if (options_mgr_->GetValue("CVMFS_TIMEOUT_DIRECT", &optarg))
2106  timeout_direct = String2Uint64(optarg);
2107  download_mgr_->SetTimeout(timeout, timeout_direct);
2108 
2109  unsigned max_retries = kDefaultRetries;
2110  unsigned backoff_init = kDefaultBackoffInitMs;
2111  unsigned backoff_max = kDefaultBackoffMaxMs;
2112  if (options_mgr_->GetValue("CVMFS_MAX_RETRIES", &optarg))
2113  max_retries = String2Uint64(optarg);
2114  if (options_mgr_->GetValue("CVMFS_BACKOFF_INIT", &optarg))
2115  backoff_init = String2Uint64(optarg) * 1000;
2116  if (options_mgr_->GetValue("CVMFS_BACKOFF_MAX", &optarg))
2117  backoff_max = String2Uint64(optarg) * 1000;
2118  download_mgr_->SetRetryParameters(max_retries, backoff_init, backoff_max);
2119 
2120  if (options_mgr_->GetValue("CVMFS_LOW_SPEED_LIMIT", &optarg))
2122  if (options_mgr_->GetValue("CVMFS_PROXY_RESET_AFTER", &optarg))
2124  if (options_mgr_->GetValue("CVMFS_HOST_RESET_AFTER", &optarg))
2126 
2127  if (options_mgr_->GetValue("CVMFS_FOLLOW_REDIRECTS", &optarg) &&
2128  options_mgr_->IsOn(optarg))
2129  {
2131  }
2132  if (options_mgr_->GetValue("CVMFS_SEND_INFO_HEADER", &optarg) &&
2133  options_mgr_->IsOn(optarg))
2134  {
2136  }
2137 }
2138 
2139 
2141  string optarg;
2142 
2143  if (file_system_->IsNfsSource()) {
2145  } else {
2147  }
2148  if (options_mgr_->GetValue("CVMFS_INITIAL_GENERATION", &optarg)) {
2150  }
2151 
2154  }
2155 }
2156 
2157 
2159  string optarg;
2160  catalog::OwnerMap uid_map;
2161  catalog::OwnerMap gid_map;
2162 
2163  if (options_mgr_->GetValue("CVMFS_UID_MAP", &optarg)) {
2164  if (!uid_map.Read(optarg)) {
2165  boot_error_ = "failed to parse uid map " + optarg;
2167  return false;
2168  }
2169  }
2170  if (options_mgr_->GetValue("CVMFS_GID_MAP", &optarg)) {
2171  if (!gid_map.Read(optarg)) {
2172  boot_error_ = "failed to parse gid map " + optarg;
2174  return false;
2175  }
2176  }
2177  catalog_mgr_->SetOwnerMaps(uid_map, gid_map);
2178 
2179  // TODO(jblomer): make local to catalog manager
2180  if (options_mgr_->GetValue("CVMFS_CLAIM_OWNERSHIP", &optarg) &&
2181  options_mgr_->IsOn(optarg))
2182  {
2183  g_claim_ownership = true;
2184  }
2185  if (options_mgr_->GetValue("CVMFS_WORLD_READABLE", &optarg) &&
2186  options_mgr_->IsOn(optarg))
2187  {
2188  g_world_readable = true;
2189  }
2190 
2191 
2192 
2193  return true;
2194 }
perf::Counter * n_eio_07_
Definition: mountpoint.h:348
IoErrorInfo io_error_info_
Definition: mountpoint.h:350
OptionsManager * options_mgr()
Definition: mountpoint.h:243
#define LogCvmfs(source, mask,...)
Definition: logging.h:25
std::string exe_path()
Definition: mountpoint.h:203
static const unsigned kDnsDefaultTimeoutMs
Definition: download.h:402
bool GetVOMSAuthz(std::string *authz) const
void SetLogSyslogFacility(const int local_facility)
Definition: logging.cc:183
FileSystem * file_system_
Definition: mountpoint.h:621
Counter * Register(const std::string &name, const std::string &desc)
Definition: statistics.cc:160
void SetCounter(perf::Counter *c)
Definition: mountpoint.cc:96
void RegisterFdMapping(int from, int to)
Definition: sqlitevfs.cc:541
std::string nfs_maps_dir_
Definition: mountpoint.h:407
void SetLogSyslogLevel(const int level)
Definition: logging.cc:151
bool IsNull() const
Definition: hash.h:383
bool foreground_
Definition: mountpoint.h:325
void Activate(const int buffer_size, const int flush_threshold, const std::string &trace_file)
Definition: tracer.cc:25
download::DownloadManager * download_mgr_
Definition: repository.h:141
struct stat64 platform_stat64
perf::Counter * n_eio_08_
Definition: mountpoint.h:349
Log2Histogram * hist_fs_forget_multi_
Definition: mountpoint.h:355
perf::Counter * n_fs_statfs_cached_
Definition: mountpoint.h:334
static FileWatcher * Create()
Definition: file_watcher.cc:44
void SetupInodeAnnotation()
Definition: mountpoint.cc:2140
bool SetupCwd()
Definition: mountpoint.cc:826
catalog::InodeAnnotation * inode_annotation_
Definition: mountpoint.h:638
static NfsMapsSqlite * Create(const std::string &db_dir, const uint64_t root_inode, const bool rebuild, perf::Statistics *statistics_)
void TearDown2ReadOnly()
Definition: cache_posix.cc:540
std::string workspace_fullpath_
Definition: mountpoint.h:377
bool DetermineNfsMode()
Definition: mountpoint.cc:370
static SqliteHistory * Open(const std::string &file_name)
void SetHostChain(const std::string &host_list)
static const unsigned kTracerFlushThreshold
Definition: mountpoint.h:583
virtual uint64_t GetSize()
Definition: quota_posix.cc:672
static Publisher * Create(const SettingsPublisher &settings)
uid_t talk_socket_uid_
Definition: mountpoint.h:669
void SetLowSpeedLimit(const unsigned low_speed_limit)
Definition: download.cc:1905
perf::Counter * n_fs_stat_stale_
Definition: mountpoint.h:332
Log2Histogram * hist_fs_opendir_
Definition: mountpoint.h:358
bool LockWorkspace()
Definition: mountpoint.cc:493
bool CreateTracer()
Definition: mountpoint.cc:1589
bool wait_workspace_
Definition: mountpoint.h:324
virtual bool Cleanup(const uint64_t leave_size)
Definition: quota_posix.cc:125
double kcache_timeout_sec_
Definition: mountpoint.h:656
signature::SignatureManager * signature_mgr_
Definition: mountpoint.h:633
void CreateFile(const std::string &path, const int mode, const bool ignore_failure)
Definition: posix.cc:277
perf::Counter * n_fs_lookup_
Definition: mountpoint.h:329
std::string name
Definition: history.h:89
AuthzSessionManager * authz_session_mgr_
Definition: mountpoint.h:630
static const unsigned kDefaultRetries
Definition: mountpoint.h:576
string Trim(const string &raw, bool trim_newline)
Definition: string.cc:428
string JoinStrings(const vector< string > &strings, const string &joint)
Definition: string.cc:325
perf::Statistics * statistics_
Definition: mountpoint.h:351
download::DownloadManager * download_mgr_
Definition: mountpoint.h:634
static const unsigned kDefaultTimeoutSec
Definition: mountpoint.h:575
std::string exe_path_
Definition: mountpoint.h:318
bool CheckInstanceName(const std::string &instance)
Definition: mountpoint.cc:108
perf::Statistics * statistics_
Definition: mountpoint.h:627
Log2Histogram * hist_fs_readlink_
Definition: mountpoint.h:357
static void SetupLoggingStandalone(const OptionsManager &options_mgr, const std::string &prefix)
Definition: mountpoint.cc:861
void SetProxyChain(const std::string &proxy_list, const std::string &fallback_proxy_list, const ProxySetModes set_mode)
Definition: download.cc:2425
static NfsMapsLeveldb * Create(const std::string &leveldb_dir, const uint64_t root_inode, const bool rebuild, perf::Statistics *statistics)
gid_t talk_socket_gid_
Definition: mountpoint.h:670
void SetupSqlite()
Definition: mountpoint.cc:1043
perf::Counter * n_eio_total_
Definition: mountpoint.h:341
bool IsOn(const std::string &param_value) const
Definition: options.cc:409
std::string GetUniqFileSuffix()
Definition: mountpoint.cc:1745
StatfsCache * statfs_cache_
Definition: mountpoint.h:650
download::DownloadManager * external_download_mgr_
Definition: mountpoint.h:635
MagicXattrManager * magic_xattr_mgr_
Definition: mountpoint.h:649
bool fuse_expire_entry_
Definition: mountpoint.h:660
CacheManager * SetupCacheMgr(const std::string &instance)
Definition: mountpoint.cc:611
const SettingsPublisher & settings() const
Definition: repository.h:317
perf::Statistics * statistics_
Definition: repository.h:139
void SetupHttpTuning()
Definition: mountpoint.cc:2096
static const char * kDefaultBlacklist
Definition: mountpoint.h:584
virtual bool AcquireQuotaManager(QuotaManager *quota_mgr)
Definition: cache_extern.cc:85
unsigned max_ttl_sec_
Definition: mountpoint.h:654
static const unsigned kDefaultMaxTtl
Definition: dns.h:182
bool CreateDownloadManagers()
Definition: mountpoint.cc:1366
std::string talk_socket_path_
Definition: mountpoint.h:668
perf::Counter * n_fs_stat_
Definition: mountpoint.h:331
perf::Counter * n_eio_05_
Definition: mountpoint.h:346
perf::Counter * no_open_files_
Definition: mountpoint.h:339
assert((mem||(size==0))&&"Out Of Memory")
void SetupLogging()
Definition: mountpoint.cc:883
static void SetupGlobalEnvironmentParams()
Definition: mountpoint.cc:856
int fd_workspace_lock_
Definition: mountpoint.h:378
void CreateStatistics()
Definition: mountpoint.cc:1508
bool UnregisterVfsRdOnly()
Definition: sqlitevfs.cc:522
CacheManager * SetupRamCacheMgr(const std::string &instance)
Definition: mountpoint.cc:714
bool cache_symlinks_
Definition: mountpoint.h:659
static void LogSqliteError(void *user_data __attribute__((unused)), int sqlite_extended_error, const char *message)
Definition: mountpoint.cc:524
static void CleanupInstance()
Definition: sqlitemem.cc:170
void EnableFuseExpireEntry()
Definition: mountpoint.cc:1234
static const unsigned kNfsMaps
Definition: mountpoint.h:182
std::string workspace_
Definition: mountpoint.h:371
string StringifyTime(const time_t seconds, const bool utc)
Definition: string.cc:105
PosixCacheSettings DeterminePosixCacheSettings(const std::string &instance)
Definition: mountpoint.cc:293
void SetDnsServer(const std::string &address)
Definition: download.cc:1834
int platform_stat(const char *path, platform_stat64 *buf)
void AssignGlobalArenas()
Definition: sqlitemem.cc:137
bool has_membership_req_
Definition: mountpoint.h:666
virtual CacheManagerIds id()=0
void SetLogMicroSyslog(const std::string &filename)
Definition: logging.cc:272
bool IsOff(const std::string &param_value) const
Definition: options.cc:416
perf::Counter * n_eio_01_
Definition: mountpoint.h:342
void ReEvaluateAuthz()
Definition: mountpoint.cc:1845
bool HasConfigRepository(const std::string &fqrn, std::string *config_path)
Definition: options.cc:256
std::set< std::string > constructed_instances_
Definition: mountpoint.h:406
bool DetermineRootHash(shash::Any *root_hash)
Definition: mountpoint.cc:1621
#define SetLogDebugFile(filename)
void SetTimeout(const unsigned seconds_proxy, const unsigned seconds_direct)
Definition: download.cc:1891
bool CreateCatalogManager()
Definition: mountpoint.cc:1309
struct cvmcache_object_info __attribute__
Definition: atomic.h:24
void RemapCatalogFd(int from, int to)
Definition: mountpoint.cc:1137
static const unsigned kNfsNone
Definition: mountpoint.h:178
std::string path_crash_guard_
Definition: mountpoint.h:384
IntegerMap< uint64_t > OwnerMap
Definition: catalog.h:41
catalog::WritableCatalogManager * catalog_mgr_
Definition: repository.h:385
bool SetupWorkspace()
Definition: mountpoint.cc:1060
static const char * kDefaultAuthzSearchPath
Definition: mountpoint.h:567
Type type()
Definition: mountpoint.h:245
FileSystem(const FileSystemInfo &fs_info)
Definition: mountpoint.cc:392
static const unsigned kLibPathCacheSize
Definition: mountpoint.h:555
std::string boot_error_
Definition: mountpoint.h:88
OptionsManager * options_mgr_
Definition: mountpoint.h:625
Log2Histogram * hist_fs_getattr_
Definition: mountpoint.h:356
bool IsValid(const std::string &input) const
Definition: sanitizer.cc:114
Log2Histogram * hist_fs_releasedir_
Definition: mountpoint.h:359
Definition: tracer.h:35
bool FileExists(const std::string &path)
Definition: posix.cc:788
bool LoadTrustedCaCrl(const std::string &path_list)
Definition: signature.cc:580
void DisableCacheSymlinks()
Definition: mountpoint.cc:1226
void SetDnsTtlLimits(const unsigned min_seconds, const unsigned max_seconds)
Definition: download.cc:1870
std::string cache_path()
Definition: cache_posix.h:105
perf::Counter * n_fs_readlink_
Definition: mountpoint.h:336
void Init(const unsigned max_pool_handles, const perf::StatisticsTemplate &statistics)
Definition: download.cc:1621
uint64_t platform_memsize()
int64_t String2Int64(const string &value)
Definition: string.cc:222
void SetMaxTtlMn(unsigned value_minutes)
Definition: mountpoint.cc:1855
string ToUpper(const string &mixed_case)
Definition: string.cc:476
static const unsigned kDefaultBackoffInitMs
Definition: mountpoint.h:577
perf::Counter * n_fs_forget_
Definition: mountpoint.h:337
MountPoint(const std::string &fqrn, FileSystem *file_system, OptionsManager *options_mgr)
Definition: mountpoint.cc:1750
void GetLimitNoFile(unsigned *soft_limit, unsigned *hard_limit)
Definition: posix.cc:1477
Tracer * tracer_
Definition: mountpoint.h:645
static TelemetryAggregator * Create(Statistics *statistics, int send_rate, OptionsManager *options_mgr, const std::string &fqrn, const TelemetrySelector type)
void SetProxyGroupResetDelay(const unsigned seconds)
Definition: download.cc:2722
static CacheManager * Create(CacheManager *upper_cache, CacheManager *lower_cache)
static bool g_alive
Definition: mountpoint.h:253
static const unsigned kDefaultNumConnections
Definition: mountpoint.h:571
void Set(const int64_t val)
Definition: statistics.h:33
void TearDown2ReadOnly()
Definition: mountpoint.cc:1124
gid_t g_gid
Definition: globals.cc:14
uuid_t uuid
Definition: uuid.h:41
static const unsigned kNfsMapsHa
Definition: mountpoint.h:186
lru::Md5PathCache * md5path_cache_
Definition: mountpoint.h:644
std::string name_
Definition: mountpoint.h:317
vector< string > SplitString(const string &str, char delim)
Definition: string.cc:290
std::string Dump()
Definition: options.cc:454
bool RegisterVfsRdOnly(CacheManager *cache_mgr, perf::Statistics *statistics, const VfsOptions options)
Definition: sqlitevfs.cc:453
static AuthzSessionManager * Create(AuthzFetcher *authz_fetcher, perf::Statistics *statistics)
std::string membership_req_
Definition: mountpoint.h:665
glue::DentryTracker * dentry_tracker_
Definition: mountpoint.h:647
perf::Counter * n_fs_read_
Definition: mountpoint.h:335
signature::SignatureManager * signature_mgr_
Definition: repository.h:140
perf::Counter * n_eio_06_
Definition: mountpoint.h:347
static const char * kDefaultCacheMgrInstance
Definition: mountpoint.h:257
bool HasSuffix(const std::string &str, const std::string &suffix, const bool ignore_case)
Definition: string.cc:281
virtual uint64_t GetCapacity()
Definition: quota_posix.cc:580
glue::PageCacheTracker * page_cache_tracker_
Definition: mountpoint.h:648
bool found_previous_crash_
Definition: mountpoint.h:390
const char kSuffixCatalog
Definition: hash.h:54
loader::Failures boot_status_
Definition: mountpoint.h:87
Failures Fetch(const std::string &base_url, const std::string &repository_name, const uint64_t minimum_timestamp, const shash::Any *base_catalog, signature::SignatureManager *signature_manager, download::DownloadManager *download_manager, ManifestEnsemble *ensemble)
bool TriageCacheMgr()
Definition: mountpoint.cc:1142
bool CheckPosixCacheSettings(const PosixCacheSettings &settings)
Definition: mountpoint.cc:125
perf::TelemetryAggregator * telemetry_aggr_
Definition: mountpoint.h:628
perf::Counter * n_fs_inode_replace_
Definition: mountpoint.h:338
bool IsNfsSource()
Definition: mountpoint.h:195
ChunkTables * chunk_tables_
Definition: mountpoint.h:640
CacheManager * cache_mgr()
Definition: mountpoint.h:201
uid_t g_uid
Definition: globals.cc:13
void SetupDnsTuning(download::DownloadManager *manager)
Definition: mountpoint.cc:1995
cvmfs::Uuid * uuid_cache_
Definition: mountpoint.h:417
unsigned GetEffectiveTtlSec()
Definition: mountpoint.cc:1723
int TryLockFile(const std::string &path)
Definition: posix.cc:910
bool MkdirDeep(const std::string &path, const mode_t mode, bool verify_writable)
Definition: posix.cc:843
int LockFile(const std::string &path)
Definition: posix.cc:968
void SetIpPreference(const dns::IpPreference preference)
Definition: download.cc:1880
static const unsigned kDefaultBackoffMaxMs
Definition: mountpoint.h:578
lru::InodeCache * inode_cache_
Definition: mountpoint.h:642
CacheManager * SetupPosixCacheMgr(const std::string &instance)
Definition: mountpoint.cc:684
perf::Counter * n_fs_open_
Definition: mountpoint.h:327
static uint64_t RoundUp8(const uint64_t size)
Definition: smalloc.h:37
bool SetupPosixQuotaMgr(const PosixCacheSettings &settings, CacheManager *cache_mgr)
Definition: mountpoint.cc:981
static SqliteMemoryManager * GetInstance()
Definition: sqlitemem.h:137
static PosixQuotaManager * Create(const std::string &cache_workspace, const uint64_t limit, const uint64_t cleanup_threshold, const bool rebuild_database)
Definition: quota_posix.cc:230
unsigned nfs_mode_
Definition: mountpoint.h:411
void SetRetryParameters(const unsigned max_retries, const unsigned backoff_init_ms, const unsigned backoff_max_ms)
Definition: download.cc:2741
string StringifyInt(const int64_t value)
Definition: string.cc:78
Type type_
Definition: mountpoint.h:319
static const unsigned kInodeCacheFactor
Definition: mountpoint.h:559
void SetMaxIpaddrPerProxy(unsigned limit)
Definition: download.cc:2752
DownloadManager * Clone(const perf::StatisticsTemplate &statistics)
Definition: download.cc:2785
std::string GetArch()
Definition: posix.cc:1443
void Inc(class Counter *counter)
Definition: statistics.h:50
bool HasPrefix(const string &str, const string &prefix, const bool ignore_case)
Definition: string.cc:267
perf::Counter * n_eio_04_
Definition: mountpoint.h:345
Log2Histogram * hist_fs_readdir_
Definition: mountpoint.h:360
bool GetValue(const std::string &key, std::string *value) const
Definition: options.cc:376
cvmfs::Uuid * uuid_
Definition: mountpoint.h:617
OptionsManager * options_mgr_
Definition: cvmfs.cc:148
bool LoadBlacklist(const std::string &path_blacklist, bool append)
Definition: signature.cc:536
BackoffThrottle * backoff_throttle_
Definition: mountpoint.h:632
AuthzAttachment * authz_attachment_
Definition: mountpoint.h:631
void SetProxyTemplates(const std::string &direct, const std::string &forced)
Definition: download.cc:2758
SimpleChunkTables * simple_chunk_tables_
Definition: mountpoint.h:641
bool InitFixed(const shash::Any &root_hash, bool alternative_path)
perf::Counter * n_eio_03_
Definition: mountpoint.h:344
static const unsigned kDefaultNfiles
Definition: mountpoint.h:256
static FileSystem * Create(const FileSystemInfo &fs_info)
Definition: mountpoint.cc:165
bool has_custom_sqlitevfs_
Definition: mountpoint.h:428
perf::Counter * n_fs_statfs_
Definition: mountpoint.h:333
bool ReloadBlacklists()
Definition: mountpoint.cc:1202
std::string name()
Definition: mountpoint.h:230
time_t IsoTimestamp2UtcTime(const std::string &iso8601)
Definition: string.cc:198
static Uuid * Create(const std::string &store_path)
Definition: uuid.cc:23
void CreateStatistics()
Definition: mountpoint.cc:209
pthread_mutex_t lock_max_ttl_
Definition: mountpoint.h:655
void CreateAuthz()
Definition: mountpoint.cc:1283
bool g_raw_symlinks
Definition: globals.cc:12
bool g_world_readable
Definition: globals.cc:15
static const char * kDefaultCacheBase
Definition: mountpoint.h:254
void SetNull()
Definition: hash.h:400
string ResolveProxyDescription(const string &cvmfs_proxies, const std::string &path_fallback_cache, DownloadManager *download_manager)
Definition: wpad.cc:206
uint64_t String2Uint64(const string &value)
Definition: string.cc:228
static const unsigned kTracerBufferSize
Definition: mountpoint.h:582
void SetupUuid()
Definition: mountpoint.cc:1109
perf::Counter * n_eio_02_
Definition: mountpoint.h:343
catalog::ClientCatalogManager * catalog_mgr_
Definition: mountpoint.h:639
bool CreateSignatureManager()
Definition: mountpoint.cc:1472
std::string fqrn_
Definition: mountpoint.h:616
unsigned GetMaxTtlMn()
Definition: mountpoint.cc:1735
std::string error_msg() const
Definition: cache_extern.h:45
bool GetUidOf(const std::string &username, uid_t *uid, gid_t *main_gid)
Definition: posix.cc:1341
bool SetupBehavior()
Definition: mountpoint.cc:1860
static ExternalCacheManager * Create(int fd_connection, unsigned max_open_fds, const std::string &ident)
AuthzFetcher * authz_fetcher_
Definition: mountpoint.h:629
bool FetchHistory(std::string *history_path)
Definition: mountpoint.cc:1689
perf::Statistics * statistics()
Definition: mountpoint.h:244
bool enforce_acls_
Definition: mountpoint.h:658
bool IsHaNfsSource()
Definition: mountpoint.h:196
void CreateTables()
Definition: mountpoint.cc:1547
shash::Any history() const
Definition: manifest.h:127
bool SetupCrashGuard()
Definition: mountpoint.cc:805
static const unsigned kDefaultQuotaLimit
Definition: mountpoint.h:255
static PosixQuotaManager * CreateShared(const std::string &exe_path, const std::string &cache_workspace, const uint64_t limit, const uint64_t cleanup_threshold, bool foreground)
Definition: quota_posix.cc:262
void ResetErrorCounters()
Definition: mountpoint.cc:594
Definition: mutex.h:42
Log2Histogram * hist_fs_lookup_
Definition: mountpoint.h:353
shash::Any root_hash
Definition: history.h:90
bool LoadPublicRsaKeys(const std::string &path_list)
Definition: signature.cc:305
void RegisterHandler(const std::string &file_path, EventHandler *handler)
Definition: file_watcher.cc:56
void GetTimeout(unsigned *seconds_proxy, unsigned *seconds_direct)
Definition: download.cc:1914
static const unsigned kDefaultMemcacheSize
Definition: mountpoint.h:563
FileSystem * file_system_
Definition: cvmfs.cc:123
std::string repository_tag_
Definition: mountpoint.h:661
static const unsigned kDefaultMinTtl
Definition: dns.h:177
bool SetupOwnerMaps()
Definition: mountpoint.cc:2158
void SetDnsParameters(const unsigned retries, const unsigned timeout_ms)
Definition: download.cc:1852
bool CreateResolvConfWatcher()
Definition: mountpoint.cc:1432
Any MkFromHexPtr(const HexPtr hex, const char suffix)
Definition: hash.cc:83
static ExternalQuotaManager * Create(ExternalCacheManager *cache_mgr)
Log2Histogram * hist_fs_read_
Definition: mountpoint.h:362
void SetCatalogWatermark(unsigned limit)
static bool g_is_enabled
Definition: algorithm.h:200
bool SetupNfsMaps()
Definition: mountpoint.cc:888
bool CheckBlacklists()
Definition: mountpoint.cc:1164
std::vector< std::string > blacklist_paths_
Definition: mountpoint.h:662
cvmfs::Fetcher * fetcher_
Definition: mountpoint.h:636
file_watcher::FileWatcher * resolv_conf_watcher_
Definition: mountpoint.h:652
static const unsigned kDnsDefaultRetries
Definition: download.h:401
bool GeoSortServers(std::vector< std::string > *servers, std::vector< uint64_t > *output_order=NULL)
Definition: download.cc:2144
bool SetupExternalDownloadMgr(bool dogeosort)
Definition: mountpoint.cc:2035
void SetOwnerMaps(const OwnerMap &uid_map, const OwnerMap &gid_map)
CacheManager * SetupExternalCacheMgr(const std::string &instance)
Definition: mountpoint.cc:645
static void CleanupInstance()
Definition: clientctx.cc:18
perf::Counter * n_fs_lookup_negative_
Definition: mountpoint.h:330
Log2Histogram * hist_fs_open_
Definition: mountpoint.h:361
virtual bool AcquireQuotaManager(QuotaManager *quota_mgr)=0
std::string cache_mgr_instance_
Definition: mountpoint.h:401
std::string path_workspace_lock_
Definition: mountpoint.h:379
virtual void IncGeneration(const uint64_t by)=0
Log2Histogram * hist_fs_release_
Definition: mountpoint.h:363
void SetLogSyslogPrefix(const std::string &prefix)
Definition: logging.cc:241
std::string MakeCanonicalPath(const std::string &path)
Definition: posix.cc:98
static const int kDefaultTelemetrySendRateSec
Definition: mountpoint.h:588
OptionsManager * options_mgr_
Definition: mountpoint.h:323
perf::Counter * no_open_dirs_
Definition: mountpoint.h:340
glue::InodeTracker * inode_tracker_
Definition: mountpoint.h:646
bool fixed_catalog_
Definition: mountpoint.h:657
static PluginHandle * CreatePlugin(const std::string &locator, const std::vector< std::string > &cmd_line)
bool IsDefined(const std::string &key)
Definition: options.cc:370
static const int kMinimumTelemetrySendRateSec
Definition: mountpoint.h:589
void GetHostInfo(std::vector< std::string > *host_chain, std::vector< int > *rtt, unsigned *current_host)
Definition: download.cc:1958
void set_membership(const std::string &m)
Definition: authz_curl.h:25
Statistics * Fork()
Definition: statistics.cc:47
CacheManager * SetupTieredCacheMgr(const std::string &instance)
Definition: mountpoint.cc:764
std::vector< std::string > FindFilesBySuffix(const std::string &dir, const std::string &suffix)
Definition: posix.cc:1121
std::string workspace()
Definition: mountpoint.h:247
bool g_claim_ownership
Definition: globals.cc:11
lru::PathCache * path_cache_
Definition: mountpoint.h:643
CacheManager * cache_mgr_
Definition: mountpoint.h:412
void SetCredentialsAttachment(CredentialsAttachment *ca)
Definition: download.cc:1818
static ClientCtx * GetInstance()
Definition: clientctx.cc:45
virtual bool AcquireQuotaManager(QuotaManager *quota_mgr)
Definition: cache_ram.cc:70
cvmfs::Uuid * uuid_cache()
Definition: mountpoint.h:246
cvmfs::Fetcher * external_fetcher_
Definition: mountpoint.h:637
perf::Counter * n_fs_dir_open_
Definition: mountpoint.h:328
static const uint64_t kSizeUnknown
Definition: cache.h:72
std::string MkCacheParm(const std::string &generic_parameter, const std::string &instance)
Definition: mountpoint.cc:559
void UnlockFile(const int filedes)
Definition: posix.cc:992
NfsMaps * nfs_maps_
Definition: mountpoint.h:423
void CreateFetchers()
Definition: mountpoint.cc:1455
void SetInodeAnnotation(InodeAnnotation *new_annotation)
int Fetch(const shash::Any &id, const uint64_t size, const std::string &name, const zlib::Algorithms compression_algorithm, const CacheManager::ObjectType object_type, const std::string &alt_url="", off_t range_offset=-1)
Definition: fetch.cc:82
virtual void SetInodeResidue(unsigned residue_class, unsigned remainder)
Definition: nfs_maps.h:31
void SetHostResetDelay(const unsigned seconds)
Definition: download.cc:2732
static MountPoint * Create(const std::string &fqrn, FileSystem *file_system, OptionsManager *options_mgr=NULL)
Definition: mountpoint.cc:1243
Log2Histogram * hist_fs_forget_
Definition: mountpoint.h:354
static PosixCacheManager * Create(const std::string &cache_path, const bool alien_cache, const RenameWorkarounds rename_workaround=kRenameNormal)
Definition: cache_posix.cc:230