CernVM-FS  2.11.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
mountpoint.cc
Go to the documentation of this file.
1 
4 #include "cvmfs_config.h"
5 #include "mountpoint.h"
6 
7 #include <errno.h>
8 #include <fcntl.h>
9 #include <inttypes.h>
10 #include <stdint.h>
11 #include <unistd.h>
12 
13 #include <algorithm>
14 #include <cassert>
15 #include <climits>
16 #include <cstring>
17 #include <vector>
18 
19 #include "duplex_fuse.h" // NOLINT
20 
21 #ifndef CVMFS_LIBCVMFS
22 #ifdef FUSE_CAP_EXPORT_SUPPORT
23 #define CVMFS_NFS_SUPPORT
24 #else
25 #warning "No NFS support, Fuse too old"
26 #endif
27 #endif
28 
29 #include "authz/authz_curl.h"
30 #include "authz/authz_fetch.h"
31 #include "authz/authz_session.h"
32 #include "backoff.h"
33 #include "cache.h"
34 #include "cache_extern.h"
35 #include "cache_posix.h"
36 #include "cache_ram.h"
37 #include "cache_tiered.h"
38 #include "catalog.h"
39 #include "catalog_mgr_client.h"
40 #include "clientctx.h"
41 #include "crypto/signature.h"
42 #include "download.h"
43 #include "duplex_sqlite3.h"
44 #include "fetch.h"
45 #include "file_chunk.h"
46 #include "globals.h"
47 #include "glue_buffer.h"
48 #include "google/protobuf/stubs/common.h"
49 #include "history.h"
50 #include "history_sqlite.h"
51 #include "lru_md.h"
52 #include "manifest.h"
53 #include "manifest_fetch.h"
54 #include "nfs_maps.h"
55 #ifdef CVMFS_NFS_SUPPORT
56 #include "nfs_maps_leveldb.h"
57 #include "nfs_maps_sqlite.h"
58 #endif
59 #include "options.h"
60 #include "quota_posix.h"
62 #include "sqlitemem.h"
63 #include "sqlitevfs.h"
64 #include "statistics.h"
65 #include "tracer.h"
66 #include "util/concurrency.h"
67 #include "util/logging.h"
68 #include "util/platform.h"
69 #include "util/pointer.h"
70 #include "util/posix.h"
71 #include "util/string.h"
72 #include "util/uuid.h"
73 #include "wpad.h"
74 
75 using namespace std; // NOLINT
76 
77 
78 bool FileSystem::g_alive = false;
79 const char *FileSystem::kDefaultCacheBase = "/var/lib/cvmfs";
80 const char *FileSystem::kDefaultCacheMgrInstance = "default";
81 
82 
83 FileSystem::IoErrorInfo::IoErrorInfo() : counter_(NULL), timestamp_last_(0) { }
84 
86  counter_->Set(0);
87  timestamp_last_ = 0;
88 }
89 
91  perf::Inc(counter_);
92  timestamp_last_ = time(NULL);
93 }
94 
96 
97 int64_t FileSystem::IoErrorInfo::count() { return counter_->Get(); }
98 
99 time_t FileSystem::IoErrorInfo::timestamp_last() { return timestamp_last_; }
100 
101 
102 
107 bool FileSystem::CheckInstanceName(const std::string &instance) {
108  if (instance.length() > 24)
109  return false;
110  sanitizer::CacheInstanceSanitizer instance_sanitizer;
111  if (!instance_sanitizer.IsValid(instance)) {
112  boot_error_ = "invalid instance name (" + instance + "), " +
113  "only characters a-z, A-Z, 0-9, _ are allowed";
115  return false;
116  }
117  return true;
118 }
119 
120 
126 {
127  if (settings.is_alien && settings.is_shared) {
128  boot_error_ = "Failure: shared local disk cache and alien cache mutually "
129  "exclusive. Please turn off shared local disk cache.";
131  return false;
132  }
133  if (settings.is_alien && settings.is_managed) {
134  boot_error_ = "Failure: quota management and alien cache mutually "
135  "exclusive. Please turn off quota limit.";
137  return false;
138  }
139 
140  if (type_ == kFsLibrary) {
141  if (settings.is_shared || settings.is_managed) {
142  boot_error_ = "Failure: libcvmfs supports only unmanaged exclusive cache "
143  "or alien cache.";
145  return false;
146  }
147  }
148 
149  if (settings.cache_base_defined && settings.cache_dir_defined) {
150  boot_error_ =
151  "'CVMFS_CACHE_BASE' and 'CVMFS_CACHE_DIR' are mutually exclusive";
153  return false;
154  }
155 
156  return true;
157 }
158 
159 
166  file_system(new FileSystem(fs_info));
167 
168  file_system->SetupGlobalEnvironmentParams();
169 
170  file_system->SetupLogging();
171  LogCvmfs(kLogCvmfs, kLogDebug, "Options:\n%s",
172  file_system->options_mgr()->Dump().c_str());
173 
174  file_system->CreateStatistics();
175  file_system->SetupSqlite();
176  if (!file_system->DetermineNfsMode())
177  return file_system.Release();
178  if (!file_system->SetupWorkspace())
179  return file_system.Release();
180 
181  // Redirect SQlite temp directory to workspace (global variable)
182  unsigned length_tempdir = file_system->workspace_.length() + 1;
183  sqlite3_temp_directory = static_cast<char *>(sqlite3_malloc(length_tempdir));
184  snprintf(sqlite3_temp_directory,
185  length_tempdir,
186  "%s",
187  file_system->workspace_.c_str());
188 
189  if (!file_system->TriageCacheMgr())
190  return file_system.Release();
191  file_system->SetupUuid();
192  if (!file_system->SetupNfsMaps())
193  return file_system.Release();
194  bool retval = sqlite::RegisterVfsRdOnly(
195  file_system->cache_mgr_,
196  file_system->statistics_,
198  assert(retval);
199  file_system->has_custom_sqlitevfs_ = true;
200 
202 
203  file_system->boot_status_ = loader::kFailOk;
204  return file_system.Release();
205 }
206 
207 
210 
211  // Register the ShortString's static counters
212  statistics_->Register("pathstring.n_instances", "Number of instances");
213  statistics_->Register("pathstring.n_overflows", "Number of overflows");
214  statistics_->Register("namestring.n_instances", "Number of instances");
215  statistics_->Register("namestring.n_overflows", "Number of overflows");
216  statistics_->Register("linkstring.n_instances", "Number of instances");
217  statistics_->Register("linkstring.n_overflows", "Number of overflows");
218 
219  // Callback counters
220  n_fs_open_ = statistics_->Register("cvmfs.n_fs_open",
221  "Overall number of file open operations");
222  n_fs_dir_open_ = statistics_->Register("cvmfs.n_fs_dir_open",
223  "Overall number of directory open operations");
224  n_fs_lookup_ = statistics_->Register("cvmfs.n_fs_lookup",
225  "Number of lookups");
226  n_fs_lookup_negative_ = statistics_->Register("cvmfs.n_fs_lookup_negative",
227  "Number of negative lookups");
228  n_fs_stat_ = statistics_->Register("cvmfs.n_fs_stat", "Number of stats");
229  n_fs_stat_stale_ = statistics_->Register("cvmfs.n_fs_stat_stale",
230  "Number of stats for stale (open, meanwhile changed) regular files");
231  n_fs_statfs_ = statistics_->Register("cvmfs.n_fs_statfs",
232  "Overall number of statsfs calls");
233  n_fs_statfs_cached_ = statistics_->Register("cvmfs.n_fs_statfs_cached",
234  "Number of statsfs calls that accessed the cached statfs info");
235  n_fs_read_ = statistics_->Register("cvmfs.n_fs_read", "Number of files read");
236  n_fs_readlink_ = statistics_->Register("cvmfs.n_fs_readlink",
237  "Number of links read");
238  n_fs_forget_ = statistics_->Register("cvmfs.n_fs_forget",
239  "Number of inode forgets");
240  n_fs_inode_replace_ = statistics_->Register("cvmfs.n_fs_inode_replace",
241  "Number of stale inodes that got replaced by an up-to-date version");
242  no_open_files_ = statistics_->Register("cvmfs.no_open_files",
243  "Number of currently opened files");
244  no_open_dirs_ = statistics_->Register("cvmfs.no_open_dirs",
245  "Number of currently opened directories");
246  io_error_info_.SetCounter(statistics_->Register("cvmfs.n_io_error",
247  "Number of I/O errors"));
248  n_eio_total_ = statistics_->Register("eio.total",
249  "EIO returned to calling process. Sum of individual eio counters");
250  n_eio_01_ = statistics_->Register("eio.01",
251  "EIO returned to calling process. cvmfs.cc:cvmfs_lookup()");
252  n_eio_02_ = statistics_->Register("eio.02",
253  "EIO returned to calling process. cvmfs.cc:ReplyNegative()");
254  n_eio_03_ = statistics_->Register("eio.03",
255  "EIO returned to calling process. cvmfs.cc:cvmfs_opendir()");
256  n_eio_04_ = statistics_->Register("eio.04",
257  "EIO returned to calling process. cvmfs.cc:cvmfs_open()");
258  n_eio_05_ = statistics_->Register("eio.05",
259  "EIO returned to calling process. cvmfs.cc:cvmfs_read()");
260  n_eio_06_ = statistics_->Register("eio.06",
261  "EIO returned to calling process. cvmfs.cc:cvmfs_open()");
262  n_eio_07_ = statistics_->Register("eio.07",
263  "EIO returned to calling process. cvmfs.cc:cvmfs_read()");
264  n_eio_08_ = statistics_->Register("eio.08",
265  "EIO returned to calling process. cvmfs.cc:cvmfs_read()");
266 
267  string optarg;
268  if (options_mgr_->GetValue("CVMFS_INSTRUMENT_FUSE", &optarg) &&
269  options_mgr_->IsOn(optarg))
270  {
272  }
273 
274  hist_fs_lookup_ = new Log2Histogram(30);
275  hist_fs_forget_ = new Log2Histogram(30);
282  hist_fs_open_ = new Log2Histogram(30);
283  hist_fs_read_ = new Log2Histogram(30);
285 }
286 
287 
293  const string &instance
294 ) {
295  string optarg;
297 
298  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SHARED", instance),
299  &optarg)
300  && options_mgr_->IsOn(optarg))
301  {
302  settings.is_shared = true;
303  }
304  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SERVER_MODE", instance),
305  &optarg)
306  && options_mgr_->IsOn(optarg))
307  {
308  settings.avoid_rename = true;
309  }
310 
311  if (type_ == kFsFuse)
312  settings.quota_limit = kDefaultQuotaLimit;
313  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_QUOTA_LIMIT", instance),
314  &optarg))
315  {
316  settings.quota_limit = String2Int64(optarg) * 1024 * 1024;
317  }
318  if (settings.quota_limit > 0)
319  settings.is_managed = true;
320 
321  settings.cache_path = kDefaultCacheBase;
322  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_BASE", instance),
323  &optarg))
324  {
325  settings.cache_path = MakeCanonicalPath(optarg);
326  settings.cache_base_defined = true;
327  }
328  if (settings.is_shared) {
329  settings.cache_path += "/shared";
330  } else {
331  settings.cache_path += "/" + name_;
332  }
333 
334  // CheckCacheMode makes sure that CVMFS_CACHE_DIR and CVMFS_CACHE_BASE are
335  // not set at the same time.
336  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_DIR", instance),
337  &optarg))
338  {
339  settings.cache_dir_defined = true;
340  settings.cache_path = optarg;
341  }
342  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_ALIEN", instance),
343  &optarg))
344  {
345  settings.is_alien = true;
346  settings.cache_path = optarg;
347  }
348  // We already changed the cwd to the workspace
349  // Which is only done if using FUSE
350  if ((type_ == kFsFuse) && (settings.cache_path == workspace_fullpath_)) {
351  settings.cache_path = ".";
352  }
353 
354  // The cache workspace usually is the cache directory, unless explicitly
355  // set otherwise
356  settings.workspace = settings.cache_path;
357  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_WORKSPACE", instance),
358  &optarg) ||
359  options_mgr_->GetValue("CVMFS_WORKSPACE", &optarg))
360  {
361  // Used for the shared quota manager
362  settings.workspace = optarg;
363  }
364 
365  return settings;
366 }
367 
368 
370  string optarg;
371 
372  if (options_mgr_->GetValue("CVMFS_NFS_SOURCE", &optarg) &&
373  options_mgr_->IsOn(optarg))
374  {
375  nfs_mode_ |= kNfsMaps;
376  if (options_mgr_->GetValue("CVMFS_NFS_SHARED", &optarg)) {
378  nfs_maps_dir_ = optarg;
379  }
380  }
381 
382  if ((type_ == kFsLibrary) && (nfs_mode_ != kNfsNone)) {
383  boot_error_ = "Failure: libcvmfs does not support NFS export.";
385  return false;
386  }
387  return true;
388 }
389 
390 
392  : name_(fs_info.name)
393  , exe_path_(fs_info.exe_path)
394  , type_(fs_info.type)
395  , options_mgr_(fs_info.options_mgr)
396  , wait_workspace_(fs_info.wait_workspace)
397  , foreground_(fs_info.foreground)
398  , n_fs_open_(NULL)
399  , n_fs_dir_open_(NULL)
400  , n_fs_lookup_(NULL)
401  , n_fs_lookup_negative_(NULL)
402  , n_fs_stat_(NULL)
403  , n_fs_stat_stale_(NULL)
404  , n_fs_statfs_(NULL)
405  , n_fs_statfs_cached_(NULL)
406  , n_fs_read_(NULL)
407  , n_fs_readlink_(NULL)
408  , n_fs_forget_(NULL)
409  , n_fs_inode_replace_(NULL)
410  , no_open_files_(NULL)
411  , no_open_dirs_(NULL)
412  , n_eio_total_(NULL)
413  , n_eio_01_(NULL)
414  , n_eio_02_(NULL)
415  , n_eio_03_(NULL)
416  , n_eio_04_(NULL)
417  , n_eio_05_(NULL)
418  , n_eio_06_(NULL)
419  , n_eio_07_(NULL)
420  , n_eio_08_(NULL)
421  , statistics_(NULL)
422  , fd_workspace_lock_(-1)
423  , found_previous_crash_(false)
425  , cache_mgr_(NULL)
426  , uuid_cache_(NULL)
427  , nfs_maps_(NULL)
428  , has_custom_sqlitevfs_(false)
429 {
430  assert(!g_alive);
431  g_alive = true;
432  g_uid = geteuid();
433  g_gid = getegid();
434 
435  string optarg;
436  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SERVER_MODE",
438  &optarg)
439  && options_mgr_->IsOn(optarg))
440  {
441  g_raw_symlinks = true;
442  }
443 }
444 
445 
448 
451 
452  delete uuid_cache_;
453  delete nfs_maps_;
454  delete cache_mgr_;
455 
456  if (sqlite3_temp_directory) {
457  sqlite3_free(sqlite3_temp_directory);
458  sqlite3_temp_directory = NULL;
459  }
460 
461  if (!path_crash_guard_.empty())
462  unlink(path_crash_guard_.c_str());
463  if (!path_workspace_lock_.empty())
464  unlink(path_workspace_lock_.c_str());
465  if (fd_workspace_lock_ >= 0)
467 
468  sqlite3_shutdown();
470 
471  delete hist_fs_lookup_;
472  delete hist_fs_forget_multi_;
473  delete hist_fs_forget_;
474  delete hist_fs_getattr_;
475  delete hist_fs_readlink_;
476  delete hist_fs_opendir_;
477  delete hist_fs_releasedir_;
478  delete hist_fs_readdir_;
479  delete hist_fs_open_;
480  delete hist_fs_read_;
481  delete hist_fs_release_;
482  delete statistics_;
483 
484  SetLogSyslogPrefix("");
485  SetLogMicroSyslog("");
486  SetLogDebugFile("");
487  google::protobuf::ShutdownProtobufLibrary();
488  g_alive = false;
489 }
490 
491 
493  path_workspace_lock_ = workspace_ + "/lock." + name_;
495  if (fd_workspace_lock_ >= 0)
496  return true;
497 
498  if (fd_workspace_lock_ == -1) {
499  boot_error_ = "could not acquire workspace lock (" +
500  StringifyInt(errno) + ")";
502  return false;
503  }
504 
505  assert(fd_workspace_lock_ == -2);
506 
507  if (!wait_workspace_) {
509  return false;
510  }
511 
513  if (fd_workspace_lock_ < 0) {
514  boot_error_ = "could not acquire workspace lock (" +
515  StringifyInt(errno) + ")";
517  return false;
518  }
519  return true;
520 }
521 
522 
524  void *user_data __attribute__((unused)),
525  int sqlite_extended_error,
526  const char *message)
527 {
528  int log_dest = kLogDebug;
529  int sqlite_error = sqlite_extended_error & 0xFF;
530  switch (sqlite_error) {
531  case SQLITE_INTERNAL:
532  case SQLITE_PERM:
533  case SQLITE_NOMEM:
534  case SQLITE_IOERR:
535  case SQLITE_CORRUPT:
536  case SQLITE_FULL:
537  case SQLITE_CANTOPEN:
538  case SQLITE_MISUSE:
539  case SQLITE_FORMAT:
540  case SQLITE_NOTADB:
541  log_dest |= kLogSyslogErr;
542  break;
543  case SQLITE_WARNING:
544  case SQLITE_NOTICE:
545  default:
546  break;
547  }
548  LogCvmfs(kLogCvmfs, log_dest, "SQlite3: %s (%d)",
549  message, sqlite_extended_error);
550 }
551 
552 
559  const string &generic_parameter,
560  const string &instance)
561 {
562  assert(HasPrefix(generic_parameter, "CVMFS_CACHE_", false));
563 
564  if (instance == kDefaultCacheMgrInstance) {
565  // Compatibility parameter names
566  if ((generic_parameter == "CVMFS_CACHE_SHARED") &&
567  !options_mgr_->IsDefined(generic_parameter))
568  {
569  return "CVMFS_SHARED_CACHE";
570  }
571  if ((generic_parameter == "CVMFS_CACHE_ALIEN") &&
572  !options_mgr_->IsDefined(generic_parameter))
573  {
574  return "CVMFS_ALIEN_CACHE";
575  }
576  if ((generic_parameter == "CVMFS_CACHE_SERVER_MODE") &&
577  !options_mgr_->IsDefined(generic_parameter))
578  {
579  return "CVMFS_SERVER_CACHE_MODE";
580  }
581  if ((generic_parameter == "CVMFS_CACHE_QUOTA_LIMIT") &&
582  !options_mgr_->IsDefined(generic_parameter))
583  {
584  return "CVMFS_QUOTA_LIMIT";
585  }
586  return generic_parameter;
587  }
588 
589  return "CVMFS_CACHE_" + instance + "_" + generic_parameter.substr(12);
590 }
591 
592 
595  n_eio_total_->Set(0);
596  n_eio_01_->Set(0);
597  n_eio_02_->Set(0);
598  n_eio_03_->Set(0);
599  n_eio_04_->Set(0);
600  n_eio_05_->Set(0);
601  n_eio_06_->Set(0);
602  n_eio_07_->Set(0);
603  n_eio_08_->Set(0);
604 }
605 
606 
610 CacheManager *FileSystem::SetupCacheMgr(const string &instance) {
611  if (constructed_instances_.find(instance) != constructed_instances_.end()) {
612  boot_error_ = "circular cache definition: " + instance;
614  return NULL;
615  }
616  constructed_instances_.insert(instance);
617 
618  LogCvmfs(kLogCvmfs, kLogDebug, "setting up cache manager instance %s",
619  instance.c_str());
620  string instance_type;
621  if (instance == kDefaultCacheMgrInstance) {
622  instance_type = "posix";
623  } else {
624  options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_TYPE", instance),
625  &instance_type);
626  }
627  if (instance_type == "posix") {
628  return SetupPosixCacheMgr(instance);
629  } else if (instance_type == "ram") {
630  return SetupRamCacheMgr(instance);
631  } else if (instance_type == "tiered") {
632  return SetupTieredCacheMgr(instance);
633  } else if (instance_type == "external") {
634  return SetupExternalCacheMgr(instance);
635  } else {
636  boot_error_ = "invalid cache manager type for '" + instance + "':" +
637  instance_type;
639  return NULL;
640  }
641 }
642 
643 
645  string optarg;
646  unsigned nfiles = kDefaultNfiles;
647  if (options_mgr_->GetValue("CVMFS_NFILES", &optarg))
648  nfiles = String2Uint64(optarg);
649  vector<string> cmd_line;
650  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_CMDLINE", instance),
651  &optarg))
652  {
653  cmd_line = SplitString(optarg, ',');
654  }
655 
656  if (!options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_LOCATOR", instance),
657  &optarg))
658  {
659  boot_error_ = MkCacheParm("CVMFS_CACHE_LOCATOR", instance) + " missing";
661  return NULL;
662  }
663 
665  ExternalCacheManager::CreatePlugin(optarg, cmd_line));
666  if (!plugin_handle->IsValid()) {
667  boot_error_ = plugin_handle->error_msg();
669  return NULL;
670  }
672  plugin_handle->fd_connection(), nfiles, name_ + ":" + instance);
673  if (cache_mgr == NULL) {
674  boot_error_ = "failed to create external cache manager for " + instance;
676  return NULL;
677  }
678  cache_mgr->AcquireQuotaManager(ExternalQuotaManager::Create(cache_mgr));
679  return cache_mgr;
680 }
681 
682 
685  if (!CheckPosixCacheSettings(settings))
686  return NULL;
687 
689  settings.cache_path,
690  settings.is_alien,
693  if (!cache_mgr.IsValid()) {
694  boot_error_ = "Failed to setup posix cache '" + instance + "' in " +
695  settings.cache_path + ": " + strerror(errno);
697  return NULL;
698  }
699 
700  // Sentinel file for future use
701  // Might be a read-only cache
702  const bool ignore_failure = settings.is_alien;
703  CreateFile(settings.cache_path + "/.cvmfscache", 0600, ignore_failure);
704 
705  if (settings.is_managed) {
706  if (!SetupPosixQuotaMgr(settings, cache_mgr.weak_ref()))
707  return NULL;
708  }
709  return cache_mgr.Release();
710 }
711 
712 
713 CacheManager *FileSystem::SetupRamCacheMgr(const string &instance) {
714  string optarg;
715  unsigned nfiles = kDefaultNfiles;
716  if (options_mgr_->GetValue("CVMFS_NFILES", &optarg)) {
717  nfiles = String2Uint64(optarg);
718  }
719  uint64_t sz_cache_bytes;
720  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SIZE", instance),
721  &optarg))
722  {
723  if (HasSuffix(optarg, "%", false)) {
724  sz_cache_bytes = platform_memsize() * String2Uint64(optarg)/100;
725  } else {
726  sz_cache_bytes = String2Uint64(optarg) * 1024 * 1024;
727  }
728  } else {
729  sz_cache_bytes = platform_memsize() >> 5; // ~3%
730  }
732  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_MALLOC", instance),
733  &optarg))
734  {
735  if (optarg == "libc") {
737  } else if (optarg == "heap") {
739  } else {
740  boot_error_ = "Failure: unknown malloc " +
741  MkCacheParm("CVMFS_CACHE_MALLOC", instance) + "=" + optarg;
743  return NULL;
744  }
745  }
746  sz_cache_bytes = RoundUp8(std::max(static_cast<uint64_t>(40 * 1024 * 1024),
747  sz_cache_bytes));
749  sz_cache_bytes,
750  nfiles,
751  alloc,
752  perf::StatisticsTemplate("cache." + instance, statistics_));
753  if (cache_mgr == NULL) {
754  boot_error_ = "failed to create ram cache manager for " + instance;
756  return NULL;
757  }
758  cache_mgr->AcquireQuotaManager(new NoopQuotaManager());
759  return cache_mgr;
760 }
761 
762 
764  string optarg;
765  if (!options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_UPPER", instance),
766  &optarg))
767  {
768  boot_error_ = MkCacheParm("CVMFS_CACHE_UPPER", instance) + " missing";
770  return NULL;
771  }
772  UniquePtr<CacheManager> upper(SetupCacheMgr(optarg));
773  if (!upper.IsValid())
774  return NULL;
775 
776  if (!options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_LOWER", instance),
777  &optarg))
778  {
779  boot_error_ = MkCacheParm("CVMFS_CACHE_LOWER", instance) + " missing";
781  return NULL;
782  }
783  UniquePtr<CacheManager> lower(SetupCacheMgr(optarg));
784  if (!lower.IsValid())
785  return NULL;
786 
787  CacheManager *tiered =
788  TieredCacheManager::Create(upper.Release(), lower.Release());
789  if (tiered == NULL) {
790  boot_error_ = "Failed to setup tiered cache manager " + instance;
792  return NULL;
793  }
794  if (options_mgr_->GetValue(
795  MkCacheParm("CVMFS_CACHE_LOWER_READONLY", instance), &optarg) &&
796  options_mgr_->IsOn(optarg))
797  {
798  static_cast<TieredCacheManager*>(tiered)->SetLowerReadOnly();
799  }
800  return tiered;
801 }
802 
803 
805  path_crash_guard_ = workspace_ + "/running." + name_;
806  platform_stat64 info;
807  int retval = platform_stat(path_crash_guard_.c_str(), &info);
808  if (retval == 0) {
809  found_previous_crash_ = true;
811  "looks like cvmfs has been crashed previously");
812  }
813  retval = open(path_crash_guard_.c_str(), O_RDONLY | O_CREAT, 0600);
814  if (retval < 0) {
815  boot_error_ = "could not open running sentinel (" +
816  StringifyInt(errno) + ")";
818  return false;
819  }
820  close(retval);
821  return true;
822 }
823 
824 
826  if (type_ == kFsFuse) {
827  // Try to jump to workspace / cache directory. This tests, if it is
828  // accessible and it brings speed later on.
829  int retval = chdir(workspace_.c_str());
830  if (retval != 0) {
831  boot_error_ = "workspace " + workspace_ + " is unavailable";
833  return false;
834  }
835  workspace_ = ".";
836  return true;
837  }
838 
839  // Note: as of version 2.4 support for CVMFS_CWD_CACHE is dropped due to
840  // disproportionate large complexity to configure correctly. This affects
841  // only libcvmfs, mostly the legacy part.
842  // string optarg;
843  // if (options_mgr_->GetValue("CVMFS_CWD_CACHE", &optarg) &&
844  // options_mgr_->IsOn(optarg))
845  // {
846  // ...
847  // }
848  return true;
849 }
850 
851 
856  setenv("CVMFS_ARCH", GetArch().c_str(), 1 /* overwrite */);
857 }
858 
859 
861  string optarg;
862  if (options_mgr_->GetValue("CVMFS_SYSLOG_LEVEL", &optarg))
864  if (options_mgr_->GetValue("CVMFS_SYSLOG_FACILITY", &optarg))
866  if (options_mgr_->GetValue("CVMFS_USYSLOG", &optarg))
867  SetLogMicroSyslog(optarg);
868  if (options_mgr_->GetValue("CVMFS_DEBUGLOG", &optarg))
869  SetLogDebugFile(optarg);
870  if (options_mgr_->GetValue("CVMFS_SYSLOG_PREFIX", &optarg)) {
871  SetLogSyslogPrefix(optarg);
872  } else {
874  }
875 }
876 
877 
879 #ifdef CVMFS_NFS_SUPPORT
880  if (!IsHaNfsSource())
882 
883  string no_nfs_sentinel;
884  if (cache_mgr_->id() == kPosixCacheManager) {
885  PosixCacheManager *posix_cache_mgr =
886  reinterpret_cast<PosixCacheManager *>(cache_mgr_);
887  no_nfs_sentinel = posix_cache_mgr->cache_path() + "/no_nfs_maps." + name_;
888  if (!IsNfsSource()) {
889  // Might be a read-only cache
890  const bool ignore_failure = posix_cache_mgr->alien_cache();
891  CreateFile(no_nfs_sentinel, 0600, ignore_failure);
892  return true;
893  }
894  } else {
895  if (IsNfsSource()) {
896  boot_error_ = "NFS source only works with POSIX cache manager.";
898  return false;
899  }
900  return true;
901  }
902 
904  assert(IsNfsSource());
905  if (!no_nfs_sentinel.empty() && FileExists(no_nfs_sentinel)) {
906  boot_error_ = "Cache was used without NFS maps before. "
907  "It has to be wiped out.";
909  return false;
910  }
911 
912  // nfs maps need to be protected by workspace lock
913  PosixCacheManager *posix_cache_mgr =
914  reinterpret_cast<PosixCacheManager *>(cache_mgr_);
915  if (posix_cache_mgr->cache_path() != workspace_) {
916  boot_error_ = "Cache directory and workspace must be identical for "
917  "NFS export";
919  return false;
920  }
921 
922  string inode_cache_dir = nfs_maps_dir_ + "/nfs_maps." + name_;
923  if (!MkdirDeep(inode_cache_dir, 0700)) {
924  boot_error_ = "Failed to initialize NFS maps";
926  return false;
927  }
928 
929  // TODO(jblomer): make this a manager class
930  if (IsHaNfsSource()) {
932  inode_cache_dir,
935  statistics_);
936  } else {
938  inode_cache_dir,
941  statistics_);
942  }
943 
944  if (nfs_maps_ == NULL) {
945  boot_error_ = "Failed to initialize NFS maps";
947  return false;
948  }
949 
950  string optarg;
951  if (options_mgr_->GetValue("CVMFS_NFS_INTERLEAVED_INODES", &optarg)) {
952  vector<string> tokens = SplitString(optarg, '%');
953  if (tokens.size() != 2) {
954  boot_error_ =
955  "invalid format for CVMFS_NFS_INTERLEAVED_INODES: " + optarg;
957  return false;
958  }
960  String2Uint64(tokens[0]));
961  }
962 
963  return true;
964 
965 #else
966  return true;
967 #endif
968 }
969 
970 
973  CacheManager *cache_mgr
974 ) {
975  assert(settings.quota_limit >= 0);
976  int64_t quota_threshold = settings.quota_limit / 2;
977  string cache_workspace = settings.cache_path;
978  if (settings.cache_path != settings.workspace) {
980  "using workspace %s to protect cache database in %s",
981  settings.workspace.c_str(), settings.cache_path.c_str());
982  cache_workspace += ":" + settings.workspace;
983  }
984  PosixQuotaManager *quota_mgr;
985 
986  if (settings.is_shared) {
988  exe_path_,
989  cache_workspace,
990  settings.quota_limit,
991  quota_threshold,
992  foreground_);
993  if (quota_mgr == NULL) {
994  boot_error_ = "Failed to initialize shared lru cache";
996  return false;
997  }
998  } else {
999  quota_mgr = PosixQuotaManager::Create(
1000  cache_workspace,
1001  settings.quota_limit,
1002  quota_threshold,
1004  if (quota_mgr == NULL) {
1005  boot_error_ = "Failed to initialize lru cache";
1007  return false;
1008  }
1009  }
1010 
1011  if (quota_mgr->GetSize() > quota_mgr->GetCapacity()) {
1013  "cache is already beyond quota size "
1014  "(size: %" PRId64 ", capacity: %" PRId64 "), cleaning up",
1015  quota_mgr->GetSize(), quota_mgr->GetCapacity());
1016  if (!quota_mgr->Cleanup(quota_threshold)) {
1017  delete quota_mgr;
1018  boot_error_ = "Failed to clean up cache";
1020  return false;
1021  }
1022  }
1023 
1024  int retval = cache_mgr->AcquireQuotaManager(quota_mgr);
1025  assert(retval);
1027  "CernVM-FS: quota initialized, current size %luMB",
1028  quota_mgr->GetSize() / (1024 * 1024));
1029  return true;
1030 }
1031 
1032 
1034  // Make sure SQlite starts clean after initialization
1035  sqlite3_shutdown();
1036 
1037  int retval;
1038  retval = sqlite3_config(SQLITE_CONFIG_LOG, FileSystem::LogSqliteError, NULL);
1039  assert(retval == SQLITE_OK);
1040  retval = sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
1041  assert(retval == SQLITE_OK);
1043 
1044  // Disable SQlite3 file locking
1045  retval = sqlite3_vfs_register(sqlite3_vfs_find("unix-none"), 1);
1046  assert(retval == SQLITE_OK);
1047 }
1048 
1049 
1051  string optarg;
1052  // This is very similar to "determine cache dir". It's for backward
1053  // compatibility with classic cache configuration where there was no
1054  // distinction between workspace and cache.
1055  // Complicated cache configurations should explicitly set CVMFS_WORKSPACE.
1057  if (options_mgr_->GetValue("CVMFS_CACHE_BASE", &optarg))
1058  workspace_ = MakeCanonicalPath(optarg);
1059  if (options_mgr_->GetValue("CVMFS_SHARED_CACHE", &optarg) &&
1060  options_mgr_->IsOn(optarg))
1061  {
1062  workspace_ += "/shared";
1063  } else {
1064  workspace_ += "/" + name_;
1065  }
1066  if (options_mgr_->GetValue("CVMFS_CACHE_DIR", &optarg)) {
1067  if (options_mgr_->IsDefined("CVMFS_CACHE_BASE")) {
1068  boot_error_ =
1069  "'CVMFS_CACHE_BASE' and 'CVMFS_CACHE_DIR' are mutually exclusive";
1071  return false;
1072  }
1073  workspace_ = optarg;
1074  }
1075  if (options_mgr_->GetValue("CVMFS_WORKSPACE", &optarg))
1076  workspace_ = optarg;
1078 
1079  // If workspace and alien cache are the same directory, we need to open
1080  // permission now to 0770 to avoid a race when fixing it later
1081  const int mode = 0770;
1082  if (!MkdirDeep(workspace_, mode, false)) {
1083  boot_error_ = "cannot create workspace directory " + workspace_;
1085  return false;
1086  }
1087 
1088  if (!LockWorkspace())
1089  return false;
1090  if (!SetupCwd())
1091  return false;
1092  if (!SetupCrashGuard())
1093  return false;
1094 
1095  return true;
1096 }
1097 
1098 
1101  if (uuid_cache_ == NULL) {
1103  "failed to load/store %s/uuid", workspace_.c_str());
1105  assert(uuid_cache_ != NULL);
1106  }
1107 }
1108 
1109 
1115  if ((cache_mgr_ != NULL) && (cache_mgr_->id() == kPosixCacheManager)) {
1116  PosixCacheManager *posix_cache_mgr =
1117  reinterpret_cast<PosixCacheManager *>(cache_mgr_);
1118  posix_cache_mgr->TearDown2ReadOnly();
1119  }
1120 
1121  unlink(path_crash_guard_.c_str());
1122  LogCvmfs(kLogCache, kLogSyslog, "switch to read-only cache mode");
1123  SetLogMicroSyslog("");
1124 }
1125 
1126 
1127 void FileSystem::RemapCatalogFd(int from, int to) {
1128  sqlite::RegisterFdMapping(from, to);
1129 }
1130 
1131 
1134  string instance;
1135  if (options_mgr_->GetValue("CVMFS_CACHE_PRIMARY", &instance) &&
1136  !instance.empty())
1137  {
1138  if (!CheckInstanceName(instance))
1139  return false;
1140  cache_mgr_instance_ = instance;
1141  }
1142 
1144  return cache_mgr_ != NULL;
1145 }
1146 
1147 
1148 //------------------------------------------------------------------------------
1149 
1150 
1151 const char *MountPoint::kDefaultAuthzSearchPath = "/usr/libexec/cvmfs/authz";
1152 const char *MountPoint::kDefaultBlacklist = "/etc/cvmfs/blacklist";
1153 
1155  blacklist_paths_.clear();
1156  string blacklist;
1157  if (!options_mgr_->GetValue("CVMFS_BLACKLIST", &blacklist))
1158  blacklist = kDefaultBlacklist;
1159  blacklist_paths_.push_back(blacklist);
1160 
1161  bool append = false;
1162  if (FileExists(blacklist)) {
1163  if (!signature_mgr_->LoadBlacklist(blacklist, append)) {
1164  boot_error_ = "failed to load blacklist " + blacklist;
1166  return false;
1167  }
1168  append = true;
1169  }
1170 
1171  string config_repository_path;
1172  if (options_mgr_->HasConfigRepository(fqrn_, &config_repository_path)) {
1173  blacklist = config_repository_path + "blacklist";
1174  blacklist_paths_.push_back(blacklist);
1175  if (FileExists(blacklist)) {
1176  if (!signature_mgr_->LoadBlacklist(blacklist, append)) {
1177  boot_error_ = "failed to load blacklist from config repository";
1179  return false;
1180  }
1181  }
1182  }
1183 
1184  return true;
1185 }
1186 
1187 
1193  bool result = true;
1194  bool append = false;
1195  for (unsigned i = 0; i < blacklist_paths_.size(); ++i) {
1196  string blacklist = blacklist_paths_[i];
1197  if (FileExists(blacklist)) {
1198  bool retval = signature_mgr_->LoadBlacklist(blacklist, append);
1199  // TODO(jblomer): this can leave us with a half-loaded blacklist
1200  if (!retval)
1201  return false;
1202  append = true;
1203  }
1204  }
1205  return result;
1206 }
1207 
1217  cache_symlinks_ = false;
1218 }
1219 
1225  fuse_expire_entry_ = true;
1226 }
1227 
1228 
1234  const string &fqrn,
1235  FileSystem *file_system,
1236  OptionsManager *options_mgr)
1237 {
1238  if (options_mgr == NULL)
1239  options_mgr = file_system->options_mgr();
1240  UniquePtr<MountPoint> mountpoint(new MountPoint(
1241  fqrn, file_system, options_mgr));
1242 
1243  // At this point, we have a repository name, the type (fuse or library) and
1244  // an options manager (which can be the same than the FileSystem's one).
1245 
1246  mountpoint->CreateStatistics();
1247  mountpoint->CreateAuthz();
1248  mountpoint->backoff_throttle_ = new BackoffThrottle();
1249 
1250  if (!mountpoint->CreateSignatureManager() || !mountpoint->CheckBlacklists())
1251  return mountpoint.Release();
1252  if (!mountpoint->CreateDownloadManagers())
1253  return mountpoint.Release();
1254  if (!mountpoint->CreateResolvConfWatcher()) {
1255  return mountpoint.Release();
1256  }
1257  mountpoint->CreateFetchers();
1258  if (!mountpoint->CreateCatalogManager())
1259  return mountpoint.Release();
1260  if (!mountpoint->CreateTracer())
1261  return mountpoint.Release();
1262 
1263  mountpoint->ReEvaluateAuthz();
1264  mountpoint->CreateTables();
1265  if (!mountpoint->SetupBehavior())
1266  return mountpoint.Release();
1267 
1268  mountpoint->boot_status_ = loader::kFailOk;
1269  return mountpoint.Release();
1270 }
1271 
1272 
1274  string optarg;
1275  string authz_helper;
1276  if (options_mgr_->GetValue("CVMFS_AUTHZ_HELPER", &optarg))
1277  authz_helper = optarg;
1278  string authz_search_path(kDefaultAuthzSearchPath);
1279  if (options_mgr_->GetValue("CVMFS_AUTHZ_SEARCH_PATH", &optarg))
1280  authz_search_path = optarg;
1281 
1283  fqrn_,
1284  authz_helper,
1285  authz_search_path,
1286  options_mgr_);
1287  assert(authz_fetcher_ != NULL);
1288 
1291  statistics_);
1292  assert(authz_session_mgr_ != NULL);
1293 
1295  assert(authz_attachment_ != NULL);
1296 }
1297 
1298 
1300  string optarg;
1301 
1303 
1305  if (!SetupOwnerMaps())
1306  return false;
1307  shash::Any root_hash;
1308  if (!DetermineRootHash(&root_hash))
1309  return false;
1310 
1311  bool retval;
1312  if (root_hash.IsNull()) {
1313  retval = catalog_mgr_->Init();
1314  } else {
1315  fixed_catalog_ = true;
1316  bool alt_root_path =
1317  options_mgr_->GetValue("CVMFS_ALT_ROOT_PATH", &optarg) &&
1318  options_mgr_->IsOn(optarg);
1319  retval = catalog_mgr_->InitFixed(root_hash, alt_root_path);
1320  }
1321  if (!retval) {
1322  boot_error_ = "Failed to initialize root file catalog";
1324  return false;
1325  }
1326 
1328  boot_error_ = "repository revision blacklisted";
1330  return false;
1331  }
1332 
1333  if (options_mgr_->GetValue("CVMFS_AUTO_UPDATE", &optarg) &&
1334  !options_mgr_->IsOn(optarg))
1335  {
1336  fixed_catalog_ = true;
1337  }
1338 
1339  if (options_mgr_->GetValue("CVMFS_CATALOG_WATERMARK", &optarg)) {
1341  } else {
1342  unsigned soft_limit;
1343  unsigned hard_limit;
1344  GetLimitNoFile(&soft_limit, &hard_limit);
1345  catalog_mgr_->SetCatalogWatermark(soft_limit / 4);
1346  }
1347 
1348  if (catalog_mgr_->volatile_flag()) {
1349  LogCvmfs(kLogCvmfs, kLogDebug, "content of repository flagged as VOLATILE");
1350  }
1351 
1352  return true;
1353 }
1354 
1355 
1357  string optarg;
1360  perf::StatisticsTemplate("download", statistics_));
1362 
1363  if (options_mgr_->GetValue("CVMFS_SERVER_URL", &optarg)) {
1364  download_mgr_->SetHostChain(optarg);
1365  }
1366 
1368  SetupHttpTuning();
1369 
1370  string forced_proxy_template;
1371  if (options_mgr_->GetValue("CVMFS_PROXY_TEMPLATE", &optarg))
1372  forced_proxy_template = optarg;
1374  forced_proxy_template);
1375 
1376  string proxies;
1377  if (options_mgr_->GetValue("CVMFS_HTTP_PROXY", &optarg))
1378  proxies = optarg;
1380  proxies,
1381  file_system_->workspace() + "/proxies" + GetUniqFileSuffix(),
1382  download_mgr_);
1383  if (proxies == "") {
1384  boot_error_ = "failed to discover HTTP proxy servers";
1386  return false;
1387  }
1388  string fallback_proxies;
1389  if (options_mgr_->GetValue("CVMFS_FALLBACK_PROXY", &optarg))
1390  fallback_proxies = optarg;
1391  download_mgr_->SetProxyChain(proxies, fallback_proxies,
1393 
1394  bool do_geosort = options_mgr_->GetValue("CVMFS_USE_GEOAPI", &optarg) &&
1395  options_mgr_->IsOn(optarg);
1396  if (do_geosort) {
1398  }
1399  if (options_mgr_->GetValue("CVMFS_MAX_SERVERS", &optarg)) {
1400  unsigned max_servers = String2Uint64(optarg);
1401  std::vector<std::string> host_chain;
1402  download_mgr_->GetHostInfo(&host_chain, NULL, NULL);
1403  if (max_servers > 0 && max_servers < host_chain.size()) {
1404  host_chain.resize(max_servers);
1405  download_mgr_->SetHostChain(host_chain);
1406  }
1407  }
1408 
1409  if (options_mgr_->GetValue("CVMFS_USE_SSL_SYSTEM_CA", &optarg) &&
1410  options_mgr_->IsOn(optarg)) {
1412  }
1413 
1414  if (options_mgr_->GetValue("CVMFS_PROXY_SHARD", &optarg) &&
1415  options_mgr_->IsOn(optarg)) {
1417  }
1418 
1419  return SetupExternalDownloadMgr(do_geosort);
1420 }
1421 
1423  std::string roaming_value;
1424  options_mgr_->GetValue("CVMFS_DNS_ROAMING", &roaming_value);
1425  if (options_mgr_->IsDefined("CVMFS_DNS_ROAMING") &&
1426  options_mgr_->IsOn(roaming_value)) {
1428  "DNS roaming is enabled for this repository.");
1429  // Create a file watcher to update the DNS settings of the download
1430  // managers when there are changes to /etc/resolv.conf
1432 
1433  if (resolv_conf_watcher_) {
1434  ResolvConfEventHandler *handler =
1436  resolv_conf_watcher_->RegisterHandler("/etc/resolv.conf", handler);
1437  }
1438  } else {
1440  "DNS roaming is disabled for this repository.");
1441  }
1442  return true;
1443 }
1444 
1446  fetcher_ = new cvmfs::Fetcher(
1448  download_mgr_,
1451 
1452  const bool is_external_data = true;
1457  perf::StatisticsTemplate("fetch-external", statistics_),
1458  is_external_data);
1459 }
1460 
1461 
1463  string optarg;
1465  signature_mgr_->Init();
1466 
1467  string public_keys;
1468  if (options_mgr_->GetValue("CVMFS_PUBLIC_KEY", &optarg)) {
1469  public_keys = optarg;
1470  } else if (options_mgr_->GetValue("CVMFS_KEYS_DIR", &optarg)) {
1471  // Collect .pub files from CVMFS_KEYS_DIR
1472  public_keys = JoinStrings(FindFilesBySuffix(optarg, ".pub"), ":");
1473  } else {
1474  public_keys =
1475  JoinStrings(FindFilesBySuffix("/etc/cvmfs/keys", ".pub"), ":");
1476  }
1477 
1478  if (!signature_mgr_->LoadPublicRsaKeys(public_keys)) {
1479  boot_error_ = "failed to load public key(s)";
1481  return false;
1482  }
1483  LogCvmfs(kLogCvmfs, kLogDebug, "CernVM-FS: using public key(s) %s",
1484  public_keys.c_str());
1485 
1486  if (options_mgr_->GetValue("CVMFS_TRUSTED_CERTS", &optarg)) {
1487  if (!signature_mgr_->LoadTrustedCaCrl(optarg)) {
1488  boot_error_ = "failed to load trusted certificates";
1490  return false;
1491  }
1492  }
1493 
1494  return true;
1495 }
1496 
1497 
1501  return;
1502 
1503  // TODO(jblomer): this should be registered by the tracker
1504  statistics_->Register("inode_tracker.n_insert",
1505  "overall number of accessed inodes");
1506  statistics_->Register("inode_tracker.n_remove",
1507  "overall number of evicted inodes");
1508  statistics_->Register("inode_tracker.no_reference",
1509  "currently active inodes");
1510  statistics_->Register("inode_tracker.n_hit_inode",
1511  "overall number of inode lookups");
1512  statistics_->Register("inode_tracker.n_hit_path",
1513  "overall number of successful path lookups");
1514  statistics_->Register("inode_tracker.n_miss_path",
1515  "overall number of unsuccessful path lookups");
1516 
1517  statistics_->Register("dentry_tracker.n_insert",
1518  "overall number of added negative cache entries");
1519  statistics_->Register("dentry_tracker.n_remove",
1520  "overall number of evicted negative cache entries");
1521  statistics_->Register("dentry_tracker.n_prune",
1522  "overall number of prune calls");
1523 
1524  statistics_->Register("page_cache_tracker.n_insert",
1525  "overall number of added page cache entries");
1526  statistics_->Register("page_cache_tracker.n_remove",
1527  "overall number of evicted page cache entries");
1528  statistics_->Register("page_cache_tracker.n_open_direct",
1529  "overall number of direct I/O open calls");
1530  statistics_->Register("page_cache_tracker.n_open_flush",
1531  "overall number of open calls where the file's page cache gets flushed");
1532  statistics_->Register("page_cache_tracker.n_open_cached",
1533  "overall number of open calls where the file's page cache is reused");
1534 }
1535 
1536 
1539  // Libcvmfs simplified tables
1542  return;
1543  }
1544 
1545  chunk_tables_ = new ChunkTables();
1546 
1547  string optarg;
1548  uint64_t mem_cache_size = kDefaultMemcacheSize;
1549  if (options_mgr_->GetValue("CVMFS_MEMCACHE_SIZE", &optarg))
1550  mem_cache_size = String2Uint64(optarg) * 1024 * 1024;
1551 
1552  const double memcache_unit_size =
1553  (static_cast<double>(kInodeCacheFactor) * lru::Md5PathCache::GetEntrySize())
1555  const unsigned memcache_num_units =
1556  mem_cache_size / static_cast<unsigned>(memcache_unit_size);
1557  // Number of cache entries must be a multiple of 64
1558  const unsigned mask_64 = ~((1 << 6) - 1);
1559  inode_cache_ = new lru::InodeCache(memcache_num_units & mask_64, statistics_);
1560  path_cache_ = new lru::PathCache(memcache_num_units & mask_64, statistics_);
1561  md5path_cache_ = new lru::Md5PathCache((memcache_num_units * 7) & mask_64,
1562  statistics_);
1563 
1567  if (file_system_->IsNfsSource())
1569 }
1570 
1580  string optarg;
1581  tracer_ = new Tracer();
1582  if (options_mgr_->GetValue("CVMFS_TRACEFILE", &optarg)) {
1584  boot_error_ = "tracer is only supported in the fuse module";
1586  return false;
1587  }
1588  string tracebuffer_file = optarg;
1589  uint64_t tracebuffer_size = kTracerBufferSize;
1590  uint64_t tracebuffer_threshold = kTracerFlushThreshold;
1591 
1592  if (options_mgr_->GetValue("CVMFS_TRACEBUFFER", &optarg)) {
1593  tracebuffer_size = String2Uint64(optarg);
1594  }
1595  if (options_mgr_->GetValue("CVMFS_TRACEBUFFER_THRESHOLD",
1596  &optarg)) {
1597  tracebuffer_threshold = String2Uint64(optarg);
1598  }
1599  assert(tracebuffer_size <= INT_MAX
1600  && tracebuffer_threshold <= INT_MAX);
1602  "Initialising tracer with buffer size %" PRIu64 " and threshold %" PRIu64,
1603  tracebuffer_size, tracebuffer_threshold);
1604  tracer_->Activate(tracebuffer_size, tracebuffer_threshold,
1605  tracebuffer_file);
1606  }
1607  return true;
1608 }
1609 
1610 
1612  string optarg;
1613  if (options_mgr_->GetValue("CVMFS_ROOT_HASH", &optarg)) {
1614  *root_hash = MkFromHexPtr(shash::HexPtr(optarg), shash::kSuffixCatalog);
1615  return true;
1616  }
1617 
1618  if (!options_mgr_->IsDefined("CVMFS_REPOSITORY_TAG") &&
1619  !options_mgr_->IsDefined("CVMFS_REPOSITORY_DATE"))
1620  {
1621  root_hash->SetNull();
1622  return true;
1623  }
1624 
1625  string history_path;
1626  if (!FetchHistory(&history_path))
1627  return false;
1628  UnlinkGuard history_file(history_path);
1630  history::SqliteHistory::Open(history_path));
1631  if (!tag_db.IsValid()) {
1633  "failed to open history database (%s)", history_path.c_str());
1634  boot_error_ = "failed to open history database";
1636  return false;
1637  }
1638 
1640  bool retval;
1641  if (!options_mgr_->GetValue("CVMFS_REPOSITORY_TAG", &repository_tag_)) {
1642  string repository_date;
1643  // options_mgr_->IsDefined("CVMFS_REPOSITORY_DATE") must be true
1644  options_mgr_->GetValue("CVMFS_REPOSITORY_DATE", &repository_date);
1645  time_t repository_utctime = IsoTimestamp2UtcTime(repository_date);
1646  if (repository_utctime == 0) {
1647  boot_error_ = "invalid timestamp in CVMFS_REPOSITORY_DATE: " +
1648  repository_date + ". Use YYYY-MM-DDTHH:MM:SSZ";
1650  return false;
1651  }
1652  retval = tag_db->GetByDate(repository_utctime, &tag);
1653  if (!retval) {
1654  boot_error_ = "no repository state as early as utc timestamp " +
1655  StringifyTime(repository_utctime, true);
1657  return false;
1658  }
1660  "time stamp %s UTC resolved to tag '%s'",
1661  StringifyTime(repository_utctime, true).c_str(),
1662  tag.name.c_str());
1663  repository_tag_ = tag.name;
1664  } else {
1665  retval = tag_db->GetByName(repository_tag_, &tag);
1666  if (!retval) {
1667  boot_error_ = "no such tag: " + repository_tag_;
1669  return false;
1670  }
1671  }
1672  LogCvmfs(kLogCvmfs, kLogDebug, "mounting tag %s", tag.name.c_str());
1673 
1674  *root_hash = tag.root_hash;
1675  return true;
1676 }
1677 
1678 
1679 bool MountPoint::FetchHistory(std::string *history_path) {
1680  manifest::Failures retval_mf;
1681  manifest::ManifestEnsemble ensemble;
1682  retval_mf = manifest::Fetch("", fqrn_, 0, NULL, signature_mgr_, download_mgr_,
1683  &ensemble);
1684  if (retval_mf != manifest::kFailOk) {
1685  boot_error_ = "Failed to fetch manifest";
1687  return false;
1688  }
1689  shash::Any history_hash = ensemble.manifest->history();
1690  if (history_hash.IsNull()) {
1691  boot_error_ = "No history";
1693  return false;
1694  }
1695 
1696  int fd = fetcher_->Fetch(
1697  history_hash,
1699  "tag database for " + fqrn_,
1702  if (fd < 0) {
1703  boot_error_ = "failed to download history: " + StringifyInt(-fd);
1705  return false;
1706  }
1707  // We have the custom sqlite vfs driver installed
1708  *history_path = "@" + StringifyInt(fd);
1709  return true;
1710 }
1711 
1712 
1714  unsigned max_ttl;
1715  {
1716  MutexLockGuard lock_guard(lock_max_ttl_);
1717  max_ttl = max_ttl_sec_;
1718  }
1719  const unsigned catalog_ttl_sec = catalog_mgr_->GetTTL();
1720 
1721  return max_ttl ? std::min(max_ttl, catalog_ttl_sec) : catalog_ttl_sec;
1722 }
1723 
1724 
1726  MutexLockGuard lock_guard(lock_max_ttl_);
1727  return max_ttl_sec_ / 60;
1728 }
1729 
1730 
1736  return "." + file_system_->name() + "-" + fqrn_;
1737 }
1738 
1739 
1741  const string &fqrn,
1742  FileSystem *file_system,
1743  OptionsManager *options_mgr)
1744  : fqrn_(fqrn)
1745  , uuid_(cvmfs::Uuid::Create(""))
1746  , file_system_(file_system)
1747  , options_mgr_(options_mgr)
1748  , statistics_(NULL)
1749  , authz_fetcher_(NULL)
1750  , authz_session_mgr_(NULL)
1751  , authz_attachment_(NULL)
1752  , backoff_throttle_(NULL)
1753  , signature_mgr_(NULL)
1754  , download_mgr_(NULL)
1755  , external_download_mgr_(NULL)
1756  , fetcher_(NULL)
1757  , external_fetcher_(NULL)
1758  , inode_annotation_(NULL)
1759  , catalog_mgr_(NULL)
1760  , chunk_tables_(NULL)
1761  , simple_chunk_tables_(NULL)
1762  , inode_cache_(NULL)
1763  , path_cache_(NULL)
1764  , md5path_cache_(NULL)
1765  , tracer_(NULL)
1766  , inode_tracker_(NULL)
1767  , dentry_tracker_(NULL)
1768  , page_cache_tracker_(NULL)
1769  , statfs_cache_(NULL)
1770  , resolv_conf_watcher_(NULL)
1771  , max_ttl_sec_(kDefaultMaxTtlSec)
1772  , kcache_timeout_sec_(static_cast<double>(kDefaultKCacheTtlSec))
1773  , fixed_catalog_(false)
1774  , enforce_acls_(false)
1775  , cache_symlinks_(false)
1776  , fuse_expire_entry_(false)
1777  , has_membership_req_(false)
1778  , talk_socket_path_(std::string("./cvmfs_io.") + fqrn)
1779  , talk_socket_uid_(0)
1780  , talk_socket_gid_(0)
1781 {
1782  int retval = pthread_mutex_init(&lock_max_ttl_, NULL);
1783  assert(retval == 0);
1784 }
1785 
1786 
1788  pthread_mutex_destroy(&lock_max_ttl_);
1789 
1790  delete page_cache_tracker_;
1791  delete dentry_tracker_;
1792  delete inode_tracker_;
1793  delete tracer_;
1794  delete md5path_cache_;
1795  delete path_cache_;
1796  delete inode_cache_;
1797  delete simple_chunk_tables_;
1798  delete chunk_tables_;
1799 
1800  delete catalog_mgr_;
1801  delete inode_annotation_;
1802  delete external_fetcher_;
1803  delete fetcher_;
1804  if (external_download_mgr_ != NULL) {
1806  delete external_download_mgr_;
1807  }
1808  if (download_mgr_ != NULL) {
1809  download_mgr_->Fini();
1810  delete download_mgr_;
1811  }
1812  if (signature_mgr_ != NULL) {
1813  signature_mgr_->Fini();
1814  delete signature_mgr_;
1815  }
1816 
1817  if (resolv_conf_watcher_ != NULL) {
1819  delete resolv_conf_watcher_;
1820  }
1821 
1822  delete backoff_throttle_;
1823  delete authz_attachment_;
1824  delete authz_session_mgr_;
1825  delete authz_fetcher_;
1826  delete statistics_;
1827  delete uuid_;
1828 
1829  delete statfs_cache_;
1830 }
1831 
1832 
1834  string old_membership_req = membership_req_;
1836  if (old_membership_req != membership_req_) {
1839  }
1840 }
1841 
1842 
1843 void MountPoint::SetMaxTtlMn(unsigned value_minutes) {
1844  MutexLockGuard lock_guard(lock_max_ttl_);
1845  max_ttl_sec_ = value_minutes * 60;
1846 }
1847 
1849  string optarg;
1850 
1851  if (options_mgr_->GetValue("CVMFS_MAX_TTL", &optarg))
1852  SetMaxTtlMn(String2Uint64(optarg));
1853 
1854  if (options_mgr_->GetValue("CVMFS_KCACHE_TIMEOUT", &optarg)) {
1855  // Can be negative and should then be interpreted as 0.0
1857  std::max(0.0, static_cast<double>(String2Int64(optarg)));
1858  }
1859  LogCvmfs(kLogCvmfs, kLogDebug, "kernel caches expire after %d seconds",
1860  static_cast<int>(kcache_timeout_sec_));
1861 
1862  uint64_t statfs_time_cache_valid = 0;
1863  if (options_mgr_->GetValue("CVMFS_STATFS_CACHE_TIMEOUT", &optarg)) {
1864  statfs_time_cache_valid = static_cast<uint64_t>(String2Uint64(optarg));
1865  }
1866  LogCvmfs(kLogCvmfs, kLogDebug, "statfs cache expires after %d seconds",
1867  static_cast<int>(statfs_time_cache_valid));
1868  statfs_cache_ = new StatfsCache(statfs_time_cache_valid);
1869 
1870  MagicXattrManager::EVisibility xattr_visibility =
1872  if (options_mgr_->GetValue("CVMFS_HIDE_MAGIC_XATTRS", &optarg)) {
1873  if (options_mgr_->IsOn(optarg))
1874  xattr_visibility = MagicXattrManager::kVisibilityNever;
1875  else if (options_mgr_->IsOff(optarg))
1876  xattr_visibility = MagicXattrManager::kVisibilityAlways;
1877  }
1878  if (options_mgr_->GetValue("CVMFS_MAGIC_XATTRS_VISIBILITY", &optarg)) {
1879  if (ToUpper(optarg) == "ROOTONLY") {
1880  xattr_visibility = MagicXattrManager::kVisibilityRootOnly;
1881  } else if (ToUpper(optarg) == "NEVER") {
1882  xattr_visibility = MagicXattrManager::kVisibilityNever;
1883  } else if (ToUpper(optarg) == "ALWAYS") {
1884  xattr_visibility = MagicXattrManager::kVisibilityAlways;
1885  } else {
1887  "unsupported setting: CVMFS_MAGIC_XATTRS_VISIBILITY=%s",
1888  optarg.c_str());
1889  }
1890  }
1891 
1892  std::set<gid_t> protected_xattr_gids;
1893  if (options_mgr_->GetValue("CVMFS_XATTR_PRIVILEGED_GIDS", &optarg)) {
1894  std::vector<string> tmp = SplitString(optarg, ',');
1895 
1896  for (size_t i = 0; i < tmp.size(); i++) {
1897  std::string trimmed = Trim(tmp[i]);
1899  "Privileged gid for xattr added: %s", trimmed.c_str());
1900  protected_xattr_gids.insert(static_cast<gid_t>(String2Uint64(trimmed)));
1901  }
1902  }
1903  std::set<std::string> protected_xattrs;
1904  if (options_mgr_->GetValue("CVMFS_XATTR_PROTECTED_XATTRS", &optarg)) {
1905  std::vector<string> tmp = SplitString(optarg, ',');
1906 
1907  for (size_t i = 0; i < tmp.size(); i++) {
1908  std::string trimmed = Trim(tmp[i]);
1910  "Protected xattr added: %s", trimmed.c_str());
1911  protected_xattrs.insert(trimmed);
1912  }
1913 
1914  // root has always access to xattr
1915  if (protected_xattr_gids.count(0) < 1) {
1916  protected_xattr_gids.insert(0);
1918  "Automatically added root to have access to protected xattrs.");
1919  }
1920  }
1921  magic_xattr_mgr_ = new MagicXattrManager(this, xattr_visibility,
1922  protected_xattrs, protected_xattr_gids);
1923 
1924 
1925  if (options_mgr_->GetValue("CVMFS_ENFORCE_ACLS", &optarg)
1926  && options_mgr_->IsOn(optarg))
1927  {
1928  enforce_acls_ = true;
1929  }
1930 
1931  if (options_mgr_->GetValue("CVMFS_CACHE_SYMLINKS", &optarg)
1932  && options_mgr_->IsOn(optarg))
1933  {
1934  cache_symlinks_ = true;
1935  }
1936 
1937 
1938 
1939  if (options_mgr_->GetValue("CVMFS_TALK_SOCKET", &optarg)) {
1940  talk_socket_path_ = optarg;
1941  }
1942  if (options_mgr_->GetValue("CVMFS_TALK_OWNER", &optarg)) {
1943  bool retval = GetUidOf(optarg, &talk_socket_uid_, &talk_socket_gid_);
1944  if (!retval) {
1945  boot_error_ = "unknown owner of cvmfs_talk socket: " + optarg;
1947  return false;
1948  }
1949  }
1950 
1951  return true;
1952 }
1953 
1954 
1960  string optarg;
1961  unsigned dns_timeout_ms = download::DownloadManager::kDnsDefaultTimeoutMs;
1962  unsigned dns_retries = download::DownloadManager::kDnsDefaultRetries;
1963  if (options_mgr_->GetValue("CVMFS_DNS_TIMEOUT", &optarg))
1964  dns_timeout_ms = String2Uint64(optarg) * 1000;
1965  if (options_mgr_->GetValue("CVMFS_DNS_RETRIES", &optarg))
1966  dns_retries = String2Uint64(optarg);
1967  manager->SetDnsParameters(dns_retries, dns_timeout_ms);
1968 
1969  // Rest has to be after SetDnsParameters because SetDnsParameters might
1970  // construct a new resolver object
1971 
1972  unsigned dns_min_ttl = dns::Resolver::kDefaultMinTtl;
1973  unsigned dns_max_ttl = dns::Resolver::kDefaultMaxTtl;
1974  if (options_mgr_->GetValue("CVMFS_DNS_MIN_TTL", &optarg))
1975  dns_min_ttl = String2Uint64(optarg);
1976  if (options_mgr_->GetValue("CVMFS_DNS_MAX_TTL", &optarg))
1977  dns_max_ttl = String2Uint64(optarg);
1978  manager->SetDnsTtlLimits(dns_min_ttl, dns_max_ttl);
1979 
1980  if (options_mgr_->GetValue("CVMFS_DNS_SERVER", &optarg)) {
1981  download_mgr_->SetDnsServer(optarg);
1982  }
1983 
1984  if (options_mgr_->GetValue("CVMFS_IPFAMILY_PREFER", &optarg)) {
1985  switch (String2Int64(optarg)) {
1986  case 4:
1988  break;
1989  case 6:
1991  break;
1992  }
1993  }
1994  if (options_mgr_->GetValue("CVMFS_MAX_IPADDR_PER_PROXY", &optarg))
1995  manager->SetMaxIpaddrPerProxy(String2Uint64(optarg));
1996 }
1997 
1998 
2000  string optarg;
2002  download_mgr_->Clone(perf::StatisticsTemplate("download-external",
2003  statistics_));
2004 
2005  unsigned timeout;
2006  unsigned timeout_direct;
2007  download_mgr_->GetTimeout(&timeout, &timeout_direct);
2008  if (options_mgr_->GetValue("CVMFS_EXTERNAL_TIMEOUT", &optarg)) {
2009  timeout = String2Uint64(optarg);
2010  }
2011  if (options_mgr_->GetValue("CVMFS_EXTERNAL_TIMEOUT_DIRECT", &optarg)) {
2012  timeout_direct = String2Uint64(optarg);
2013  }
2014  external_download_mgr_->SetTimeout(timeout, timeout_direct);
2015 
2016  if (options_mgr_->GetValue("CVMFS_EXTERNAL_URL", &optarg)) {
2018  if (dogeosort) {
2019  std::vector<std::string> host_chain;
2020  external_download_mgr_->GetHostInfo(&host_chain, NULL, NULL);
2021  download_mgr_->GeoSortServers(&host_chain);
2022  external_download_mgr_->SetHostChain(host_chain);
2023  }
2024  } else {
2026  }
2027 
2028  if (options_mgr_->GetValue("CVMFS_EXTERNAL_MAX_SERVERS", &optarg)) {
2029  unsigned max_servers = String2Uint64(optarg);
2030  std::vector<std::string> host_chain;
2031  external_download_mgr_->GetHostInfo(&host_chain, NULL, NULL);
2032  if (max_servers > 0 && max_servers < host_chain.size()) {
2033  host_chain.resize(max_servers);
2034  external_download_mgr_->SetHostChain(host_chain);
2035  }
2036  }
2037 
2038  string proxies = "DIRECT";
2039  if (options_mgr_->GetValue("CVMFS_EXTERNAL_HTTP_PROXY", &optarg)) {
2041  optarg,
2042  file_system_->workspace() + "/proxies-external" + GetUniqFileSuffix(),
2044  if (proxies == "") {
2045  boot_error_ = "failed to discover external HTTP proxy servers";
2047  return false;
2048  }
2049  }
2050  string fallback_proxies;
2051  if (options_mgr_->GetValue("CVMFS_EXTERNAL_FALLBACK_PROXY", &optarg))
2052  fallback_proxies = optarg;
2054  proxies, fallback_proxies, download::DownloadManager::kSetProxyBoth);
2055 
2056  return true;
2057 }
2058 
2059 
2061  string optarg;
2062 
2063  // TODO(jblomer): avoid double default settings
2064 
2065  unsigned timeout = kDefaultTimeoutSec;
2066  unsigned timeout_direct = kDefaultTimeoutSec;
2067  if (options_mgr_->GetValue("CVMFS_TIMEOUT", &optarg))
2068  timeout = String2Uint64(optarg);
2069  if (options_mgr_->GetValue("CVMFS_TIMEOUT_DIRECT", &optarg))
2070  timeout_direct = String2Uint64(optarg);
2071  download_mgr_->SetTimeout(timeout, timeout_direct);
2072 
2073  unsigned max_retries = kDefaultRetries;
2074  unsigned backoff_init = kDefaultBackoffInitMs;
2075  unsigned backoff_max = kDefaultBackoffMaxMs;
2076  if (options_mgr_->GetValue("CVMFS_MAX_RETRIES", &optarg))
2077  max_retries = String2Uint64(optarg);
2078  if (options_mgr_->GetValue("CVMFS_BACKOFF_INIT", &optarg))
2079  backoff_init = String2Uint64(optarg) * 1000;
2080  if (options_mgr_->GetValue("CVMFS_BACKOFF_MAX", &optarg))
2081  backoff_max = String2Uint64(optarg) * 1000;
2082  download_mgr_->SetRetryParameters(max_retries, backoff_init, backoff_max);
2083 
2084  if (options_mgr_->GetValue("CVMFS_LOW_SPEED_LIMIT", &optarg))
2086  if (options_mgr_->GetValue("CVMFS_PROXY_RESET_AFTER", &optarg))
2088  if (options_mgr_->GetValue("CVMFS_HOST_RESET_AFTER", &optarg))
2090 
2091  if (options_mgr_->GetValue("CVMFS_FOLLOW_REDIRECTS", &optarg) &&
2092  options_mgr_->IsOn(optarg))
2093  {
2095  }
2096  if (options_mgr_->GetValue("CVMFS_SEND_INFO_HEADER", &optarg) &&
2097  options_mgr_->IsOn(optarg))
2098  {
2100  }
2101 }
2102 
2103 
2105  string optarg;
2106 
2107  if (file_system_->IsNfsSource()) {
2109  } else {
2111  }
2112  if (options_mgr_->GetValue("CVMFS_INITIAL_GENERATION", &optarg)) {
2114  }
2115 
2118  }
2119 }
2120 
2121 
2123  string optarg;
2124  catalog::OwnerMap uid_map;
2125  catalog::OwnerMap gid_map;
2126 
2127  if (options_mgr_->GetValue("CVMFS_UID_MAP", &optarg)) {
2128  if (!uid_map.Read(optarg)) {
2129  boot_error_ = "failed to parse uid map " + optarg;
2131  return false;
2132  }
2133  }
2134  if (options_mgr_->GetValue("CVMFS_GID_MAP", &optarg)) {
2135  if (!gid_map.Read(optarg)) {
2136  boot_error_ = "failed to parse gid map " + optarg;
2138  return false;
2139  }
2140  }
2141  catalog_mgr_->SetOwnerMaps(uid_map, gid_map);
2142 
2143  // TODO(jblomer): make local to catalog manager
2144  if (options_mgr_->GetValue("CVMFS_CLAIM_OWNERSHIP", &optarg) &&
2145  options_mgr_->IsOn(optarg))
2146  {
2147  g_claim_ownership = true;
2148  }
2149  if (options_mgr_->GetValue("CVMFS_WORLD_READABLE", &optarg) &&
2150  options_mgr_->IsOn(optarg))
2151  {
2152  g_world_readable = true;
2153  }
2154 
2155 
2156 
2157  return true;
2158 }
perf::Counter * n_eio_07_
Definition: mountpoint.h:343
IoErrorInfo io_error_info_
Definition: mountpoint.h:345
OptionsManager * options_mgr()
Definition: mountpoint.h:238
#define LogCvmfs(source, mask,...)
Definition: logging.h:22
std::string exe_path()
Definition: mountpoint.h:198
static const unsigned kDnsDefaultTimeoutMs
Definition: download.h:398
bool GetVOMSAuthz(std::string *authz) const
void SetLogSyslogFacility(const int local_facility)
Definition: logging.cc:183
FileSystem * file_system_
Definition: mountpoint.h:610
Counter * Register(const std::string &name, const std::string &desc)
Definition: statistics.cc:139
void SetCounter(perf::Counter *c)
Definition: mountpoint.cc:95
void RegisterFdMapping(int from, int to)
Definition: sqlitevfs.cc:541
std::string nfs_maps_dir_
Definition: mountpoint.h:402
void SetLogSyslogLevel(const int level)
Definition: logging.cc:151
bool IsNull() const
Definition: hash.h:383
bool foreground_
Definition: mountpoint.h:320
void Activate(const int buffer_size, const int flush_threshold, const std::string &trace_file)
Definition: tracer.cc:25
download::DownloadManager * download_mgr_
Definition: repository.h:141
struct stat64 platform_stat64
perf::Counter * n_eio_08_
Definition: mountpoint.h:344
Log2Histogram * hist_fs_forget_multi_
Definition: mountpoint.h:350
perf::Counter * n_fs_statfs_cached_
Definition: mountpoint.h:329
static FileWatcher * Create()
Definition: file_watcher.cc:44
void SetupInodeAnnotation()
Definition: mountpoint.cc:2104
bool SetupCwd()
Definition: mountpoint.cc:825
catalog::InodeAnnotation * inode_annotation_
Definition: mountpoint.h:626
static NfsMapsSqlite * Create(const std::string &db_dir, const uint64_t root_inode, const bool rebuild, perf::Statistics *statistics_)
void TearDown2ReadOnly()
Definition: cache_posix.cc:540
std::string workspace_fullpath_
Definition: mountpoint.h:372
bool DetermineNfsMode()
Definition: mountpoint.cc:369
static SqliteHistory * Open(const std::string &file_name)
void SetHostChain(const std::string &host_list)
static const unsigned kTracerFlushThreshold
Definition: mountpoint.h:577
virtual uint64_t GetSize()
Definition: quota_posix.cc:674
static Publisher * Create(const SettingsPublisher &settings)
uid_t talk_socket_uid_
Definition: mountpoint.h:657
void SetLowSpeedLimit(const unsigned low_speed_limit)
Definition: download.cc:1901
perf::Counter * n_fs_stat_stale_
Definition: mountpoint.h:327
Log2Histogram * hist_fs_opendir_
Definition: mountpoint.h:353
bool LockWorkspace()
Definition: mountpoint.cc:492
bool CreateTracer()
Definition: mountpoint.cc:1579
bool wait_workspace_
Definition: mountpoint.h:319
virtual bool Cleanup(const uint64_t leave_size)
Definition: quota_posix.cc:125
double kcache_timeout_sec_
Definition: mountpoint.h:644
signature::SignatureManager * signature_mgr_
Definition: mountpoint.h:621
void CreateFile(const std::string &path, const int mode, const bool ignore_failure)
Definition: posix.cc:276
perf::Counter * n_fs_lookup_
Definition: mountpoint.h:324
std::string name
Definition: history.h:89
AuthzSessionManager * authz_session_mgr_
Definition: mountpoint.h:618
static const unsigned kDefaultRetries
Definition: mountpoint.h:570
string Trim(const string &raw, bool trim_newline)
Definition: string.cc:428
string JoinStrings(const vector< string > &strings, const string &joint)
Definition: string.cc:325
perf::Statistics * statistics_
Definition: mountpoint.h:346
download::DownloadManager * download_mgr_
Definition: mountpoint.h:622
static const unsigned kDefaultTimeoutSec
Definition: mountpoint.h:569
std::string exe_path_
Definition: mountpoint.h:313
bool CheckInstanceName(const std::string &instance)
Definition: mountpoint.cc:107
perf::Statistics * statistics_
Definition: mountpoint.h:616
Log2Histogram * hist_fs_readlink_
Definition: mountpoint.h:352
void SetProxyChain(const std::string &proxy_list, const std::string &fallback_proxy_list, const ProxySetModes set_mode)
Definition: download.cc:2421
static NfsMapsLeveldb * Create(const std::string &leveldb_dir, const uint64_t root_inode, const bool rebuild, perf::Statistics *statistics)
gid_t talk_socket_gid_
Definition: mountpoint.h:658
void SetupSqlite()
Definition: mountpoint.cc:1033
perf::Counter * n_eio_total_
Definition: mountpoint.h:336
bool IsOn(const std::string &param_value) const
Definition: options.cc:409
std::string GetUniqFileSuffix()
Definition: mountpoint.cc:1735
StatfsCache * statfs_cache_
Definition: mountpoint.h:638
download::DownloadManager * external_download_mgr_
Definition: mountpoint.h:623
MagicXattrManager * magic_xattr_mgr_
Definition: mountpoint.h:637
bool fuse_expire_entry_
Definition: mountpoint.h:648
CacheManager * SetupCacheMgr(const std::string &instance)
Definition: mountpoint.cc:610
const SettingsPublisher & settings() const
Definition: repository.h:317
perf::Statistics * statistics_
Definition: repository.h:139
void SetupHttpTuning()
Definition: mountpoint.cc:2060
static const char * kDefaultBlacklist
Definition: mountpoint.h:578
virtual bool AcquireQuotaManager(QuotaManager *quota_mgr)
Definition: cache_extern.cc:85
unsigned max_ttl_sec_
Definition: mountpoint.h:642
static const unsigned kDefaultMaxTtl
Definition: dns.h:182
bool CreateDownloadManagers()
Definition: mountpoint.cc:1356
std::string talk_socket_path_
Definition: mountpoint.h:656
perf::Counter * n_fs_stat_
Definition: mountpoint.h:326
perf::Counter * n_eio_05_
Definition: mountpoint.h:341
perf::Counter * no_open_files_
Definition: mountpoint.h:334
assert((mem||(size==0))&&"Out Of Memory")
void SetupLogging()
Definition: mountpoint.cc:860
void SetupGlobalEnvironmentParams()
Definition: mountpoint.cc:855
int fd_workspace_lock_
Definition: mountpoint.h:373
void CreateStatistics()
Definition: mountpoint.cc:1498
bool UnregisterVfsRdOnly()
Definition: sqlitevfs.cc:522
CacheManager * SetupRamCacheMgr(const std::string &instance)
Definition: mountpoint.cc:713
bool cache_symlinks_
Definition: mountpoint.h:647
static void LogSqliteError(void *user_data __attribute__((unused)), int sqlite_extended_error, const char *message)
Definition: mountpoint.cc:523
static void CleanupInstance()
Definition: sqlitemem.cc:170
void EnableFuseExpireEntry()
Definition: mountpoint.cc:1224
static const unsigned kNfsMaps
Definition: mountpoint.h:181
std::string workspace_
Definition: mountpoint.h:366
string StringifyTime(const time_t seconds, const bool utc)
Definition: string.cc:105
PosixCacheSettings DeterminePosixCacheSettings(const std::string &instance)
Definition: mountpoint.cc:292
void SetDnsServer(const std::string &address)
Definition: download.cc:1830
int platform_stat(const char *path, platform_stat64 *buf)
void AssignGlobalArenas()
Definition: sqlitemem.cc:137
bool has_membership_req_
Definition: mountpoint.h:654
virtual CacheManagerIds id()=0
void SetLogMicroSyslog(const std::string &filename)
Definition: logging.cc:272
bool IsOff(const std::string &param_value) const
Definition: options.cc:416
perf::Counter * n_eio_01_
Definition: mountpoint.h:337
void ReEvaluateAuthz()
Definition: mountpoint.cc:1833
bool HasConfigRepository(const std::string &fqrn, std::string *config_path)
Definition: options.cc:256
std::set< std::string > constructed_instances_
Definition: mountpoint.h:401
bool DetermineRootHash(shash::Any *root_hash)
Definition: mountpoint.cc:1611
#define SetLogDebugFile(filename)
void SetTimeout(const unsigned seconds_proxy, const unsigned seconds_direct)
Definition: download.cc:1887
bool CreateCatalogManager()
Definition: mountpoint.cc:1299
struct cvmcache_object_info __attribute__
Definition: atomic.h:24
void RemapCatalogFd(int from, int to)
Definition: mountpoint.cc:1127
static const unsigned kNfsNone
Definition: mountpoint.h:177
std::string path_crash_guard_
Definition: mountpoint.h:379
IntegerMap< uint64_t > OwnerMap
Definition: catalog.h:41
catalog::WritableCatalogManager * catalog_mgr_
Definition: repository.h:385
bool SetupWorkspace()
Definition: mountpoint.cc:1050
static const char * kDefaultAuthzSearchPath
Definition: mountpoint.h:561
Type type()
Definition: mountpoint.h:240
FileSystem(const FileSystemInfo &fs_info)
Definition: mountpoint.cc:391
static const unsigned kLibPathCacheSize
Definition: mountpoint.h:549
std::string boot_error_
Definition: mountpoint.h:87
OptionsManager * options_mgr_
Definition: mountpoint.h:614
Log2Histogram * hist_fs_getattr_
Definition: mountpoint.h:351
bool IsValid(const std::string &input) const
Definition: sanitizer.cc:114
Log2Histogram * hist_fs_releasedir_
Definition: mountpoint.h:354
Definition: tracer.h:35
bool FileExists(const std::string &path)
Definition: posix.cc:787
bool LoadTrustedCaCrl(const std::string &path_list)
Definition: signature.cc:580
void DisableCacheSymlinks()
Definition: mountpoint.cc:1216
void SetDnsTtlLimits(const unsigned min_seconds, const unsigned max_seconds)
Definition: download.cc:1866
std::string cache_path()
Definition: cache_posix.h:105
perf::Counter * n_fs_readlink_
Definition: mountpoint.h:331
void Init(const unsigned max_pool_handles, const perf::StatisticsTemplate &statistics)
Definition: download.cc:1614
uint64_t platform_memsize()
int64_t String2Int64(const string &value)
Definition: string.cc:222
void SetMaxTtlMn(unsigned value_minutes)
Definition: mountpoint.cc:1843
string ToUpper(const string &mixed_case)
Definition: string.cc:476
static const unsigned kDefaultBackoffInitMs
Definition: mountpoint.h:571
perf::Counter * n_fs_forget_
Definition: mountpoint.h:332
MountPoint(const std::string &fqrn, FileSystem *file_system, OptionsManager *options_mgr)
Definition: mountpoint.cc:1740
void GetLimitNoFile(unsigned *soft_limit, unsigned *hard_limit)
Definition: posix.cc:1476
Tracer * tracer_
Definition: mountpoint.h:633
void SetProxyGroupResetDelay(const unsigned seconds)
Definition: download.cc:2718
static CacheManager * Create(CacheManager *upper_cache, CacheManager *lower_cache)
static bool g_alive
Definition: mountpoint.h:248
static const unsigned kDefaultNumConnections
Definition: mountpoint.h:565
void Set(const int64_t val)
Definition: statistics.h:33
void TearDown2ReadOnly()
Definition: mountpoint.cc:1114
gid_t g_gid
Definition: globals.cc:14
uuid_t uuid
Definition: uuid.h:41
static const unsigned kNfsMapsHa
Definition: mountpoint.h:185
lru::Md5PathCache * md5path_cache_
Definition: mountpoint.h:632
std::string name_
Definition: mountpoint.h:312
vector< string > SplitString(const string &str, char delim)
Definition: string.cc:290
std::string Dump()
Definition: options.cc:454
bool RegisterVfsRdOnly(CacheManager *cache_mgr, perf::Statistics *statistics, const VfsOptions options)
Definition: sqlitevfs.cc:453
static AuthzSessionManager * Create(AuthzFetcher *authz_fetcher, perf::Statistics *statistics)
std::string membership_req_
Definition: mountpoint.h:653
glue::DentryTracker * dentry_tracker_
Definition: mountpoint.h:635
perf::Counter * n_fs_read_
Definition: mountpoint.h:330
signature::SignatureManager * signature_mgr_
Definition: repository.h:140
perf::Counter * n_eio_06_
Definition: mountpoint.h:342
static const char * kDefaultCacheMgrInstance
Definition: mountpoint.h:252
bool HasSuffix(const std::string &str, const std::string &suffix, const bool ignore_case)
Definition: string.cc:281
virtual uint64_t GetCapacity()
Definition: quota_posix.cc:582
glue::PageCacheTracker * page_cache_tracker_
Definition: mountpoint.h:636
bool found_previous_crash_
Definition: mountpoint.h:385
const char kSuffixCatalog
Definition: hash.h:54
loader::Failures boot_status_
Definition: mountpoint.h:86
Failures Fetch(const std::string &base_url, const std::string &repository_name, const uint64_t minimum_timestamp, const shash::Any *base_catalog, signature::SignatureManager *signature_manager, download::DownloadManager *download_manager, ManifestEnsemble *ensemble)
bool TriageCacheMgr()
Definition: mountpoint.cc:1132
bool CheckPosixCacheSettings(const PosixCacheSettings &settings)
Definition: mountpoint.cc:124
perf::Counter * n_fs_inode_replace_
Definition: mountpoint.h:333
bool IsNfsSource()
Definition: mountpoint.h:190
ChunkTables * chunk_tables_
Definition: mountpoint.h:628
CacheManager * cache_mgr()
Definition: mountpoint.h:196
uid_t g_uid
Definition: globals.cc:13
void SetupDnsTuning(download::DownloadManager *manager)
Definition: mountpoint.cc:1959
cvmfs::Uuid * uuid_cache_
Definition: mountpoint.h:412
unsigned GetEffectiveTtlSec()
Definition: mountpoint.cc:1713
int TryLockFile(const std::string &path)
Definition: posix.cc:909
bool MkdirDeep(const std::string &path, const mode_t mode, bool verify_writable)
Definition: posix.cc:842
int LockFile(const std::string &path)
Definition: posix.cc:967
void SetIpPreference(const dns::IpPreference preference)
Definition: download.cc:1876
static const unsigned kDefaultBackoffMaxMs
Definition: mountpoint.h:572
lru::InodeCache * inode_cache_
Definition: mountpoint.h:630
CacheManager * SetupPosixCacheMgr(const std::string &instance)
Definition: mountpoint.cc:683
perf::Counter * n_fs_open_
Definition: mountpoint.h:322
static uint64_t RoundUp8(const uint64_t size)
Definition: smalloc.h:37
bool SetupPosixQuotaMgr(const PosixCacheSettings &settings, CacheManager *cache_mgr)
Definition: mountpoint.cc:971
static SqliteMemoryManager * GetInstance()
Definition: sqlitemem.h:137
static PosixQuotaManager * Create(const std::string &cache_workspace, const uint64_t limit, const uint64_t cleanup_threshold, const bool rebuild_database)
Definition: quota_posix.cc:230
unsigned nfs_mode_
Definition: mountpoint.h:406
void SetRetryParameters(const unsigned max_retries, const unsigned backoff_init_ms, const unsigned backoff_max_ms)
Definition: download.cc:2737
string StringifyInt(const int64_t value)
Definition: string.cc:78
Type type_
Definition: mountpoint.h:314
static const unsigned kInodeCacheFactor
Definition: mountpoint.h:553
void SetMaxIpaddrPerProxy(unsigned limit)
Definition: download.cc:2748
DownloadManager * Clone(const perf::StatisticsTemplate &statistics)
Definition: download.cc:2781
std::string GetArch()
Definition: posix.cc:1442
void Inc(class Counter *counter)
Definition: statistics.h:50
bool HasPrefix(const string &str, const string &prefix, const bool ignore_case)
Definition: string.cc:267
perf::Counter * n_eio_04_
Definition: mountpoint.h:340
Log2Histogram * hist_fs_readdir_
Definition: mountpoint.h:355
bool GetValue(const std::string &key, std::string *value) const
Definition: options.cc:376
cvmfs::Uuid * uuid_
Definition: mountpoint.h:606
OptionsManager * options_mgr_
Definition: cvmfs.cc:147
bool LoadBlacklist(const std::string &path_blacklist, bool append)
Definition: signature.cc:536
BackoffThrottle * backoff_throttle_
Definition: mountpoint.h:620
AuthzAttachment * authz_attachment_
Definition: mountpoint.h:619
void SetProxyTemplates(const std::string &direct, const std::string &forced)
Definition: download.cc:2754
SimpleChunkTables * simple_chunk_tables_
Definition: mountpoint.h:629
bool InitFixed(const shash::Any &root_hash, bool alternative_path)
perf::Counter * n_eio_03_
Definition: mountpoint.h:339
static const unsigned kDefaultNfiles
Definition: mountpoint.h:251
static FileSystem * Create(const FileSystemInfo &fs_info)
Definition: mountpoint.cc:164
bool has_custom_sqlitevfs_
Definition: mountpoint.h:423
perf::Counter * n_fs_statfs_
Definition: mountpoint.h:328
bool ReloadBlacklists()
Definition: mountpoint.cc:1192
std::string name()
Definition: mountpoint.h:225
time_t IsoTimestamp2UtcTime(const std::string &iso8601)
Definition: string.cc:198
static Uuid * Create(const std::string &store_path)
Definition: uuid.cc:23
void CreateStatistics()
Definition: mountpoint.cc:208
pthread_mutex_t lock_max_ttl_
Definition: mountpoint.h:643
void CreateAuthz()
Definition: mountpoint.cc:1273
bool g_raw_symlinks
Definition: globals.cc:12
bool g_world_readable
Definition: globals.cc:15
static const char * kDefaultCacheBase
Definition: mountpoint.h:249
void SetNull()
Definition: hash.h:400
string ResolveProxyDescription(const string &cvmfs_proxies, const std::string &path_fallback_cache, DownloadManager *download_manager)
Definition: wpad.cc:206
uint64_t String2Uint64(const string &value)
Definition: string.cc:228
static const unsigned kTracerBufferSize
Definition: mountpoint.h:576
void SetupUuid()
Definition: mountpoint.cc:1099
perf::Counter * n_eio_02_
Definition: mountpoint.h:338
catalog::ClientCatalogManager * catalog_mgr_
Definition: mountpoint.h:627
bool CreateSignatureManager()
Definition: mountpoint.cc:1462
std::string fqrn_
Definition: mountpoint.h:605
unsigned GetMaxTtlMn()
Definition: mountpoint.cc:1725
std::string error_msg() const
Definition: cache_extern.h:45
bool GetUidOf(const std::string &username, uid_t *uid, gid_t *main_gid)
Definition: posix.cc:1340
bool SetupBehavior()
Definition: mountpoint.cc:1848
static ExternalCacheManager * Create(int fd_connection, unsigned max_open_fds, const std::string &ident)
AuthzFetcher * authz_fetcher_
Definition: mountpoint.h:617
bool FetchHistory(std::string *history_path)
Definition: mountpoint.cc:1679
perf::Statistics * statistics()
Definition: mountpoint.h:239
bool enforce_acls_
Definition: mountpoint.h:646
bool IsHaNfsSource()
Definition: mountpoint.h:191
void CreateTables()
Definition: mountpoint.cc:1537
shash::Any history() const
Definition: manifest.h:127
bool SetupCrashGuard()
Definition: mountpoint.cc:804
static const unsigned kDefaultQuotaLimit
Definition: mountpoint.h:250
static PosixQuotaManager * CreateShared(const std::string &exe_path, const std::string &cache_workspace, const uint64_t limit, const uint64_t cleanup_threshold, bool foreground)
Definition: quota_posix.cc:262
void ResetErrorCounters()
Definition: mountpoint.cc:593
Definition: mutex.h:42
Log2Histogram * hist_fs_lookup_
Definition: mountpoint.h:348
shash::Any root_hash
Definition: history.h:90
bool LoadPublicRsaKeys(const std::string &path_list)
Definition: signature.cc:305
void RegisterHandler(const std::string &file_path, EventHandler *handler)
Definition: file_watcher.cc:56
void GetTimeout(unsigned *seconds_proxy, unsigned *seconds_direct)
Definition: download.cc:1910
static const unsigned kDefaultMemcacheSize
Definition: mountpoint.h:557
FileSystem * file_system_
Definition: cvmfs.cc:122
std::string repository_tag_
Definition: mountpoint.h:649
static const unsigned kDefaultMinTtl
Definition: dns.h:177
bool SetupOwnerMaps()
Definition: mountpoint.cc:2122
void SetDnsParameters(const unsigned retries, const unsigned timeout_ms)
Definition: download.cc:1848
bool CreateResolvConfWatcher()
Definition: mountpoint.cc:1422
Any MkFromHexPtr(const HexPtr hex, const char suffix)
Definition: hash.cc:83
static ExternalQuotaManager * Create(ExternalCacheManager *cache_mgr)
Log2Histogram * hist_fs_read_
Definition: mountpoint.h:357
void SetCatalogWatermark(unsigned limit)
static bool g_is_enabled
Definition: algorithm.h:200
bool SetupNfsMaps()
Definition: mountpoint.cc:878
bool CheckBlacklists()
Definition: mountpoint.cc:1154
std::vector< std::string > blacklist_paths_
Definition: mountpoint.h:650
cvmfs::Fetcher * fetcher_
Definition: mountpoint.h:624
file_watcher::FileWatcher * resolv_conf_watcher_
Definition: mountpoint.h:640
static const unsigned kDnsDefaultRetries
Definition: download.h:397
bool GeoSortServers(std::vector< std::string > *servers, std::vector< uint64_t > *output_order=NULL)
Definition: download.cc:2140
bool SetupExternalDownloadMgr(bool dogeosort)
Definition: mountpoint.cc:1999
void SetOwnerMaps(const OwnerMap &uid_map, const OwnerMap &gid_map)
CacheManager * SetupExternalCacheMgr(const std::string &instance)
Definition: mountpoint.cc:644
static void CleanupInstance()
Definition: clientctx.cc:18
perf::Counter * n_fs_lookup_negative_
Definition: mountpoint.h:325
Log2Histogram * hist_fs_open_
Definition: mountpoint.h:356
virtual bool AcquireQuotaManager(QuotaManager *quota_mgr)=0
std::string cache_mgr_instance_
Definition: mountpoint.h:396
std::string path_workspace_lock_
Definition: mountpoint.h:374
virtual void IncGeneration(const uint64_t by)=0
Log2Histogram * hist_fs_release_
Definition: mountpoint.h:358
void SetLogSyslogPrefix(const std::string &prefix)
Definition: logging.cc:241
std::string MakeCanonicalPath(const std::string &path)
Definition: posix.cc:97
OptionsManager * options_mgr_
Definition: mountpoint.h:318
perf::Counter * no_open_dirs_
Definition: mountpoint.h:335
glue::InodeTracker * inode_tracker_
Definition: mountpoint.h:634
bool fixed_catalog_
Definition: mountpoint.h:645
static PluginHandle * CreatePlugin(const std::string &locator, const std::vector< std::string > &cmd_line)
bool IsDefined(const std::string &key)
Definition: options.cc:370
void GetHostInfo(std::vector< std::string > *host_chain, std::vector< int > *rtt, unsigned *current_host)
Definition: download.cc:1954
void set_membership(const std::string &m)
Definition: authz_curl.h:25
Statistics * Fork()
Definition: statistics.cc:47
CacheManager * SetupTieredCacheMgr(const std::string &instance)
Definition: mountpoint.cc:763
std::vector< std::string > FindFilesBySuffix(const std::string &dir, const std::string &suffix)
Definition: posix.cc:1120
std::string workspace()
Definition: mountpoint.h:242
bool g_claim_ownership
Definition: globals.cc:11
lru::PathCache * path_cache_
Definition: mountpoint.h:631
CacheManager * cache_mgr_
Definition: mountpoint.h:407
void SetCredentialsAttachment(CredentialsAttachment *ca)
Definition: download.cc:1814
static ClientCtx * GetInstance()
Definition: clientctx.cc:45
virtual bool AcquireQuotaManager(QuotaManager *quota_mgr)
Definition: cache_ram.cc:70
cvmfs::Uuid * uuid_cache()
Definition: mountpoint.h:241
cvmfs::Fetcher * external_fetcher_
Definition: mountpoint.h:625
perf::Counter * n_fs_dir_open_
Definition: mountpoint.h:323
static const uint64_t kSizeUnknown
Definition: cache.h:72
std::string MkCacheParm(const std::string &generic_parameter, const std::string &instance)
Definition: mountpoint.cc:558
void UnlockFile(const int filedes)
Definition: posix.cc:991
NfsMaps * nfs_maps_
Definition: mountpoint.h:418
void CreateFetchers()
Definition: mountpoint.cc:1445
void SetInodeAnnotation(InodeAnnotation *new_annotation)
int Fetch(const shash::Any &id, const uint64_t size, const std::string &name, const zlib::Algorithms compression_algorithm, const CacheManager::ObjectType object_type, const std::string &alt_url="", off_t range_offset=-1)
Definition: fetch.cc:82
virtual void SetInodeResidue(unsigned residue_class, unsigned remainder)
Definition: nfs_maps.h:31
void SetHostResetDelay(const unsigned seconds)
Definition: download.cc:2728
static MountPoint * Create(const std::string &fqrn, FileSystem *file_system, OptionsManager *options_mgr=NULL)
Definition: mountpoint.cc:1233
Log2Histogram * hist_fs_forget_
Definition: mountpoint.h:349
static PosixCacheManager * Create(const std::string &cache_path, const bool alien_cache, const RenameWorkarounds rename_workaround=kRenameNormal)
Definition: cache_posix.cc:230