CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
mountpoint.cc
Go to the documentation of this file.
1 
4 #include "cvmfs_config.h"
5 #include "mountpoint.h"
6 
7 #include <errno.h>
8 #include <fcntl.h>
9 #include <inttypes.h>
10 #include <stdint.h>
11 #include <unistd.h>
12 
13 #include <algorithm>
14 #include <cassert>
15 #include <climits>
16 #include <cstring>
17 #include <vector>
18 
19 #include "duplex_fuse.h" // NOLINT
20 
21 #ifndef CVMFS_LIBCVMFS
22 #ifdef FUSE_CAP_EXPORT_SUPPORT
23 #define CVMFS_NFS_SUPPORT
24 #else
25 #warning "No NFS support, Fuse too old"
26 #endif
27 #endif
28 
29 #include "authz/authz_curl.h"
30 #include "authz/authz_fetch.h"
31 #include "authz/authz_session.h"
32 #include "backoff.h"
33 #include "cache.h"
34 #include "cache_extern.h"
35 #include "cache_posix.h"
36 #include "cache_ram.h"
37 #include "cache_tiered.h"
38 #include "catalog.h"
39 #include "catalog_mgr_client.h"
40 #include "clientctx.h"
41 #include "download.h"
42 #include "duplex_sqlite3.h"
43 #include "fetch.h"
44 #include "file_chunk.h"
45 #include "globals.h"
46 #include "glue_buffer.h"
47 #include "google/protobuf/stubs/common.h"
48 #include "history.h"
49 #include "history_sqlite.h"
50 #include "logging.h"
51 #include "lru_md.h"
52 #include "manifest.h"
53 #include "manifest_fetch.h"
54 #include "nfs_maps.h"
55 #ifdef CVMFS_NFS_SUPPORT
56 #include "nfs_maps_leveldb.h"
57 #include "nfs_maps_sqlite.h"
58 #endif
59 #include "options.h"
60 #include "platform.h"
61 #include "quota_posix.h"
63 #include "signature.h"
64 #include "sqlitemem.h"
65 #include "sqlitevfs.h"
66 #include "statistics.h"
67 #include "tracer.h"
68 #include "util/pointer.h"
69 #include "util/posix.h"
70 #include "util/string.h"
71 #include "util_concurrency.h"
72 #include "uuid.h"
73 #include "wpad.h"
74 
75 using namespace std; // NOLINT
76 
77 
78 bool FileSystem::g_alive = false;
79 const char *FileSystem::kDefaultCacheBase = "/var/lib/cvmfs";
80 const char *FileSystem::kDefaultCacheMgrInstance = "default";
81 
82 
87 bool FileSystem::CheckInstanceName(const std::string &instance) {
88  if (instance.length() > 24)
89  return false;
90  sanitizer::CacheInstanceSanitizer instance_sanitizer;
91  if (!instance_sanitizer.IsValid(instance)) {
92  boot_error_ = "invalid instance name (" + instance + "), " +
93  "only characters a-z, A-Z, 0-9, _ are allowed";
94  boot_status_ = loader::kFailCacheDir;
95  return false;
96  }
97  return true;
98 }
99 
100 
106 {
107  if (settings.is_alien && settings.is_shared) {
108  boot_error_ = "Failure: shared local disk cache and alien cache mutually "
109  "exclusive. Please turn off shared local disk cache.";
110  boot_status_ = loader::kFailOptions;
111  return false;
112  }
113  if (settings.is_alien && settings.is_managed) {
114  boot_error_ = "Failure: quota management and alien cache mutually "
115  "exclusive. Please turn off quota limit.";
116  boot_status_ = loader::kFailOptions;
117  return false;
118  }
119 
120  if (type_ == kFsLibrary) {
121  if (settings.is_shared || settings.is_managed) {
122  boot_error_ = "Failure: libcvmfs supports only unmanaged exclusive cache "
123  "or alien cache.";
124  boot_status_ = loader::kFailOptions;
125  return false;
126  }
127  }
128 
129  if (settings.cache_base_defined && settings.cache_dir_defined) {
130  boot_error_ =
131  "'CVMFS_CACHE_BASE' and 'CVMFS_CACHE_DIR' are mutually exclusive";
132  boot_status_ = loader::kFailOptions;
133  return false;
134  }
135 
136  return true;
137 }
138 
139 
146  file_system(new FileSystem(fs_info));
147 
148  file_system->SetupLogging();
149  LogCvmfs(kLogCvmfs, kLogDebug, "Options:\n%s",
150  file_system->options_mgr()->Dump().c_str());
151 
152  file_system->CreateStatistics();
153  file_system->SetupSqlite();
154  if (!file_system->DetermineNfsMode())
155  return file_system.Release();
156  if (!file_system->SetupWorkspace())
157  return file_system.Release();
158 
159  // Redirect SQlite temp directory to workspace (global variable)
160  unsigned length_tempdir = file_system->workspace_.length() + 1;
161  sqlite3_temp_directory = static_cast<char *>(sqlite3_malloc(length_tempdir));
162  snprintf(sqlite3_temp_directory,
163  length_tempdir,
164  "%s",
165  file_system->workspace_.c_str());
166 
167  if (!file_system->TriageCacheMgr())
168  return file_system.Release();
169  file_system->SetupUuid();
170  if (!file_system->SetupNfsMaps())
171  return file_system.Release();
172  bool retval = sqlite::RegisterVfsRdOnly(
173  file_system->cache_mgr_,
174  file_system->statistics_,
176  assert(retval);
177  file_system->has_custom_sqlitevfs_ = true;
178 
180 
181  file_system->boot_status_ = loader::kFailOk;
182  return file_system.Release();
183 }
184 
185 
188 
189  // Register the ShortString's static counters
190  statistics_->Register("pathstring.n_instances", "Number of instances");
191  statistics_->Register("pathstring.n_overflows", "Number of overflows");
192  statistics_->Register("namestring.n_instances", "Number of instances");
193  statistics_->Register("namestring.n_overflows", "Number of overflows");
194  statistics_->Register("linkstring.n_instances", "Number of instances");
195  statistics_->Register("linkstring.n_overflows", "Number of overflows");
196 
197  // Callback counters
198  n_fs_open_ = statistics_->Register("cvmfs.n_fs_open",
199  "Overall number of file open operations");
200  n_fs_dir_open_ = statistics_->Register("cvmfs.n_fs_dir_open",
201  "Overall number of directory open operations");
202  n_fs_lookup_ = statistics_->Register("cvmfs.n_fs_lookup",
203  "Number of lookups");
204  n_fs_lookup_negative_ = statistics_->Register("cvmfs.n_fs_lookup_negative",
205  "Number of negative lookups");
206  n_fs_stat_ = statistics_->Register("cvmfs.n_fs_stat", "Number of stats");
207  n_fs_read_ = statistics_->Register("cvmfs.n_fs_read", "Number of files read");
208  n_fs_readlink_ = statistics_->Register("cvmfs.n_fs_readlink",
209  "Number of links read");
210  n_fs_forget_ = statistics_->Register("cvmfs.n_fs_forget",
211  "Number of inode forgets");
212  n_io_error_ = statistics_->Register("cvmfs.n_io_error",
213  "Number of I/O errors");
214  no_open_files_ = statistics_->Register("cvmfs.no_open_files",
215  "Number of currently opened files");
216  no_open_dirs_ = statistics_->Register("cvmfs.no_open_dirs",
217  "Number of currently opened directories");
218 
219  string optarg;
220  if (options_mgr_->GetValue("CVMFS_INSTRUMENT_FUSE", &optarg) &&
221  options_mgr_->IsOn(optarg))
222  {
224  }
225 
226  hist_fs_lookup_ = new Log2Histogram(30);
227  hist_fs_forget_ = new Log2Histogram(30);
228  hist_fs_forget_multi_ = new Log2Histogram(30);
229  hist_fs_getattr_ = new Log2Histogram(30);
230  hist_fs_readlink_ = new Log2Histogram(30);
231  hist_fs_opendir_ = new Log2Histogram(30);
232  hist_fs_releasedir_ = new Log2Histogram(30);
233  hist_fs_readdir_ = new Log2Histogram(30);
234  hist_fs_open_ = new Log2Histogram(30);
235  hist_fs_read_ = new Log2Histogram(30);
236  hist_fs_release_ = new Log2Histogram(30);
237 }
238 
239 
245  const string &instance
246 ) {
247  string optarg;
249 
250  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SHARED", instance),
251  &optarg)
252  && options_mgr_->IsOn(optarg))
253  {
254  settings.is_shared = true;
255  }
256  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SERVER_MODE", instance),
257  &optarg)
258  && options_mgr_->IsOn(optarg))
259  {
260  settings.avoid_rename = true;
261  }
262 
263  if (type_ == kFsFuse)
264  settings.quota_limit = kDefaultQuotaLimit;
265  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_QUOTA_LIMIT", instance),
266  &optarg))
267  {
268  settings.quota_limit = String2Int64(optarg) * 1024 * 1024;
269  }
270  if (settings.quota_limit > 0)
271  settings.is_managed = true;
272 
273  settings.cache_path = kDefaultCacheBase;
274  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_BASE", instance),
275  &optarg))
276  {
277  settings.cache_path = MakeCanonicalPath(optarg);
278  settings.cache_base_defined = true;
279  }
280  if (settings.is_shared) {
281  settings.cache_path += "/shared";
282  } else {
283  settings.cache_path += "/" + name_;
284  }
285 
286  // CheckCacheMode makes sure that CVMFS_CACHE_DIR and CVMFS_CACHE_BASE are
287  // not set at the same time.
288  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_DIR", instance),
289  &optarg))
290  {
291  settings.cache_dir_defined = true;
292  settings.cache_path = optarg;
293  }
294  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_ALIEN", instance),
295  &optarg))
296  {
297  settings.is_alien = true;
298  settings.cache_path = optarg;
299  }
300  // We already changed the cwd to the workspace
301  // Which is only done if using FUSE
302  if ((type_ == kFsFuse) && (settings.cache_path == workspace_fullpath_)) {
303  settings.cache_path = ".";
304  }
305 
306  // The cache workspace usually is the cache directory, unless explicitly
307  // set otherwise
308  settings.workspace = settings.cache_path;
309  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_WORKSPACE", instance),
310  &optarg) ||
311  options_mgr_->GetValue("CVMFS_WORKSPACE", &optarg))
312  {
313  // Used for the shared quota manager
314  settings.workspace = optarg;
315  }
316 
317  return settings;
318 }
319 
320 
322  string optarg;
323 
324  if (options_mgr_->GetValue("CVMFS_NFS_SOURCE", &optarg) &&
325  options_mgr_->IsOn(optarg))
326  {
327  nfs_mode_ |= kNfsMaps;
328  if (options_mgr_->GetValue("CVMFS_NFS_SHARED", &optarg)) {
329  nfs_mode_ |= kNfsMapsHa;
330  nfs_maps_dir_ = optarg;
331  }
332  }
333 
334  if ((type_ == kFsLibrary) && (nfs_mode_ != kNfsNone)) {
335  boot_error_ = "Failure: libcvmfs does not support NFS export.";
336  boot_status_ = loader::kFailOptions;
337  return false;
338  }
339  return true;
340 }
341 
342 
344  : name_(fs_info.name)
345  , exe_path_(fs_info.exe_path)
346  , type_(fs_info.type)
347  , options_mgr_(fs_info.options_mgr)
348  , wait_workspace_(fs_info.wait_workspace)
349  , foreground_(fs_info.foreground)
350  , n_fs_open_(NULL)
351  , n_fs_dir_open_(NULL)
352  , n_fs_lookup_(NULL)
353  , n_fs_lookup_negative_(NULL)
354  , n_fs_stat_(NULL)
355  , n_fs_read_(NULL)
356  , n_fs_readlink_(NULL)
357  , n_fs_forget_(NULL)
358  , n_io_error_(NULL)
359  , no_open_files_(NULL)
360  , no_open_dirs_(NULL)
361  , statistics_(NULL)
362  , fd_workspace_lock_(-1)
363  , found_previous_crash_(false)
364  , nfs_mode_(kNfsNone)
365  , cache_mgr_(NULL)
366  , uuid_cache_(NULL)
367  , nfs_maps_(NULL)
368  , has_custom_sqlitevfs_(false)
369 {
370  assert(!g_alive);
371  g_alive = true;
372  g_uid = geteuid();
373  g_gid = getegid();
374 
375  string optarg;
376  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SERVER_MODE",
378  &optarg)
379  && options_mgr_->IsOn(optarg))
380  {
381  g_raw_symlinks = true;
382  }
383 }
384 
385 
388 
391 
392  delete uuid_cache_;
393  delete nfs_maps_;
394  delete cache_mgr_;
395 
396  if (sqlite3_temp_directory) {
397  sqlite3_free(sqlite3_temp_directory);
398  sqlite3_temp_directory = NULL;
399  }
400 
401  if (!path_crash_guard_.empty())
402  unlink(path_crash_guard_.c_str());
403  if (!path_workspace_lock_.empty())
404  unlink(path_workspace_lock_.c_str());
405  if (fd_workspace_lock_ >= 0)
407 
408  sqlite3_shutdown();
410 
411  delete hist_fs_lookup_;
412  delete hist_fs_forget_multi_;
413  delete hist_fs_forget_;
414  delete hist_fs_getattr_;
415  delete hist_fs_readlink_;
416  delete hist_fs_opendir_;
417  delete hist_fs_releasedir_;
418  delete hist_fs_readdir_;
419  delete hist_fs_open_;
420  delete hist_fs_read_;
421  delete hist_fs_release_;
422  delete statistics_;
423 
424  SetLogSyslogPrefix("");
425  SetLogMicroSyslog("");
426  SetLogDebugFile("");
427  google::protobuf::ShutdownProtobufLibrary();
428  g_alive = false;
429 }
430 
431 
433  path_workspace_lock_ = workspace_ + "/lock." + name_;
435  if (fd_workspace_lock_ >= 0)
436  return true;
437 
438  if (fd_workspace_lock_ == -1) {
439  boot_error_ = "could not acquire workspace lock (" +
440  StringifyInt(errno) + ")";
442  return false;
443  }
444 
445  assert(fd_workspace_lock_ == -2);
446 
447  if (!wait_workspace_) {
449  return false;
450  }
451 
453  if (fd_workspace_lock_ < 0) {
454  boot_error_ = "could not acquire workspace lock (" +
455  StringifyInt(errno) + ")";
457  return false;
458  }
459  return true;
460 }
461 
462 
464  void *user_data __attribute__((unused)),
465  int sqlite_extended_error,
466  const char *message)
467 {
468  int log_dest = kLogDebug;
469  int sqlite_error = sqlite_extended_error & 0xFF;
470  switch (sqlite_error) {
471  case SQLITE_INTERNAL:
472  case SQLITE_PERM:
473  case SQLITE_NOMEM:
474  case SQLITE_IOERR:
475  case SQLITE_CORRUPT:
476  case SQLITE_FULL:
477  case SQLITE_CANTOPEN:
478  case SQLITE_MISUSE:
479  case SQLITE_FORMAT:
480  case SQLITE_NOTADB:
481  log_dest |= kLogSyslogErr;
482  break;
483  case SQLITE_WARNING:
484  case SQLITE_NOTICE:
485  default:
486  break;
487  }
488  LogCvmfs(kLogCvmfs, log_dest, "SQlite3: %s (%d)",
489  message, sqlite_extended_error);
490 }
491 
492 
499  const string &generic_parameter,
500  const string &instance)
501 {
502  assert(HasPrefix(generic_parameter, "CVMFS_CACHE_", false));
503 
504  if (instance == kDefaultCacheMgrInstance) {
505  // Compatibility parameter names
506  if ((generic_parameter == "CVMFS_CACHE_SHARED") &&
507  !options_mgr_->IsDefined(generic_parameter))
508  {
509  return "CVMFS_SHARED_CACHE";
510  }
511  if ((generic_parameter == "CVMFS_CACHE_ALIEN") &&
512  !options_mgr_->IsDefined(generic_parameter))
513  {
514  return "CVMFS_ALIEN_CACHE";
515  }
516  if ((generic_parameter == "CVMFS_CACHE_SERVER_MODE") &&
517  !options_mgr_->IsDefined(generic_parameter))
518  {
519  return "CVMFS_SERVER_CACHE_MODE";
520  }
521  if ((generic_parameter == "CVMFS_CACHE_QUOTA_LIMIT") &&
522  !options_mgr_->IsDefined(generic_parameter))
523  {
524  return "CVMFS_QUOTA_LIMIT";
525  }
526  return generic_parameter;
527  }
528 
529  return "CVMFS_CACHE_" + instance + "_" + generic_parameter.substr(12);
530 }
531 
532 
534  n_io_error_->Set(0);
535 }
536 
537 
541 CacheManager *FileSystem::SetupCacheMgr(const string &instance) {
542  if (constructed_instances_.find(instance) != constructed_instances_.end()) {
543  boot_error_ = "circular cache definition: " + instance;
545  return NULL;
546  }
547  constructed_instances_.insert(instance);
548 
549  LogCvmfs(kLogCvmfs, kLogDebug, "setting up cache manager instance %s",
550  instance.c_str());
551  string instance_type;
552  if (instance == kDefaultCacheMgrInstance) {
553  instance_type = "posix";
554  } else {
555  options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_TYPE", instance),
556  &instance_type);
557  }
558  if (instance_type == "posix") {
559  return SetupPosixCacheMgr(instance);
560  } else if (instance_type == "ram") {
561  return SetupRamCacheMgr(instance);
562  } else if (instance_type == "tiered") {
563  return SetupTieredCacheMgr(instance);
564  } else if (instance_type == "external") {
565  return SetupExternalCacheMgr(instance);
566  } else {
567  boot_error_ = "invalid cache manager type for '" + instance + "':" +
568  instance_type;
570  return NULL;
571  }
572 }
573 
574 
576  string optarg;
577  unsigned nfiles = kDefaultNfiles;
578  if (options_mgr_->GetValue("CVMFS_NFILES", &optarg))
579  nfiles = String2Uint64(optarg);
580  vector<string> cmd_line;
581  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_CMDLINE", instance),
582  &optarg))
583  {
584  cmd_line = SplitString(optarg, ',');
585  }
586 
587  if (!options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_LOCATOR", instance),
588  &optarg))
589  {
590  boot_error_ = MkCacheParm("CVMFS_CACHE_LOCATOR", instance) + " missing";
592  return NULL;
593  }
594 
596  ExternalCacheManager::CreatePlugin(optarg, cmd_line));
597  if (!plugin_handle->IsValid()) {
598  boot_error_ = plugin_handle->error_msg();
600  return NULL;
601  }
603  plugin_handle->fd_connection(), nfiles, name_ + ":" + instance);
604  if (cache_mgr == NULL) {
605  boot_error_ = "failed to create external cache manager for " + instance;
607  return NULL;
608  }
609  cache_mgr->AcquireQuotaManager(ExternalQuotaManager::Create(cache_mgr));
610  return cache_mgr;
611 }
612 
613 
616  if (!CheckPosixCacheSettings(settings))
617  return NULL;
618 
620  settings.cache_path,
621  settings.is_alien,
624  if (!cache_mgr.IsValid()) {
625  boot_error_ = "Failed to setup posix cache '" + instance + "' in " +
626  settings.cache_path + ": " + strerror(errno);
628  return NULL;
629  }
630 
631  // Sentinel file for future use
632  // Might be a read-only cache
633  const bool ignore_failure = settings.is_alien;
634  CreateFile(settings.cache_path + "/.cvmfscache", 0600, ignore_failure);
635 
636  if (settings.is_managed) {
637  if (!SetupPosixQuotaMgr(settings, cache_mgr.weak_ref()))
638  return NULL;
639  }
640  return cache_mgr.Release();
641 }
642 
643 
644 CacheManager *FileSystem::SetupRamCacheMgr(const string &instance) {
645  string optarg;
646  unsigned nfiles = kDefaultNfiles;
647  if (options_mgr_->GetValue("CVMFS_NFILES", &optarg)) {
648  nfiles = String2Uint64(optarg);
649  }
650  uint64_t sz_cache_bytes;
651  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SIZE", instance),
652  &optarg))
653  {
654  if (HasSuffix(optarg, "%", false)) {
655  sz_cache_bytes = platform_memsize() * String2Uint64(optarg)/100;
656  } else {
657  sz_cache_bytes = String2Uint64(optarg) * 1024 * 1024;
658  }
659  } else {
660  sz_cache_bytes = platform_memsize() >> 5; // ~3%
661  }
663  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_MALLOC", instance),
664  &optarg))
665  {
666  if (optarg == "libc") {
668  } else if (optarg == "heap") {
670  } else {
671  boot_error_ = "Failure: unknown malloc " +
672  MkCacheParm("CVMFS_CACHE_MALLOC", instance) + "=" + optarg;
674  return NULL;
675  }
676  }
677  sz_cache_bytes = RoundUp8(std::max(static_cast<uint64_t>(40 * 1024 * 1024),
678  sz_cache_bytes));
680  sz_cache_bytes,
681  nfiles,
682  alloc,
683  perf::StatisticsTemplate("cache." + instance, statistics_));
684  if (cache_mgr == NULL) {
685  boot_error_ = "failed to create ram cache manager for " + instance;
687  return NULL;
688  }
689  cache_mgr->AcquireQuotaManager(new NoopQuotaManager());
690  return cache_mgr;
691 }
692 
693 
695  string optarg;
696  if (!options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_UPPER", instance),
697  &optarg))
698  {
699  boot_error_ = MkCacheParm("CVMFS_CACHE_UPPER", instance) + " missing";
701  return NULL;
702  }
703  UniquePtr<CacheManager> upper(SetupCacheMgr(optarg));
704  if (!upper.IsValid())
705  return NULL;
706 
707  if (!options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_LOWER", instance),
708  &optarg))
709  {
710  boot_error_ = MkCacheParm("CVMFS_CACHE_LOWER", instance) + " missing";
712  return NULL;
713  }
714  UniquePtr<CacheManager> lower(SetupCacheMgr(optarg));
715  if (!lower.IsValid())
716  return NULL;
717 
718  CacheManager *tiered =
719  TieredCacheManager::Create(upper.Release(), lower.Release());
720  if (tiered == NULL) {
721  boot_error_ = "Failed to setup tiered cache manager " + instance;
723  return NULL;
724  }
725  if (options_mgr_->GetValue(
726  MkCacheParm("CVMFS_CACHE_LOWER_READONLY", instance), &optarg) &&
727  options_mgr_->IsOn(optarg))
728  {
729  static_cast<TieredCacheManager*>(tiered)->SetLowerReadOnly();
730  }
731  return tiered;
732 }
733 
734 
736  path_crash_guard_ = workspace_ + "/running." + name_;
737  platform_stat64 info;
738  int retval = platform_stat(path_crash_guard_.c_str(), &info);
739  if (retval == 0) {
740  found_previous_crash_ = true;
742  "looks like cvmfs has been crashed previously");
743  }
744  retval = open(path_crash_guard_.c_str(), O_RDONLY | O_CREAT, 0600);
745  if (retval < 0) {
746  boot_error_ = "could not open running sentinel (" +
747  StringifyInt(errno) + ")";
749  return false;
750  }
751  close(retval);
752  return true;
753 }
754 
755 
757  if (type_ == kFsFuse) {
758  // Try to jump to workspace / cache directory. This tests, if it is
759  // accessible and it brings speed later on.
760  int retval = chdir(workspace_.c_str());
761  if (retval != 0) {
762  boot_error_ = "workspace " + workspace_ + " is unavailable";
764  return false;
765  }
766  workspace_ = ".";
767  return true;
768  }
769 
770  // Note: as of version 2.4 support for CVMFS_CWD_CACHE is dropped due to
771  // disproportionate large complexity to configure correctly. This affects
772  // only libcvmfs, mostly the legacy part.
773  // string optarg;
774  // if (options_mgr_->GetValue("CVMFS_CWD_CACHE", &optarg) &&
775  // options_mgr_->IsOn(optarg))
776  // {
777  // ...
778  // }
779  return true;
780 }
781 
782 
784  string optarg;
785  if (options_mgr_->GetValue("CVMFS_SYSLOG_LEVEL", &optarg))
787  if (options_mgr_->GetValue("CVMFS_SYSLOG_FACILITY", &optarg))
789  if (options_mgr_->GetValue("CVMFS_USYSLOG", &optarg))
790  SetLogMicroSyslog(optarg);
791  if (options_mgr_->GetValue("CVMFS_DEBUGLOG", &optarg))
792  SetLogDebugFile(optarg);
793  if (options_mgr_->GetValue("CVMFS_SYSLOG_PREFIX", &optarg)) {
794  SetLogSyslogPrefix(optarg);
795  } else {
797  }
798 }
799 
800 
802 #ifdef CVMFS_NFS_SUPPORT
803  if (!IsHaNfsSource())
805 
806  string no_nfs_sentinel;
807  if (cache_mgr_->id() == kPosixCacheManager) {
808  PosixCacheManager *posix_cache_mgr =
809  reinterpret_cast<PosixCacheManager *>(cache_mgr_);
810  no_nfs_sentinel = posix_cache_mgr->cache_path() + "/no_nfs_maps." + name_;
811  if (!IsNfsSource()) {
812  // Might be a read-only cache
813  const bool ignore_failure = posix_cache_mgr->alien_cache();
814  CreateFile(no_nfs_sentinel, 0600, ignore_failure);
815  return true;
816  }
817  } else {
818  if (IsNfsSource()) {
819  boot_error_ = "NFS source only works with POSIX cache manager.";
821  return false;
822  }
823  return true;
824  }
825 
827  assert(IsNfsSource());
828  if (!no_nfs_sentinel.empty() && FileExists(no_nfs_sentinel)) {
829  boot_error_ = "Cache was used without NFS maps before. "
830  "It has to be wiped out.";
832  return false;
833  }
834 
835  // nfs maps need to be protected by workspace lock
836  PosixCacheManager *posix_cache_mgr =
837  reinterpret_cast<PosixCacheManager *>(cache_mgr_);
838  if (posix_cache_mgr->cache_path() != workspace_) {
839  boot_error_ = "Cache directory and workspace must be identical for "
840  "NFS export";
842  return false;
843  }
844 
845  string inode_cache_dir = nfs_maps_dir_ + "/nfs_maps." + name_;
846  if (!MkdirDeep(inode_cache_dir, 0700)) {
847  boot_error_ = "Failed to initialize NFS maps";
849  return false;
850  }
851 
852  // TODO(jblomer): make this a manager class
853  if (IsHaNfsSource()) {
855  inode_cache_dir,
858  statistics_);
859  } else {
861  inode_cache_dir,
864  statistics_);
865  }
866 
867  if (nfs_maps_ == NULL) {
868  boot_error_ = "Failed to initialize NFS maps";
870  return false;
871  }
872 
873  string optarg;
874  if (options_mgr_->GetValue("CVMFS_NFS_INTERLEAVED_INODES", &optarg)) {
875  vector<string> tokens = SplitString(optarg, '%');
876  if (tokens.size() != 2) {
877  boot_error_ =
878  "invalid format for CVMFS_NFS_INTERLEAVED_INODES: " + optarg;
880  return false;
881  }
883  String2Uint64(tokens[0]));
884  }
885 
886  return true;
887 
888 #else
889  return true;
890 #endif
891 }
892 
893 
896  CacheManager *cache_mgr
897 ) {
898  assert(settings.quota_limit >= 0);
899  int64_t quota_threshold = settings.quota_limit / 2;
900  string cache_workspace = settings.cache_path;
901  if (settings.cache_path != settings.workspace) {
903  "using workspace %s to protect cache database in %s",
904  settings.workspace.c_str(), settings.cache_path.c_str());
905  cache_workspace += ":" + settings.workspace;
906  }
907  PosixQuotaManager *quota_mgr;
908 
909  if (settings.is_shared) {
911  exe_path_,
912  cache_workspace,
913  settings.quota_limit,
914  quota_threshold,
915  foreground_);
916  if (quota_mgr == NULL) {
917  boot_error_ = "Failed to initialize shared lru cache";
919  return false;
920  }
921  } else {
922  quota_mgr = PosixQuotaManager::Create(
923  cache_workspace,
924  settings.quota_limit,
925  quota_threshold,
927  if (quota_mgr == NULL) {
928  boot_error_ = "Failed to initialize lru cache";
930  return false;
931  }
932  }
933 
934  if (quota_mgr->GetSize() > quota_mgr->GetCapacity()) {
936  "cache is already beyond quota size "
937  "(size: %" PRId64 ", capacity: %" PRId64 "), cleaning up",
938  quota_mgr->GetSize(), quota_mgr->GetCapacity());
939  if (!quota_mgr->Cleanup(quota_threshold)) {
940  delete quota_mgr;
941  boot_error_ = "Failed to clean up cache";
943  return false;
944  }
945  }
946 
947  int retval = cache_mgr->AcquireQuotaManager(quota_mgr);
948  assert(retval);
950  "CernVM-FS: quota initialized, current size %luMB",
951  quota_mgr->GetSize() / (1024 * 1024));
952  return true;
953 }
954 
955 
957  // Make sure SQlite starts clean after initialization
958  sqlite3_shutdown();
959 
960  int retval;
961  retval = sqlite3_config(SQLITE_CONFIG_LOG, FileSystem::LogSqliteError, NULL);
962  assert(retval == SQLITE_OK);
963  retval = sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
964  assert(retval == SQLITE_OK);
966 
967  // Disable SQlite3 file locking
968  retval = sqlite3_vfs_register(sqlite3_vfs_find("unix-none"), 1);
969  assert(retval == SQLITE_OK);
970 }
971 
972 
974  string optarg;
975  // This is very similar to "determine cache dir". It's for backward
976  // compatibility with classic cache configuration where there was no
977  // distinction between workspace and cache.
978  // Complicated cache configurations should explicitly set CVMFS_WORKSPACE.
980  if (options_mgr_->GetValue("CVMFS_CACHE_BASE", &optarg))
981  workspace_ = MakeCanonicalPath(optarg);
982  if (options_mgr_->GetValue("CVMFS_SHARED_CACHE", &optarg) &&
983  options_mgr_->IsOn(optarg))
984  {
985  workspace_ += "/shared";
986  } else {
987  workspace_ += "/" + name_;
988  }
989  if (options_mgr_->GetValue("CVMFS_CACHE_DIR", &optarg)) {
990  if (options_mgr_->IsDefined("CVMFS_CACHE_BASE")) {
991  boot_error_ =
992  "'CVMFS_CACHE_BASE' and 'CVMFS_CACHE_DIR' are mutually exclusive";
994  return false;
995  }
996  workspace_ = optarg;
997  }
998  if (options_mgr_->GetValue("CVMFS_WORKSPACE", &optarg))
999  workspace_ = optarg;
1001 
1002  // If workspace and alien cache are the same directory, we need to open
1003  // permission now to 0770 to avoid a race when fixing it later
1004  const int mode = 0770;
1005  if (!MkdirDeep(workspace_, mode, false)) {
1006  boot_error_ = "cannot create workspace directory " + workspace_;
1008  return false;
1009  }
1010 
1011  if (!LockWorkspace())
1012  return false;
1013  if (!SetupCwd())
1014  return false;
1015  if (!SetupCrashGuard())
1016  return false;
1017 
1018  return true;
1019 }
1020 
1021 
1024  if (uuid_cache_ == NULL) {
1026  "failed to load/store %s/uuid", workspace_.c_str());
1028  assert(uuid_cache_ != NULL);
1029  }
1030 }
1031 
1032 
1038  if ((cache_mgr_ != NULL) && (cache_mgr_->id() == kPosixCacheManager)) {
1039  PosixCacheManager *posix_cache_mgr =
1040  reinterpret_cast<PosixCacheManager *>(cache_mgr_);
1041  posix_cache_mgr->TearDown2ReadOnly();
1042  }
1043 
1044  unlink(path_crash_guard_.c_str());
1045  LogCvmfs(kLogCache, kLogSyslog, "switch to read-only cache mode");
1046  SetLogMicroSyslog("");
1047 }
1048 
1049 
1050 void FileSystem::RemapCatalogFd(int from, int to) {
1051  sqlite::RegisterFdMapping(from, to);
1052 }
1053 
1054 
1057  string instance;
1058  if (options_mgr_->GetValue("CVMFS_CACHE_PRIMARY", &instance) &&
1059  !instance.empty())
1060  {
1061  if (!CheckInstanceName(instance))
1062  return false;
1063  cache_mgr_instance_ = instance;
1064  }
1065 
1067  return cache_mgr_ != NULL;
1068 }
1069 
1070 
1071 //------------------------------------------------------------------------------
1072 
1073 
1074 const char *MountPoint::kDefaultAuthzSearchPath = "/usr/libexec/cvmfs/authz";
1075 const char *MountPoint::kDefaultBlacklist = "/etc/cvmfs/blacklist";
1076 
1078  blacklist_paths_.clear();
1079  string blacklist;
1080  if (!options_mgr_->GetValue("CVMFS_BLACKLIST", &blacklist))
1081  blacklist = kDefaultBlacklist;
1082  blacklist_paths_.push_back(blacklist);
1083 
1084  bool append = false;
1085  if (FileExists(blacklist)) {
1086  if (!signature_mgr_->LoadBlacklist(blacklist, append)) {
1087  boot_error_ = "failed to load blacklist " + blacklist;
1089  return false;
1090  }
1091  append = true;
1092  }
1093 
1094  string config_repository_path;
1095  if (options_mgr_->HasConfigRepository(fqrn_, &config_repository_path)) {
1096  blacklist = config_repository_path + "blacklist";
1097  blacklist_paths_.push_back(blacklist);
1098  if (FileExists(blacklist)) {
1099  if (!signature_mgr_->LoadBlacklist(blacklist, append)) {
1100  boot_error_ = "failed to load blacklist from config repository";
1102  return false;
1103  }
1104  }
1105  }
1106 
1107  return true;
1108 }
1109 
1110 
1116  bool result = true;
1117  bool append = false;
1118  for (unsigned i = 0; i < blacklist_paths_.size(); ++i) {
1119  string blacklist = blacklist_paths_[i];
1120  if (FileExists(blacklist)) {
1121  bool retval = signature_mgr_->LoadBlacklist(blacklist, append);
1122  // TODO(jblomer): this can leave us with a half-loaded blacklist
1123  if (!retval)
1124  return false;
1125  append = true;
1126  }
1127  }
1128  return result;
1129 }
1130 
1131 
1137  const string &fqrn,
1138  FileSystem *file_system,
1139  OptionsManager *options_mgr)
1140 {
1141  if (options_mgr == NULL)
1142  options_mgr = file_system->options_mgr();
1143  UniquePtr<MountPoint> mountpoint(new MountPoint(
1144  fqrn, file_system, options_mgr));
1145 
1146  // At this point, we have a repository name, the type (fuse or library) and
1147  // an options manager (which can be the same than the FileSystem's one).
1148 
1149  mountpoint->CreateStatistics();
1150  mountpoint->CreateAuthz();
1151  mountpoint->backoff_throttle_ = new BackoffThrottle();
1152 
1153  if (!mountpoint->CreateSignatureManager() || !mountpoint->CheckBlacklists())
1154  return mountpoint.Release();
1155  if (!mountpoint->CreateDownloadManagers())
1156  return mountpoint.Release();
1157  if (!mountpoint->CreateResolvConfWatcher()) {
1158  return mountpoint.Release();
1159  }
1160  mountpoint->CreateFetchers();
1161  if (!mountpoint->CreateCatalogManager())
1162  return mountpoint.Release();
1163  if (!mountpoint->CreateTracer())
1164  return mountpoint.Release();
1165 
1166  mountpoint->ReEvaluateAuthz();
1167  mountpoint->CreateTables();
1168  if (!mountpoint->SetupBehavior())
1169  return mountpoint.Release();
1170 
1171  mountpoint->boot_status_ = loader::kFailOk;
1172  return mountpoint.Release();
1173 }
1174 
1175 
1177  string optarg;
1178  string authz_helper;
1179  if (options_mgr_->GetValue("CVMFS_AUTHZ_HELPER", &optarg))
1180  authz_helper = optarg;
1181  string authz_search_path(kDefaultAuthzSearchPath);
1182  if (options_mgr_->GetValue("CVMFS_AUTHZ_SEARCH_PATH", &optarg))
1183  authz_search_path = optarg;
1184 
1186  fqrn_,
1187  authz_helper,
1188  authz_search_path,
1189  options_mgr_);
1190  assert(authz_fetcher_ != NULL);
1191 
1194  statistics_);
1195  assert(authz_session_mgr_ != NULL);
1196 
1198  assert(authz_attachment_ != NULL);
1199 }
1200 
1201 
1203  string optarg;
1204 
1206 
1208  if (!SetupOwnerMaps())
1209  return false;
1210  shash::Any root_hash;
1211  if (!DetermineRootHash(&root_hash))
1212  return false;
1213 
1214  bool retval;
1215  if (root_hash.IsNull()) {
1216  retval = catalog_mgr_->Init();
1217  } else {
1218  fixed_catalog_ = true;
1219  bool alt_root_path =
1220  options_mgr_->GetValue("CVMFS_ALT_ROOT_PATH", &optarg) &&
1221  options_mgr_->IsOn(optarg);
1222  retval = catalog_mgr_->InitFixed(root_hash, alt_root_path);
1223  }
1224  if (!retval) {
1225  boot_error_ = "Failed to initialize root file catalog";
1227  return false;
1228  }
1229 
1231  boot_error_ = "repository revision blacklisted";
1233  return false;
1234  }
1235 
1236  if (options_mgr_->GetValue("CVMFS_AUTO_UPDATE", &optarg) &&
1237  !options_mgr_->IsOn(optarg))
1238  {
1239  fixed_catalog_ = true;
1240  }
1241 
1242  if (options_mgr_->GetValue("CVMFS_CATALOG_WATERMARK", &optarg)) {
1244  } else {
1245  unsigned soft_limit;
1246  unsigned hard_limit;
1247  GetLimitNoFile(&soft_limit, &hard_limit);
1248  catalog_mgr_->SetCatalogWatermark(soft_limit / 4);
1249  }
1250 
1251  if (catalog_mgr_->volatile_flag()) {
1252  LogCvmfs(kLogCvmfs, kLogDebug, "content of repository flagged as VOLATILE");
1253  }
1254 
1255  return true;
1256 }
1257 
1258 
1260  string optarg;
1262  const bool use_system_proxy = false;
1263  download_mgr_->Init(kDefaultNumConnections, use_system_proxy,
1264  perf::StatisticsTemplate("download", statistics_));
1266 
1267  if (options_mgr_->GetValue("CVMFS_SERVER_URL", &optarg)) {
1268  download_mgr_->SetHostChain(optarg);
1269  }
1270 
1272  SetupHttpTuning();
1273 
1274  string forced_proxy_template;
1275  if (options_mgr_->GetValue("CVMFS_PROXY_TEMPLATE", &optarg))
1276  forced_proxy_template = optarg;
1278  forced_proxy_template);
1279 
1280  string proxies;
1281  if (options_mgr_->GetValue("CVMFS_HTTP_PROXY", &optarg))
1282  proxies = optarg;
1284  proxies,
1285  file_system_->workspace() + "/proxies" + GetUniqFileSuffix(),
1286  download_mgr_);
1287  if (proxies == "") {
1288  boot_error_ = "failed to discover HTTP proxy servers";
1290  return false;
1291  }
1292  string fallback_proxies;
1293  if (options_mgr_->GetValue("CVMFS_FALLBACK_PROXY", &optarg))
1294  fallback_proxies = optarg;
1295  download_mgr_->SetProxyChain(proxies, fallback_proxies,
1297 
1298  bool do_geosort = options_mgr_->GetValue("CVMFS_USE_GEOAPI", &optarg) &&
1299  options_mgr_->IsOn(optarg);
1300  if (do_geosort) {
1302  }
1303  if (options_mgr_->GetValue("CVMFS_MAX_SERVERS", &optarg)) {
1304  unsigned max_servers = String2Uint64(optarg);
1305  std::vector<std::string> host_chain;
1306  download_mgr_->GetHostInfo(&host_chain, NULL, NULL);
1307  if (max_servers > 0 && max_servers < host_chain.size()) {
1308  host_chain.resize(max_servers);
1309  download_mgr_->SetHostChain(host_chain);
1310  }
1311  }
1312 
1313  if (options_mgr_->GetValue("CVMFS_USE_SSL_SYSTEM_CA", &optarg) &&
1314  options_mgr_->IsOn(optarg)) {
1316  }
1317 
1318  return SetupExternalDownloadMgr(do_geosort);
1319 }
1320 
1322  std::string roaming_value;
1323  options_mgr_->GetValue("CVMFS_DNS_ROAMING", &roaming_value);
1324  if (options_mgr_->IsDefined("CVMFS_DNS_ROAMING") &&
1325  options_mgr_->IsOn(roaming_value)) {
1327  "DNS roaming is enabled for this repository.");
1328  // Create a file watcher to update the DNS settings of the download
1329  // managers when there are changes to /etc/resolv.conf
1331 
1332  if (resolv_conf_watcher_) {
1333  ResolvConfEventHandler *handler =
1335  resolv_conf_watcher_->RegisterHandler("/etc/resolv.conf", handler);
1336  }
1337  } else {
1339  "DNS roaming is disabled for this repository.");
1340  }
1341  return true;
1342 }
1343 
1345  fetcher_ = new cvmfs::Fetcher(
1347  download_mgr_,
1350 
1351  const bool is_external_data = true;
1356  perf::StatisticsTemplate("fetch-external", statistics_),
1357  is_external_data);
1358 }
1359 
1360 
1362  string optarg;
1364  signature_mgr_->Init();
1365 
1366  string public_keys;
1367  if (options_mgr_->GetValue("CVMFS_PUBLIC_KEY", &optarg)) {
1368  public_keys = optarg;
1369  } else if (options_mgr_->GetValue("CVMFS_KEYS_DIR", &optarg)) {
1370  // Collect .pub files from CVMFS_KEYS_DIR
1371  public_keys = JoinStrings(FindFilesBySuffix(optarg, ".pub"), ":");
1372  } else {
1373  public_keys =
1374  JoinStrings(FindFilesBySuffix("/etc/cvmfs/keys", ".pub"), ":");
1375  }
1376 
1377  if (!signature_mgr_->LoadPublicRsaKeys(public_keys)) {
1378  boot_error_ = "failed to load public key(s)";
1380  return false;
1381  }
1382  LogCvmfs(kLogCvmfs, kLogDebug, "CernVM-FS: using public key(s) %s",
1383  public_keys.c_str());
1384 
1385  if (options_mgr_->GetValue("CVMFS_TRUSTED_CERTS", &optarg)) {
1386  if (!signature_mgr_->LoadTrustedCaCrl(optarg)) {
1387  boot_error_ = "failed to load trusted certificates";
1389  return false;
1390  }
1391  }
1392 
1393  return true;
1394 }
1395 
1396 
1400  return;
1401 
1402  // TODO(jblomer): this should be registered by the tracker
1403  statistics_->Register("inode_tracker.n_insert",
1404  "overall number of accessed inodes");
1405  statistics_->Register("inode_tracker.n_remove",
1406  "overall number of evicted inodes");
1407  statistics_->Register("inode_tracker.no_reference",
1408  "currently active inodes");
1409  statistics_->Register("inode_tracker.n_hit_inode",
1410  "overall number of inode lookups");
1411  statistics_->Register("inode_tracker.n_hit_path",
1412  "overall number of successful path lookups");
1413  statistics_->Register("inode_tracker.n_miss_path",
1414  "overall number of unsuccessful path lookups");
1415 
1416  statistics_->Register("nentry_tracker.n_insert",
1417  "overall number of added negative cache entries");
1418  statistics_->Register("nentry_tracker.n_remove",
1419  "overall number of evicted negative cache entries");
1420  statistics_->Register("nentry_tracker.n_prune",
1421  "overall number of prune calls");
1422 }
1423 
1424 
1427  // Libcvmfs simplified tables
1430  return;
1431  }
1432 
1433  chunk_tables_ = new ChunkTables();
1434 
1435  string optarg;
1436  uint64_t mem_cache_size = kDefaultMemcacheSize;
1437  if (options_mgr_->GetValue("CVMFS_MEMCACHE_SIZE", &optarg))
1438  mem_cache_size = String2Uint64(optarg) * 1024 * 1024;
1439 
1440  const double memcache_unit_size =
1441  (static_cast<double>(kInodeCacheFactor) * lru::Md5PathCache::GetEntrySize())
1443  const unsigned memcache_num_units =
1444  mem_cache_size / static_cast<unsigned>(memcache_unit_size);
1445  // Number of cache entries must be a multiple of 64
1446  const unsigned mask_64 = ~((1 << 6) - 1);
1447  inode_cache_ = new lru::InodeCache(memcache_num_units & mask_64, statistics_);
1448  path_cache_ = new lru::PathCache(memcache_num_units & mask_64, statistics_);
1449  md5path_cache_ = new lru::Md5PathCache((memcache_num_units * 7) & mask_64,
1450  statistics_);
1451 
1454 }
1455 
1465  string optarg;
1466  tracer_ = new Tracer();
1467  if (options_mgr_->GetValue("CVMFS_TRACEFILE", &optarg)) {
1469  boot_error_ = "tracer is only supported in the fuse module";
1471  return false;
1472  }
1473  string tracebuffer_file = optarg;
1474  uint64_t tracebuffer_size = kTracerBufferSize;
1475  uint64_t tracebuffer_threshold = kTracerFlushThreshold;
1476 
1477  if (options_mgr_->GetValue("CVMFS_TRACEBUFFER", &optarg)) {
1478  tracebuffer_size = String2Uint64(optarg);
1479  }
1480  if (options_mgr_->GetValue("CVMFS_TRACEBUFFER_THRESHOLD",
1481  &optarg)) {
1482  tracebuffer_threshold = String2Uint64(optarg);
1483  }
1484  assert(tracebuffer_size <= INT_MAX
1485  && tracebuffer_threshold <= INT_MAX);
1487  "Initialising tracer with buffer size %" PRIu64 " and threshold %" PRIu64,
1488  tracebuffer_size, tracebuffer_threshold);
1489  tracer_->Activate(tracebuffer_size, tracebuffer_threshold,
1490  tracebuffer_file);
1491  }
1492  return true;
1493 }
1494 
1495 
1497  string optarg;
1498  if (options_mgr_->GetValue("CVMFS_ROOT_HASH", &optarg)) {
1499  *root_hash = MkFromHexPtr(shash::HexPtr(optarg), shash::kSuffixCatalog);
1500  return true;
1501  }
1502 
1503  if (!options_mgr_->IsDefined("CVMFS_REPOSITORY_TAG") &&
1504  !options_mgr_->IsDefined("CVMFS_REPOSITORY_DATE"))
1505  {
1506  root_hash->SetNull();
1507  return true;
1508  }
1509 
1510  string history_path;
1511  if (!FetchHistory(&history_path))
1512  return false;
1513  UnlinkGuard history_file(history_path);
1515  history::SqliteHistory::Open(history_path));
1516  if (!tag_db.IsValid()) {
1518  "failed to open history database (%s)", history_path.c_str());
1519  boot_error_ = "failed to open history database";
1521  return false;
1522  }
1523 
1525  bool retval;
1526  if (!options_mgr_->GetValue("CVMFS_REPOSITORY_TAG", &repository_tag_)) {
1527  string repository_date;
1528  // options_mgr_->IsDefined("CVMFS_REPOSITORY_DATE") must be true
1529  options_mgr_->GetValue("CVMFS_REPOSITORY_DATE", &repository_date);
1530  time_t repository_utctime = IsoTimestamp2UtcTime(repository_date);
1531  if (repository_utctime == 0) {
1532  boot_error_ = "invalid timestamp in CVMFS_REPOSITORY_DATE: " +
1533  repository_date + ". Use YYYY-MM-DDTHH:MM:SSZ";
1535  return false;
1536  }
1537  retval = tag_db->GetByDate(repository_utctime, &tag);
1538  if (!retval) {
1539  boot_error_ = "no repository state as early as utc timestamp " +
1540  StringifyTime(repository_utctime, true);
1542  return false;
1543  }
1545  "time stamp %s UTC resolved to tag '%s'",
1546  StringifyTime(repository_utctime, true).c_str(),
1547  tag.name.c_str());
1548  repository_tag_ = tag.name;
1549  } else {
1550  retval = tag_db->GetByName(repository_tag_, &tag);
1551  if (!retval) {
1552  boot_error_ = "no such tag: " + repository_tag_;
1554  return false;
1555  }
1556  }
1557  LogCvmfs(kLogCvmfs, kLogDebug, "mounting tag %s", tag.name.c_str());
1558 
1559  *root_hash = tag.root_hash;
1560  return true;
1561 }
1562 
1563 
1564 bool MountPoint::FetchHistory(std::string *history_path) {
1565  manifest::Failures retval_mf;
1566  manifest::ManifestEnsemble ensemble;
1567  retval_mf = manifest::Fetch("", fqrn_, 0, NULL, signature_mgr_, download_mgr_,
1568  &ensemble);
1569  if (retval_mf != manifest::kFailOk) {
1570  boot_error_ = "Failed to fetch manifest";
1572  return false;
1573  }
1574  shash::Any history_hash = ensemble.manifest->history();
1575  if (history_hash.IsNull()) {
1576  boot_error_ = "No history";
1578  return false;
1579  }
1580 
1581  int fd = fetcher_->Fetch(
1582  history_hash,
1584  "tag database for " + fqrn_,
1587  if (fd < 0) {
1588  boot_error_ = "failed to download history: " + StringifyInt(-fd);
1590  return false;
1591  }
1592  // We have the custom sqlite vfs driver installed
1593  *history_path = "@" + StringifyInt(fd);
1594  return true;
1595 }
1596 
1597 
1599  unsigned max_ttl;
1600  {
1601  MutexLockGuard lock_guard(lock_max_ttl_);
1602  max_ttl = max_ttl_sec_;
1603  }
1604  const unsigned catalog_ttl_sec = catalog_mgr_->GetTTL();
1605 
1606  return max_ttl ? std::min(max_ttl, catalog_ttl_sec) : catalog_ttl_sec;
1607 }
1608 
1609 
1611  MutexLockGuard lock_guard(lock_max_ttl_);
1612  return max_ttl_sec_ / 60;
1613 }
1614 
1615 
1621  return "." + file_system_->name() + "-" + fqrn_;
1622 }
1623 
1624 
1626  const string &fqrn,
1627  FileSystem *file_system,
1628  OptionsManager *options_mgr)
1629  : fqrn_(fqrn)
1630  , uuid_(cvmfs::Uuid::Create(""))
1631  , file_system_(file_system)
1632  , options_mgr_(options_mgr)
1633  , statistics_(NULL)
1634  , authz_fetcher_(NULL)
1635  , authz_session_mgr_(NULL)
1636  , authz_attachment_(NULL)
1637  , backoff_throttle_(NULL)
1638  , signature_mgr_(NULL)
1639  , download_mgr_(NULL)
1640  , external_download_mgr_(NULL)
1641  , fetcher_(NULL)
1642  , external_fetcher_(NULL)
1643  , inode_annotation_(NULL)
1644  , catalog_mgr_(NULL)
1645  , chunk_tables_(NULL)
1646  , simple_chunk_tables_(NULL)
1647  , inode_cache_(NULL)
1648  , path_cache_(NULL)
1649  , md5path_cache_(NULL)
1650  , tracer_(NULL)
1651  , inode_tracker_(NULL)
1652  , nentry_tracker_(NULL)
1653  , resolv_conf_watcher_(NULL)
1654  , max_ttl_sec_(kDefaultMaxTtlSec)
1655  , kcache_timeout_sec_(static_cast<double>(kDefaultKCacheTtlSec))
1656  , fixed_catalog_(false)
1657  , enforce_acls_(false)
1658  , has_membership_req_(false)
1659  , talk_socket_path_(std::string("./cvmfs_io.") + fqrn)
1660  , talk_socket_uid_(0)
1661  , talk_socket_gid_(0)
1662 {
1663  int retval = pthread_mutex_init(&lock_max_ttl_, NULL);
1664  assert(retval == 0);
1665 }
1666 
1667 
1669  pthread_mutex_destroy(&lock_max_ttl_);
1670 
1671  delete nentry_tracker_;
1672  delete inode_tracker_;
1673  delete tracer_;
1674  delete md5path_cache_;
1675  delete path_cache_;
1676  delete inode_cache_;
1677  delete simple_chunk_tables_;
1678  delete chunk_tables_;
1679 
1680  delete catalog_mgr_;
1681  delete inode_annotation_;
1682  delete external_fetcher_;
1683  delete fetcher_;
1684  if (external_download_mgr_ != NULL) {
1686  delete external_download_mgr_;
1687  }
1688  if (download_mgr_ != NULL) {
1689  download_mgr_->Fini();
1690  delete download_mgr_;
1691  }
1692  if (signature_mgr_ != NULL) {
1693  signature_mgr_->Fini();
1694  delete signature_mgr_;
1695  }
1696 
1697  if (resolv_conf_watcher_ != NULL) {
1699  delete resolv_conf_watcher_;
1700  }
1701 
1702  delete backoff_throttle_;
1703  delete authz_attachment_;
1704  delete authz_session_mgr_;
1705  delete authz_fetcher_;
1706  delete statistics_;
1707  delete uuid_;
1708 }
1709 
1710 
1712  string old_membership_req = membership_req_;
1714  if (old_membership_req != membership_req_) {
1717  }
1718 }
1719 
1720 
1721 void MountPoint::SetMaxTtlMn(unsigned value_minutes) {
1722  MutexLockGuard lock_guard(lock_max_ttl_);
1723  max_ttl_sec_ = value_minutes * 60;
1724 }
1725 
1726 
1728  string optarg;
1729 
1730  if (options_mgr_->GetValue("CVMFS_MAX_TTL", &optarg))
1731  SetMaxTtlMn(String2Uint64(optarg));
1732 
1733  if (options_mgr_->GetValue("CVMFS_KCACHE_TIMEOUT", &optarg)) {
1734  // Can be negative and should then be interpreted as 0.0
1736  std::max(0.0, static_cast<double>(String2Int64(optarg)));
1737  }
1738  LogCvmfs(kLogCvmfs, kLogDebug, "kernel caches expire after %d seconds",
1739  static_cast<int>(kcache_timeout_sec_));
1740 
1741  bool hide_magic_xattrs = false;
1742  if (options_mgr_->GetValue("CVMFS_HIDE_MAGIC_XATTRS", &optarg)
1743  && options_mgr_->IsOn(optarg))
1744  {
1745  hide_magic_xattrs = true;
1746  }
1747  magic_xattr_mgr_ = new MagicXattrManager(this, hide_magic_xattrs);
1748 
1749 
1750  if (options_mgr_->GetValue("CVMFS_ENFORCE_ACLS", &optarg)
1751  && options_mgr_->IsOn(optarg))
1752  {
1753  enforce_acls_ = true;
1754  }
1755 
1756  if (options_mgr_->GetValue("CVMFS_TALK_SOCKET", &optarg)) {
1757  talk_socket_path_ = optarg;
1758  }
1759  if (options_mgr_->GetValue("CVMFS_TALK_OWNER", &optarg)) {
1760  bool retval = GetUidOf(optarg, &talk_socket_uid_, &talk_socket_gid_);
1761  if (!retval) {
1762  boot_error_ = "unknown owner of cvmfs_talk socket: " + optarg;
1764  return false;
1765  }
1766  }
1767 
1768  return true;
1769 }
1770 
1771 
1777  string optarg;
1778  unsigned dns_timeout_ms = download::DownloadManager::kDnsDefaultTimeoutMs;
1779  unsigned dns_retries = download::DownloadManager::kDnsDefaultRetries;
1780  if (options_mgr_->GetValue("CVMFS_DNS_TIMEOUT", &optarg))
1781  dns_timeout_ms = String2Uint64(optarg) * 1000;
1782  if (options_mgr_->GetValue("CVMFS_DNS_RETRIES", &optarg))
1783  dns_retries = String2Uint64(optarg);
1784  manager->SetDnsParameters(dns_retries, dns_timeout_ms);
1785 
1786  // Rest has to be after SetDnsParameters because SetDnsParameters might
1787  // construct a new resolver object
1788 
1789  unsigned dns_min_ttl = dns::Resolver::kDefaultMinTtl;
1790  unsigned dns_max_ttl = dns::Resolver::kDefaultMaxTtl;
1791  if (options_mgr_->GetValue("CVMFS_DNS_MIN_TTL", &optarg))
1792  dns_min_ttl = String2Uint64(optarg);
1793  if (options_mgr_->GetValue("CVMFS_DNS_MAX_TTL", &optarg))
1794  dns_max_ttl = String2Uint64(optarg);
1795  manager->SetDnsTtlLimits(dns_min_ttl, dns_max_ttl);
1796 
1797  if (options_mgr_->GetValue("CVMFS_DNS_SERVER", &optarg)) {
1798  download_mgr_->SetDnsServer(optarg);
1799  }
1800 
1801  if (options_mgr_->GetValue("CVMFS_IPFAMILY_PREFER", &optarg)) {
1802  switch (String2Int64(optarg)) {
1803  case 4:
1805  break;
1806  case 6:
1808  break;
1809  }
1810  }
1811  if (options_mgr_->GetValue("CVMFS_MAX_IPADDR_PER_PROXY", &optarg))
1812  manager->SetMaxIpaddrPerProxy(String2Uint64(optarg));
1813 }
1814 
1815 
1817  string optarg;
1819  download_mgr_->Clone(perf::StatisticsTemplate("download-external",
1820  statistics_));
1821 
1822  unsigned timeout;
1823  unsigned timeout_direct;
1824  download_mgr_->GetTimeout(&timeout, &timeout_direct);
1825  if (options_mgr_->GetValue("CVMFS_EXTERNAL_TIMEOUT", &optarg)) {
1826  timeout = String2Uint64(optarg);
1827  }
1828  if (options_mgr_->GetValue("CVMFS_EXTERNAL_TIMEOUT_DIRECT", &optarg)) {
1829  timeout_direct = String2Uint64(optarg);
1830  }
1831  external_download_mgr_->SetTimeout(timeout, timeout_direct);
1832 
1833  if (options_mgr_->GetValue("CVMFS_EXTERNAL_URL", &optarg)) {
1835  if (dogeosort) {
1836  std::vector<std::string> host_chain;
1837  external_download_mgr_->GetHostInfo(&host_chain, NULL, NULL);
1838  download_mgr_->GeoSortServers(&host_chain);
1839  external_download_mgr_->SetHostChain(host_chain);
1840  }
1841  } else {
1843  }
1844 
1845  if (options_mgr_->GetValue("CVMFS_EXTERNAL_MAX_SERVERS", &optarg)) {
1846  unsigned max_servers = String2Uint64(optarg);
1847  std::vector<std::string> host_chain;
1848  external_download_mgr_->GetHostInfo(&host_chain, NULL, NULL);
1849  if (max_servers > 0 && max_servers < host_chain.size()) {
1850  host_chain.resize(max_servers);
1851  external_download_mgr_->SetHostChain(host_chain);
1852  }
1853  }
1854 
1855  string proxies = "DIRECT";
1856  if (options_mgr_->GetValue("CVMFS_EXTERNAL_HTTP_PROXY", &optarg)) {
1858  optarg,
1859  file_system_->workspace() + "/proxies-external" + GetUniqFileSuffix(),
1861  if (proxies == "") {
1862  boot_error_ = "failed to discover external HTTP proxy servers";
1864  return false;
1865  }
1866  }
1867  string fallback_proxies;
1868  if (options_mgr_->GetValue("CVMFS_EXTERNAL_FALLBACK_PROXY", &optarg))
1869  fallback_proxies = optarg;
1871  proxies, fallback_proxies, download::DownloadManager::kSetProxyBoth);
1872 
1873  return true;
1874 }
1875 
1876 
1878  string optarg;
1879 
1880  // TODO(jblomer): avoid double default settings
1881 
1882  unsigned timeout = kDefaultTimeoutSec;
1883  unsigned timeout_direct = kDefaultTimeoutSec;
1884  if (options_mgr_->GetValue("CVMFS_TIMEOUT", &optarg))
1885  timeout = String2Uint64(optarg);
1886  if (options_mgr_->GetValue("CVMFS_TIMEOUT_DIRECT", &optarg))
1887  timeout_direct = String2Uint64(optarg);
1888  download_mgr_->SetTimeout(timeout, timeout_direct);
1889 
1890  unsigned max_retries = kDefaultRetries;
1891  unsigned backoff_init = kDefaultBackoffInitMs;
1892  unsigned backoff_max = kDefaultBackoffMaxMs;
1893  if (options_mgr_->GetValue("CVMFS_MAX_RETRIES", &optarg))
1894  max_retries = String2Uint64(optarg);
1895  if (options_mgr_->GetValue("CVMFS_BACKOFF_INIT", &optarg))
1896  backoff_init = String2Uint64(optarg) * 1000;
1897  if (options_mgr_->GetValue("CVMFS_BACKOFF_MAX", &optarg))
1898  backoff_max = String2Uint64(optarg) * 1000;
1899  download_mgr_->SetRetryParameters(max_retries, backoff_init, backoff_max);
1900 
1901  if (options_mgr_->GetValue("CVMFS_LOW_SPEED_LIMIT", &optarg))
1903  if (options_mgr_->GetValue("CVMFS_PROXY_RESET_AFTER", &optarg))
1905  if (options_mgr_->GetValue("CVMFS_HOST_RESET_AFTER", &optarg))
1907 
1908  if (options_mgr_->GetValue("CVMFS_FOLLOW_REDIRECTS", &optarg) &&
1909  options_mgr_->IsOn(optarg))
1910  {
1912  }
1913  if (options_mgr_->GetValue("CVMFS_SEND_INFO_HEADER", &optarg) &&
1914  options_mgr_->IsOn(optarg))
1915  {
1917  }
1918 }
1919 
1920 
1922  string optarg;
1923 
1924  if (file_system_->IsNfsSource()) {
1926  } else {
1928  }
1929  if (options_mgr_->GetValue("CVMFS_INITIAL_GENERATION", &optarg)) {
1931  }
1932 
1935  }
1936 }
1937 
1938 
1940  string optarg;
1941  catalog::OwnerMap uid_map;
1942  catalog::OwnerMap gid_map;
1943 
1944  if (options_mgr_->GetValue("CVMFS_UID_MAP", &optarg)) {
1945  if (!uid_map.Read(optarg)) {
1946  boot_error_ = "failed to parse uid map " + optarg;
1948  return false;
1949  }
1950  }
1951  if (options_mgr_->GetValue("CVMFS_GID_MAP", &optarg)) {
1952  if (!gid_map.Read(optarg)) {
1953  boot_error_ = "failed to parse gid map " + optarg;
1955  return false;
1956  }
1957  }
1958  catalog_mgr_->SetOwnerMaps(uid_map, gid_map);
1959 
1960  // TODO(jblomer): make local to catalog manager
1961  if (options_mgr_->GetValue("CVMFS_CLAIM_OWNERSHIP", &optarg) &&
1962  options_mgr_->IsOn(optarg))
1963  {
1964  g_claim_ownership = true;
1965  }
1966 
1967  return true;
1968 }
OptionsManager * options_mgr()
Definition: mountpoint.h:203
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
static const unsigned kDnsDefaultTimeoutMs
Definition: download.h:391
bool GetVOMSAuthz(std::string *authz) const
void SetLogSyslogFacility(const int local_facility)
Definition: logging.cc:129
FileSystem * file_system_
Definition: mountpoint.h:522
Counter * Register(const std::string &name, const std::string &desc)
Definition: statistics.cc:139
void RegisterFdMapping(int from, int to)
Definition: sqlitevfs.cc:541
std::string nfs_maps_dir_
Definition: mountpoint.h:353
void SetLogSyslogLevel(const int level)
Definition: logging.cc:97
bool IsNull() const
Definition: hash.h:379
bool foreground_
Definition: mountpoint.h:284
void Activate(const int buffer_size, const int flush_threshold, const std::string &trace_file)
Definition: tracer.cc:25
download::DownloadManager * download_mgr_
Definition: repository.h:140
struct stat64 platform_stat64
Log2Histogram * hist_fs_forget_multi_
Definition: mountpoint.h:301
void SetupInodeAnnotation()
Definition: mountpoint.cc:1921
bool GetValue(const std::string &key, std::string *value)
Definition: options.cc:376
bool SetupCwd()
Definition: mountpoint.cc:756
catalog::InodeAnnotation * inode_annotation_
Definition: mountpoint.h:538
static NfsMapsSqlite * Create(const std::string &db_dir, const uint64_t root_inode, const bool rebuild, perf::Statistics *statistics_)
void TearDown2ReadOnly()
Definition: cache_posix.cc:539
std::string workspace_fullpath_
Definition: mountpoint.h:323
bool DetermineNfsMode()
Definition: mountpoint.cc:321
static SqliteHistory * Open(const std::string &file_name)
void SetHostChain(const std::string &host_list)
static const unsigned kTracerFlushThreshold
Definition: mountpoint.h:489
vector< string > SplitString(const string &str, const char delim, const unsigned max_chunks)
Definition: string.cc:288
virtual uint64_t GetSize()
Definition: quota_posix.cc:674
static Publisher * Create(const SettingsPublisher &settings)
uid_t talk_socket_uid_
Definition: mountpoint.h:565
void SetLowSpeedLimit(const unsigned low_speed_limit)
Definition: download.cc:1886
Log2Histogram * hist_fs_opendir_
Definition: mountpoint.h:304
bool LockWorkspace()
Definition: mountpoint.cc:432
bool CreateTracer()
Definition: mountpoint.cc:1464
bool wait_workspace_
Definition: mountpoint.h:283
virtual bool Cleanup(const uint64_t leave_size)
Definition: quota_posix.cc:125
double kcache_timeout_sec_
Definition: mountpoint.h:554
signature::SignatureManager * signature_mgr_
Definition: mountpoint.h:533
void CreateFile(const std::string &path, const int mode, const bool ignore_failure)
Definition: posix.cc:313
std::string name
Definition: history.h:113
AuthzSessionManager * authz_session_mgr_
Definition: mountpoint.h:530
static const unsigned kDefaultRetries
Definition: mountpoint.h:482
string JoinStrings(const vector< string > &strings, const string &joint)
Definition: string.cc:318
perf::Statistics * statistics_
Definition: mountpoint.h:297
download::DownloadManager * download_mgr_
Definition: mountpoint.h:534
static const unsigned kDefaultTimeoutSec
Definition: mountpoint.h:481
std::string exe_path_
Definition: mountpoint.h:277
bool CheckInstanceName(const std::string &instance)
Definition: mountpoint.cc:87
perf::Statistics * statistics_
Definition: mountpoint.h:528
Log2Histogram * hist_fs_readlink_
Definition: mountpoint.h:303
void SetProxyChain(const std::string &proxy_list, const std::string &fallback_proxy_list, const ProxySetModes set_mode)
Definition: download.cc:2430
static NfsMapsLeveldb * Create(const std::string &leveldb_dir, const uint64_t root_inode, const bool rebuild, perf::Statistics *statistics)
gid_t talk_socket_gid_
Definition: mountpoint.h:566
void SetupSqlite()
Definition: mountpoint.cc:956
bool foreground_
Definition: loader.cc:135
std::string GetUniqFileSuffix()
Definition: mountpoint.cc:1620
download::DownloadManager * external_download_mgr_
Definition: mountpoint.h:535
MagicXattrManager * magic_xattr_mgr_
Definition: mountpoint.h:548
CacheManager * SetupCacheMgr(const std::string &instance)
Definition: mountpoint.cc:541
const SettingsPublisher & settings() const
Definition: repository.h:306
perf::Statistics * statistics_
Definition: repository.h:138
void SetupHttpTuning()
Definition: mountpoint.cc:1877
static const char * kDefaultBlacklist
Definition: mountpoint.h:490
virtual bool AcquireQuotaManager(QuotaManager *quota_mgr)
Definition: cache_extern.cc:85
unsigned max_ttl_sec_
Definition: mountpoint.h:552
static const unsigned kDefaultMaxTtl
Definition: dns.h:182
bool CreateDownloadManagers()
Definition: mountpoint.cc:1259
std::string talk_socket_path_
Definition: mountpoint.h:564
assert((mem||(size==0))&&"Out Of Memory")
void SetupLogging()
Definition: mountpoint.cc:783
int fd_workspace_lock_
Definition: mountpoint.h:324
void CreateStatistics()
Definition: mountpoint.cc:1397
bool UnregisterVfsRdOnly()
Definition: sqlitevfs.cc:522
CacheManager * SetupRamCacheMgr(const std::string &instance)
Definition: mountpoint.cc:644
static void LogSqliteError(void *user_data __attribute__((unused)), int sqlite_extended_error, const char *message)
Definition: mountpoint.cc:463
static void CleanupInstance()
Definition: sqlitemem.cc:170
std::string workspace_
Definition: mountpoint.h:317
string StringifyTime(const time_t seconds, const bool utc)
Definition: string.cc:105
PosixCacheSettings DeterminePosixCacheSettings(const std::string &instance)
Definition: mountpoint.cc:244
void SetDnsServer(const std::string &address)
Definition: download.cc:1815
int platform_stat(const char *path, platform_stat64 *buf)
void AssignGlobalArenas()
Definition: sqlitemem.cc:137
bool has_membership_req_
Definition: mountpoint.h:562
virtual CacheManagerIds id()=0
void SetLogMicroSyslog(const std::string &filename)
Definition: logging.cc:214
void ReEvaluateAuthz()
Definition: mountpoint.cc:1711
file_watcher::FileWatcher * platform_file_watcher()
bool HasConfigRepository(const std::string &fqrn, std::string *config_path)
Definition: options.cc:256
std::set< std::string > constructed_instances_
Definition: mountpoint.h:352
bool DetermineRootHash(shash::Any *root_hash)
Definition: mountpoint.cc:1496
#define SetLogDebugFile(filename)
void SetTimeout(const unsigned seconds_proxy, const unsigned seconds_direct)
Definition: download.cc:1872
bool CreateCatalogManager()
Definition: mountpoint.cc:1202
glue::NentryTracker * nentry_tracker_
Definition: mountpoint.h:547
struct cvmcache_object_info __attribute__
Definition: atomic.h:24
void RemapCatalogFd(int from, int to)
Definition: mountpoint.cc:1050
std::string path_crash_guard_
Definition: mountpoint.h:330
IntegerMap< uint64_t > OwnerMap
Definition: catalog.h:41
catalog::WritableCatalogManager * catalog_mgr_
Definition: repository.h:379
bool SetupWorkspace()
Definition: mountpoint.cc:973
static const char * kDefaultAuthzSearchPath
Definition: mountpoint.h:473
Type type()
Definition: mountpoint.h:205
FileSystem(const FileSystemInfo &fs_info)
Definition: mountpoint.cc:343
static const unsigned kLibPathCacheSize
Definition: mountpoint.h:461
std::string boot_error_
Definition: mountpoint.h:85
OptionsManager * options_mgr_
Definition: mountpoint.h:526
Log2Histogram * hist_fs_getattr_
Definition: mountpoint.h:302
bool IsValid(const std::string &input) const
Definition: sanitizer.cc:114
Log2Histogram * hist_fs_releasedir_
Definition: mountpoint.h:305
Definition: tracer.h:35
bool FileExists(const std::string &path)
Definition: posix.cc:816
bool LoadTrustedCaCrl(const std::string &path_list)
Definition: signature.cc:552
void SetDnsTtlLimits(const unsigned min_seconds, const unsigned max_seconds)
Definition: download.cc:1851
std::string cache_path()
Definition: cache_posix.h:105
uint64_t platform_memsize()
int64_t String2Int64(const string &value)
Definition: string.cc:222
void SetMaxTtlMn(unsigned value_minutes)
Definition: mountpoint.cc:1721
static const unsigned kDefaultBackoffInitMs
Definition: mountpoint.h:483
MountPoint(const std::string &fqrn, FileSystem *file_system, OptionsManager *options_mgr)
Definition: mountpoint.cc:1625
void GetLimitNoFile(unsigned *soft_limit, unsigned *hard_limit)
Definition: posix.cc:1495
Tracer * tracer_
Definition: mountpoint.h:545
void SetProxyGroupResetDelay(const unsigned seconds)
Definition: download.cc:2664
static CacheManager * Create(CacheManager *upper_cache, CacheManager *lower_cache)
static bool g_alive
Definition: mountpoint.h:213
static const unsigned kDefaultNumConnections
Definition: mountpoint.h:477
void Set(const int64_t val)
Definition: statistics.h:33
void TearDown2ReadOnly()
Definition: mountpoint.cc:1037
gid_t g_gid
Definition: globals.cc:14
uuid_t uuid
Definition: uuid.h:39
lru::Md5PathCache * md5path_cache_
Definition: mountpoint.h:544
std::string name_
Definition: mountpoint.h:276
std::string Dump()
Definition: options.cc:447
bool RegisterVfsRdOnly(CacheManager *cache_mgr, perf::Statistics *statistics, const VfsOptions options)
Definition: sqlitevfs.cc:453
static AuthzSessionManager * Create(AuthzFetcher *authz_fetcher, perf::Statistics *statistics)
std::string membership_req_
Definition: mountpoint.h:561
signature::SignatureManager * signature_mgr_
Definition: repository.h:139
static const char * kDefaultCacheMgrInstance
Definition: mountpoint.h:217
bool HasSuffix(const std::string &str, const std::string &suffix, const bool ignore_case)
Definition: string.cc:279
virtual uint64_t GetCapacity()
Definition: quota_posix.cc:582
bool found_previous_crash_
Definition: mountpoint.h:336
const char kSuffixCatalog
Definition: hash.h:52
loader::Failures boot_status_
Definition: mountpoint.h:84
Failures Fetch(const std::string &base_url, const std::string &repository_name, const uint64_t minimum_timestamp, const shash::Any *base_catalog, signature::SignatureManager *signature_manager, download::DownloadManager *download_manager, ManifestEnsemble *ensemble)
bool TriageCacheMgr()
Definition: mountpoint.cc:1055
bool CheckPosixCacheSettings(const PosixCacheSettings &settings)
Definition: mountpoint.cc:104
bool IsNfsSource()
Definition: mountpoint.h:168
ChunkTables * chunk_tables_
Definition: mountpoint.h:540
CacheManager * cache_mgr()
Definition: mountpoint.h:174
DownloadManager * Clone(perf::StatisticsTemplate statistics)
Definition: download.cc:2727
uid_t g_uid
Definition: globals.cc:13
void SetupDnsTuning(download::DownloadManager *manager)
Definition: mountpoint.cc:1776
cvmfs::Uuid * uuid_cache_
Definition: mountpoint.h:363
unsigned GetEffectiveTtlSec()
Definition: mountpoint.cc:1598
int TryLockFile(const std::string &path)
Definition: posix.cc:938
bool MkdirDeep(const std::string &path, const mode_t mode, bool verify_writable)
Definition: posix.cc:871
Any MkFromHexPtr(const HexPtr hex, const char suffix)
Definition: hash.cc:83
int LockFile(const std::string &path)
Definition: posix.cc:996
void SetIpPreference(const dns::IpPreference preference)
Definition: download.cc:1861
static const unsigned kDefaultBackoffMaxMs
Definition: mountpoint.h:484
lru::InodeCache * inode_cache_
Definition: mountpoint.h:542
CacheManager * SetupPosixCacheMgr(const std::string &instance)
Definition: mountpoint.cc:614
static uint64_t RoundUp8(const uint64_t size)
Definition: smalloc.h:37
bool SetupPosixQuotaMgr(const PosixCacheSettings &settings, CacheManager *cache_mgr)
Definition: mountpoint.cc:894
static SqliteMemoryManager * GetInstance()
Definition: sqlitemem.h:137
static PosixQuotaManager * Create(const std::string &cache_workspace, const uint64_t limit, const uint64_t cleanup_threshold, const bool rebuild_database)
Definition: quota_posix.cc:230
void SetRetryParameters(const unsigned max_retries, const unsigned backoff_init_ms, const unsigned backoff_max_ms)
Definition: download.cc:2683
string StringifyInt(const int64_t value)
Definition: string.cc:78
Type type_
Definition: mountpoint.h:278
static const unsigned kInodeCacheFactor
Definition: mountpoint.h:465
perf::Counter * n_io_error_
Definition: mountpoint.h:294
void SetMaxIpaddrPerProxy(unsigned limit)
Definition: download.cc:2694
bool HasPrefix(const string &str, const string &prefix, const bool ignore_case)
Definition: string.cc:265
Log2Histogram * hist_fs_readdir_
Definition: mountpoint.h:306
cvmfs::Uuid * uuid_
Definition: mountpoint.h:518
OptionsManager * options_mgr_
Definition: cvmfs.cc:145
bool LoadBlacklist(const std::string &path_blacklist, bool append)
Definition: signature.cc:508
BackoffThrottle * backoff_throttle_
Definition: mountpoint.h:532
AuthzAttachment * authz_attachment_
Definition: mountpoint.h:531
void SetProxyTemplates(const std::string &direct, const std::string &forced)
Definition: download.cc:2700
SimpleChunkTables * simple_chunk_tables_
Definition: mountpoint.h:541
bool InitFixed(const shash::Any &root_hash, bool alternative_path)
static const unsigned kDefaultNfiles
Definition: mountpoint.h:216
static FileSystem * Create(const FileSystemInfo &fs_info)
Definition: mountpoint.cc:144
bool has_custom_sqlitevfs_
Definition: mountpoint.h:374
bool ReloadBlacklists()
Definition: mountpoint.cc:1115
std::string name()
Definition: mountpoint.h:199
time_t IsoTimestamp2UtcTime(const std::string &iso8601)
Definition: string.cc:198
static Uuid * Create(const std::string &store_path)
Definition: uuid.cc:23
void CreateStatistics()
Definition: mountpoint.cc:186
pthread_mutex_t lock_max_ttl_
Definition: mountpoint.h:553
void CreateAuthz()
Definition: mountpoint.cc:1176
bool g_raw_symlinks
Definition: globals.cc:12
static const char * kDefaultCacheBase
Definition: mountpoint.h:214
void SetNull()
Definition: hash.h:387
string ResolveProxyDescription(const string &cvmfs_proxies, const std::string &path_fallback_cache, DownloadManager *download_manager)
Definition: wpad.cc:206
uint64_t String2Uint64(const string &value)
Definition: string.cc:228
static const unsigned kTracerBufferSize
Definition: mountpoint.h:488
void SetupUuid()
Definition: mountpoint.cc:1022
catalog::ClientCatalogManager * catalog_mgr_
Definition: mountpoint.h:539
bool CreateSignatureManager()
Definition: mountpoint.cc:1361
std::string fqrn_
Definition: mountpoint.h:517
unsigned GetMaxTtlMn()
Definition: mountpoint.cc:1610
std::string error_msg() const
Definition: cache_extern.h:45
bool GetUidOf(const std::string &username, uid_t *uid, gid_t *main_gid)
Definition: posix.cc:1369
bool SetupBehavior()
Definition: mountpoint.cc:1727
static ExternalCacheManager * Create(int fd_connection, unsigned max_open_fds, const std::string &ident)
AuthzFetcher * authz_fetcher_
Definition: mountpoint.h:529
bool FetchHistory(std::string *history_path)
Definition: mountpoint.cc:1564
perf::Statistics * statistics()
Definition: mountpoint.h:204
bool enforce_acls_
Definition: mountpoint.h:556
bool IsHaNfsSource()
Definition: mountpoint.h:169
void CreateTables()
Definition: mountpoint.cc:1425
shash::Any history() const
Definition: manifest.h:127
bool SetupCrashGuard()
Definition: mountpoint.cc:735
static PosixQuotaManager * CreateShared(const std::string &exe_path, const std::string &cache_workspace, const uint64_t limit, const uint64_t cleanup_threshold, bool foreground)
Definition: quota_posix.cc:262
void ResetErrorCounters()
Definition: mountpoint.cc:533
Log2Histogram * hist_fs_lookup_
Definition: mountpoint.h:299
shash::Any root_hash
Definition: history.h:114
bool LoadPublicRsaKeys(const std::string &path_list)
Definition: signature.cc:277
void RegisterHandler(const std::string &file_path, EventHandler *handler)
Definition: file_watcher.cc:35
void GetTimeout(unsigned *seconds_proxy, unsigned *seconds_direct)
Definition: download.cc:1895
static const unsigned kDefaultMemcacheSize
Definition: mountpoint.h:469
FileSystem * file_system_
Definition: cvmfs.cc:120
std::string repository_tag_
Definition: mountpoint.h:557
static const unsigned kDefaultMinTtl
Definition: dns.h:177
bool SetupOwnerMaps()
Definition: mountpoint.cc:1939
void SetDnsParameters(const unsigned retries, const unsigned timeout_ms)
Definition: download.cc:1833
bool CreateResolvConfWatcher()
Definition: mountpoint.cc:1321
static ExternalQuotaManager * Create(ExternalCacheManager *cache_mgr)
Log2Histogram * hist_fs_read_
Definition: mountpoint.h:308
void SetCatalogWatermark(unsigned limit)
static bool g_is_enabled
Definition: algorithm.h:184
bool SetupNfsMaps()
Definition: mountpoint.cc:801
bool CheckBlacklists()
Definition: mountpoint.cc:1077
void Init(const unsigned max_pool_handles, const bool use_system_proxy, perf::StatisticsTemplate statistics)
Definition: download.cc:1594
std::vector< std::string > blacklist_paths_
Definition: mountpoint.h:558
cvmfs::Fetcher * fetcher_
Definition: mountpoint.h:536
file_watcher::FileWatcher * resolv_conf_watcher_
Definition: mountpoint.h:550
static const unsigned kDnsDefaultRetries
Definition: download.h:390
bool GeoSortServers(std::vector< std::string > *servers, std::vector< uint64_t > *output_order=NULL)
Definition: download.cc:2138
bool SetupExternalDownloadMgr(bool dogeosort)
Definition: mountpoint.cc:1816
void SetOwnerMaps(const OwnerMap &uid_map, const OwnerMap &gid_map)
CacheManager * SetupExternalCacheMgr(const std::string &instance)
Definition: mountpoint.cc:575
static void CleanupInstance()
Definition: clientctx.cc:17
Log2Histogram * hist_fs_open_
Definition: mountpoint.h:307
virtual bool AcquireQuotaManager(QuotaManager *quota_mgr)=0
std::string cache_mgr_instance_
Definition: mountpoint.h:347
std::string path_workspace_lock_
Definition: mountpoint.h:325
virtual void IncGeneration(const uint64_t by)=0
Log2Histogram * hist_fs_release_
Definition: mountpoint.h:309
void SetLogSyslogPrefix(const std::string &prefix)
Definition: logging.cc:187
std::string MakeCanonicalPath(const std::string &path)
Definition: posix.cc:96
OptionsManager * options_mgr_
Definition: mountpoint.h:282
glue::InodeTracker * inode_tracker_
Definition: mountpoint.h:546
bool fixed_catalog_
Definition: mountpoint.h:555
static PluginHandle * CreatePlugin(const std::string &locator, const std::vector< std::string > &cmd_line)
bool IsDefined(const std::string &key)
Definition: options.cc:370
void GetHostInfo(std::vector< std::string > *host_chain, std::vector< int > *rtt, unsigned *current_host)
Definition: download.cc:1939
void set_membership(const std::string &m)
Definition: authz_curl.h:25
Statistics * Fork()
Definition: statistics.cc:47
CacheManager * SetupTieredCacheMgr(const std::string &instance)
Definition: mountpoint.cc:694
std::vector< std::string > FindFilesBySuffix(const std::string &dir, const std::string &suffix)
Definition: posix.cc:1149
std::string workspace()
Definition: mountpoint.h:207
bool g_claim_ownership
Definition: globals.cc:11
lru::PathCache * path_cache_
Definition: mountpoint.h:543
bool IsOn(const std::string &param_value)
Definition: options.cc:409
CacheManager * cache_mgr_
Definition: mountpoint.h:358
void SetCredentialsAttachment(CredentialsAttachment *ca)
Definition: download.cc:1799
static ClientCtx * GetInstance()
Definition: clientctx.cc:44
virtual bool AcquireQuotaManager(QuotaManager *quota_mgr)
Definition: cache_ram.cc:70
cvmfs::Uuid * uuid_cache()
Definition: mountpoint.h:206
cvmfs::Fetcher * external_fetcher_
Definition: mountpoint.h:537
static const uint64_t kSizeUnknown
Definition: cache.h:72
std::string MkCacheParm(const std::string &generic_parameter, const std::string &instance)
Definition: mountpoint.cc:498
void UnlockFile(const int filedes)
Definition: posix.cc:1020
NfsMaps * nfs_maps_
Definition: mountpoint.h:369
void CreateFetchers()
Definition: mountpoint.cc:1344
void SetInodeAnnotation(InodeAnnotation *new_annotation)
int Fetch(const shash::Any &id, const uint64_t size, const std::string &name, const zlib::Algorithms compression_algorithm, const CacheManager::ObjectType object_type, const std::string &alt_url="", off_t range_offset=-1)
Definition: fetch.cc:81
virtual void SetInodeResidue(unsigned residue_class, unsigned remainder)
Definition: nfs_maps.h:31
void SetHostResetDelay(const unsigned seconds)
Definition: download.cc:2674
static MountPoint * Create(const std::string &fqrn, FileSystem *file_system, OptionsManager *options_mgr=NULL)
Definition: mountpoint.cc:1136
Log2Histogram * hist_fs_forget_
Definition: mountpoint.h:300
static PosixCacheManager * Create(const std::string &cache_path, const bool alien_cache, const RenameWorkarounds rename_workaround=kRenameNormal)
Definition: cache_posix.cc:230