mountpoint.cc

Go to the documentation of this file.

#include "cvmfs_config.h"
#include "mountpoint.h"

#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <stdint.h>
#include <unistd.h>

#include <algorithm>
#include <cassert>
#include <climits>
#include <cstring>
#include <vector>

#include "duplex_fuse.h"  // NOLINT

#ifndef CVMFS_LIBCVMFS
#ifdef FUSE_CAP_EXPORT_SUPPORT
#define CVMFS_NFS_SUPPORT
#else
#warning "No NFS support, Fuse too old"
#endif
#endif

#include "authz/authz_curl.h"
#include "authz/authz_fetch.h"
#include "authz/authz_session.h"
#include "backoff.h"
#include "cache.h"
#include "cache_extern.h"
#include "cache_posix.h"
#include "cache_ram.h"
#include "cache_tiered.h"
#include "catalog.h"
#include "catalog_mgr_client.h"
#include "clientctx.h"
#include "crypto/signature.h"
#include "download.h"
#include "duplex_sqlite3.h"
#include "fetch.h"
#include "file_chunk.h"
#include "globals.h"
#include "glue_buffer.h"
#include "google/protobuf/stubs/common.h"
#include "history.h"
#include "history_sqlite.h"
#include "lru_md.h"
#include "manifest.h"
#include "manifest_fetch.h"
#include "nfs_maps.h"
#ifdef CVMFS_NFS_SUPPORT
#include "nfs_maps_leveldb.h"
#include "nfs_maps_sqlite.h"
#endif
#include "options.h"
#include "quota_posix.h"
#include "resolv_conf_event_handler.h"
#include "sqlitemem.h"
#include "sqlitevfs.h"
#include "statistics.h"
#include "tracer.h"
#include "util/concurrency.h"
#include "util/logging.h"
#include "util/platform.h"
#include "util/pointer.h"
#include "util/posix.h"
#include "util/string.h"
#include "util/uuid.h"
#include "wpad.h"

using namespace std;  // NOLINT


bool FileSystem::g_alive = false;
const char *FileSystem::kDefaultCacheBase = "/var/lib/cvmfs";
const char *FileSystem::kDefaultCacheMgrInstance = "default";


FileSystem::IoErrorInfo::IoErrorInfo() : counter_(NULL), timestamp_last_(0) { }

void FileSystem::IoErrorInfo::Reset() {
  counter_->Set(0);
  timestamp_last_ = 0;
}

void FileSystem::IoErrorInfo::AddIoError() {
  perf::Inc(counter_);
  timestamp_last_ = time(NULL);
}

void FileSystem::IoErrorInfo::SetCounter(perf::Counter *c) { counter_ = c; }

int64_t FileSystem::IoErrorInfo::count() { return counter_->Get(); }

time_t FileSystem::IoErrorInfo::timestamp_last() { return timestamp_last_; }



bool FileSystem::CheckInstanceName(const std::string &instance) {
  if (instance.length() > 24)
    return false;
  sanitizer::CacheInstanceSanitizer instance_sanitizer;
  if (!instance_sanitizer.IsValid(instance)) {
    boot_error_ = "invalid instance name (" + instance + "), " +
                  "only characters a-z, A-Z, 0-9, _ are allowed";
    boot_status_ = loader::kFailCacheDir;
    return false;
  }
  return true;
}


bool FileSystem::CheckPosixCacheSettings(
  const FileSystem::PosixCacheSettings &settings)
{
  if (settings.is_alien && settings.is_shared) {
    boot_error_ = "Failure: shared local disk cache and alien cache mutually "
                  "exclusive. Please turn off shared local disk cache.";
    boot_status_ = loader::kFailOptions;
    return false;
  }
  if (settings.is_alien && settings.is_managed) {
    boot_error_ = "Failure: quota management and alien cache mutually "
                  "exclusive. Please turn off quota limit.";
    boot_status_ = loader::kFailOptions;
    return false;
  }

  if (type_ == kFsLibrary) {
    if (settings.is_shared || settings.is_managed) {
      boot_error_ = "Failure: libcvmfs supports only unmanaged exclusive cache "
                    "or alien cache.";
      boot_status_ = loader::kFailOptions;
      return false;
    }
  }

  if (settings.cache_base_defined && settings.cache_dir_defined) {
    boot_error_ =
      "'CVMFS_CACHE_BASE' and 'CVMFS_CACHE_DIR' are mutually exclusive";
    boot_status_ = loader::kFailOptions;
    return false;
  }

  return true;
}


FileSystem *FileSystem::Create(const FileSystem::FileSystemInfo &fs_info) {
  UniquePtr<FileSystem>
    file_system(new FileSystem(fs_info));

  file_system->SetupLogging();
  LogCvmfs(kLogCvmfs, kLogDebug, "Options:\n%s",
           file_system->options_mgr()->Dump().c_str());

  file_system->CreateStatistics();
  file_system->SetupSqlite();
  if (!file_system->DetermineNfsMode())
    return file_system.Release();
  if (!file_system->SetupWorkspace())
    return file_system.Release();

  // Redirect SQlite temp directory to workspace (global variable)
  unsigned length_tempdir = file_system->workspace_.length() + 1;
  sqlite3_temp_directory = static_cast<char *>(sqlite3_malloc(length_tempdir));
  snprintf(sqlite3_temp_directory,
           length_tempdir,
           "%s",
           file_system->workspace_.c_str());

  if (!file_system->TriageCacheMgr())
    return file_system.Release();
  file_system->SetupUuid();
  if (!file_system->SetupNfsMaps())
    return file_system.Release();
  bool retval = sqlite::RegisterVfsRdOnly(
    file_system->cache_mgr_,
    file_system->statistics_,
    sqlite::kVfsOptDefault);
  assert(retval);
  file_system->has_custom_sqlitevfs_ = true;

  ClientCtx::GetInstance();

  file_system->boot_status_ = loader::kFailOk;
  return file_system.Release();
}


void FileSystem::CreateStatistics() {
  statistics_ = new perf::Statistics();

  // Register the ShortString's static counters
  statistics_->Register("pathstring.n_instances", "Number of instances");
  statistics_->Register("pathstring.n_overflows", "Number of overflows");
  statistics_->Register("namestring.n_instances", "Number of instances");
  statistics_->Register("namestring.n_overflows", "Number of overflows");
  statistics_->Register("linkstring.n_instances", "Number of instances");
  statistics_->Register("linkstring.n_overflows", "Number of overflows");

  // Callback counters
  n_fs_open_ = statistics_->Register("cvmfs.n_fs_open",
                                     "Overall number of file open operations");
  n_fs_dir_open_ = statistics_->Register("cvmfs.n_fs_dir_open",
                   "Overall number of directory open operations");
  n_fs_lookup_ = statistics_->Register("cvmfs.n_fs_lookup",
                                       "Number of lookups");
  n_fs_lookup_negative_ = statistics_->Register("cvmfs.n_fs_lookup_negative",
                                                "Number of negative lookups");
  n_fs_stat_ = statistics_->Register("cvmfs.n_fs_stat", "Number of stats");
  n_fs_stat_stale_ = statistics_->Register("cvmfs.n_fs_stat_stale",
    "Number of stats for stale (open, meanwhile changed) regular files");
  n_fs_read_ = statistics_->Register("cvmfs.n_fs_read", "Number of files read");
  n_fs_readlink_ = statistics_->Register("cvmfs.n_fs_readlink",
                                         "Number of links read");
  n_fs_forget_ = statistics_->Register("cvmfs.n_fs_forget",
                                       "Number of inode forgets");
  n_fs_inode_replace_ = statistics_->Register("cvmfs.n_fs_inode_replace",
    "Number of stale inodes that got replaced by an up-to-date version");
  no_open_files_ = statistics_->Register("cvmfs.no_open_files",
                                         "Number of currently opened files");
  no_open_dirs_ = statistics_->Register("cvmfs.no_open_dirs",
                  "Number of currently opened directories");
  io_error_info_.SetCounter(statistics_->Register("cvmfs.n_io_error",
                                                  "Number of I/O errors"));

  string optarg;
  if (options_mgr_->GetValue("CVMFS_INSTRUMENT_FUSE", &optarg) &&
      options_mgr_->IsOn(optarg))
  {
    HighPrecisionTimer::g_is_enabled = true;
  }

  hist_fs_lookup_ = new Log2Histogram(30);
  hist_fs_forget_ = new Log2Histogram(30);
  hist_fs_forget_multi_ = new Log2Histogram(30);
  hist_fs_getattr_ = new Log2Histogram(30);
  hist_fs_readlink_ = new Log2Histogram(30);
  hist_fs_opendir_ = new Log2Histogram(30);
  hist_fs_releasedir_ = new Log2Histogram(30);
  hist_fs_readdir_ = new Log2Histogram(30);
  hist_fs_open_ = new Log2Histogram(30);
  hist_fs_read_ = new Log2Histogram(30);
  hist_fs_release_ = new Log2Histogram(30);
}


FileSystem::PosixCacheSettings FileSystem::DeterminePosixCacheSettings(
  const string &instance
) {
  string optarg;
  PosixCacheSettings settings;

  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SHARED", instance),
                             &optarg)
      && options_mgr_->IsOn(optarg))
  {
    settings.is_shared = true;
  }
  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SERVER_MODE", instance),
                             &optarg)
      && options_mgr_->IsOn(optarg))
  {
    settings.avoid_rename = true;
  }

  if (type_ == kFsFuse)
    settings.quota_limit = kDefaultQuotaLimit;
  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_QUOTA_LIMIT", instance),
                             &optarg))
  {
    settings.quota_limit = String2Int64(optarg) * 1024 * 1024;
  }
  if (settings.quota_limit > 0)
    settings.is_managed = true;

  settings.cache_path = kDefaultCacheBase;
  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_BASE", instance),
                             &optarg))
  {
    settings.cache_path = MakeCanonicalPath(optarg);
    settings.cache_base_defined = true;
  }
  if (settings.is_shared) {
    settings.cache_path += "/shared";
  } else {
    settings.cache_path += "/" + name_;
  }

  // CheckCacheMode makes sure that CVMFS_CACHE_DIR and CVMFS_CACHE_BASE are
  // not set at the same time.
  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_DIR", instance),
                             &optarg))
  {
    settings.cache_dir_defined = true;
    settings.cache_path = optarg;
  }
  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_ALIEN", instance),
                             &optarg))
  {
    settings.is_alien = true;
    settings.cache_path = optarg;
  }
  // We already changed the cwd to the workspace
  // Which is only done if using FUSE
  if ((type_ == kFsFuse) && (settings.cache_path == workspace_fullpath_)) {
    settings.cache_path = ".";
  }

  // The cache workspace usually is the cache directory, unless explicitly
  // set otherwise
  settings.workspace = settings.cache_path;
  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_WORKSPACE", instance),
                             &optarg) ||
      options_mgr_->GetValue("CVMFS_WORKSPACE", &optarg))
  {
    // Used for the shared quota manager
    settings.workspace = optarg;
  }

  return settings;
}


bool FileSystem::DetermineNfsMode() {
  string optarg;

  if (options_mgr_->GetValue("CVMFS_NFS_SOURCE", &optarg) &&
      options_mgr_->IsOn(optarg))
  {
    nfs_mode_ |= kNfsMaps;
    if (options_mgr_->GetValue("CVMFS_NFS_SHARED", &optarg)) {
      nfs_mode_ |= kNfsMapsHa;
      nfs_maps_dir_ = optarg;
    }
  }

  if ((type_ == kFsLibrary) && (nfs_mode_ != kNfsNone)) {
    boot_error_ = "Failure: libcvmfs does not support NFS export.";
    boot_status_ = loader::kFailOptions;
    return false;
  }
  return true;
}


FileSystem::FileSystem(const FileSystem::FileSystemInfo &fs_info)
  : name_(fs_info.name)
  , exe_path_(fs_info.exe_path)
  , type_(fs_info.type)
  , options_mgr_(fs_info.options_mgr)
  , wait_workspace_(fs_info.wait_workspace)
  , foreground_(fs_info.foreground)
  , n_fs_open_(NULL)
  , n_fs_dir_open_(NULL)
  , n_fs_lookup_(NULL)
  , n_fs_lookup_negative_(NULL)
  , n_fs_stat_(NULL)
  , n_fs_stat_stale_(NULL)
  , n_fs_read_(NULL)
  , n_fs_readlink_(NULL)
  , n_fs_forget_(NULL)
  , n_fs_inode_replace_(NULL)
  , no_open_files_(NULL)
  , no_open_dirs_(NULL)
  , statistics_(NULL)
  , fd_workspace_lock_(-1)
  , found_previous_crash_(false)
  , nfs_mode_(kNfsNone)
  , cache_mgr_(NULL)
  , uuid_cache_(NULL)
  , nfs_maps_(NULL)
  , has_custom_sqlitevfs_(false)
{
  assert(!g_alive);
  g_alive = true;
  g_uid = geteuid();
  g_gid = getegid();

  string optarg;
  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SERVER_MODE",
                                         kDefaultCacheMgrInstance),
                             &optarg)
      && options_mgr_->IsOn(optarg))
  {
    g_raw_symlinks = true;
  }
}


FileSystem::~FileSystem() {
  ClientCtx::CleanupInstance();

  if (has_custom_sqlitevfs_)
    sqlite::UnregisterVfsRdOnly();

  delete uuid_cache_;
  delete nfs_maps_;
  delete cache_mgr_;

  if (sqlite3_temp_directory) {
    sqlite3_free(sqlite3_temp_directory);
    sqlite3_temp_directory = NULL;
  }

  if (!path_crash_guard_.empty())
    unlink(path_crash_guard_.c_str());
  if (!path_workspace_lock_.empty())
    unlink(path_workspace_lock_.c_str());
  if (fd_workspace_lock_ >= 0)
    UnlockFile(fd_workspace_lock_);

  sqlite3_shutdown();
  SqliteMemoryManager::CleanupInstance();

  delete hist_fs_lookup_;
  delete hist_fs_forget_multi_;
  delete hist_fs_forget_;
  delete hist_fs_getattr_;
  delete hist_fs_readlink_;
  delete hist_fs_opendir_;
  delete hist_fs_releasedir_;
  delete hist_fs_readdir_;
  delete hist_fs_open_;
  delete hist_fs_read_;
  delete hist_fs_release_;
  delete statistics_;

  SetLogSyslogPrefix("");
  SetLogMicroSyslog("");
  SetLogDebugFile("");
  google::protobuf::ShutdownProtobufLibrary();
  g_alive = false;
}


bool FileSystem::LockWorkspace() {
  path_workspace_lock_ = workspace_ + "/lock." + name_;
  fd_workspace_lock_ = TryLockFile(path_workspace_lock_);
  if (fd_workspace_lock_ >= 0)
    return true;

  if (fd_workspace_lock_ == -1) {
    boot_error_ = "could not acquire workspace lock (" +
                 StringifyInt(errno) + ")";
    boot_status_ = loader::kFailCacheDir;
    return false;
  }

  assert(fd_workspace_lock_ == -2);

  if (!wait_workspace_) {
    boot_status_ = loader::kFailLockWorkspace;
    return false;
  }

  fd_workspace_lock_ = LockFile(path_workspace_lock_);
  if (fd_workspace_lock_ < 0) {
    boot_error_ = "could not acquire workspace lock (" +
                   StringifyInt(errno) + ")";
    boot_status_ = loader::kFailCacheDir;
    return false;
  }
  return true;
}


void FileSystem::LogSqliteError(
  void *user_data __attribute__((unused)),
  int sqlite_extended_error,
  const char *message)
{
  int log_dest = kLogDebug;
  int sqlite_error = sqlite_extended_error & 0xFF;
  switch (sqlite_error) {
    case SQLITE_INTERNAL:
    case SQLITE_PERM:
    case SQLITE_NOMEM:
    case SQLITE_IOERR:
    case SQLITE_CORRUPT:
    case SQLITE_FULL:
    case SQLITE_CANTOPEN:
    case SQLITE_MISUSE:
    case SQLITE_FORMAT:
    case SQLITE_NOTADB:
      log_dest |= kLogSyslogErr;
      break;
    case SQLITE_WARNING:
    case SQLITE_NOTICE:
    default:
      break;
  }
  LogCvmfs(kLogCvmfs, log_dest, "SQlite3: %s (%d)",
           message, sqlite_extended_error);
}


string FileSystem::MkCacheParm(
  const string &generic_parameter,
  const string &instance)
{
  assert(HasPrefix(generic_parameter, "CVMFS_CACHE_", false));

  if (instance == kDefaultCacheMgrInstance) {
    // Compatibility parameter names
    if ((generic_parameter == "CVMFS_CACHE_SHARED") &&
        !options_mgr_->IsDefined(generic_parameter))
    {
      return "CVMFS_SHARED_CACHE";
    }
    if ((generic_parameter == "CVMFS_CACHE_ALIEN") &&
        !options_mgr_->IsDefined(generic_parameter))
    {
      return "CVMFS_ALIEN_CACHE";
    }
    if ((generic_parameter == "CVMFS_CACHE_SERVER_MODE") &&
        !options_mgr_->IsDefined(generic_parameter))
    {
      return "CVMFS_SERVER_CACHE_MODE";
    }
    if ((generic_parameter == "CVMFS_CACHE_QUOTA_LIMIT") &&
        !options_mgr_->IsDefined(generic_parameter))
    {
      return "CVMFS_QUOTA_LIMIT";
    }
    return generic_parameter;
  }

  return "CVMFS_CACHE_" + instance + "_" + generic_parameter.substr(12);
}


void FileSystem::ResetErrorCounters() {
  io_error_info_.Reset();
}


CacheManager *FileSystem::SetupCacheMgr(const string &instance) {
  if (constructed_instances_.find(instance) != constructed_instances_.end()) {
    boot_error_ = "circular cache definition: " + instance;
    boot_status_ = loader::kFailCacheDir;
    return NULL;
  }
  constructed_instances_.insert(instance);

  LogCvmfs(kLogCvmfs, kLogDebug, "setting up cache manager instance %s",
           instance.c_str());
  string instance_type;
  if (instance == kDefaultCacheMgrInstance) {
    instance_type = "posix";
  } else {
    options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_TYPE", instance),
                           &instance_type);
  }
  if (instance_type == "posix") {
    return SetupPosixCacheMgr(instance);
  } else if (instance_type == "ram") {
    return SetupRamCacheMgr(instance);
  } else if (instance_type == "tiered") {
    return SetupTieredCacheMgr(instance);
  } else if (instance_type == "external") {
    return SetupExternalCacheMgr(instance);
  } else {
    boot_error_ = "invalid cache manager type for '" + instance +  "':" +
      instance_type;
    boot_status_ = loader::kFailCacheDir;
    return NULL;
  }
}


CacheManager *FileSystem::SetupExternalCacheMgr(const string &instance) {
  string optarg;
  unsigned nfiles = kDefaultNfiles;
  if (options_mgr_->GetValue("CVMFS_NFILES", &optarg))
    nfiles = String2Uint64(optarg);
  vector<string> cmd_line;
  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_CMDLINE", instance),
      &optarg))
  {
    cmd_line = SplitString(optarg, ',');
  }

  if (!options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_LOCATOR", instance),
      &optarg))
  {
    boot_error_ = MkCacheParm("CVMFS_CACHE_LOCATOR", instance) + " missing";
    boot_status_ = loader::kFailCacheDir;
    return NULL;
  }

  UniquePtr<ExternalCacheManager::PluginHandle> plugin_handle(
    ExternalCacheManager::CreatePlugin(optarg, cmd_line));
  if (!plugin_handle->IsValid()) {
    boot_error_ = plugin_handle->error_msg();
    boot_status_ = loader::kFailCacheDir;
    return NULL;
  }
  ExternalCacheManager *cache_mgr = ExternalCacheManager::Create(
    plugin_handle->fd_connection(), nfiles, name_ + ":" + instance);
  if (cache_mgr == NULL) {
    boot_error_ = "failed to create external cache manager for " + instance;
    boot_status_ = loader::kFailCacheDir;
    return NULL;
  }
  cache_mgr->AcquireQuotaManager(ExternalQuotaManager::Create(cache_mgr));
  return cache_mgr;
}


CacheManager *FileSystem::SetupPosixCacheMgr(const string &instance) {
  PosixCacheSettings settings = DeterminePosixCacheSettings(instance);
  if (!CheckPosixCacheSettings(settings))
    return NULL;

  UniquePtr<PosixCacheManager> cache_mgr(PosixCacheManager::Create(
    settings.cache_path,
    settings.is_alien,
    settings.avoid_rename ? PosixCacheManager::kRenameLink
                          : PosixCacheManager::kRenameNormal));
  if (!cache_mgr.IsValid()) {
    boot_error_ = "Failed to setup posix cache '" + instance + "' in " +
                  settings.cache_path + ": " + strerror(errno);
    boot_status_ = loader::kFailCacheDir;
    return NULL;
  }

  // Sentinel file for future use
  // Might be a read-only cache
  const bool ignore_failure = settings.is_alien;
  CreateFile(settings.cache_path + "/.cvmfscache", 0600, ignore_failure);

  if (settings.is_managed) {
    if (!SetupPosixQuotaMgr(settings, cache_mgr.weak_ref()))
      return NULL;
  }
  return cache_mgr.Release();
}


CacheManager *FileSystem::SetupRamCacheMgr(const string &instance) {
  string optarg;
  unsigned nfiles = kDefaultNfiles;
  if (options_mgr_->GetValue("CVMFS_NFILES", &optarg)) {
    nfiles = String2Uint64(optarg);
  }
  uint64_t sz_cache_bytes;
  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_SIZE", instance),
                             &optarg))
  {
    if (HasSuffix(optarg, "%", false)) {
      sz_cache_bytes = platform_memsize() * String2Uint64(optarg)/100;
    } else {
      sz_cache_bytes = String2Uint64(optarg) * 1024 * 1024;
    }
  } else {
    sz_cache_bytes = platform_memsize() >> 5;  // ~3%
  }
  MemoryKvStore::MemoryAllocator alloc = MemoryKvStore::kMallocHeap;
  if (options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_MALLOC", instance),
                             &optarg))
  {
    if (optarg == "libc") {
      alloc = MemoryKvStore::kMallocLibc;
    } else if (optarg == "heap") {
      alloc = MemoryKvStore::kMallocHeap;
    } else {
      boot_error_ = "Failure: unknown malloc " +
                    MkCacheParm("CVMFS_CACHE_MALLOC", instance) + "=" + optarg;
      boot_status_ = loader::kFailOptions;
      return NULL;
    }
  }
  sz_cache_bytes = RoundUp8(std::max(static_cast<uint64_t>(40 * 1024 * 1024),
                                     sz_cache_bytes));
  RamCacheManager *cache_mgr = new RamCacheManager(
        sz_cache_bytes,
        nfiles,
        alloc,
        perf::StatisticsTemplate("cache." + instance, statistics_));
  if (cache_mgr == NULL) {
    boot_error_ = "failed to create ram cache manager for " + instance;
    boot_status_ = loader::kFailCacheDir;
    return NULL;
  }
  cache_mgr->AcquireQuotaManager(new NoopQuotaManager());
  return cache_mgr;
}


CacheManager *FileSystem::SetupTieredCacheMgr(const string &instance) {
  string optarg;
  if (!options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_UPPER", instance),
                              &optarg))
  {
    boot_error_ = MkCacheParm("CVMFS_CACHE_UPPER", instance) + " missing";
    boot_status_ = loader::kFailOptions;
    return NULL;
  }
  UniquePtr<CacheManager> upper(SetupCacheMgr(optarg));
  if (!upper.IsValid())
    return NULL;

  if (!options_mgr_->GetValue(MkCacheParm("CVMFS_CACHE_LOWER", instance),
                              &optarg))
  {
    boot_error_ = MkCacheParm("CVMFS_CACHE_LOWER", instance) + " missing";
    boot_status_ = loader::kFailOptions;
    return NULL;
  }
  UniquePtr<CacheManager> lower(SetupCacheMgr(optarg));
  if (!lower.IsValid())
    return NULL;

  CacheManager *tiered =
    TieredCacheManager::Create(upper.Release(), lower.Release());
  if (tiered == NULL) {
    boot_error_ = "Failed to setup tiered cache manager " + instance;
    boot_status_ = loader::kFailCacheDir;
    return NULL;
  }
  if (options_mgr_->GetValue(
        MkCacheParm("CVMFS_CACHE_LOWER_READONLY", instance), &optarg) &&
      options_mgr_->IsOn(optarg))
  {
    static_cast<TieredCacheManager*>(tiered)->SetLowerReadOnly();
  }
  return tiered;
}


bool FileSystem::SetupCrashGuard() {
  path_crash_guard_ = workspace_ + "/running." + name_;
  platform_stat64 info;
  int retval = platform_stat(path_crash_guard_.c_str(), &info);
  if (retval == 0) {
    found_previous_crash_ = true;
    LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogWarn,
             "looks like cvmfs has been crashed previously");
  }
  retval = open(path_crash_guard_.c_str(), O_RDONLY | O_CREAT, 0600);
  if (retval < 0) {
    boot_error_ = "could not open running sentinel (" +
                  StringifyInt(errno) + ")";
    boot_status_ = loader::kFailCacheDir;
    return false;
  }
  close(retval);
  return true;
}


bool FileSystem::SetupCwd() {
  if (type_ == kFsFuse) {
    // Try to jump to workspace / cache directory.  This tests, if it is
    // accessible and it brings speed later on.
    int retval = chdir(workspace_.c_str());
    if (retval != 0) {
      boot_error_ = "workspace " + workspace_ + " is unavailable";
      boot_status_ = loader::kFailCacheDir;
      return false;
    }
    workspace_ = ".";
    return true;
  }

  // Note: as of version 2.4 support for CVMFS_CWD_CACHE is dropped due to
  // disproportionate large complexity to configure correctly.  This affects
  // only libcvmfs, mostly the legacy part.
  // string optarg;
  // if (options_mgr_->GetValue("CVMFS_CWD_CACHE", &optarg) &&
  //    options_mgr_->IsOn(optarg))
  // {
  //  ...
  // }
  return true;
}


void FileSystem::SetupLogging() {
  string optarg;
  if (options_mgr_->GetValue("CVMFS_SYSLOG_LEVEL", &optarg))
    SetLogSyslogLevel(String2Uint64(optarg));
  if (options_mgr_->GetValue("CVMFS_SYSLOG_FACILITY", &optarg))
    SetLogSyslogFacility(String2Int64(optarg));
  if (options_mgr_->GetValue("CVMFS_USYSLOG", &optarg))
    SetLogMicroSyslog(optarg);
  if (options_mgr_->GetValue("CVMFS_DEBUGLOG", &optarg))
    SetLogDebugFile(optarg);
  if (options_mgr_->GetValue("CVMFS_SYSLOG_PREFIX", &optarg)) {
    SetLogSyslogPrefix(optarg);
  } else {
    SetLogSyslogPrefix(name_);
  }
}


bool FileSystem::SetupNfsMaps() {
#ifdef CVMFS_NFS_SUPPORT
  if (!IsHaNfsSource())
    nfs_maps_dir_ = workspace_;

  string no_nfs_sentinel;
  if (cache_mgr_->id() == kPosixCacheManager) {
    PosixCacheManager *posix_cache_mgr =
        reinterpret_cast<PosixCacheManager *>(cache_mgr_);
    no_nfs_sentinel = posix_cache_mgr->cache_path() + "/no_nfs_maps." + name_;
    if (!IsNfsSource()) {
      // Might be a read-only cache
      const bool ignore_failure = posix_cache_mgr->alien_cache();
      CreateFile(no_nfs_sentinel, 0600, ignore_failure);
      return true;
    }
  } else {
    if (IsNfsSource()) {
      boot_error_ = "NFS source only works with POSIX cache manager.";
      boot_status_ = loader::kFailNfsMaps;
      return false;
    }
    return true;
  }

  assert(cache_mgr_->id() == kPosixCacheManager);
  assert(IsNfsSource());
  if (!no_nfs_sentinel.empty() && FileExists(no_nfs_sentinel)) {
    boot_error_ = "Cache was used without NFS maps before. "
                  "It has to be wiped out.";
    boot_status_ = loader::kFailNfsMaps;
    return false;
  }

  // nfs maps need to be protected by workspace lock
  PosixCacheManager *posix_cache_mgr =
        reinterpret_cast<PosixCacheManager *>(cache_mgr_);
  if (posix_cache_mgr->cache_path() != workspace_) {
    boot_error_ = "Cache directory and workspace must be identical for "
                  "NFS export";
    boot_status_ = loader::kFailNfsMaps;
    return false;
  }

  string inode_cache_dir = nfs_maps_dir_ + "/nfs_maps." + name_;
  if (!MkdirDeep(inode_cache_dir, 0700)) {
    boot_error_ = "Failed to initialize NFS maps";
    boot_status_ = loader::kFailNfsMaps;
    return false;
  }

  // TODO(jblomer): make this a manager class
  if (IsHaNfsSource()) {
    nfs_maps_ = NfsMapsSqlite::Create(
      inode_cache_dir,
      catalog::ClientCatalogManager::kInodeOffset + 1,
      found_previous_crash_,
      statistics_);
  } else {
    nfs_maps_ = NfsMapsLeveldb::Create(
      inode_cache_dir,
      catalog::ClientCatalogManager::kInodeOffset + 1,
      found_previous_crash_,
      statistics_);
  }

  if (nfs_maps_ == NULL) {
    boot_error_ = "Failed to initialize NFS maps";
    boot_status_ = loader::kFailNfsMaps;
    return false;
  }

  string optarg;
  if (options_mgr_->GetValue("CVMFS_NFS_INTERLEAVED_INODES", &optarg)) {
    vector<string> tokens = SplitString(optarg, '%');
    if (tokens.size() != 2) {
      boot_error_ =
        "invalid format for CVMFS_NFS_INTERLEAVED_INODES: " + optarg;
      boot_status_ = loader::kFailNfsMaps;
      return false;
    }
    nfs_maps_->SetInodeResidue(String2Uint64(tokens[1]),
                               String2Uint64(tokens[0]));
  }

  return true;

#else
  return true;
#endif
}


bool FileSystem::SetupPosixQuotaMgr(
  const FileSystem::PosixCacheSettings &settings,
  CacheManager *cache_mgr
) {
  assert(settings.quota_limit >= 0);
  int64_t quota_threshold = settings.quota_limit / 2;
  string cache_workspace = settings.cache_path;
  if (settings.cache_path != settings.workspace) {
    LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
             "using workspace %s to protect cache database in %s",
             settings.workspace.c_str(), settings.cache_path.c_str());
    cache_workspace += ":" + settings.workspace;
  }
  PosixQuotaManager *quota_mgr;

  if (settings.is_shared) {
    quota_mgr = PosixQuotaManager::CreateShared(
                  exe_path_,
                  cache_workspace,
                  settings.quota_limit,
                  quota_threshold,
                  foreground_);
    if (quota_mgr == NULL) {
      boot_error_ = "Failed to initialize shared lru cache";
      boot_status_ = loader::kFailQuota;
      return false;
    }
  } else {
    quota_mgr = PosixQuotaManager::Create(
                  cache_workspace,
                  settings.quota_limit,
                  quota_threshold,
                  found_previous_crash_);
    if (quota_mgr == NULL) {
      boot_error_ = "Failed to initialize lru cache";
      boot_status_ = loader::kFailQuota;
      return false;
    }
  }

  if (quota_mgr->GetSize() > quota_mgr->GetCapacity()) {
    LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslog,
             "cache is already beyond quota size "
             "(size: %" PRId64 ", capacity: %" PRId64 "), cleaning up",
             quota_mgr->GetSize(), quota_mgr->GetCapacity());
    if (!quota_mgr->Cleanup(quota_threshold)) {
      delete quota_mgr;
      boot_error_ = "Failed to clean up cache";
      boot_status_ = loader::kFailQuota;
      return false;
    }
  }

  int retval = cache_mgr->AcquireQuotaManager(quota_mgr);
  assert(retval);
  LogCvmfs(kLogCvmfs, kLogDebug,
           "CernVM-FS: quota initialized, current size %luMB",
           quota_mgr->GetSize() / (1024 * 1024));
  return true;
}


void FileSystem::SetupSqlite() {
  // Make sure SQlite starts clean after initialization
  sqlite3_shutdown();

  int retval;
  retval = sqlite3_config(SQLITE_CONFIG_LOG, FileSystem::LogSqliteError, NULL);
  assert(retval == SQLITE_OK);
  retval = sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
  assert(retval == SQLITE_OK);
  SqliteMemoryManager::GetInstance()->AssignGlobalArenas();

  // Disable SQlite3 file locking
  retval = sqlite3_vfs_register(sqlite3_vfs_find("unix-none"), 1);
  assert(retval == SQLITE_OK);
}


bool FileSystem::SetupWorkspace() {
  string optarg;
  // This is very similar to "determine cache dir".  It's for backward
  // compatibility with classic cache configuration where there was no
  // distinction between workspace and cache.
  // Complicated cache configurations should explicitly set CVMFS_WORKSPACE.
  workspace_ = kDefaultCacheBase;
  if (options_mgr_->GetValue("CVMFS_CACHE_BASE", &optarg))
    workspace_ = MakeCanonicalPath(optarg);
  if (options_mgr_->GetValue("CVMFS_SHARED_CACHE", &optarg) &&
      options_mgr_->IsOn(optarg))
  {
    workspace_ += "/shared";
  } else {
    workspace_ += "/" + name_;
  }
  if (options_mgr_->GetValue("CVMFS_CACHE_DIR", &optarg)) {
    if (options_mgr_->IsDefined("CVMFS_CACHE_BASE")) {
      boot_error_ =
        "'CVMFS_CACHE_BASE' and 'CVMFS_CACHE_DIR' are mutually exclusive";
      boot_status_ = loader::kFailOptions;
      return false;
    }
    workspace_ = optarg;
  }
  if (options_mgr_->GetValue("CVMFS_WORKSPACE", &optarg))
    workspace_ = optarg;
  workspace_fullpath_ = workspace_;

  // If workspace and alien cache are the same directory, we need to open
  // permission now to 0770 to avoid a race when fixing it later
  const int mode = 0770;
  if (!MkdirDeep(workspace_, mode, false)) {
    boot_error_ = "cannot create workspace directory " + workspace_;
    boot_status_ = loader::kFailCacheDir;
    return false;
  }

  if (!LockWorkspace())
    return false;
  if (!SetupCwd())
    return false;
  if (!SetupCrashGuard())
    return false;

  return true;
}


void FileSystem::SetupUuid() {
  uuid_cache_ = cvmfs::Uuid::Create(workspace_ + "/uuid");
  if (uuid_cache_ == NULL) {
    LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogWarn,
             "failed to load/store %s/uuid", workspace_.c_str());
    uuid_cache_ = cvmfs::Uuid::Create("");
    assert(uuid_cache_ != NULL);
  }
}


void FileSystem::TearDown2ReadOnly() {
  if ((cache_mgr_ != NULL) && (cache_mgr_->id() == kPosixCacheManager)) {
    PosixCacheManager *posix_cache_mgr =
      reinterpret_cast<PosixCacheManager *>(cache_mgr_);
    posix_cache_mgr->TearDown2ReadOnly();
  }

  unlink(path_crash_guard_.c_str());
  LogCvmfs(kLogCache, kLogSyslog, "switch to read-only cache mode");
  SetLogMicroSyslog("");
}


void FileSystem::RemapCatalogFd(int from, int to) {
  sqlite::RegisterFdMapping(from, to);
}


bool FileSystem::TriageCacheMgr() {
  cache_mgr_instance_ = kDefaultCacheMgrInstance;
  string instance;
  if (options_mgr_->GetValue("CVMFS_CACHE_PRIMARY", &instance) &&
      !instance.empty())
  {
    if (!CheckInstanceName(instance))
      return false;
    cache_mgr_instance_ = instance;
  }

  cache_mgr_ = SetupCacheMgr(cache_mgr_instance_);
  return cache_mgr_ != NULL;
}


//------------------------------------------------------------------------------


const char *MountPoint::kDefaultAuthzSearchPath = "/usr/libexec/cvmfs/authz";
const char *MountPoint::kDefaultBlacklist = "/etc/cvmfs/blacklist";

bool MountPoint::CheckBlacklists() {
  blacklist_paths_.clear();
  string blacklist;
  if (!options_mgr_->GetValue("CVMFS_BLACKLIST", &blacklist))
    blacklist = kDefaultBlacklist;
  blacklist_paths_.push_back(blacklist);

  bool append = false;
  if (FileExists(blacklist)) {
    if (!signature_mgr_->LoadBlacklist(blacklist, append)) {
      boot_error_ = "failed to load blacklist " + blacklist;
      boot_status_ = loader::kFailSignature;
      return false;
    }
    append = true;
  }

  string config_repository_path;
  if (options_mgr_->HasConfigRepository(fqrn_, &config_repository_path)) {
    blacklist = config_repository_path + "blacklist";
    blacklist_paths_.push_back(blacklist);
    if (FileExists(blacklist)) {
      if (!signature_mgr_->LoadBlacklist(blacklist, append)) {
        boot_error_ = "failed to load blacklist from config repository";
        boot_status_ = loader::kFailSignature;
        return false;
      }
    }
  }

  return true;
}


bool MountPoint::ReloadBlacklists() {
  bool result = true;
  bool append = false;
  for (unsigned i = 0; i < blacklist_paths_.size(); ++i) {
    string blacklist = blacklist_paths_[i];
    if (FileExists(blacklist)) {
      bool retval = signature_mgr_->LoadBlacklist(blacklist, append);
      // TODO(jblomer): this can leave us with a half-loaded blacklist
      if (!retval)
        return false;
      append = true;
    }
  }
  return result;
}


MountPoint *MountPoint::Create(
  const string &fqrn,
  FileSystem *file_system,
  OptionsManager *options_mgr)
{
  if (options_mgr == NULL)
    options_mgr = file_system->options_mgr();
  UniquePtr<MountPoint> mountpoint(new MountPoint(
    fqrn, file_system, options_mgr));

  // At this point, we have a repository name, the type (fuse or library) and
  // an options manager (which can be the same than the FileSystem's one).

  mountpoint->CreateStatistics();
  mountpoint->CreateAuthz();
  mountpoint->backoff_throttle_ = new BackoffThrottle();

  if (!mountpoint->CreateSignatureManager() || !mountpoint->CheckBlacklists())
    return mountpoint.Release();
  if (!mountpoint->CreateDownloadManagers())
    return mountpoint.Release();
  if (!mountpoint->CreateResolvConfWatcher()) {
    return mountpoint.Release();
  }
  mountpoint->CreateFetchers();
  if (!mountpoint->CreateCatalogManager())
    return mountpoint.Release();
  if (!mountpoint->CreateTracer())
    return mountpoint.Release();

  mountpoint->ReEvaluateAuthz();
  mountpoint->CreateTables();
  if (!mountpoint->SetupBehavior())
    return mountpoint.Release();

  mountpoint->boot_status_ = loader::kFailOk;
  return mountpoint.Release();
}


void MountPoint::CreateAuthz() {
  string optarg;
  string authz_helper;
  if (options_mgr_->GetValue("CVMFS_AUTHZ_HELPER", &optarg))
    authz_helper = optarg;
  string authz_search_path(kDefaultAuthzSearchPath);
  if (options_mgr_->GetValue("CVMFS_AUTHZ_SEARCH_PATH", &optarg))
    authz_search_path = optarg;

  authz_fetcher_ = new AuthzExternalFetcher(
    fqrn_,
    authz_helper,
    authz_search_path,
    options_mgr_);
  assert(authz_fetcher_ != NULL);

  authz_session_mgr_ = AuthzSessionManager::Create(
    authz_fetcher_,
    statistics_);
  assert(authz_session_mgr_ != NULL);

  authz_attachment_ = new AuthzAttachment(authz_session_mgr_);
  assert(authz_attachment_ != NULL);
}


bool MountPoint::CreateCatalogManager() {
  string optarg;

  catalog_mgr_ = new catalog::ClientCatalogManager(this);

  SetupInodeAnnotation();
  if (!SetupOwnerMaps())
    return false;
  shash::Any root_hash;
  if (!DetermineRootHash(&root_hash))
    return false;

  bool retval;
  if (root_hash.IsNull()) {
    retval = catalog_mgr_->Init();
  } else {
    fixed_catalog_ = true;
    bool alt_root_path =
      options_mgr_->GetValue("CVMFS_ALT_ROOT_PATH", &optarg) &&
      options_mgr_->IsOn(optarg);
    retval = catalog_mgr_->InitFixed(root_hash, alt_root_path);
  }
  if (!retval) {
    boot_error_ = "Failed to initialize root file catalog";
    boot_status_ = loader::kFailCatalog;
    return false;
  }

  if (catalog_mgr_->IsRevisionBlacklisted()) {
    boot_error_ = "repository revision blacklisted";
    boot_status_ = loader::kFailRevisionBlacklisted;
    return false;
  }

  if (options_mgr_->GetValue("CVMFS_AUTO_UPDATE", &optarg) &&
      !options_mgr_->IsOn(optarg))
  {
    fixed_catalog_ = true;
  }

  if (options_mgr_->GetValue("CVMFS_CATALOG_WATERMARK", &optarg)) {
    catalog_mgr_->SetCatalogWatermark(String2Uint64(optarg));
  } else {
    unsigned soft_limit;
    unsigned hard_limit;
    GetLimitNoFile(&soft_limit, &hard_limit);
    catalog_mgr_->SetCatalogWatermark(soft_limit / 4);
  }

  if (catalog_mgr_->volatile_flag()) {
    LogCvmfs(kLogCvmfs, kLogDebug, "content of repository flagged as VOLATILE");
  }

  return true;
}


bool MountPoint::CreateDownloadManagers() {
  string optarg;
  download_mgr_ = new download::DownloadManager();
  download_mgr_->Init(kDefaultNumConnections,
                      perf::StatisticsTemplate("download", statistics_));
  download_mgr_->SetCredentialsAttachment(authz_attachment_);

  if (options_mgr_->GetValue("CVMFS_SERVER_URL", &optarg)) {
    download_mgr_->SetHostChain(optarg);
  }

  SetupDnsTuning(download_mgr_);
  SetupHttpTuning();

  string forced_proxy_template;
  if (options_mgr_->GetValue("CVMFS_PROXY_TEMPLATE", &optarg))
    forced_proxy_template = optarg;
  download_mgr_->SetProxyTemplates(file_system_->uuid_cache()->uuid(),
                                   forced_proxy_template);

  string proxies;
  if (options_mgr_->GetValue("CVMFS_HTTP_PROXY", &optarg))
    proxies = optarg;
  proxies = download::ResolveProxyDescription(
    proxies,
    file_system_->workspace() + "/proxies" + GetUniqFileSuffix(),
    download_mgr_);
  if (proxies == "") {
    boot_error_ = "failed to discover HTTP proxy servers";
    boot_status_ = loader::kFailWpad;
    return false;
  }
  string fallback_proxies;
  if (options_mgr_->GetValue("CVMFS_FALLBACK_PROXY", &optarg))
    fallback_proxies = optarg;
  download_mgr_->SetProxyChain(proxies, fallback_proxies,
                               download::DownloadManager::kSetProxyBoth);

  bool do_geosort = options_mgr_->GetValue("CVMFS_USE_GEOAPI", &optarg) &&
                    options_mgr_->IsOn(optarg);
  if (do_geosort) {
    download_mgr_->ProbeGeo();
  }
  if (options_mgr_->GetValue("CVMFS_MAX_SERVERS", &optarg)) {
    unsigned max_servers = String2Uint64(optarg);
    std::vector<std::string> host_chain;
    download_mgr_->GetHostInfo(&host_chain, NULL, NULL);
    if (max_servers > 0 && max_servers < host_chain.size()) {
      host_chain.resize(max_servers);
      download_mgr_->SetHostChain(host_chain);
    }
  }

  if (options_mgr_->GetValue("CVMFS_USE_SSL_SYSTEM_CA", &optarg) &&
      options_mgr_->IsOn(optarg)) {
    download_mgr_->UseSystemCertificatePath();
  }

  if (options_mgr_->GetValue("CVMFS_PROXY_SHARD", &optarg) &&
      options_mgr_->IsOn(optarg)) {
    download_mgr_->ShardProxies();
  }

  return SetupExternalDownloadMgr(do_geosort);
}

bool MountPoint::CreateResolvConfWatcher() {
  std::string roaming_value;
  options_mgr_->GetValue("CVMFS_DNS_ROAMING", &roaming_value);
  if (options_mgr_->IsDefined("CVMFS_DNS_ROAMING") &&
      options_mgr_->IsOn(roaming_value)) {
    LogCvmfs(kLogCvmfs, kLogDebug,
             "DNS roaming is enabled for this repository.");
    // Create a file watcher to update the DNS settings of the download
    // managers when there are changes to /etc/resolv.conf
    resolv_conf_watcher_ = file_watcher::FileWatcher::Create();

    if (resolv_conf_watcher_) {
      ResolvConfEventHandler *handler =
          new ResolvConfEventHandler(download_mgr_, external_download_mgr_);
      resolv_conf_watcher_->RegisterHandler("/etc/resolv.conf", handler);
    }
  } else {
    LogCvmfs(kLogCvmfs, kLogDebug,
             "DNS roaming is disabled for this repository.");
  }
  return true;
}

void MountPoint::CreateFetchers() {
  fetcher_ = new cvmfs::Fetcher(
    file_system_->cache_mgr(),
    download_mgr_,
    backoff_throttle_,
    perf::StatisticsTemplate("fetch", statistics_));

  const bool is_external_data = true;
  external_fetcher_ = new cvmfs::Fetcher(
    file_system_->cache_mgr(),
    external_download_mgr_,
    backoff_throttle_,
    perf::StatisticsTemplate("fetch-external", statistics_),
    is_external_data);
}


bool MountPoint::CreateSignatureManager() {
  string optarg;
  signature_mgr_ = new signature::SignatureManager();
  signature_mgr_->Init();

  string public_keys;
  if (options_mgr_->GetValue("CVMFS_PUBLIC_KEY", &optarg)) {
    public_keys = optarg;
  } else if (options_mgr_->GetValue("CVMFS_KEYS_DIR", &optarg)) {
    // Collect .pub files from CVMFS_KEYS_DIR
    public_keys = JoinStrings(FindFilesBySuffix(optarg, ".pub"), ":");
  } else {
    public_keys =
      JoinStrings(FindFilesBySuffix("/etc/cvmfs/keys", ".pub"), ":");
  }

  if (!signature_mgr_->LoadPublicRsaKeys(public_keys)) {
    boot_error_ = "failed to load public key(s)";
    boot_status_ = loader::kFailSignature;
    return false;
  }
  LogCvmfs(kLogCvmfs, kLogDebug, "CernVM-FS: using public key(s) %s",
           public_keys.c_str());

  if (options_mgr_->GetValue("CVMFS_TRUSTED_CERTS", &optarg)) {
    if (!signature_mgr_->LoadTrustedCaCrl(optarg)) {
      boot_error_ = "failed to load trusted certificates";
      boot_status_ = loader::kFailSignature;
      return false;
    }
  }

  return true;
}


void MountPoint::CreateStatistics() {
  statistics_ = file_system_->statistics()->Fork();
  if (file_system_->type() != FileSystem::kFsFuse)
    return;

  // TODO(jblomer): this should be registered by the tracker
  statistics_->Register("inode_tracker.n_insert",
                        "overall number of accessed inodes");
  statistics_->Register("inode_tracker.n_remove",
                        "overall number of evicted inodes");
  statistics_->Register("inode_tracker.no_reference",
                        "currently active inodes");
  statistics_->Register("inode_tracker.n_hit_inode",
                        "overall number of inode lookups");
  statistics_->Register("inode_tracker.n_hit_path",
                        "overall number of successful path lookups");
  statistics_->Register("inode_tracker.n_miss_path",
                        "overall number of unsuccessful path lookups");

  statistics_->Register("dentry_tracker.n_insert",
                        "overall number of added negative cache entries");
  statistics_->Register("dentry_tracker.n_remove",
                        "overall number of evicted negative cache entries");
  statistics_->Register("dentry_tracker.n_prune",
                        "overall number of prune calls");

  statistics_->Register("page_cache_tracker.n_insert",
                        "overall number of added page cache entries");
  statistics_->Register("page_cache_tracker.n_remove",
                        "overall number of evicted page cache entries");
  statistics_->Register("page_cache_tracker.n_open_direct",
                        "overall number of direct I/O open calls");
  statistics_->Register("page_cache_tracker.n_open_flush",
    "overall number of open calls where the file's page cache gets flushed");
  statistics_->Register("page_cache_tracker.n_open_cached",
    "overall number of open calls where the file's page cache is reused");
}


void MountPoint::CreateTables() {
  if (file_system_->type() != FileSystem::kFsFuse) {
    // Libcvmfs simplified tables
    md5path_cache_ = new lru::Md5PathCache(kLibPathCacheSize, statistics_);
    simple_chunk_tables_ = new SimpleChunkTables();
    return;
  }

  chunk_tables_ = new ChunkTables();

  string optarg;
  uint64_t mem_cache_size = kDefaultMemcacheSize;
  if (options_mgr_->GetValue("CVMFS_MEMCACHE_SIZE", &optarg))
    mem_cache_size = String2Uint64(optarg) * 1024 * 1024;

  const double memcache_unit_size =
    (static_cast<double>(kInodeCacheFactor) * lru::Md5PathCache::GetEntrySize())
    + lru::InodeCache::GetEntrySize() + lru::PathCache::GetEntrySize();
  const unsigned memcache_num_units =
    mem_cache_size / static_cast<unsigned>(memcache_unit_size);
  // Number of cache entries must be a multiple of 64
  const unsigned mask_64 = ~((1 << 6) - 1);
  inode_cache_ = new lru::InodeCache(memcache_num_units & mask_64, statistics_);
  path_cache_ = new lru::PathCache(memcache_num_units & mask_64, statistics_);
  md5path_cache_ = new lru::Md5PathCache((memcache_num_units * 7) & mask_64,
                                         statistics_);

  inode_tracker_ = new glue::InodeTracker();
  dentry_tracker_ = new glue::DentryTracker();
  page_cache_tracker_ = new glue::PageCacheTracker();
  if (file_system_->IsNfsSource())
    page_cache_tracker_->Disable();
}

bool MountPoint::CreateTracer() {
  string optarg;
  tracer_ = new Tracer();
  if (options_mgr_->GetValue("CVMFS_TRACEFILE", &optarg)) {
    if (file_system_->type() != FileSystem::kFsFuse) {
      boot_error_ = "tracer is only supported in the fuse module";
      boot_status_ = loader::kFailOptions;
      return false;
    }
    string tracebuffer_file = optarg;
    uint64_t tracebuffer_size = kTracerBufferSize;
    uint64_t tracebuffer_threshold = kTracerFlushThreshold;

    if (options_mgr_->GetValue("CVMFS_TRACEBUFFER", &optarg)) {
      tracebuffer_size = String2Uint64(optarg);
    }
    if (options_mgr_->GetValue("CVMFS_TRACEBUFFER_THRESHOLD",
      &optarg)) {
      tracebuffer_threshold = String2Uint64(optarg);
    }
    assert(tracebuffer_size <= INT_MAX
      && tracebuffer_threshold <= INT_MAX);
    LogCvmfs(kLogCvmfs, kLogDebug,
      "Initialising tracer with buffer size %" PRIu64 " and threshold %" PRIu64,
      tracebuffer_size, tracebuffer_threshold);
    tracer_->Activate(tracebuffer_size, tracebuffer_threshold,
      tracebuffer_file);
  }
  return true;
}


bool MountPoint::DetermineRootHash(shash::Any *root_hash) {
  string optarg;
  if (options_mgr_->GetValue("CVMFS_ROOT_HASH", &optarg)) {
    *root_hash = MkFromHexPtr(shash::HexPtr(optarg), shash::kSuffixCatalog);
    return true;
  }

  if (!options_mgr_->IsDefined("CVMFS_REPOSITORY_TAG") &&
      !options_mgr_->IsDefined("CVMFS_REPOSITORY_DATE"))
  {
    root_hash->SetNull();
    return true;
  }

  string history_path;
  if (!FetchHistory(&history_path))
    return false;
  UnlinkGuard history_file(history_path);
  UniquePtr<history::History> tag_db(
    history::SqliteHistory::Open(history_path));
  if (!tag_db.IsValid()) {
    LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslog,
             "failed to open history database (%s)", history_path.c_str());
    boot_error_ = "failed to open history database";
    boot_status_ = loader::kFailHistory;
    return false;
  }

  history::History::Tag tag;
  bool retval;
  if (!options_mgr_->GetValue("CVMFS_REPOSITORY_TAG", &repository_tag_)) {
    string repository_date;
    // options_mgr_->IsDefined("CVMFS_REPOSITORY_DATE") must be true
    options_mgr_->GetValue("CVMFS_REPOSITORY_DATE", &repository_date);
    time_t repository_utctime = IsoTimestamp2UtcTime(repository_date);
    if (repository_utctime == 0) {
      boot_error_ = "invalid timestamp in CVMFS_REPOSITORY_DATE: " +
                    repository_date + ". Use YYYY-MM-DDTHH:MM:SSZ";
      boot_status_ = loader::kFailHistory;
      return false;
    }
    retval = tag_db->GetByDate(repository_utctime, &tag);
    if (!retval) {
      boot_error_ = "no repository state as early as utc timestamp " +
                     StringifyTime(repository_utctime, true);
      boot_status_ = loader::kFailHistory;
      return false;
    }
    LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslog,
             "time stamp %s UTC resolved to tag '%s'",
             StringifyTime(repository_utctime, true).c_str(),
             tag.name.c_str());
    repository_tag_ = tag.name;
  } else {
    retval = tag_db->GetByName(repository_tag_, &tag);
    if (!retval) {
      boot_error_ = "no such tag: " + repository_tag_;
      boot_status_ = loader::kFailHistory;
      return false;
    }
  }
  LogCvmfs(kLogCvmfs, kLogDebug, "mounting tag %s", tag.name.c_str());

  *root_hash = tag.root_hash;
  return true;
}


bool MountPoint::FetchHistory(std::string *history_path) {
  manifest::Failures retval_mf;
  manifest::ManifestEnsemble ensemble;
  retval_mf = manifest::Fetch("", fqrn_, 0, NULL, signature_mgr_, download_mgr_,
                              &ensemble);
  if (retval_mf != manifest::kFailOk) {
    boot_error_ = "Failed to fetch manifest";
    boot_status_ = loader::kFailHistory;
    return false;
  }
  shash::Any history_hash = ensemble.manifest->history();
  if (history_hash.IsNull()) {
    boot_error_ = "No history";
    boot_status_ = loader::kFailHistory;
    return false;
  }

  int fd = fetcher_->Fetch(
    history_hash,
    CacheManager::kSizeUnknown,
    "tag database for " + fqrn_,
    zlib::kZlibDefault,
    CacheManager::kTypeRegular);
  if (fd < 0) {
    boot_error_ = "failed to download history: " + StringifyInt(-fd);
    boot_status_ = loader::kFailHistory;
    return false;
  }
  // We have the custom sqlite vfs driver installed
  *history_path = "@" + StringifyInt(fd);
  return true;
}


unsigned MountPoint::GetEffectiveTtlSec() {
  unsigned max_ttl;
  {
    MutexLockGuard lock_guard(lock_max_ttl_);
    max_ttl = max_ttl_sec_;
  }
  const unsigned catalog_ttl_sec = catalog_mgr_->GetTTL();

  return max_ttl ? std::min(max_ttl, catalog_ttl_sec) : catalog_ttl_sec;
}


unsigned MountPoint::GetMaxTtlMn() {
  MutexLockGuard lock_guard(lock_max_ttl_);
  return max_ttl_sec_ / 60;
}


string MountPoint::GetUniqFileSuffix() {
  return "." + file_system_->name() + "-" + fqrn_;
}


MountPoint::MountPoint(
  const string &fqrn,
  FileSystem *file_system,
  OptionsManager *options_mgr)
  : fqrn_(fqrn)
  , uuid_(cvmfs::Uuid::Create(""))
  , file_system_(file_system)
  , options_mgr_(options_mgr)
  , statistics_(NULL)
  , authz_fetcher_(NULL)
  , authz_session_mgr_(NULL)
  , authz_attachment_(NULL)
  , backoff_throttle_(NULL)
  , signature_mgr_(NULL)
  , download_mgr_(NULL)
  , external_download_mgr_(NULL)
  , fetcher_(NULL)
  , external_fetcher_(NULL)
  , inode_annotation_(NULL)
  , catalog_mgr_(NULL)
  , chunk_tables_(NULL)
  , simple_chunk_tables_(NULL)
  , inode_cache_(NULL)
  , path_cache_(NULL)
  , md5path_cache_(NULL)
  , tracer_(NULL)
  , inode_tracker_(NULL)
  , dentry_tracker_(NULL)
  , page_cache_tracker_(NULL)
  , resolv_conf_watcher_(NULL)
  , max_ttl_sec_(kDefaultMaxTtlSec)
  , kcache_timeout_sec_(static_cast<double>(kDefaultKCacheTtlSec))
  , fixed_catalog_(false)
  , enforce_acls_(false)
  , has_membership_req_(false)
  , talk_socket_path_(std::string("./cvmfs_io.") + fqrn)
  , talk_socket_uid_(0)
  , talk_socket_gid_(0)
{
  int retval = pthread_mutex_init(&lock_max_ttl_, NULL);
  assert(retval == 0);
}


MountPoint::~MountPoint() {
  pthread_mutex_destroy(&lock_max_ttl_);

  delete page_cache_tracker_;
  delete dentry_tracker_;
  delete inode_tracker_;
  delete tracer_;
  delete md5path_cache_;
  delete path_cache_;
  delete inode_cache_;
  delete simple_chunk_tables_;
  delete chunk_tables_;

  delete catalog_mgr_;
  delete inode_annotation_;
  delete external_fetcher_;
  delete fetcher_;
  if (external_download_mgr_ != NULL) {
    external_download_mgr_->Fini();
    delete external_download_mgr_;
  }
  if (download_mgr_ != NULL) {
    download_mgr_->Fini();
    delete download_mgr_;
  }
  if (signature_mgr_ != NULL) {
    signature_mgr_->Fini();
    delete signature_mgr_;
  }

  if (resolv_conf_watcher_ != NULL) {
    resolv_conf_watcher_->Stop();
    delete resolv_conf_watcher_;
  }

  delete backoff_throttle_;
  delete authz_attachment_;
  delete authz_session_mgr_;
  delete authz_fetcher_;
  delete statistics_;
  delete uuid_;
}


void MountPoint::ReEvaluateAuthz() {
  string old_membership_req = membership_req_;
  has_membership_req_ = catalog_mgr_->GetVOMSAuthz(&membership_req_);
  if (old_membership_req != membership_req_) {
    authz_session_mgr_->ClearSessionCache();
    authz_attachment_->set_membership(membership_req_);
  }
}


void MountPoint::SetMaxTtlMn(unsigned value_minutes) {
  MutexLockGuard lock_guard(lock_max_ttl_);
  max_ttl_sec_ = value_minutes * 60;
}


bool MountPoint::SetupBehavior() {
  string optarg;

  if (options_mgr_->GetValue("CVMFS_MAX_TTL", &optarg))
    SetMaxTtlMn(String2Uint64(optarg));

  if (options_mgr_->GetValue("CVMFS_KCACHE_TIMEOUT", &optarg)) {
    // Can be negative and should then be interpreted as 0.0
    kcache_timeout_sec_ =
      std::max(0.0, static_cast<double>(String2Int64(optarg)));
  }
  LogCvmfs(kLogCvmfs, kLogDebug, "kernel caches expire after %d seconds",
           static_cast<int>(kcache_timeout_sec_));

  MagicXattrManager::EVisibility xattr_visibility =
    MagicXattrManager::kVisibilityRootOnly;
  if (options_mgr_->GetValue("CVMFS_HIDE_MAGIC_XATTRS", &optarg)) {
    if (options_mgr_->IsOn(optarg))
      xattr_visibility = MagicXattrManager::kVisibilityNever;
    else if (options_mgr_->IsOff(optarg))
      xattr_visibility = MagicXattrManager::kVisibilityAlways;
  }
  if (options_mgr_->GetValue("CVMFS_MAGIC_XATTRS_VISIBILITY", &optarg)) {
    if (ToUpper(optarg) == "ROOTONLY") {
      xattr_visibility = MagicXattrManager::kVisibilityRootOnly;
    } else if (ToUpper(optarg) == "NEVER") {
      xattr_visibility = MagicXattrManager::kVisibilityNever;
    } else if (ToUpper(optarg) == "ALWAYS") {
      xattr_visibility = MagicXattrManager::kVisibilityAlways;
    } else {
      LogCvmfs(kLogCvmfs, kLogSyslogWarn | kLogDebug,
               "unsupported setting: CVMFS_MAGIC_XATTRS_VISIBILITY=%s",
               optarg.c_str());
    }
  }
  magic_xattr_mgr_ = new MagicXattrManager(this, xattr_visibility);


  if (options_mgr_->GetValue("CVMFS_ENFORCE_ACLS", &optarg)
      && options_mgr_->IsOn(optarg))
  {
    enforce_acls_ = true;
  }

  if (options_mgr_->GetValue("CVMFS_TALK_SOCKET", &optarg)) {
    talk_socket_path_ = optarg;
  }
  if (options_mgr_->GetValue("CVMFS_TALK_OWNER", &optarg)) {
    bool retval = GetUidOf(optarg, &talk_socket_uid_, &talk_socket_gid_);
    if (!retval) {
      boot_error_ = "unknown owner of cvmfs_talk socket: " + optarg;
      boot_status_ = loader::kFailOptions;
      return false;
    }
  }

  return true;
}


void MountPoint::SetupDnsTuning(download::DownloadManager *manager) {
  string optarg;
  unsigned dns_timeout_ms = download::DownloadManager::kDnsDefaultTimeoutMs;
  unsigned dns_retries = download::DownloadManager::kDnsDefaultRetries;
  if (options_mgr_->GetValue("CVMFS_DNS_TIMEOUT", &optarg))
    dns_timeout_ms = String2Uint64(optarg) * 1000;
  if (options_mgr_->GetValue("CVMFS_DNS_RETRIES", &optarg))
    dns_retries = String2Uint64(optarg);
  manager->SetDnsParameters(dns_retries, dns_timeout_ms);

  // Rest has to be after SetDnsParameters because SetDnsParameters might
  // construct a new resolver object

  unsigned dns_min_ttl = dns::Resolver::kDefaultMinTtl;
  unsigned dns_max_ttl = dns::Resolver::kDefaultMaxTtl;
  if (options_mgr_->GetValue("CVMFS_DNS_MIN_TTL", &optarg))
    dns_min_ttl = String2Uint64(optarg);
  if (options_mgr_->GetValue("CVMFS_DNS_MAX_TTL", &optarg))
    dns_max_ttl = String2Uint64(optarg);
  manager->SetDnsTtlLimits(dns_min_ttl, dns_max_ttl);

  if (options_mgr_->GetValue("CVMFS_DNS_SERVER", &optarg)) {
    download_mgr_->SetDnsServer(optarg);
  }

  if (options_mgr_->GetValue("CVMFS_IPFAMILY_PREFER", &optarg)) {
    switch (String2Int64(optarg)) {
      case 4:
        manager->SetIpPreference(dns::kIpPreferV4);
        break;
      case 6:
        manager->SetIpPreference(dns::kIpPreferV6);
        break;
    }
  }
  if (options_mgr_->GetValue("CVMFS_MAX_IPADDR_PER_PROXY", &optarg))
    manager->SetMaxIpaddrPerProxy(String2Uint64(optarg));
}


bool MountPoint::SetupExternalDownloadMgr(bool dogeosort) {
  string optarg;
  external_download_mgr_ =
    download_mgr_->Clone(perf::StatisticsTemplate("download-external",
      statistics_));

  unsigned timeout;
  unsigned timeout_direct;
  download_mgr_->GetTimeout(&timeout, &timeout_direct);
  if (options_mgr_->GetValue("CVMFS_EXTERNAL_TIMEOUT", &optarg)) {
    timeout = String2Uint64(optarg);
  }
  if (options_mgr_->GetValue("CVMFS_EXTERNAL_TIMEOUT_DIRECT", &optarg)) {
    timeout_direct = String2Uint64(optarg);
  }
  external_download_mgr_->SetTimeout(timeout, timeout_direct);

  if (options_mgr_->GetValue("CVMFS_EXTERNAL_URL", &optarg)) {
    external_download_mgr_->SetHostChain(optarg);
    if (dogeosort) {
      std::vector<std::string> host_chain;
      external_download_mgr_->GetHostInfo(&host_chain, NULL, NULL);
      download_mgr_->GeoSortServers(&host_chain);
      external_download_mgr_->SetHostChain(host_chain);
    }
  } else {
    external_download_mgr_->SetHostChain("");
  }

  if (options_mgr_->GetValue("CVMFS_EXTERNAL_MAX_SERVERS", &optarg)) {
    unsigned max_servers = String2Uint64(optarg);
    std::vector<std::string> host_chain;
    external_download_mgr_->GetHostInfo(&host_chain, NULL, NULL);
    if (max_servers > 0 && max_servers < host_chain.size()) {
      host_chain.resize(max_servers);
      external_download_mgr_->SetHostChain(host_chain);
    }
  }

  string proxies = "DIRECT";
  if (options_mgr_->GetValue("CVMFS_EXTERNAL_HTTP_PROXY", &optarg)) {
    proxies = download::ResolveProxyDescription(
      optarg,
      file_system_->workspace() + "/proxies-external" + GetUniqFileSuffix(),
      external_download_mgr_);
    if (proxies == "") {
      boot_error_ = "failed to discover external HTTP proxy servers";
      boot_status_ = loader::kFailWpad;
      return false;
    }
  }
  string fallback_proxies;
  if (options_mgr_->GetValue("CVMFS_EXTERNAL_FALLBACK_PROXY", &optarg))
    fallback_proxies = optarg;
  external_download_mgr_->SetProxyChain(
    proxies, fallback_proxies, download::DownloadManager::kSetProxyBoth);

  return true;
}


void MountPoint::SetupHttpTuning() {
  string optarg;

  // TODO(jblomer): avoid double default settings

  unsigned timeout = kDefaultTimeoutSec;
  unsigned timeout_direct = kDefaultTimeoutSec;
  if (options_mgr_->GetValue("CVMFS_TIMEOUT", &optarg))
    timeout = String2Uint64(optarg);
  if (options_mgr_->GetValue("CVMFS_TIMEOUT_DIRECT", &optarg))
    timeout_direct = String2Uint64(optarg);
  download_mgr_->SetTimeout(timeout, timeout_direct);

  unsigned max_retries = kDefaultRetries;
  unsigned backoff_init = kDefaultBackoffInitMs;
  unsigned backoff_max = kDefaultBackoffMaxMs;
  if (options_mgr_->GetValue("CVMFS_MAX_RETRIES", &optarg))
    max_retries = String2Uint64(optarg);
  if (options_mgr_->GetValue("CVMFS_BACKOFF_INIT", &optarg))
    backoff_init = String2Uint64(optarg) * 1000;
  if (options_mgr_->GetValue("CVMFS_BACKOFF_MAX", &optarg))
    backoff_max = String2Uint64(optarg) * 1000;
  download_mgr_->SetRetryParameters(max_retries, backoff_init, backoff_max);

  if (options_mgr_->GetValue("CVMFS_LOW_SPEED_LIMIT", &optarg))
    download_mgr_->SetLowSpeedLimit(String2Uint64(optarg));
  if (options_mgr_->GetValue("CVMFS_PROXY_RESET_AFTER", &optarg))
    download_mgr_->SetProxyGroupResetDelay(String2Uint64(optarg));
  if (options_mgr_->GetValue("CVMFS_HOST_RESET_AFTER", &optarg))
    download_mgr_->SetHostResetDelay(String2Uint64(optarg));

  if (options_mgr_->GetValue("CVMFS_FOLLOW_REDIRECTS", &optarg) &&
      options_mgr_->IsOn(optarg))
  {
    download_mgr_->EnableRedirects();
  }
  if (options_mgr_->GetValue("CVMFS_SEND_INFO_HEADER", &optarg) &&
      options_mgr_->IsOn(optarg))
  {
    download_mgr_->EnableInfoHeader();
  }
}


void MountPoint::SetupInodeAnnotation() {
  string optarg;

  if (file_system_->IsNfsSource()) {
    inode_annotation_ = new catalog::InodeNfsGenerationAnnotation();
  } else {
    inode_annotation_ = new catalog::InodeGenerationAnnotation();
  }
  if (options_mgr_->GetValue("CVMFS_INITIAL_GENERATION", &optarg)) {
    inode_annotation_->IncGeneration(String2Uint64(optarg));
  }

  if (file_system_->type() == FileSystem::kFsFuse) {
    catalog_mgr_->SetInodeAnnotation(inode_annotation_);
  }
}


bool MountPoint::SetupOwnerMaps() {
  string optarg;
  catalog::OwnerMap uid_map;
  catalog::OwnerMap gid_map;

  if (options_mgr_->GetValue("CVMFS_UID_MAP", &optarg)) {
    if (!uid_map.Read(optarg)) {
      boot_error_ = "failed to parse uid map " + optarg;
      boot_status_ = loader::kFailOptions;
      return false;
    }
  }
  if (options_mgr_->GetValue("CVMFS_GID_MAP", &optarg)) {
    if (!gid_map.Read(optarg)) {
      boot_error_ = "failed to parse gid map " + optarg;
      boot_status_ = loader::kFailOptions;
      return false;
    }
  }
  catalog_mgr_->SetOwnerMaps(uid_map, gid_map);

  // TODO(jblomer): make local to catalog manager
  if (options_mgr_->GetValue("CVMFS_CLAIM_OWNERSHIP", &optarg) &&
      options_mgr_->IsOn(optarg))
  {
    g_claim_ownership = true;
  }

  return true;
}