catalog_mgr_client.cc

Go to the documentation of this file.

#include "catalog_mgr_client.h"

#include <string>
#include <vector>

#include "cache_posix.h"
#include "crypto/signature.h"
#include "fetch.h"
#include "manifest.h"
#include "mountpoint.h"
#include "network/download.h"
#include "quota.h"
#include "statistics.h"
#include "util/posix.h"
#include "util/string.h"

using namespace std;  // NOLINT

namespace catalog {

void ClientCatalogManager::ActivateCatalog(Catalog *catalog) {
  const Counters &counters = const_cast<const Catalog *>(catalog)
                                 ->GetCounters();
  if (catalog->IsRoot()) {
    all_inodes_ = counters.GetAllEntries();
  }
  loaded_inodes_ += counters.GetSelfEntries();
}


ClientCatalogManager::ClientCatalogManager(MountPoint *mountpoint)
    : AbstractCatalogManager<Catalog>(mountpoint->statistics())
    , repo_name_(mountpoint->fqrn())
    , fetcher_(mountpoint->fetcher())
    , signature_mgr_(mountpoint->signature_mgr())
    , workspace_(mountpoint->file_system()->workspace())
    , offline_mode_(false)
    , all_inodes_(0)
    , loaded_inodes_(0)
    , fixed_root_catalog_()
    , fixed_alt_root_catalog_(false)
    , root_fd_(-1) {
  LogCvmfs(kLogCatalog, kLogDebug, "constructing client catalog manager");
  n_certificate_hits_ = mountpoint->statistics()->Register(
      "cache.n_certificate_hits", "Number of certificate hits");
  n_certificate_misses_ = mountpoint->statistics()->Register(
      "cache.n_certificate_misses", "Number of certificate misses");
}


ClientCatalogManager::~ClientCatalogManager() {
  LogCvmfs(kLogCache, kLogDebug, "unpinning / unloading all catalogs");

  for (map<PathString, shash::Any>::iterator i = mounted_catalogs_.begin(),
                                             iend = mounted_catalogs_.end();
       i != iend;
       ++i) {
    fetcher_->cache_mgr()->quota_mgr()->Unpin(i->second);
  }
}


Catalog *ClientCatalogManager::CreateCatalog(const PathString &mountpoint,
                                             const shash::Any &catalog_hash,
                                             catalog::Catalog *parent_catalog) {
  mounted_catalogs_[mountpoint] = loaded_catalogs_[mountpoint];
  loaded_catalogs_.erase(mountpoint);
  return new Catalog(mountpoint, catalog_hash, parent_catalog);
}


shash::Any ClientCatalogManager::GetRootHash() {
  ReadLock();
  shash::Any result = mounted_catalogs_[PathString("", 0)];
  Unlock();
  return result;
}


bool ClientCatalogManager::InitFixed(const shash::Any &root_hash,
                                     bool alternative_path) {
  LogCvmfs(kLogCatalog, kLogDebug, "Initialize catalog with fixed root hash %s",
           root_hash.ToString().c_str());
  WriteLock();
  fixed_alt_root_catalog_ = alternative_path;
  fixed_root_catalog_ = root_hash;

  const bool attached = MountCatalog(PathString("", 0), root_hash, NULL);
  Unlock();

  if (!attached) {
    LogCvmfs(kLogCatalog, kLogDebug, "failed to initialize fixed root catalog");
  }

  return attached;
}

LoadReturn ClientCatalogManager::GetNewRootCatalogContext(
    CatalogContext *result) {
  result->SetMountpoint(PathString("", 0));

  // quick escape if we have a fixed catalog
  if (!fixed_root_catalog_.IsNull()) {
    result->SetHash(fixed_root_catalog_);
    result->SetRootCtlgRevision(GetRevisionNoLock());

    // it might or might not be already mounted, but we do not care
    // as we do no need to download and save the manifest
    // (see LoadCatalogByHash()) as such we must set the location to this
    result->SetRootCtlgLocation(kCtlgLocationMounted);
    offline_mode_ = false;

    // we can do this here as the very first time fixed catalog is loaded it
    // call directly MountCatalog() and will skip the call to this function
    //  here
    return catalog::kLoadUp2Date;
  }

  // 1) Get alien cache root catalog (local)

  // Happens only on init/remount, i.e. quota won't delete a cached catalog
  shash::Any local_newest_hash(shash::kSha1, shash::kSuffixCatalog);
  shash::Any mounted_hash(shash::kSha1, shash::kSuffixCatalog);
  uint64_t local_newest_timestamp = 0;
  uint64_t local_newest_revision = manifest::Breadcrumb::kInvalidRevision;

  const manifest::Breadcrumb breadcrumb =
      fetcher_->cache_mgr()->LoadBreadcrumb(repo_name_);
  if (breadcrumb.IsValid()) {
    local_newest_hash = breadcrumb.catalog_hash;
    local_newest_timestamp = breadcrumb.timestamp;
    local_newest_revision = breadcrumb.revision;
    LogCvmfs(kLogCache, kLogDebug,
             "Cached copy publish date %s (hash %s, revision %" PRIu64 ")",
             StringifyTime(static_cast<int64_t>(local_newest_timestamp), true)
                 .c_str(),
             local_newest_hash.ToString().c_str(), breadcrumb.revision);
  } else {
    LogCvmfs(kLogCache, kLogDebug, "Unable to read local checksum %s",
             breadcrumb.ToString().c_str());
  }

  // 2) Select local newest catalog: mounted vs alien

  result->SetRootCtlgLocation(kCtlgLocationBreadcrumb);
  LoadReturn success_code = catalog::kLoadNew;

  if (mounted_catalogs_.size() > 0) {
    const std::map<PathString, shash::Any>::iterator
        curr_hash_itr = mounted_catalogs_.find(PathString("", 0));
    mounted_hash = curr_hash_itr->second;
  }

  // We only look for currently loaded catalog if the revision is newer than
  // the breadcrumb revision and both revision numbers are valid (!= -1ul).
  if ((local_newest_revision <= GetRevisionNoLock()
       || local_newest_revision == manifest::Breadcrumb::kInvalidRevision)
      && mounted_catalogs_.size() > 0) {
    local_newest_hash = mounted_hash;
    local_newest_revision = GetRevisionNoLock();
    // if needed for integration test 707: breadcrumb_timestamp_newer()
    local_newest_timestamp = GetTimestampNoLock() > local_newest_timestamp
                                 ? GetTimestampNoLock()
                                 : local_newest_timestamp;
    result->SetRootCtlgLocation(kCtlgLocationMounted);
    success_code = catalog::kLoadUp2Date;
  } else if (local_newest_revision == 0 && mounted_catalogs_.size() > 0) {
    // breadcrumb has no revision
    // TODO(heretherebedragons) this branch can be removed in future versions

    // revisions are better, but if we dont have any we need to compare by
    // timestamp (you can have multiple revisions in the same timestamp)
    if (local_newest_timestamp < GetTimestampNoLock()) {
      local_newest_hash = mounted_hash;
      local_newest_revision = GetRevisionNoLock();
      local_newest_timestamp = GetTimestampNoLock();
      result->SetRootCtlgLocation(kCtlgLocationMounted);
      success_code = catalog::kLoadUp2Date;
    }
  }

  // 3) Get remote root catalog (fails if remote catalog is older)
  manifest::Failures manifest_failure;
  UniquePtr<CachedManifestEnsemble> ensemble(
      new CachedManifestEnsemble(fetcher_->cache_mgr(), this));
  manifest_failure = manifest::Fetch(
      "", repo_name_, local_newest_timestamp, &local_newest_hash,
      signature_mgr_, fetcher_->download_mgr(), ensemble.weak_ref());

  if (manifest_failure == manifest::kFailOk) {
    // server has newest revision or no valid local revision
    if (ensemble->manifest->revision() > local_newest_revision
        || local_newest_revision == manifest::Breadcrumb::kInvalidRevision
        // if revision is 0 both local and server, load catalog from server
        // as local is most likely just "initialized" without valid value
        || (ensemble->manifest->revision() == 0
            && local_newest_revision == 0)) {
      result->SetHash(ensemble->manifest->catalog_hash());
      result->SetRootCtlgRevision(ensemble->manifest->revision());
      result->SetRootCtlgLocation(kCtlgLocationServer);
      fixed_alt_root_catalog_ = ensemble->manifest->has_alt_catalog_path();

      result->TakeManifestEnsemble(
          static_cast<manifest::ManifestEnsemble *>(ensemble.Release()));
      offline_mode_ = false;

      return catalog::kLoadNew;
    }
  }
  LogCvmfs(kLogCache, kLogDebug,
           "Failed fetch manifest from server: "
           "manifest too old or server unreachable (%d - %s)",
           manifest_failure, manifest::Code2Ascii(manifest_failure));

  // total failure: server not reachable and no valid local hash
  if ((manifest_failure != manifest::kFailOk) && local_newest_hash.IsNull()) {
    LogCvmfs(kLogCache, kLogDebug, "No valid root catalog found!");
    return catalog::kLoadFail;
  }

  if (manifest_failure == manifest::kFailOk
      && ensemble->manifest->revision() == local_newest_revision) {
    offline_mode_ = false;
  } else {
    offline_mode_ = true;
  }
  result->SetHash(local_newest_hash);
  result->SetRootCtlgRevision(local_newest_revision);

  // for integration test 707: breadcrumb_revision_large()
  if (breadcrumb.IsValid() && breadcrumb.catalog_hash == mounted_hash) {
    success_code = catalog::kLoadUp2Date;
  }

  return success_code;
}

std::string ClientCatalogManager::GetCatalogDescription(
    const PathString &mountpoint, const shash::Any &hash) {
  return "file catalog at " + repo_name_ + ":"
         + (mountpoint.IsEmpty()
                ? "/"
                : string(mountpoint.GetChars(), mountpoint.GetLength()))
         + " (" + hash.ToString() + ")";
}

LoadReturn ClientCatalogManager::LoadCatalogByHash(
    CatalogContext *ctlg_context) {
  const string catalog_descr =
      GetCatalogDescription(ctlg_context->mountpoint(), ctlg_context->hash());
  string alt_root_catalog_path = "";

  // root catalog needs special handling because of alt_root_catalog_path
  if (ctlg_context->IsRootCatalog() && fixed_alt_root_catalog_) {
    alt_root_catalog_path = ctlg_context->hash().MakeAlternativePath();
  }

  const LoadReturn load_ret = FetchCatalogByHash(
      ctlg_context->hash(), catalog_descr, alt_root_catalog_path,
      ctlg_context->GetSqlitePathPtr());
  if (load_ret == catalog::kLoadNew) {
    loaded_catalogs_[ctlg_context->mountpoint()] = ctlg_context->hash();

    if (ctlg_context->IsRootCatalog()) {
      if (ctlg_context->root_ctlg_location() == kCtlgLocationMounted) {
        return kLoadUp2Date;
      }

      // if coming from server: update breadcrumb
      if (ctlg_context->root_ctlg_location() == kCtlgLocationServer) {
        // Store new manifest and certificate
        CacheManager::Label label;
        label.path = repo_name_;
        label.flags |= CacheManager::kLabelCertificate;
        if (ctlg_context->manifest_ensemble()->cert_size > 0) {
          fetcher_->cache_mgr()->CommitFromMem(
              CacheManager::LabeledObject(
                  ctlg_context->manifest_ensemble()->manifest->certificate(),
                  label),
              ctlg_context->manifest_ensemble()->cert_buf,
              ctlg_context->manifest_ensemble()->cert_size);
        }
        fetcher_->cache_mgr()->StoreBreadcrumb(
            *ctlg_context->manifest_ensemble()->manifest);
      }
    }
  }

  return load_ret;
}

LoadReturn ClientCatalogManager::FetchCatalogByHash(
    const shash::Any &hash,
    const string &name,
    const std::string &alt_root_catalog_path,
    std::string *sqlite_path) {
  assert(hash.suffix == shash::kSuffixCatalog);
  CacheManager::Label label;
  label.path = name;
  label.flags = CacheManager::kLabelCatalog;
  const int fd = fetcher_->Fetch(CacheManager::LabeledObject(hash, label),
                                 alt_root_catalog_path);
  if (fd >= 0) {
    if (root_fd_ < 0) {
      root_fd_ = fd;
    }

    LogCvmfs(kLogCatalog, kLogDebug, "FetchCatalogByHash filedescriptor %d",
             fd);
    *sqlite_path = "@" + StringifyInt(fd);
    return kLoadNew;
  }

  if (fd == -ENOSPC)
    return kLoadNoSpace;

  return kLoadFail;
}

void ClientCatalogManager::StageNestedCatalogByHash(
    const shash::Any &hash, const PathString &mountpoint) {
  assert(hash.suffix == shash::kSuffixCatalog);

  CacheManager::Label label;
  label.path = GetCatalogDescription(mountpoint, hash);
  label.flags = CacheManager::kLabelCatalog;
  const int fd = fetcher_->Fetch(CacheManager::LabeledObject(hash, label));
  if (fd >= 0)
    fetcher_->cache_mgr()->Close(fd);
}

void ClientCatalogManager::UnloadCatalog(const Catalog *catalog) {
  LogCvmfs(kLogCache, kLogDebug, "unloading catalog %s",
           catalog->mountpoint().c_str());

  const map<PathString, shash::Any>::iterator iter =
      mounted_catalogs_.find(catalog->mountpoint());
  assert(iter != mounted_catalogs_.end());
  fetcher_->cache_mgr()->quota_mgr()->Unpin(iter->second);
  mounted_catalogs_.erase(iter);
  const catalog::Counters &counters = catalog->GetCounters();
  loaded_inodes_ -= counters.GetSelfEntries();
}


bool ClientCatalogManager::IsRevisionBlacklisted() {
  const uint64_t revision = GetRevision();

  LogCvmfs(kLogCache, kLogDebug,
           "checking if %s revision %" PRIu64 " is blacklisted",
           repo_name_.c_str(), revision);

  vector<string> blacklist = signature_mgr_->GetBlacklist();
  for (unsigned i = 0; i < blacklist.size(); ++i) {
    std::string line = blacklist[i];
    if (line[0] != '<')
      continue;
    unsigned idx = repo_name_.length() + 1;
    if (line.length() <= idx)
      continue;
    if ((line[idx] != ' ') && (line[idx] != '\t'))
      continue;
    if (line.substr(1, idx - 1) != repo_name_)
      continue;
    ++idx;
    while ((line[idx] == ' ') || (line[idx] == '\t'))
      ++idx;
    if (idx >= line.length())
      continue;
    uint64_t rev;
    if (!String2Uint64Parse(line.substr(idx), &rev))
      continue;
    if (revision < rev)
      return true;
  }

  return false;
}


//------------------------------------------------------------------------------


void CachedManifestEnsemble::FetchCertificate(const shash::Any &hash) {
  CacheManager::Label label;
  label.flags |= CacheManager::kLabelCertificate;
  label.path = catalog_mgr_->repo_name();
  uint64_t size;
  const bool retval = cache_mgr_->Open2Mem(
      CacheManager::LabeledObject(hash, label), &cert_buf, &size);
  cert_size = size;
  if (retval)
    perf::Inc(catalog_mgr_->n_certificate_hits_);
  else
    perf::Inc(catalog_mgr_->n_certificate_misses_);
}

}  // namespace catalog