swissknife_reflog.cc

Go to the documentation of this file.

#include "swissknife_reflog.h"

#include <cassert>

#include <string>
#include <vector>

#include "manifest.h"
#include "object_fetcher.h"
#include "upload_facility.h"
#include "util/exception.h"

namespace swissknife {

typedef HttpObjectFetcher<> ObjectFetcher;

class RootChainWalker {
 public:
  typedef ObjectFetcher::CatalogTN CatalogTN;
  typedef ObjectFetcher::HistoryTN HistoryTN;

 public:
  RootChainWalker(const manifest::Manifest *manifest,
                  ObjectFetcher            *object_fetcher,
                  manifest::Reflog         *reflog)
    : object_fetcher_(object_fetcher)
    , reflog_(reflog)
    , manifest_(manifest) {}

  void FindObjectsAndPopulateReflog();

 protected:
  typedef std::vector<shash::Any> CatalogList;

 protected:
  CatalogTN* FetchCatalog(const shash::Any catalog_hash);
  HistoryTN* FetchHistory(const shash::Any history_hash);

  void WalkRootCatalogs(const shash::Any &root_catalog_hash);
  void WalkHistories(const shash::Any &history_hash);

  bool WalkCatalogsInHistory(const HistoryTN *history);
  void WalkListedCatalogs(const CatalogList &catalog_list);

  template <class DatabaseT>
  DatabaseT* ReturnOrAbort(const ObjectFetcherFailures::Failures  failure,
                           const shash::Any                      &content_hash,
                           DatabaseT                             *database);

 private:
  ObjectFetcher            *object_fetcher_;
  manifest::Reflog         *reflog_;
  const manifest::Manifest *manifest_;
};


ParameterList CommandReconstructReflog::GetParams() const {
  ParameterList r;
  r.push_back(Parameter::Mandatory('r', "repository url"));
  r.push_back(Parameter::Mandatory('u', "spooler definition string"));
  r.push_back(Parameter::Mandatory('n', "fully qualified repository name"));
  r.push_back(Parameter::Mandatory('t', "temporary directory"));
  r.push_back(Parameter::Mandatory('k', "repository keychain"));
  r.push_back(Parameter::Mandatory('R', "path to reflog.chksum file"));
  r.push_back(Parameter::Optional('@', "proxy url"));
  return r;
}


int CommandReconstructReflog::Main(const ArgumentList &args) {
  const std::string &repo_url           = *args.find('r')->second;
  const std::string &spooler            = *args.find('u')->second;
  const std::string &repo_name          = *args.find('n')->second;
  const std::string &tmp_dir            = *args.find('t')->second;
  const std::string &repo_keys          = *args.find('k')->second;
  const std::string &reflog_chksum_path = *args.find('R')->second;

  const bool follow_redirects = false;
  const std::string proxy = ((args.count('@') > 0) ?
                             *args.find('@')->second : "");
  if (!this->InitDownloadManager(follow_redirects, proxy) ||
      !this->InitSignatureManager(repo_keys)) {
    LogCvmfs(kLogCvmfs, kLogStderr, "failed to init repo connection");
    return 1;
  }

  ObjectFetcher object_fetcher(repo_name,
                               repo_url,
                               tmp_dir,
                               download_manager(),
                               signature_manager());

  UniquePtr<manifest::Manifest> manifest;
  ObjectFetcher::Failures retval = object_fetcher.FetchManifest(&manifest);
  if (retval != ObjectFetcher::kFailOk) {
    LogCvmfs(kLogCvmfs, kLogStderr, "failed to load repository manifest "
                                    "(%d - %s)",
                                    retval, Code2Ascii(retval));
    return 1;
  }

  const upload::SpoolerDefinition spooler_definition(spooler, shash::kAny);
  UniquePtr<upload::AbstractUploader> uploader(
                       upload::AbstractUploader::Construct(spooler_definition));

  if (!uploader.IsValid()) {
    LogCvmfs(kLogCvmfs, kLogStderr, "failed to initialize spooler for '%s'",
             spooler.c_str());
    return 1;
  }

  UniquePtr<manifest::Reflog> reflog(CreateEmptyReflog(tmp_dir, repo_name));
  reflog->TakeDatabaseFileOwnership();

  reflog->BeginTransaction();
  AddStaticManifestObjects(reflog.weak_ref(), manifest.weak_ref());
  RootChainWalker walker(manifest.weak_ref(),
                         &object_fetcher,
                         reflog.weak_ref());
  walker.FindObjectsAndPopulateReflog();
  reflog->CommitTransaction();

  LogCvmfs(kLogCvmfs, kLogStdout, "found %lu entries", reflog->CountEntries());

  reflog->DropDatabaseFileOwnership();
  const std::string reflog_db = reflog->database_file();
  reflog.Destroy();
  uploader->UploadFile(reflog_db, ".cvmfsreflog");
  shash::Any reflog_hash(manifest->GetHashAlgorithm());
  manifest::Reflog::HashDatabase(reflog_db, &reflog_hash);
  uploader->WaitForUpload();
  unlink(reflog_db.c_str());

  const int errors = uploader->GetNumberOfErrors();
  if (errors > 0) {
    LogCvmfs(kLogCvmfs, kLogStderr, "failed to upload generated Reflog");
  }

  uploader->TearDown();

  manifest::Reflog::WriteChecksum(reflog_chksum_path, reflog_hash);

  return (errors == 0) ? 0 : 1;
}


void CommandReconstructReflog::AddStaticManifestObjects(
                                          manifest::Reflog    *reflog,
                                          manifest::Manifest  *manifest) const {
  const shash::Any certificate = manifest->certificate();
  const shash::Any meta_info   = manifest->meta_info();
  assert(!certificate.IsNull());

  bool success = reflog->AddCertificate(certificate);
  assert(success);
  LogCvmfs(kLogCvmfs, kLogStdout, "Certificate: %s",
           certificate.ToString().c_str());

  if (!meta_info.IsNull()) {
    success = reflog->AddMetainfo(meta_info);
    assert(success);
    LogCvmfs(kLogCvmfs, kLogStdout, "Metainfo: %s",
             meta_info.ToString().c_str());
  }
}


void RootChainWalker::FindObjectsAndPopulateReflog() {
  const shash::Any root_catalog = manifest_->catalog_hash();
  const shash::Any history      = manifest_->history();

  assert(!root_catalog.IsNull());
  WalkRootCatalogs(root_catalog);

  if (!history.IsNull()) {
    WalkHistories(history);
  }
}


void RootChainWalker::WalkRootCatalogs(const shash::Any &root_catalog_hash) {
  shash::Any           current_hash = root_catalog_hash;
  UniquePtr<CatalogTN> current_catalog;

  while (!current_hash.IsNull()                  &&
         !reflog_->ContainsCatalog(current_hash) &&
         (current_catalog = FetchCatalog(current_hash)).IsValid()) {
    LogCvmfs(kLogCvmfs, kLogStdout, "Catalog: %s Revision: %" PRIu64,
             current_hash.ToString().c_str(), current_catalog->GetRevision());

    const bool success = reflog_->AddCatalog(current_hash);
    assert(success);

    current_hash = current_catalog->GetPreviousRevision();
  }
}


void RootChainWalker::WalkHistories(const shash::Any &history_hash) {
  shash::Any           current_hash = history_hash;
  UniquePtr<HistoryTN> current_history;

  while (!current_hash.IsNull()                  &&
         !reflog_->ContainsHistory(current_hash) &&
         (current_history = FetchHistory(current_hash)).IsValid()) {
    LogCvmfs(kLogCvmfs, kLogStdout, "History: %s",
             current_hash.ToString().c_str());

    bool cancel = WalkCatalogsInHistory(current_history.weak_ref());
    const bool success = reflog_->AddHistory(current_hash);
    assert(success);

    if (cancel) {
      current_hash = shash::Any(current_hash.algorithm);
    } else {
      current_hash = current_history->previous_revision();
    }
  }
}


bool RootChainWalker::WalkCatalogsInHistory(const HistoryTN *history) {
  CatalogList tag_hashes;
  const bool list_success = history->GetHashes(&tag_hashes);
  assert(list_success);

  CatalogList bin_hashes;
  const bool bin_success = history->ListRecycleBin(&bin_hashes);
  if (!bin_success) {
    LogCvmfs(kLogCvmfs, kLogStderr, "  Warning: 'recycle bin' table missing");
  }

  WalkListedCatalogs(tag_hashes);
  WalkListedCatalogs(bin_hashes);

  return !bin_success;
}


void RootChainWalker::WalkListedCatalogs(
                             const RootChainWalker::CatalogList &catalog_list) {
  CatalogList::const_iterator i    = catalog_list.begin();
  CatalogList::const_iterator iend = catalog_list.end();
  for (; i != iend; ++i) {
    WalkRootCatalogs(*i);
  }
}


RootChainWalker::CatalogTN* RootChainWalker::FetchCatalog(
                                                const shash::Any catalog_hash) {
  CatalogTN *catalog = NULL;
  const char *root_path = "";
  ObjectFetcherFailures::Failures failure =
    object_fetcher_->FetchCatalog(catalog_hash, root_path, &catalog);

  return ReturnOrAbort(failure, catalog_hash, catalog);
}


RootChainWalker::HistoryTN* RootChainWalker::FetchHistory(
                                                const shash::Any history_hash) {
  HistoryTN *history = NULL;
  ObjectFetcherFailures::Failures failure =
    object_fetcher_->FetchHistory(&history, history_hash);

  return ReturnOrAbort(failure, history_hash, history);
}


template <class DatabaseT>
DatabaseT* RootChainWalker::ReturnOrAbort(
                            const ObjectFetcherFailures::Failures  failure,
                            const shash::Any                      &content_hash,
                            DatabaseT                             *database) {
  switch (failure) {
    case ObjectFetcherFailures::kFailOk:
      return database;
    case ObjectFetcherFailures::kFailNotFound:
      return NULL;
    default:
      PANIC(kLogStderr, "Failed to load object '%s' (%d - %s)",
            content_hash.ToStringWithSuffix().c_str(), failure,
            Code2Ascii(failure));
  }
}

}  // namespace swissknife