object_fetcher.h

Go to the documentation of this file.

#ifndef CVMFS_OBJECT_FETCHER_H_
#define CVMFS_OBJECT_FETCHER_H_

#include <unistd.h>

#include <string>

#include "catalog.h"
#include "crypto/signature.h"
#include "history_sqlite.h"
#include "manifest.h"
#include "manifest_fetch.h"
#include "network/download.h"
#include "reflog.h"
#include "util/posix.h"

template <class ConcreteObjectFetcherT>
struct object_fetcher_traits;

struct ObjectFetcherFailures {
  enum Failures {
    kFailOk,
    kFailNotFound,
    kFailLocalIO,
    kFailNetwork,
    kFailDecompression,
    kFailManifestNameMismatch,
    kFailManifestSignatureMismatch,
    kFailBadData,
    kFailUnknown,

    kFailNumEntries
  };
};

inline const char* Code2Ascii(const ObjectFetcherFailures::Failures error) {
  const char *texts[ObjectFetcherFailures::kFailNumEntries + 1];
  texts[0] = "OK";
  texts[1] = "object not found";
  texts[2] = "local I/O failure";
  texts[3] = "network failure";
  texts[4] = "decompression failed";
  texts[5] = "manifest name doesn't match";
  texts[6] = "manifest signature is invalid";
  texts[7] = "bad data received";
  texts[8] = "no text";
  return texts[error];
}

template <class DerivedT>
class AbstractObjectFetcher : public ObjectFetcherFailures {
 public:
  typedef typename object_fetcher_traits<DerivedT>::CatalogTN CatalogTN;
  typedef typename object_fetcher_traits<DerivedT>::HistoryTN HistoryTN;
  typedef typename object_fetcher_traits<DerivedT>::ReflogTN  ReflogTN;

  typedef ObjectFetcherFailures::Failures Failures;

  static const std::string kManifestFilename;
  static const std::string kReflogFilename;

 public:
  Failures FetchManifest(manifest::Manifest** manifest) {
    return static_cast<DerivedT*>(this)->FetchManifest(manifest);
  }

  Failures FetchHistory(HistoryTN        **history,
                        const shash::Any  &history_hash = shash::Any()) {
    // retrieve the current HEAD history hash (if nothing else given)
    shash::Any effective_history_hash = (!history_hash.IsNull())
            ? history_hash
            : GetHistoryHash();
    if (effective_history_hash.IsNull()) {
      return kFailNotFound;
    }
    assert(history_hash.suffix == shash::kSuffixHistory ||
           history_hash.IsNull());

    // download the history hash
    std::string path;
    const Failures retval = Fetch(effective_history_hash, &path);
    if (retval != kFailOk) {
      return retval;
    }

    // open the history file
    *history = HistoryTN::Open(path);
    if (NULL == *history) {
      return kFailLocalIO;
    }

    (*history)->TakeDatabaseFileOwnership();
    return kFailOk;
  }

  Failures FetchCatalog(const shash::Any   &catalog_hash,
                        const std::string  &catalog_path,
                              CatalogTN   **catalog,
                        const bool          is_nested = false,
                              CatalogTN    *parent    = NULL) {
    assert(!catalog_hash.IsNull());
    assert(catalog_hash.suffix == shash::kSuffixCatalog);

    std::string path;
    const Failures retval = Fetch(catalog_hash, &path);
    if (retval != kFailOk) {
      return retval;
    }

    *catalog = CatalogTN::AttachFreely(catalog_path,
                                       path,
                                       catalog_hash,
                                       parent,
                                       is_nested);
    if (NULL == *catalog) {
      return kFailLocalIO;
    }

    (*catalog)->TakeDatabaseFileOwnership();
    return kFailOk;
  }

  Failures FetchReflog(const shash::Any &reflog_hash, ReflogTN **reflog) {
    assert(!reflog_hash.IsNull());
    assert(reflog_hash.suffix == shash::kSuffixNone);

    std::string tmp_path;
    const bool decompress = false;
    const bool nocache = true;
    Failures failure = Fetch(kReflogFilename, decompress, nocache, &tmp_path);
    if (failure != kFailOk) {
      return failure;
    }

    // Ensure data integrity
    shash::Any computed_hash(reflog_hash.algorithm);
    ReflogTN::HashDatabase(tmp_path, &computed_hash);
    if (computed_hash != reflog_hash) {
      unlink(tmp_path.c_str());
      return kFailBadData;
    }

    *reflog = ReflogTN::Open(tmp_path);
    if (NULL == *reflog) {
      return kFailLocalIO;
    }

    (*reflog)->TakeDatabaseFileOwnership();
    return kFailOk;
  }

  Failures FetchManifest(UniquePtr<manifest::Manifest> *manifest) {
    manifest::Manifest *raw_manifest_ptr = NULL;
    Failures failure = FetchManifest(&raw_manifest_ptr);
    if (failure == kFailOk) *manifest = raw_manifest_ptr;
    return failure;
  }

  Failures FetchHistory(UniquePtr<HistoryTN>  *history,
                        const shash::Any      &history_hash = shash::Any()) {
    HistoryTN *raw_history_ptr = NULL;
    Failures failure = FetchHistory(&raw_history_ptr, history_hash);
    if (failure == kFailOk) *history = raw_history_ptr;
    return failure;
  }

  Failures FetchCatalog(const shash::Any            &catalog_hash,
                        const std::string           &catalog_path,
                              UniquePtr<CatalogTN>  *catalog,
                        const bool                   is_nested = false,
                              CatalogTN             *parent    = NULL) {
    CatalogTN *raw_catalog_ptr = NULL;
    Failures failure = FetchCatalog(catalog_hash,
                                    catalog_path,
                                    &raw_catalog_ptr,
                                    is_nested,
                                    parent);
    if (failure == kFailOk) *catalog = raw_catalog_ptr;
    return failure;
  }

  Failures FetchReflog(const shash::Any &reflog_hash,
                       UniquePtr<ReflogTN> *reflog)
  {
    ReflogTN *raw_reflog_ptr = NULL;
    Failures failure = FetchReflog(reflog_hash, &raw_reflog_ptr);
    if (failure == kFailOk) *reflog = raw_reflog_ptr;
    return failure;
  }

  std::string GetUrl(const shash::Any &hash) const {
    return static_cast<DerivedT*>(this)->GetUrl(hash);
  }

  bool HasHistory() {
    shash::Any history_hash = GetHistoryHash();
    return !history_hash.IsNull();
  }

  const std::string& temporary_directory() const {
    return temporary_directory_;
  }

 protected:
  explicit AbstractObjectFetcher(const std::string &temp_dir)
    : temporary_directory_(temp_dir) {}

  Failures Fetch(const shash::Any &object_hash, std::string *file_path) {
    return static_cast<DerivedT*>(this)->Fetch(object_hash, file_path);
  }

  Failures Fetch(const std::string &relative_path,
                 const bool         decompress,
                 const bool         nocache,
                       std::string *file_path) {
    return static_cast<DerivedT*>(this)->Fetch(relative_path,
                                               decompress,
                                               nocache,
                                               file_path);
  }

  shash::Any GetHistoryHash() {
    UniquePtr<manifest::Manifest> manifest;
    const Failures retval = FetchManifest(&manifest);

    if (retval != kFailOk   ||
        !manifest.IsValid() ||
        manifest->history().IsNull()) {
      return shash::Any();
    }

    return manifest->history();
  }

 private:
  const std::string temporary_directory_;
};

template <class DerivedT>
const std::string AbstractObjectFetcher<DerivedT>::kManifestFilename =
                                                              ".cvmfspublished";
template <class DerivedT>
const std::string AbstractObjectFetcher<DerivedT>::kReflogFilename =
                                                                 ".cvmfsreflog";


template <class CatalogT = catalog::Catalog,
          class HistoryT = history::SqliteHistory,
          class ReflogT  = manifest::Reflog>
class LocalObjectFetcher :
  public AbstractObjectFetcher<LocalObjectFetcher<CatalogT, HistoryT, ReflogT> >
{
 protected:
  typedef LocalObjectFetcher<CatalogT, HistoryT, ReflogT> ThisTN;
  typedef AbstractObjectFetcher<ThisTN>                   BaseTN;

 public:
  typedef typename BaseTN::Failures Failures;

 public:
  LocalObjectFetcher(const std::string &base_path,
                     const std::string &temp_dir)
    : BaseTN(temp_dir)
    , base_path_(base_path) {}

  using BaseTN::FetchManifest;  // un-hiding convenience overload
  Failures FetchManifest(manifest::Manifest** manifest) {
    const std::string path = BuildPath(BaseTN::kManifestFilename);
    if (!FileExists(path)) {
      return BaseTN::kFailNotFound;
    }

    *manifest = manifest::Manifest::LoadFile(path);
    return (*manifest != NULL) ? BaseTN::kFailOk
                               : BaseTN::kFailUnknown;
  }

  std::string GetUrl(const shash::Any &hash) const {
    return "file://" + BuildPath(BuildRelativePath(hash));
  }

  Failures Fetch(const shash::Any &object_hash, std::string *file_path) {
    assert(file_path != NULL);
    file_path->clear();

    const std::string relative_path = BuildRelativePath(object_hash);
    const bool        decompress    = true;
    const bool        nocache       = false;
    return Fetch(relative_path, decompress, nocache, file_path);
  }


  Failures Fetch(const std::string &relative_path,
                 const bool         decompress,
                 const bool         /* nocache */,
                       std::string *file_path) {
    assert(file_path != NULL);
    file_path->clear();

    // check if the requested file object is available locally
    const std::string source = BuildPath(relative_path);
    if (!FileExists(source)) {
      LogCvmfs(kLogDownload, kLogDebug, "failed to locate file '%s'",
               relative_path.c_str());
      return BaseTN::kFailNotFound;
    }

    // create a temporary file to store the (decompressed) object file
    const std::string tmp_path = BaseTN::temporary_directory() + "/" +
                                 GetFileName(relative_path);
    FILE *f = CreateTempFile(tmp_path, 0600, "w", file_path);
    if (NULL == f) {
      LogCvmfs(kLogDownload, kLogStderr,
               "failed to create temp file '%s' (errno: %d)",
               tmp_path.c_str(), errno);
      return BaseTN::kFailLocalIO;
    }

    // decompress or copy the requested object file
    const bool success = (decompress)
      ? zlib::DecompressPath2File(source, f)
      : CopyPath2File(source, f);
    fclose(f);

    // check the decompression success and remove the temporary file otherwise
    if (!success) {
      LogCvmfs(kLogDownload, kLogDebug, "failed to fetch file from '%s' "
                                        "to '%s' (errno: %d)",
               source.c_str(), file_path->c_str(), errno);
      unlink(file_path->c_str());
      file_path->clear();
      return BaseTN::kFailDecompression;
    }

    return BaseTN::kFailOk;
  }

 protected:
  std::string BuildPath(const std::string &relative_path) const {
    return base_path_ + "/" + relative_path;
  }

  std::string BuildRelativePath(const shash::Any &hash) const {
    return "data/" + hash.MakePath();
  }

 private:
  const std::string base_path_;
};

template <class CatalogT, class HistoryT, class ReflogT>
struct object_fetcher_traits<LocalObjectFetcher<CatalogT, HistoryT, ReflogT> > {
    typedef CatalogT CatalogTN;
    typedef HistoryT HistoryTN;
    typedef ReflogT  ReflogTN;
};


template <class CatalogT = catalog::Catalog,
          class HistoryT = history::SqliteHistory,
          class ReflogT  = manifest::Reflog>
class HttpObjectFetcher :
  public AbstractObjectFetcher<HttpObjectFetcher<CatalogT, HistoryT, ReflogT> >
{
 protected:
  typedef HttpObjectFetcher<CatalogT, HistoryT, ReflogT>  ThisTN;
  typedef AbstractObjectFetcher<ThisTN>                   BaseTN;

 public:
  typedef typename BaseTN::Failures Failures;

 public:
  HttpObjectFetcher(const std::string           &repo_name,
                    const std::string           &repo_url,
                    const std::string           &temp_dir,
                    download::DownloadManager   *download_mgr,
                    signature::SignatureManager *signature_mgr)
    : BaseTN(temp_dir)
    , repo_url_(repo_url)
    , repo_name_(repo_name)
    , download_manager_(download_mgr)
    , signature_manager_(signature_mgr) {}

 public:
  using BaseTN::FetchManifest;  // un-hiding convenience overload
  Failures FetchManifest(manifest::Manifest** manifest) {
    const std::string url = BuildUrl(BaseTN::kManifestFilename);

    // Download manifest file
    struct manifest::ManifestEnsemble manifest_ensemble;
    manifest::Failures retval = manifest::Fetch(
                                  repo_url_,
                                  repo_name_,
                                  0,
                                  NULL,
                                  signature_manager_,
                                  download_manager_,
                                  &manifest_ensemble);

    // Check if manifest was loaded correctly
    switch (retval) {
      case manifest::kFailOk:
        break;

      case manifest::kFailNameMismatch:
        LogCvmfs(kLogDownload, kLogDebug,
                 "repository name mismatch. No name provided?");
        return BaseTN::kFailManifestNameMismatch;

      case manifest::kFailBadSignature:
      case manifest::kFailBadCertificate:
      case manifest::kFailBadWhitelist:
        LogCvmfs(kLogDownload, kLogDebug,
                 "repository signature mismatch. No key(s) provided?");
        return BaseTN::kFailManifestSignatureMismatch;

      default:
        LogCvmfs(kLogDownload, kLogDebug,
                 "failed to load manifest (%d - %s)",
                 retval, Code2Ascii(retval));
        return BaseTN::kFailUnknown;
    }

    assert(retval == manifest::kFailOk);
    *manifest = new manifest::Manifest(*manifest_ensemble.manifest);
    return (*manifest != NULL) ? BaseTN::kFailOk
                               : BaseTN::kFailUnknown;
  }

  std::string GetUrl(const shash::Any &hash) const {
    return BuildUrl(BuildRelativeUrl(hash));
  }

  Failures Fetch(const shash::Any &object_hash, std::string *object_file) {
    assert(object_file != NULL);
    assert(!object_hash.IsNull());

    const bool decompress = true;
    const bool nocache = false;
    const std::string url = BuildRelativeUrl(object_hash);
    return Download(url, decompress, nocache, &object_hash, object_file);
  }

  Failures Fetch(const std::string &relative_path,
                 const bool         decompress,
                 const bool         nocache,
                       std::string *file_path) {
    const shash::Any *expected_hash = NULL;
    return Download(relative_path, decompress, nocache, expected_hash,
                    file_path);
  }

 protected:
  std::string BuildUrl(const std::string &relative_path) const {
    return repo_url_ + "/" + relative_path;
  }

  std::string BuildRelativeUrl(const shash::Any &hash) const {
    return "data/" + hash.MakePath();
  }

  Failures Download(const std::string &relative_path,
                    const bool         decompress,
                    const bool         nocache,
                    const shash::Any  *expected_hash,
                          std::string *file_path) {
    file_path->clear();

    // create temporary file to host the fetching result
    const std::string tmp_path = BaseTN::temporary_directory() + "/" +
                                 GetFileName(relative_path);
    FILE *f = CreateTempFile(tmp_path, 0600, "w", file_path);
    if (NULL == f) {
      LogCvmfs(kLogDownload, kLogStderr,
               "failed to create temp file '%s' (errno: %d)",
               tmp_path.c_str(), errno);
      return BaseTN::kFailLocalIO;
    }

    // fetch and decompress the requested object
    const std::string url = BuildUrl(relative_path);
    const bool probe_hosts = false;
    cvmfs::FileSink filesink(f);
    download::JobInfo download_job(&url, decompress, probe_hosts, expected_hash,
                                   &filesink);
    download_job.SetForceNocache(nocache);
    download::Failures retval = download_manager_->Fetch(&download_job);
    const bool success = (retval == download::kFailOk);
    fclose(f);

    // check if download worked and remove temporary file if not
    if (!success) {
      LogCvmfs(kLogDownload, kLogDebug, "failed to download file "
                                        "%s to '%s' (%d - %s)",
               relative_path.c_str(), file_path->c_str(),
               retval, Code2Ascii(retval));
      unlink(file_path->c_str());
      file_path->clear();

      // hand out the error status
      switch (retval) {
        case download::kFailLocalIO:
          return BaseTN::kFailLocalIO;

        case download::kFailBadUrl:
        case download::kFailProxyResolve:
        case download::kFailHostResolve:
        case download::kFailUnsupportedProtocol:
          LogCvmfs(kLogDownload, kLogDebug | kLogStderr,
                   "HTTP connection error %d: %s", retval, url.c_str());
          return BaseTN::kFailNetwork;

        case download::kFailProxyHttp:
        case download::kFailHostHttp:
          if (download_job.http_code() == 404)
            return BaseTN::kFailNotFound;
          LogCvmfs(kLogDownload, kLogDebug | kLogStderr,
                   "HTTP protocol error %d: %s (%d)",
                   download_job.http_code(), url.c_str(), retval);
          return BaseTN::kFailNetwork;

        case download::kFailBadData:
        case download::kFailTooBig:
          return BaseTN::kFailBadData;

        default:
          if (download::IsProxyTransferError(retval) ||
              download::IsHostTransferError(retval))
          {
            LogCvmfs(kLogDownload, kLogDebug | kLogStderr,
                     "HTTP transfer error %d (HTTP code %d): %s",
                     retval, download_job.http_code(), url.c_str());
            return BaseTN::kFailNetwork;
          }
          return BaseTN::kFailUnknown;
      }
    }

    assert(success);
    return BaseTN::kFailOk;
  }

 private:
  const std::string            repo_url_;
  const std::string            repo_name_;
  download::DownloadManager   *download_manager_;
  signature::SignatureManager *signature_manager_;
};

template <class CatalogT, class HistoryT, class ReflogT>
struct object_fetcher_traits<HttpObjectFetcher<CatalogT, HistoryT, ReflogT> > {
    typedef CatalogT CatalogTN;
    typedef HistoryT HistoryTN;
    typedef ReflogT  ReflogTN;
};

#endif  // CVMFS_OBJECT_FETCHER_H_