CernVM-FS  2.10.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
object_fetcher.h
Go to the documentation of this file.
1 
5 #ifndef CVMFS_OBJECT_FETCHER_H_
6 #define CVMFS_OBJECT_FETCHER_H_
7 
8 #include <unistd.h>
9 
10 #include <string>
11 
12 #include "catalog.h"
13 #include "download.h"
14 #include "history_sqlite.h"
15 #include "manifest.h"
16 #include "manifest_fetch.h"
17 #include "reflog.h"
18 #include "signature.h"
19 #include "util/posix.h"
20 
31 template <class ConcreteObjectFetcherT>
33 
35  enum Failures {
45 
47  };
48 };
49 
50 inline const char* Code2Ascii(const ObjectFetcherFailures::Failures error) {
51  const char *texts[ObjectFetcherFailures::kFailNumEntries + 1];
52  texts[0] = "OK";
53  texts[1] = "object not found";
54  texts[2] = "local I/O failure";
55  texts[3] = "network failure";
56  texts[4] = "decompression failed";
57  texts[5] = "manifest name doesn't match";
58  texts[6] = "manifest signature is invalid";
59  texts[7] = "bad data received";
60  texts[8] = "no text";
61  return texts[error];
62 }
63 
80 template <class DerivedT>
82  public:
86 
88 
89  static const std::string kManifestFilename;
90  static const std::string kReflogFilename;
91 
92  public:
101  return static_cast<DerivedT*>(this)->FetchManifest(manifest);
102  }
103 
116  const shash::Any &history_hash = shash::Any()) {
117  // retrieve the current HEAD history hash (if nothing else given)
118  shash::Any effective_history_hash = (!history_hash.IsNull())
119  ? history_hash
120  : GetHistoryHash();
121  if (effective_history_hash.IsNull()) {
122  return kFailNotFound;
123  }
124  assert(history_hash.suffix == shash::kSuffixHistory ||
125  history_hash.IsNull());
126 
127  // download the history hash
128  std::string path;
129  const Failures retval = Fetch(effective_history_hash, &path);
130  if (retval != kFailOk) {
131  return retval;
132  }
133 
134  // open the history file
135  *history = HistoryTN::Open(path);
136  if (NULL == *history) {
137  return kFailLocalIO;
138  }
139 
140  (*history)->TakeDatabaseFileOwnership();
141  return kFailOk;
142  }
143 
155  Failures FetchCatalog(const shash::Any &catalog_hash,
156  const std::string &catalog_path,
157  CatalogTN **catalog,
158  const bool is_nested = false,
159  CatalogTN *parent = NULL) {
160  assert(!catalog_hash.IsNull());
161  assert(catalog_hash.suffix == shash::kSuffixCatalog);
162 
163  std::string path;
164  const Failures retval = Fetch(catalog_hash, &path);
165  if (retval != kFailOk) {
166  return retval;
167  }
168 
169  *catalog = CatalogTN::AttachFreely(catalog_path,
170  path,
171  catalog_hash,
172  parent,
173  is_nested);
174  if (NULL == *catalog) {
175  return kFailLocalIO;
176  }
177 
178  (*catalog)->TakeDatabaseFileOwnership();
179  return kFailOk;
180  }
181 
182  Failures FetchReflog(const shash::Any &reflog_hash, ReflogTN **reflog) {
183  assert(!reflog_hash.IsNull());
184  assert(reflog_hash.suffix == shash::kSuffixNone);
185 
186  std::string tmp_path;
187  const bool decompress = false;
188  const bool nocache = true;
189  Failures failure = Fetch(kReflogFilename, decompress, nocache, &tmp_path);
190  if (failure != kFailOk) {
191  return failure;
192  }
193 
194  // Ensure data integrity
195  shash::Any computed_hash(reflog_hash.algorithm);
196  ReflogTN::HashDatabase(tmp_path, &computed_hash);
197  if (computed_hash != reflog_hash) {
198  unlink(tmp_path.c_str());
199  return kFailBadData;
200  }
201 
202  *reflog = ReflogTN::Open(tmp_path);
203  if (NULL == *reflog) {
204  return kFailLocalIO;
205  }
206 
207  (*reflog)->TakeDatabaseFileOwnership();
208  return kFailOk;
209  }
210 
212  manifest::Manifest *raw_manifest_ptr = NULL;
213  Failures failure = FetchManifest(&raw_manifest_ptr);
214  if (failure == kFailOk) *manifest = raw_manifest_ptr;
215  return failure;
216  }
217 
219  const shash::Any &history_hash = shash::Any()) {
220  HistoryTN *raw_history_ptr = NULL;
221  Failures failure = FetchHistory(&raw_history_ptr, history_hash);
222  if (failure == kFailOk) *history = raw_history_ptr;
223  return failure;
224  }
225 
226  Failures FetchCatalog(const shash::Any &catalog_hash,
227  const std::string &catalog_path,
228  UniquePtr<CatalogTN> *catalog,
229  const bool is_nested = false,
230  CatalogTN *parent = NULL) {
231  CatalogTN *raw_catalog_ptr = NULL;
232  Failures failure = FetchCatalog(catalog_hash,
233  catalog_path,
234  &raw_catalog_ptr,
235  is_nested,
236  parent);
237  if (failure == kFailOk) *catalog = raw_catalog_ptr;
238  return failure;
239  }
240 
241  Failures FetchReflog(const shash::Any &reflog_hash,
243  {
244  ReflogTN *raw_reflog_ptr = NULL;
245  Failures failure = FetchReflog(reflog_hash, &raw_reflog_ptr);
246  if (failure == kFailOk) *reflog = raw_reflog_ptr;
247  return failure;
248  }
249 
250  std::string GetUrl(const shash::Any &hash) const {
251  return static_cast<DerivedT*>(this)->GetUrl(hash);
252  }
253 
254  bool HasHistory() {
255  shash::Any history_hash = GetHistoryHash();
256  return !history_hash.IsNull();
257  }
258 
259  const std::string& temporary_directory() const {
260  return temporary_directory_;
261  }
262 
263  protected:
264  explicit AbstractObjectFetcher(const std::string &temp_dir)
265  : temporary_directory_(temp_dir) {}
266 
276  Failures Fetch(const shash::Any &object_hash, std::string *file_path) {
277  return static_cast<DerivedT*>(this)->Fetch(object_hash, file_path);
278  }
279 
280  Failures Fetch(const std::string &relative_path,
281  const bool decompress,
282  const bool nocache,
283  std::string *file_path) {
284  return static_cast<DerivedT*>(this)->Fetch(relative_path,
285  decompress,
286  nocache,
287  file_path);
288  }
289 
298  const Failures retval = FetchManifest(&manifest);
299 
300  if (retval != kFailOk ||
301  !manifest.IsValid() ||
302  manifest->history().IsNull()) {
303  return shash::Any();
304  }
305 
306  return manifest->history();
307  }
308 
309  private:
310  const std::string temporary_directory_;
311 };
312 
313 template <class DerivedT>
315  ".cvmfspublished";
316 template <class DerivedT>
318  ".cvmfsreflog";
319 
320 
326 template <class CatalogT = catalog::Catalog,
327  class HistoryT = history::SqliteHistory,
328  class ReflogT = manifest::Reflog>
330  public AbstractObjectFetcher<LocalObjectFetcher<CatalogT, HistoryT, ReflogT> >
331 {
332  protected:
335 
336  public:
337  typedef typename BaseTN::Failures Failures;
338 
339  public:
346  LocalObjectFetcher(const std::string &base_path,
347  const std::string &temp_dir)
348  : BaseTN(temp_dir)
349  , base_path_(base_path) {}
350 
351  using BaseTN::FetchManifest; // un-hiding convenience overload
353  const std::string path = BuildPath(BaseTN::kManifestFilename);
354  if (!FileExists(path)) {
355  return BaseTN::kFailNotFound;
356  }
357 
358  *manifest = manifest::Manifest::LoadFile(path);
359  return (*manifest != NULL) ? BaseTN::kFailOk
361  }
362 
363  std::string GetUrl(const shash::Any &hash) const {
364  return "file://" + BuildPath(BuildRelativePath(hash));
365  }
366 
367  Failures Fetch(const shash::Any &object_hash, std::string *file_path) {
368  assert(file_path != NULL);
369  file_path->clear();
370 
371  const std::string relative_path = BuildRelativePath(object_hash);
372  const bool decompress = true;
373  const bool nocache = false;
374  return Fetch(relative_path, decompress, nocache, file_path);
375  }
376 
377 
378  Failures Fetch(const std::string &relative_path,
379  const bool decompress,
380  const bool /* nocache */,
381  std::string *file_path) {
382  assert(file_path != NULL);
383  file_path->clear();
384 
385  // check if the requested file object is available locally
386  const std::string source = BuildPath(relative_path);
387  if (!FileExists(source)) {
388  LogCvmfs(kLogDownload, kLogDebug, "failed to locate file '%s'",
389  relative_path.c_str());
390  return BaseTN::kFailNotFound;
391  }
392 
393  // create a temporary file to store the (decompressed) object file
394  const std::string tmp_path = BaseTN::temporary_directory() + "/" +
395  GetFileName(relative_path);
396  FILE *f = CreateTempFile(tmp_path, 0600, "w", file_path);
397  if (NULL == f) {
399  "failed to create temp file '%s' (errno: %d)",
400  tmp_path.c_str(), errno);
401  return BaseTN::kFailLocalIO;
402  }
403 
404  // decompress or copy the requested object file
405  const bool success = (decompress)
406  ? zlib::DecompressPath2File(source, f)
407  : CopyPath2File(source, f);
408  fclose(f);
409 
410  // check the decompression success and remove the temporary file otherwise
411  if (!success) {
412  LogCvmfs(kLogDownload, kLogDebug, "failed to fetch file from '%s' "
413  "to '%s' (errno: %d)",
414  source.c_str(), file_path->c_str(), errno);
415  unlink(file_path->c_str());
416  file_path->clear();
418  }
419 
420  return BaseTN::kFailOk;
421  }
422 
423  protected:
424  std::string BuildPath(const std::string &relative_path) const {
425  return base_path_ + "/" + relative_path;
426  }
427 
428  std::string BuildRelativePath(const shash::Any &hash) const {
429  return "data/" + hash.MakePath();
430  }
431 
432  private:
433  const std::string base_path_;
434 };
435 
436 template <class CatalogT, class HistoryT, class ReflogT>
437 struct object_fetcher_traits<LocalObjectFetcher<CatalogT, HistoryT, ReflogT> > {
438  typedef CatalogT CatalogTN;
439  typedef HistoryT HistoryTN;
440  typedef ReflogT ReflogTN;
441 };
442 
443 
449 template <class CatalogT = catalog::Catalog,
450  class HistoryT = history::SqliteHistory,
451  class ReflogT = manifest::Reflog>
453  public AbstractObjectFetcher<HttpObjectFetcher<CatalogT, HistoryT, ReflogT> >
454 {
455  protected:
458 
459  public:
460  typedef typename BaseTN::Failures Failures;
461 
462  public:
475  HttpObjectFetcher(const std::string &repo_name,
476  const std::string &repo_url,
477  const std::string &temp_dir,
478  download::DownloadManager *download_mgr,
480  : BaseTN(temp_dir)
481  , repo_url_(repo_url)
482  , repo_name_(repo_name)
483  , download_manager_(download_mgr)
484  , signature_manager_(signature_mgr) {}
485 
486  public:
487  using BaseTN::FetchManifest; // un-hiding convenience overload
489  const std::string url = BuildUrl(BaseTN::kManifestFilename);
490 
491  // Download manifest file
492  struct manifest::ManifestEnsemble manifest_ensemble;
494  repo_url_,
495  repo_name_,
496  0,
497  NULL,
500  &manifest_ensemble);
501 
502  // Check if manifest was loaded correctly
503  switch (retval) {
504  case manifest::kFailOk:
505  break;
506 
509  "repository name mismatch. No name provided?");
511 
516  "repository signature mismatch. No key(s) provided?");
518 
519  default:
521  "failed to load manifest (%d - %s)",
522  retval, Code2Ascii(retval));
523  return BaseTN::kFailUnknown;
524  }
525 
526  assert(retval == manifest::kFailOk);
527  *manifest = new manifest::Manifest(*manifest_ensemble.manifest);
528  return (*manifest != NULL) ? BaseTN::kFailOk
530  }
531 
532  std::string GetUrl(const shash::Any &hash) const {
533  return BuildUrl(BuildRelativeUrl(hash));
534  }
535 
536  Failures Fetch(const shash::Any &object_hash, std::string *object_file) {
537  assert(object_file != NULL);
538  assert(!object_hash.IsNull());
539 
540  const bool decompress = true;
541  const bool nocache = false;
542  const std::string url = BuildRelativeUrl(object_hash);
543  return Download(url, decompress, nocache, &object_hash, object_file);
544  }
545 
546  Failures Fetch(const std::string &relative_path,
547  const bool decompress,
548  const bool nocache,
549  std::string *file_path) {
550  const shash::Any *expected_hash = NULL;
551  return Download(relative_path, decompress, nocache, expected_hash,
552  file_path);
553  }
554 
555  protected:
556  std::string BuildUrl(const std::string &relative_path) const {
557  return repo_url_ + "/" + relative_path;
558  }
559 
560  std::string BuildRelativeUrl(const shash::Any &hash) const {
561  return "data/" + hash.MakePath();
562  }
563 
564  Failures Download(const std::string &relative_path,
565  const bool decompress,
566  const bool nocache,
567  const shash::Any *expected_hash,
568  std::string *file_path) {
569  file_path->clear();
570 
571  // create temporary file to host the fetching result
572  const std::string tmp_path = BaseTN::temporary_directory() + "/" +
573  GetFileName(relative_path);
574  FILE *f = CreateTempFile(tmp_path, 0600, "w", file_path);
575  if (NULL == f) {
577  "failed to create temp file '%s' (errno: %d)",
578  tmp_path.c_str(), errno);
579  return BaseTN::kFailLocalIO;
580  }
581 
582  // fetch and decompress the requested object
583  const std::string url = BuildUrl(relative_path);
584  const bool probe_hosts = false;
585  download::JobInfo download_job(&url,
586  decompress,
587  probe_hosts,
588  f,
589  expected_hash);
590  download_job.force_nocache = nocache;
591  download::Failures retval = download_manager_->Fetch(&download_job);
592  const bool success = (retval == download::kFailOk);
593  fclose(f);
594 
595  // check if download worked and remove temporary file if not
596  if (!success) {
597  LogCvmfs(kLogDownload, kLogDebug, "failed to download file "
598  "%s to '%s' (%d - %s)",
599  relative_path.c_str(), file_path->c_str(),
600  retval, Code2Ascii(retval));
601  unlink(file_path->c_str());
602  file_path->clear();
603 
604  // hand out the error status
605  switch (retval) {
607  return BaseTN::kFailLocalIO;
608 
614  "HTTP connection error %d: %s", retval, url.c_str());
615  return BaseTN::kFailNetwork;
616 
619  if (download_job.http_code == 404)
620  return BaseTN::kFailNotFound;
622  "HTTP protocol error %d: %s (%d)",
623  download_job.http_code, url.c_str(), retval);
624  return BaseTN::kFailNetwork;
625 
628  return BaseTN::kFailBadData;
629 
630  default:
631  if (download::IsProxyTransferError(retval) ||
633  {
635  "HTTP transfer error %d (HTTP code %d): %s",
636  retval, download_job.http_code, url.c_str());
637  return BaseTN::kFailNetwork;
638  }
639  return BaseTN::kFailUnknown;
640  }
641  }
642 
643  assert(success);
644  return BaseTN::kFailOk;
645  }
646 
647  private:
648  const std::string repo_url_;
649  const std::string repo_name_;
652 };
653 
654 template <class CatalogT, class HistoryT, class ReflogT>
655 struct object_fetcher_traits<HttpObjectFetcher<CatalogT, HistoryT, ReflogT> > {
656  typedef CatalogT CatalogTN;
657  typedef HistoryT HistoryTN;
658  typedef ReflogT ReflogTN;
659 };
660 
661 #endif // CVMFS_OBJECT_FETCHER_H_
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
const char * Code2Ascii(const ObjectFetcherFailures::Failures error)
bool IsNull() const
Definition: hash.h:382
const manifest::Manifest * manifest() const
Definition: repository.h:123
std::string GetFileName(const std::string &path)
Definition: posix.cc:162
ObjectFetcherFailures::Failures Failures
Failures Fetch(const shash::Any &object_hash, std::string *file_path)
FILE * CreateTempFile(const std::string &path_prefix, const int mode, const char *open_flags, std::string *final_path)
Definition: posix.cc:1030
void Open()
const history::History * history() const
Failures FetchReflog(const shash::Any &reflog_hash, ReflogTN **reflog)
std::string BuildRelativeUrl(const shash::Any &hash) const
Failures FetchHistory(HistoryTN **history, const shash::Any &history_hash=shash::Any())
const std::string temporary_directory_
Failures FetchManifest(UniquePtr< manifest::Manifest > *manifest)
Failures FetchManifest(manifest::Manifest **manifest)
assert((mem||(size==0))&&"Out Of Memory")
Failures FetchReflog(const shash::Any &reflog_hash, UniquePtr< ReflogTN > *reflog)
Algorithms algorithm
Definition: hash.h:124
shash::Any GetHistoryHash()
object_fetcher_traits< DerivedT >::ReflogTN ReflogTN
std::string BuildUrl(const std::string &relative_path) const
std::string BuildPath(const std::string &relative_path) const
bool CopyPath2File(const std::string &src, FILE *fdest)
Definition: compression.cc:46
const std::string repo_name_
bool FileExists(const std::string &path)
Definition: posix.cc:816
AbstractObjectFetcher< ThisTN > BaseTN
BaseTN::Failures Failures
object_fetcher_traits< DerivedT >::CatalogTN CatalogTN
Failures Fetch(const std::string &relative_path, const bool decompress, const bool nocache, std::string *file_path)
std::string GetUrl(const shash::Any &hash) const
signature::SignatureManager * signature_manager_
const char kSuffixCatalog
Definition: hash.h:53
Failures FetchCatalog(const shash::Any &catalog_hash, const std::string &catalog_path, CatalogTN **catalog, const bool is_nested=false, CatalogTN *parent=NULL)
Failures Fetch(const std::string &base_url, const std::string &repository_name, const uint64_t minimum_timestamp, const shash::Any *base_catalog, signature::SignatureManager *signature_manager, download::DownloadManager *download_manager, ManifestEnsemble *ensemble)
bool DecompressPath2File(const string &src, FILE *fdest)
Definition: compression.cc:667
const signature::SignatureManager * signature_mgr() const
Definition: repository.h:119
const std::string repo_url_
bool IsProxyTransferError(const Failures error)
Definition: download.h:76
download::DownloadManager * download_manager_
object_fetcher_traits< DerivedT >::HistoryTN HistoryTN
HttpObjectFetcher< CatalogT, HistoryT, ReflogT > ThisTN
BaseTN::Failures Failures
HttpObjectFetcher(const std::string &repo_name, const std::string &repo_url, const std::string &temp_dir, download::DownloadManager *download_mgr, signature::SignatureManager *signature_mgr)
Failures FetchCatalog(const shash::Any &catalog_hash, const std::string &catalog_path, UniquePtr< CatalogTN > *catalog, const bool is_nested=false, CatalogTN *parent=NULL)
bool IsValid() const
Definition: pointer.h:43
const std::string base_path_
const char kSuffixHistory
Definition: hash.h:54
AbstractObjectFetcher< ThisTN > BaseTN
LocalObjectFetcher< CatalogT, HistoryT, ReflogT > ThisTN
static const std::string kReflogFilename
Failures Fetch(JobInfo *info)
Definition: download.cc:1716
LocalObjectFetcher(const std::string &base_path, const std::string &temp_dir)
Failures FetchManifest(manifest::Manifest **manifest)
Failures FetchHistory(UniquePtr< HistoryTN > *history, const shash::Any &history_hash=shash::Any())
Failures FetchManifest(manifest::Manifest **manifest)
Failures Download(const std::string &relative_path, const bool decompress, const bool nocache, const shash::Any *expected_hash, std::string *file_path)
bool IsHostTransferError(const Failures error)
Definition: download.h:64
Failures Fetch(const shash::Any &object_hash, std::string *object_file)
Failures Fetch(const std::string &relative_path, const bool decompress, const bool, std::string *file_path)
std::string GetUrl(const shash::Any &hash) const
static const std::string kManifestFilename
Suffix suffix
Definition: hash.h:125
std::string MakePath() const
Definition: hash.h:315
Failures Fetch(const std::string &relative_path, const bool decompress, const bool nocache, std::string *file_path)
static Manifest * LoadFile(const std::string &path)
Definition: manifest.cc:78
Failures Fetch(const shash::Any &object_hash, std::string *file_path)
const std::string & temporary_directory() const
AbstractObjectFetcher(const std::string &temp_dir)
const char kSuffixNone
Definition: hash.h:52
std::string BuildRelativePath(const shash::Any &hash) const
std::string GetUrl(const shash::Any &hash) const