CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
repository.cc
Go to the documentation of this file.
1 
5 #include "cvmfs_config.h"
6 #include "publish/repository.h"
7 
8 #include <cassert>
9 #include <cstddef>
10 #include <cstdlib>
11 
12 #include "catalog_mgr_ro.h"
13 #include "catalog_mgr_rw.h"
14 #include "download.h"
15 #include "gateway_util.h"
16 #include "hash.h"
17 #include "history_sqlite.h"
19 #include "logging.h"
20 #include "manifest.h"
21 #include "manifest_fetch.h"
22 #include "publish/except.h"
24 #include "publish/settings.h"
25 #include "reflog.h"
26 #include "signature.h"
27 #include "statistics.h"
28 #include "sync_mediator.h"
29 #include "sync_union_aufs.h"
30 #include "sync_union_overlayfs.h"
31 #include "sync_union_tarball.h"
32 #include "upload.h"
34 #include "util/pointer.h"
35 #include "whitelist.h"
36 
37 // TODO(jblomer): Remove Me
38 namespace swissknife {
39 class CommandTag {
40  static const std::string kHeadTag;
41  static const std::string kPreviousHeadTag;
42 };
43 const std::string CommandTag::kHeadTag = "trunk";
44 const std::string CommandTag::kPreviousHeadTag = "trunk-previous";
45 }
46 
47 namespace publish {
48 
50  : settings_("")
51  , statistics_(new perf::Statistics())
52  , signature_mgr_(new signature::SignatureManager())
53  , download_mgr_(NULL)
54  , simple_catalog_mgr_(NULL)
55  , whitelist_(NULL)
56  , reflog_(NULL)
57  , manifest_(NULL)
58  , history_(NULL)
59 {
61 }
62 
64  : settings_(settings)
65  , statistics_(new perf::Statistics())
66  , signature_mgr_(new signature::SignatureManager())
67  , download_mgr_(NULL)
68  , simple_catalog_mgr_(NULL)
69  , whitelist_(NULL)
70  , reflog_(NULL)
71  , manifest_(NULL)
72  , history_(NULL)
73 {
75 
76  int rvb;
77  std::string keys = JoinStrings(FindFilesBySuffix(
78  settings.keychain().keychain_dir(), ".pub"), ":");
79  rvb = signature_mgr_->LoadPublicRsaKeys(keys);
80  if (!rvb) {
82  delete signature_mgr_;
83  delete statistics_;
84  throw EPublish("cannot load public rsa key");
85  }
86 
88  download_mgr_->Init(16, false,
90  try {
91  DownloadRootObjects(settings.url(), settings.fqrn(), settings.tmp_dir());
92  } catch (const EPublish& e) {
95  delete signature_mgr_;
96  delete download_mgr_;
97  delete statistics_;
98  throw;
99  }
100 }
101 
103  if (signature_mgr_ != NULL) signature_mgr_->Fini();
104  if (download_mgr_ != NULL) download_mgr_->Fini();
105 
106  delete history_;
107  delete manifest_;
108  delete reflog_;
109  delete whitelist_;
110  delete signature_mgr_;
111  delete download_mgr_;
112  delete simple_catalog_mgr_;
113  delete statistics_;
114 }
115 
116 const history::History *Repository::history() const { return history_; }
117 
119  if (simple_catalog_mgr_ != NULL) return simple_catalog_mgr_;
120 
123  settings_.url(),
124  settings_.tmp_dir(),
126  statistics_,
127  true /* manage_catalog_files */);
129  return simple_catalog_mgr_;
130 }
131 
132 
134  const std::string &url, const std::string &fqrn, const std::string &tmp_dir)
135 {
136  delete whitelist_;
138  whitelist::Failures rv_whitelist = whitelist_->LoadUrl(url);
140  throw EPublish(std::string("cannot load whitelist [") +
141  whitelist::Code2Ascii(rv_whitelist) + "]");
142  }
143 
145  const uint64_t minimum_timestamp = 0;
146  const shash::Any *base_catalog = NULL;
147  manifest::Failures rv_manifest = manifest::Fetch(
148  url, fqrn, minimum_timestamp, base_catalog, signature_mgr_, download_mgr_,
149  &ensemble);
150  if (rv_manifest != manifest::kFailOk) throw EPublish("cannot load manifest");
151  delete manifest_;
152  manifest_ = new manifest::Manifest(*ensemble.manifest);
153 
154  std::string reflog_path;
155  FILE *reflog_fd =
156  CreateTempFile(tmp_dir + "/reflog", kPrivateFileMode, "w", &reflog_path);
157  std::string reflog_url = url + "/.cvmfsreflog";
158  // TODO(jblomer): verify reflog hash
159  // shash::Any reflog_hash(manifest_->GetHashAlgorithm());
160  download::JobInfo download_reflog(
161  &reflog_url,
162  false /* compressed */,
163  false /* probe hosts */,
164  reflog_fd,
165  NULL);
166  download::Failures rv_dl = download_mgr_->Fetch(&download_reflog);
167  fclose(reflog_fd);
168  if (rv_dl == download::kFailOk) {
169  delete reflog_;
170  reflog_ = manifest::Reflog::Open(reflog_path);
171  if (reflog_ == NULL) throw EPublish("cannot open reflog");
173  } else {
174  if (!download_reflog.IsFileNotFound()) {
175  throw EPublish(std::string("cannot load reflog [") +
176  download::Code2Ascii(rv_dl) + "]");
177  }
178  assert(reflog_ == NULL);
179  }
180 
181  std::string tags_path;
182  FILE *tags_fd =
183  CreateTempFile(tmp_dir + "/tags", kPrivateFileMode, "w", &tags_path);
184  if (!manifest_->history().IsNull()) {
185  std::string tags_url = url + "/data/" + manifest_->history().MakePath();
186  shash::Any tags_hash(manifest_->history());
187  download::JobInfo download_tags(
188  &tags_url,
189  true /* compressed */,
190  true /* probe hosts */,
191  tags_fd,
192  &tags_hash);
193  rv_dl = download_mgr_->Fetch(&download_tags);
194  fclose(tags_fd);
195  if (rv_dl != download::kFailOk) throw EPublish("cannot load tag database");
196  delete history_;
198  if (history_ == NULL) throw EPublish("cannot open tag database");
199  } else {
200  fclose(tags_fd);
201  delete history_;
202  history_ = history::SqliteHistory::Create(tags_path, fqrn);
203  if (history_ == NULL) throw EPublish("cannot create tag database");
204  }
206 
207  if (!manifest_->meta_info().IsNull()) {
208  shash::Any info_hash(manifest_->meta_info());
209  std::string info_url = url + "/data/" + info_hash.MakePath();
210  download::JobInfo download_info(
211  &info_url,
212  true /* compressed */,
213  true /* probe_hosts */,
214  &info_hash);
215  download::Failures rv_info = download_mgr_->Fetch(&download_info);
216  if (rv_info != download::kFailOk) {
217  throw EPublish(std::string("cannot load meta info [") +
218  download::Code2Ascii(rv_info) + "]");
219  }
220  meta_info_ = std::string(download_info.destination_mem.data,
221  download_info.destination_mem.pos);
222  } else {
223  meta_info_ = "n/a";
224  }
225 }
226 
227 
228 std::string Repository::GetFqrnFromUrl(const std::string &url) {
229  return GetFileName(MakeCanonicalPath(url));
230 }
231 
232 
234  std::string url = settings_.url() + "/.cvmfs_master_replica";
235  download::JobInfo head(&url, false /* probe_hosts */);
236  download::Failures retval = download_mgr_->Fetch(&head);
237  if (retval == download::kFailOk) return true;
238  if (head.IsFileNotFound()) return false;
239 
240  throw EPublish(std::string("error looking for .cvmfs_master_replica [") +
241  download::Code2Ascii(retval) + "]");
242 }
243 
244 
245 //------------------------------------------------------------------------------
246 
247 
249  if ((spooler_files_ != NULL) && (spooler_catalogs_ != NULL))
250  return;
251  assert((spooler_files_ == NULL) && (spooler_catalogs_ == NULL));
252 
254  settings_.storage().GetLocator(),
255  settings_.transaction().hash_algorithm(),
256  settings_.transaction().compression_algorithm());
257  sd.session_token_file =
258  settings_.transaction().spool_area().gw_session_token();
259  sd.key_file = settings_.keychain().gw_key_path();
260 
261  spooler_files_ = upload::Spooler::Construct(sd, statistics_publish_);
262  if (spooler_files_ == NULL)
263  throw EPublish("could not initialize file spooler");
264 
265  upload::SpoolerDefinition sd_catalogs(sd.Dup2DefaultCompression());
267  upload::Spooler::Construct(sd_catalogs, statistics_publish_);
268  if (spooler_catalogs_ == NULL) {
269  delete spooler_files_;
270  throw EPublish("could not initialize catalog spooler");
271  }
272 }
273 
274 
277  throw EPublish("dangling master key pair");
278  }
280  throw EPublish("dangling repository keys");
281  }
286 
288  std::string whitelist_str = whitelist::Whitelist::CreateString(
289  settings_.fqrn(),
290  settings_.whitelist_validity_days(),
291  settings_.transaction().hash_algorithm(),
293  whitelist::Failures rv_wl = whitelist_->LoadMem(whitelist_str);
294  if (rv_wl != whitelist::kFailOk)
295  throw EPublish("whitelist generation failed");
296 }
297 
298 
300  // Reflog
301  const std::string reflog_path = CreateTempPath(
302  settings_.transaction().spool_area().tmp_dir() + "/cvmfs_reflog", 0600);
304  if (reflog_ == NULL) throw EPublish("could not create reflog");
305  reflog_->TakeDatabaseFileOwnership();
306 
307  // Root file catalog and initial manifest
309  settings_.transaction().spool_area().tmp_dir(),
310  settings_.transaction().is_volatile(),
311  settings_.transaction().voms_authz(),
313  spooler_catalogs_->WaitForUpload();
314  if (manifest_ == NULL)
315  throw EPublish("could not create initial file catalog");
316  reflog_->AddCatalog(manifest_->catalog_hash());
317 
319  manifest_->set_ttl(settings_.transaction().ttl_second());
320  const bool needs_bootstrap_shortcuts =
321  !settings_.transaction().voms_authz().empty();
322  manifest_->set_has_alt_catalog_path(needs_bootstrap_shortcuts);
324  settings_.transaction().is_garbage_collectable());
325 
326  // Tag database
327  const std::string tags_path = CreateTempPath(
328  settings_.transaction().spool_area().tmp_dir() + "/cvmfs_tags", 0600);
330  if (history_ == NULL) throw EPublish("could not create tag database");
331  history_->TakeDatabaseFileOwnership();
332  history::History::Tag tag_trunk(
333  "trunk",
336  "empty repository", "" /* branch */);
337  history_->Insert(tag_trunk);
338 
339  // Meta information, TODO(jblomer)
340  meta_info_ = "{}";
341 }
342 
343 
346  if (!spooler_files_->Create())
347  throw EPublish("could not initialize repository storage area");
348 }
349 
350 
352  upload::Spooler::CallbackPtr callback =
353  spooler_files_->RegisterListener(&Publisher::OnProcessCertificate, this);
354  spooler_files_->ProcessCertificate(
356  spooler_files_->WaitForUpload();
357  spooler_files_->UnregisterListener(callback);
358 }
359 
360 
362  assert(history_ != NULL);
364  const string history_path = history_->filename();
366  delete history_;
367 
368  upload::Spooler::CallbackPtr callback =
369  spooler_files_->RegisterListener(&Publisher::OnProcessHistory, this);
370  spooler_files_->ProcessHistory(history_path);
371  spooler_files_->WaitForUpload();
372  spooler_files_->UnregisterListener(callback);
373 
375  assert(history_ != NULL);
377 }
378 
379 
381  upload::Spooler::CallbackPtr callback =
382  spooler_files_->RegisterListener(&Publisher::OnProcessMetainfo, this);
383  spooler_files_->ProcessMetainfo(new StringIngestionSource(meta_info_));
384  spooler_files_->WaitForUpload();
385  spooler_files_->UnregisterListener(callback);
386 }
387 
388 
390  std::string signed_manifest = manifest_->ExportString();
391  shash::Any manifest_hash(settings_.transaction().hash_algorithm());
393  reinterpret_cast<const unsigned char *>(signed_manifest.data()),
394  signed_manifest.length(), &manifest_hash);
395  signed_manifest += "--\n" + manifest_hash.ToString() + "\n";
396  unsigned char *signature;
397  unsigned sig_size;
398  bool rvb = signature_mgr_->Sign(
399  reinterpret_cast<const unsigned char *>(manifest_hash.ToString().data()),
400  manifest_hash.GetHexSize(), &signature, &sig_size);
401  if (!rvb) throw EPublish("cannot sign manifest");
402  signed_manifest += std::string(reinterpret_cast<char *>(signature), sig_size);
403  free(signature);
404 
405  // Create alternative bootstrapping symlinks for VOMS secured repos
407  rvb =
408  spooler_files_->PlaceBootstrappingShortcut(manifest_->certificate()) &&
409  spooler_files_->PlaceBootstrappingShortcut(manifest_->catalog_hash()) &&
410  (manifest_->history().IsNull() ||
411  spooler_files_->PlaceBootstrappingShortcut(manifest_->history())) &&
412  (manifest_->meta_info().IsNull() ||
413  spooler_files_->PlaceBootstrappingShortcut(manifest_->meta_info()));
414  if (!rvb) EPublish("cannot place VOMS bootstrapping symlinks");
415  }
416 
417  upload::Spooler::CallbackPtr callback =
418  spooler_files_->RegisterListener(&Publisher::OnUploadManifest, this);
419  spooler_files_->Upload(".cvmfspublished",
420  new StringIngestionSource(signed_manifest));
421  spooler_files_->WaitForUpload();
422  spooler_files_->UnregisterListener(callback);
423 }
424 
425 
427  const string reflog_path = reflog_->database_file();
429  delete reflog_;
430 
431  shash::Any hash_reflog(settings_.transaction().hash_algorithm());
432  manifest::Reflog::HashDatabase(reflog_path, &hash_reflog);
433 
434  upload::Spooler::CallbackPtr callback =
435  spooler_files_->RegisterListener(&Publisher::OnUploadReflog, this);
436  spooler_files_->UploadReflog(reflog_path);
437  spooler_files_->WaitForUpload();
438  spooler_files_->UnregisterListener(callback);
439 
440  manifest_->set_reflog_hash(hash_reflog);
441 
442  reflog_ = manifest::Reflog::Open(reflog_path);
443  assert(reflog_ != NULL);
445 }
446 
447 
449  // TODO(jblomer): PKCS7 handling
450  upload::Spooler::CallbackPtr callback =
451  spooler_files_->RegisterListener(&Publisher::OnUploadWhitelist, this);
452  spooler_files_->Upload(".cvmfswhitelist",
454  spooler_files_->WaitForUpload();
455  spooler_files_->UnregisterListener(callback);
456 }
457 
458 
459 Publisher *Publisher::Create(const SettingsPublisher &settings) {
460  UniquePtr<Publisher> publisher(new Publisher());
461 
462  publisher->settings_ = settings;
463  if (settings.is_silent())
464  publisher->llvl_ = kLogNone;
465  publisher->signature_mgr_ = new signature::SignatureManager();
466  publisher->signature_mgr_->Init();
467 
468  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout | kLogNoLinebreak,
469  "Creating Key Chain... ");
470  publisher->CreateKeychain();
471  publisher->ExportKeychain();
472  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout, "done");
473 
474  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout | kLogNoLinebreak,
475  "Creating Backend Storage... ");
476  publisher->CreateStorage();
477  publisher->PushWhitelist();
478  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout, "done");
479 
480  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout | kLogNoLinebreak,
481  "Creating Initial Repository... ");
482  publisher->InitSpoolArea();
483  publisher->CreateRootObjects();
484  publisher->PushHistory();
485  publisher->PushCertificate();
486  publisher->PushMetainfo();
487  publisher->PushReflog();
488  publisher->PushManifest();
489  // TODO(jblomer): meta-info
490 
491  // Re-create from empty repository in order to properly initialize
492  // parent Repository object
493  publisher = new Publisher(settings);
494 
495  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout, "done");
496 
497  return publisher.Release();
498 }
499 
502 
503  bool rvb;
506  if (!rvb) throw EPublish("cannot export public master key");
509  if (!rvb) throw EPublish("cannot export certificate");
510 
513  if (!rvb) throw EPublish("cannot export private certificate key");
516  if (!rvb) throw EPublish("cannot export private master key");
517 
518  int rvi;
519  rvi = chown(settings_.keychain().master_public_key_path().c_str(),
520  settings_.owner_uid(), settings_.owner_gid());
521  if (rvi != 0) throw EPublish("cannot set key file ownership");
522  rvi = chown(settings_.keychain().certificate_path().c_str(),
523  settings_.owner_uid(), settings_.owner_gid());
524  if (rvi != 0) throw EPublish("cannot set key file ownership");
525  rvi = chown(settings_.keychain().private_key_path().c_str(),
526  settings_.owner_uid(), settings_.owner_gid());
527  if (rvi != 0) throw EPublish("cannot set key file ownership");
528  rvi = chown(settings_.keychain().master_private_key_path().c_str(),
529  settings_.owner_uid(), settings_.owner_gid());
530  if (rvi != 0) throw EPublish("cannot set key file ownership");
531 }
532 
534  if (result.return_code != 0) {
535  throw EPublish("cannot write certificate to storage");
536  }
539 }
540 
542  if (result.return_code != 0) {
543  throw EPublish("cannot write tag database to storage");
544  }
547 }
548 
550  if (result.return_code != 0) {
551  throw EPublish("cannot write repository meta info to storage");
552  }
555 }
556 
558  if (result.return_code != 0) {
559  throw EPublish("cannot write manifest to storage");
560  }
561 }
562 
564  if (result.return_code != 0) {
565  throw EPublish("cannot write reflog to storage");
566  }
567 }
568 
570  if (result.return_code != 0) {
571  throw EPublish("cannot write whitelist to storage");
572  }
573 }
574 
575 void Publisher::CreateDirectoryAsOwner(const std::string &path, int mode)
576 {
577  bool rvb = MkdirDeep(path, kPrivateDirMode);
578  if (!rvb) throw EPublish("cannot create directory " + path);
579  int rvi = chown(path.c_str(), settings_.owner_uid(), settings_.owner_gid());
580  if (rvi != 0) throw EPublish("cannot set ownership on directory " + path);
581 }
582 
584  CreateDirectoryAsOwner(settings_.transaction().spool_area().workspace(),
586  CreateDirectoryAsOwner(settings_.transaction().spool_area().tmp_dir(),
588  CreateDirectoryAsOwner(settings_.transaction().spool_area().cache_dir(),
590  CreateDirectoryAsOwner(settings_.transaction().spool_area().scratch_dir(),
592  CreateDirectoryAsOwner(settings_.transaction().spool_area().ovl_work_dir(),
594 
595  // On a managed node, the mount points are already mounted
596  if (!DirectoryExists(settings_.transaction().spool_area().readonly_mnt())) {
597  CreateDirectoryAsOwner(settings_.transaction().spool_area().readonly_mnt(),
599  }
600  if (!DirectoryExists(settings_.transaction().spool_area().union_mnt())) {
602  settings_.transaction().spool_area().union_mnt(), kPrivateDirMode);
603  }
604 }
605 
607  : settings_("invalid.cvmfs.io")
608  , statistics_publish_(new perf::StatisticsTemplate("publish", statistics_))
609  , llvl_(kLogNormal)
610  , in_transaction_(false)
611  , spooler_files_(NULL)
612  , spooler_catalogs_(NULL)
613  , catalog_mgr_(NULL)
614  , sync_parameters_(NULL)
615  , sync_mediator_(NULL)
616  , sync_union_(NULL)
617 {
618 }
619 
621  : Repository(SettingsRepository(settings))
622  , settings_(settings)
623  , statistics_publish_(new perf::StatisticsTemplate("publish", statistics_))
624  , llvl_(settings.is_silent() ? kLogNone : kLogNormal)
625  , in_transaction_(false)
626  , spooler_files_(NULL)
627  , spooler_catalogs_(NULL)
628  , catalog_mgr_(NULL)
629  , sync_parameters_(NULL)
630  , sync_mediator_(NULL)
631  , sync_union_(NULL)
632 {
634  unsigned layout_revision = settings.transaction().layout_revision();
635  throw EPublish(
636  "This repository uses layout revision " + StringifyInt(layout_revision)
637  + ".\n"
638  "This version of CernVM-FS requires layout revision " + StringifyInt(
639  kRequiredLayoutRevision) + ", which is\n"
640  "incompatible to " + StringifyInt(layout_revision) + ".\n\n"
641  "Please run `cvmfs_server migrate` to update your repository before "
642  "proceeding.",
644  }
645 
646  CreateDirectoryAsOwner(settings_.transaction().spool_area().tmp_dir(),
648 
649  bool check_keys_match = true;
650  int rvb;
651  rvb =
653  if (!rvb) {
654  check_keys_match = false;
656  "Warning: cannot load certificate, thus cannot commit changes");
657  }
659  settings.keychain().private_key_path(), "");
660  if (!rvb) {
661  check_keys_match = false;
663  "Warning: cannot load private key, thus cannot commit changes");
664  }
665  // The private master key might be on a key card instead
666  if (FileExists(settings.keychain().master_private_key_path())) {
668  settings.keychain().master_private_key_path());
669  if (!rvb) throw EPublish("cannot load private master key");
670  }
671  if (check_keys_match && !signature_mgr_->KeysMatch())
672  throw EPublish("corrupted keychain");
673 
674  if (settings.storage().type() == upload::SpoolerDefinition::Gateway) {
675  if (!settings.keychain().HasGatewayKey()) {
676  throw EPublish("gateway key missing: " +
677  settings.keychain().gw_key_path());
678  }
680  if (!gw_key_.IsValid()) {
681  throw EPublish("cannot read gateway key: " +
682  settings.keychain().gw_key_path());
683  }
684  }
685 
686  if (settings.is_managed())
687  managed_node_ = new ManagedNode(this);
689  if (in_transaction_)
691 }
692 
694  delete sync_union_;
695  delete sync_mediator_;
696  delete sync_parameters_;
697  delete catalog_mgr_;
698  delete spooler_catalogs_;
699  delete spooler_files_;
700 }
701 
702 
705 
706  if (catalog_mgr_ == NULL) {
708  settings_.transaction().base_hash(),
709  settings_.url(),
710  settings_.transaction().spool_area().tmp_dir(),
713  settings_.transaction().enforce_limits(),
714  settings_.transaction().limit_nested_catalog_kentries(),
715  settings_.transaction().limit_root_catalog_kentries(),
716  settings_.transaction().limit_file_size_mb(),
717  statistics_,
718  settings_.transaction().use_catalog_autobalance(),
719  settings_.transaction().autobalance_max_weight(),
720  settings_.transaction().autobalance_min_weight());
721  catalog_mgr_->Init();
722  }
723 
724  if (sync_parameters_ == NULL) {
725  SyncParameters *p = new SyncParameters();
726  p->spooler = spooler_files_;
727  p->repo_name = settings_.fqrn();
728  p->dir_union = settings_.transaction().spool_area().union_mnt();
729  p->dir_scratch = settings_.transaction().spool_area().scratch_dir();
730  p->dir_rdonly = settings_.transaction().spool_area().readonly_mnt();
731  p->dir_temp = settings_.transaction().spool_area().tmp_dir();
732  p->base_hash = settings_.transaction().base_hash();
733  p->stratum0 = settings_.url();
734  // p->manifest_path = SHOULD NOT BE NEEDED
735  // p->spooler_definition = SHOULD NOT BE NEEDED;
736  // p->union_fs_type = SHOULD NOT BE NEEDED
737  p->print_changeset = settings_.transaction().print_changeset();
738  p->dry_run = settings_.transaction().dry_run();
739  sync_parameters_ = p;
740  }
741 
742  if (sync_mediator_ == NULL) {
745  }
746 
747  if (sync_union_ == NULL) {
748  switch (settings_.transaction().union_fs()) {
749  case kUnionFsAufs:
752  settings_.transaction().spool_area().readonly_mnt(),
753  settings_.transaction().spool_area().union_mnt(),
754  settings_.transaction().spool_area().scratch_dir());
755  break;
756  case kUnionFsOverlay:
759  settings_.transaction().spool_area().readonly_mnt(),
760  settings_.transaction().spool_area().union_mnt(),
761  settings_.transaction().spool_area().scratch_dir());
762  break;
763  case kUnionFsTarball:
766  settings_.transaction().spool_area().readonly_mnt(),
767  // TODO(jblomer): get from settings
768  "tar_file",
769  "base_directory",
770  "to_delete",
771  false /* create_catalog */);
772  break;
773  default:
774  throw EPublish("unknown union file system");
775  }
776  bool rvb = sync_union_->Initialize();
777  if (!rvb) {
778  delete sync_union_;
779  sync_union_ = NULL;
780  throw EPublish("cannot initialize union file system engine");
781  }
782  }
783 }
784 
786  if (settings_.transaction().dry_run()) {
787  SyncImpl();
788  return;
789  }
790 
791  const std::string publishing_lock =
792  settings_.transaction().spool_area().publishing_lock();
793 
794  ServerLockFile::Acquire(publishing_lock, false /* ignore_stale */);
795  try {
796  SyncImpl();
797  ServerLockFile::Release(publishing_lock);
798  } catch (...) {
799  ServerLockFile::Release(publishing_lock);
800  throw;
801  }
802 }
803 
806 
808  bool rvb = sync_mediator_->Commit(manifest_);
809  if (!rvb) throw EPublish("cannot write change set to storage");
810 
811  if (!settings_.transaction().dry_run()) {
812  spooler_files_->WaitForUpload();
813  spooler_catalogs_->WaitForUpload();
814  spooler_files_->FinalizeSession(false /* commit */);
815 
816  const std::string old_root_hash =
817  settings_.transaction().base_hash().ToString(true /* with_suffix */);
818  const std::string new_root_hash =
819  manifest_->catalog_hash().ToString(true /* with_suffix */);
820  rvb = spooler_catalogs_->FinalizeSession(true /* commit */,
821  old_root_hash, new_root_hash,
822  /* TODO(jblomer) */ sync_parameters_->repo_tag);
823  if (!rvb)
824  throw EPublish("failed to commit transaction");
825 
826  // Reset to the new catalog root hash
827  settings_.GetTransaction()->SetBaseHash(manifest_->catalog_hash());
828  // TODO(jblomer): think about how to deal with the scratch area at
829  // this point
830  // WipeScratchArea();
831  }
832 
833  delete sync_union_;
834  delete sync_mediator_;
835  delete sync_parameters_;
836  delete catalog_mgr_;
837  sync_union_ = NULL;
838  sync_mediator_ = NULL;
839  sync_parameters_ = NULL;
840  catalog_mgr_ = NULL;
841 
842  if (!settings_.transaction().dry_run()) {
843  LogCvmfs(kLogCvmfs, kLogStdout, "New revision: %d", manifest_->revision());
845  }
846 }
847 
849  if (!in_transaction_) throw EPublish("cannot publish outside transaction");
850 
851  PushReflog();
852  PushManifest();
853  in_transaction_ = false;
854 }
855 
856 
857 void Publisher::MarkReplicatible(bool value) {
859 
860  if (value) {
861  spooler_files_->Upload("/dev/null", "/.cvmfs_master_replica");
862  } else {
863  spooler_files_->RemoveAsync("/.cvmfs_master_replica");
864  }
865  spooler_files_->WaitForUpload();
866  if (spooler_files_->GetNumberOfErrors() > 0)
867  throw EPublish("cannot set replication mode");
868 }
869 
871 void Publisher::Migrate() {}
873 void Publisher::Rollback() {}
875 
876 
877 //------------------------------------------------------------------------------
878 
879 
881 }
882 
883 
885 }
886 
887 } // namespace publish
bool Commit(manifest::Manifest *manifest)
int return_code
the return value of the spooler operation
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
void CheckTransactionStatus()
void set_repository_name(const std::string &repository_name)
Definition: manifest.h:93
void TakeDatabaseFileOwnership()
Definition: reflog.cc:307
const SettingsRepository settings_
Definition: repository.h:136
std::string database_file() const
Definition: reflog.cc:337
void CreateRootObjects()
Definition: repository.cc:299
manifest::Reflog * reflog_
Definition: repository.h:146
bool IsNull() const
Definition: hash.h:379
download::DownloadManager * download_mgr_
Definition: repository.h:140
bool HasGatewayKey() const
Definition: settings.cc:272
std::string stratum0
bool AddHistory(const shash::Any &history)
Definition: reflog.cc:134
void ConstructSyncManagers()
Definition: repository.cc:703
void OnUploadReflog(const upload::SpoolerResult &result)
Definition: repository.cc:563
catalog::SimpleCatalogManager * GetSimpleCatalogManager()
Definition: repository.cc:118
UniquePtr< ManagedNode > managed_node_
Definition: repository.h:375
std::string GetFileName(const std::string &path)
Definition: posix.cc:159
Repository(const SettingsRepository &settings)
Definition: repository.cc:63
const int kPrivateDirMode
Definition: posix.h:35
void Ingest()
Definition: repository.cc:870
UniquePtr< perf::StatisticsTemplate > statistics_publish_
Definition: repository.h:359
static Publisher * Create(const SettingsPublisher &settings)
std::string keychain_dir() const
Definition: settings.h:329
std::string GetCertificate() const
Definition: signature.cc:370
static const std::string kPreviousHeadTag
Definition: repository.cc:41
Failures LoadUrl(const std::string &base_url)
Definition: whitelist.cc:216
std::string gw_key_path() const
Definition: settings.h:336
FILE * CreateTempFile(const std::string &path_prefix, const int mode, const char *open_flags, std::string *final_path)
Definition: posix.cc:939
string JoinStrings(const vector< string > &strings, const string &joint)
Definition: string.cc:317
std::string ToString(const bool with_suffix=false) const
Definition: hash.h:245
void OnProcessCertificate(const upload::SpoolerResult &result)
Definition: repository.cc:533
virtual bool Initialize()
Definition: sync_union.cc:24
void set_certificate(const shash::Any &certificate)
Definition: manifest.h:87
const history::History * history() const
bool IsFileNotFound()
Definition: download.cc:364
void CreateDirectoryAsOwner(const std::string &path, int mode)
Definition: repository.cc:575
void PushCertificate()
Definition: repository.cc:351
unsigned layout_revision() const
Definition: settings.h:192
void OnUploadWhitelist(const upload::SpoolerResult &result)
Definition: repository.cc:569
void CreateStorage()
Definition: repository.cc:344
bool IsMasterReplica()
Definition: repository.cc:233
std::string private_key_path() const
Definition: settings.h:334
const SettingsPublisher & settings() const
Definition: repository.h:306
perf::Statistics * statistics_
Definition: repository.h:138
Status status() const
Definition: whitelist.h:103
std::string CreateTempPath(const std::string &path_prefix, const int mode)
Definition: posix.cc:968
void set_meta_info(const shash::Any &meta_info)
Definition: manifest.h:111
std::string GetPrivateKey()
Definition: signature.cc:386
std::string ExportString() const
Definition: manifest.cc:174
virtual ~Replica()
Definition: repository.cc:884
std::string repo_name
assert((mem||(size==0))&&"Out Of Memory")
bool has_alt_catalog_path() const
Definition: manifest.h:130
upload::Spooler * spooler_catalogs_
Definition: repository.h:378
gateway::GatewayKey gw_key_
Definition: repository.h:366
bool SafeWriteToFile(const std::string &content, const std::string &path, int mode)
Definition: posix.cc:1930
bool in_transaction_
Definition: repository.h:364
void PushMetainfo()
Definition: repository.cc:380
bool LoadPrivateKeyPath(const std::string &file_pem, const std::string &password)
Definition: signature.cc:171
class __attribute__((visibility("default"))) Publisher ManagedNode(Publisher *p)
Definition: repository.h:173
void PushReflog()
Definition: repository.cc:426
uint64_t catalog_size() const
Definition: manifest.h:125
bool AddCatalog(const shash::Any &catalog)
Definition: reflog.cc:128
static Reflog * Create(const std::string &database_path, const std::string &repo_name)
Definition: reflog.cc:32
catalog::WritableCatalogManager * catalog_mgr_
Definition: repository.h:379
std::string dir_scratch
bool HasMasterKeys() const
Definition: settings.cc:253
void GenerateCertificate(const std::string &cn)
Definition: signature.cc:452
uint64_t revision() const
Definition: manifest.h:121
static Reflog * Open(const std::string &database_path)
Definition: reflog.cc:17
static SqliteHistory * OpenWritable(const std::string &file_name)
std::string master_public_key_path() const
Definition: settings.h:333
bool FileExists(const std::string &path)
Definition: posix.cc:725
std::string GetActivePubkeys() const
Definition: signature.cc:357
std::string dir_temp
const int kPrivateFileMode
Definition: posix.h:34
void OnProcessMetainfo(const upload::SpoolerResult &result)
Definition: repository.cc:549
const char * Code2Ascii(const Failures error)
Definition: download.h:86
static manifest::Manifest * CreateRepository(const std::string &dir_temp, const bool volatile_content, const std::string &voms_authz, upload::Spooler *spooler)
bool HasRepositoryKeys() const
Definition: settings.cc:267
std::string meta_info_
Definition: repository.h:150
void PushHistory()
Definition: repository.cc:361
const SettingsKeychain & keychain() const
Definition: settings.h:422
publish::SyncUnion * sync_union_
Definition: repository.h:382
uint64_t publish_timestamp() const
Definition: manifest.h:128
bool AddMetainfo(const shash::Any &metainfo)
Definition: reflog.cc:140
const int kDefaultDirMode
Definition: posix.h:32
bool Sign(const unsigned char *buffer, const unsigned buffer_size, unsigned char **signature, unsigned *signature_size)
Definition: signature.cc:730
void OnUploadManifest(const upload::SpoolerResult &result)
Definition: repository.cc:557
std::string dir_rdonly
signature::SignatureManager * signature_mgr_
Definition: repository.h:139
Failures Fetch(const std::string &base_url, const std::string &repository_name, const uint64_t minimum_timestamp, const shash::Any *base_catalog, signature::SignatureManager *signature_manager, download::DownloadManager *download_manager, ManifestEnsemble *ensemble)
void UpdateMetaInfo()
Definition: repository.cc:874
std::string master_private_key_path() const
Definition: settings.h:330
void DownloadRootObjects(const std::string &url, const std::string &fqrn, const std::string &tmp_dir)
Definition: repository.cc:133
shash::Any base_hash
static const std::string kHeadTag
Definition: repository.cc:40
shash::Any certificate() const
Definition: manifest.h:126
virtual ~Publisher()
upload::Spooler * spooler
bool MkdirDeep(const std::string &path, const mode_t mode, bool verify_writable)
Definition: posix.cc:780
bool LoadCertificatePath(const std::string &file_pem)
Definition: signature.cc:217
const SettingsTransaction & transaction() const
Definition: settings.h:421
bool IsValid() const
Definition: gateway_util.h:17
const char * Code2Ascii(const Failures error)
Definition: whitelist.h:49
std::string dir_union
void Sync()
Definition: repository.cc:785
void InitSpoolArea()
Definition: repository.cc:583
static void HashDatabase(const std::string &database_path, shash::Any *hash_reflog)
Definition: reflog.cc:322
void PushManifest()
Definition: repository.cc:389
shash::Any catalog_hash() const
Definition: manifest.h:124
string StringifyInt(const int64_t value)
Definition: string.cc:77
std::string certificate_path() const
Definition: settings.h:335
catalog::SimpleCatalogManager * simple_catalog_mgr_
Definition: repository.h:144
history::SqliteHistory * history_
Definition: repository.h:148
void Resign()
Definition: repository.cc:872
bool is_managed() const
Definition: settings.h:418
void Publish()
const SettingsStorage & storage() const
Definition: settings.h:420
void DropDatabaseFileOwnership()
Definition: reflog.cc:313
bool SetPreviousRevision(const shash::Any &history_hash)
bool LoadPrivateMasterKeyPath(const std::string &file_pem)
Definition: signature.cc:153
bool DirectoryExists(const std::string &path)
Definition: posix.cc:747
void set_history(const shash::Any &history_db)
Definition: manifest.h:90
void set_garbage_collectability(const bool garbage_collectable)
Definition: manifest.h:105
bool AddCertificate(const shash::Any &certificate)
Definition: reflog.cc:121
std::string filename() const
manifest::Manifest * manifest_
Definition: repository.h:147
void OnProcessHistory(const upload::SpoolerResult &result)
Definition: repository.cc:541
void set_ttl(const uint32_t ttl)
Definition: manifest.h:85
upload::Spooler * spooler_files_
Definition: repository.h:377
void HashMem(const unsigned char *buffer, const unsigned buffer_size, Any *any_digest)
Definition: hash.cc:255
void SyncImpl()
Definition: repository.cc:804
void MarkReplicatible(bool value)
Definition: repository.cc:857
Publisher(const SettingsPublisher &settings)
Definition: repository.cc:620
class __attribute__((visibility("default"))) Replica Replica(const SettingsReplica &settings)
Definition: repository.cc:880
const SettingsKeychain & keychain() const
Definition: settings.h:371
std::string ExportString() const
Definition: whitelist.cc:77
virtual void Traverse()=0
std::string GetPrivateMasterKey()
Definition: signature.cc:402
void Migrate(InodeTracker *old_tracker, glue::InodeTracker *new_tracker)
Definition: compat.cc:83
Failures Fetch(JobInfo *info)
Definition: download.cc:1719
struct download::JobInfo::@3 destination_mem
shash::Any history() const
Definition: manifest.h:127
static std::string CreateString(const std::string &fqrn, int validity_days, shash::Algorithms hash_algorithm, signature::SignatureManager *signature_manager)
Definition: whitelist.cc:47
bool LoadPublicRsaKeys(const std::string &path_list)
Definition: signature.cc:277
whitelist::Whitelist * whitelist_
Definition: repository.h:145
GatewayKey ReadGatewayKey(const std::string &key_file_name)
Definition: gateway_util.cc:28
static SqliteHistory * Create(const std::string &file_name, const std::string &fqrn)
bool HasDanglingRepositoryKeys() const
Definition: settings.cc:259
Failures LoadMem(const std::string &whitelist)
Definition: whitelist.cc:196
RepositoryTag repo_tag
SyncMediator * sync_mediator_
Definition: repository.h:381
std::string fqrn() const
Definition: settings.h:367
void ExportKeychain()
Definition: repository.cc:500
void set_has_alt_catalog_path(const bool &has_alt_path)
Definition: manifest.h:108
void Init(const unsigned max_pool_handles, const bool use_system_proxy, perf::StatisticsTemplate statistics)
Definition: download.cc:1594
int llvl_
Definition: repository.h:363
upload::SpoolerDefinition::DriverType type() const
Definition: settings.h:298
std::string MakePath() const
Definition: hash.h:312
std::string MakeCanonicalPath(const std::string &path)
Definition: posix.cc:95
virtual ~Repository()
Definition: repository.cc:102
void set_reflog_hash(const shash::Any &checksum)
Definition: manifest.h:117
std::string url() const
Definition: settings.h:368
void ConstructSpoolers()
Definition: repository.cc:248
void Rollback()
void CreateKeychain()
Definition: repository.cc:275
void PushWhitelist()
Definition: repository.cc:448
std::vector< std::string > FindFilesBySuffix(const std::string &dir, const std::string &suffix)
Definition: posix.cc:1058
shash::Any meta_info() const
Definition: manifest.h:131
std::string tmp_dir() const
Definition: settings.h:369
static const unsigned kRequiredLayoutRevision
Definition: repository.h:276
SyncParameters * sync_parameters_
Definition: repository.h:380
class static __attribute__((visibility("default"))) Repository std::string GetFqrnFromUrl(const std::string &url)
Definition: repository.cc:228
bool HasDanglingMasterKeys() const
Definition: settings.cc:245