CernVM-FS  2.10.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
repository.cc
Go to the documentation of this file.
1 
5 #include "cvmfs_config.h"
6 #include "publish/repository.h"
7 
8 #include <cassert>
9 #include <cstddef>
10 #include <cstdlib>
11 
12 #include "catalog_mgr_ro.h"
13 #include "catalog_mgr_rw.h"
14 #include "download.h"
15 #include "gateway_util.h"
16 #include "hash.h"
17 #include "history_sqlite.h"
19 #include "logging.h"
20 #include "manifest.h"
21 #include "manifest_fetch.h"
22 #include "publish/except.h"
24 #include "publish/settings.h"
25 #include "reflog.h"
26 #include "signature.h"
27 #include "statistics.h"
28 #include "sync_mediator.h"
29 #include "sync_union_aufs.h"
30 #include "sync_union_overlayfs.h"
31 #include "sync_union_tarball.h"
32 #include "upload.h"
34 #include "util/pointer.h"
35 #include "whitelist.h"
36 
37 // TODO(jblomer): Remove Me
38 namespace swissknife {
39 class CommandTag {
40  static const std::string kHeadTag;
41  static const std::string kPreviousHeadTag;
42 };
43 const std::string CommandTag::kHeadTag = "trunk";
44 const std::string CommandTag::kPreviousHeadTag = "trunk-previous";
45 }
46 
47 namespace publish {
48 
50  : settings_("")
51  , statistics_(new perf::Statistics())
52  , signature_mgr_(new signature::SignatureManager())
53  , download_mgr_(NULL)
54  , simple_catalog_mgr_(NULL)
55  , whitelist_(NULL)
56  , reflog_(NULL)
57  , manifest_(NULL)
58  , history_(NULL)
59 {
61 }
62 
64  : settings_(settings)
65  , statistics_(new perf::Statistics())
66  , signature_mgr_(new signature::SignatureManager())
67  , download_mgr_(NULL)
68  , simple_catalog_mgr_(NULL)
69  , whitelist_(NULL)
70  , reflog_(NULL)
71  , manifest_(NULL)
72  , history_(NULL)
73 {
75 
76  int rvb;
77  std::string keys = JoinStrings(FindFilesBySuffix(
78  settings.keychain().keychain_dir(), ".pub"), ":");
79  rvb = signature_mgr_->LoadPublicRsaKeys(keys);
80  if (!rvb) {
82  delete signature_mgr_;
83  delete statistics_;
84  throw EPublish("cannot load public rsa key");
85  }
86 
87  if (!settings.cert_bundle().empty()) {
88  int rvi = setenv("X509_CERT_BUNDLE", settings.cert_bundle().c_str(),
89  1 /* overwrite */);
90  if (rvi != 0)
91  throw EPublish("cannot set X509_CERT_BUNDLE environment variable");
92  }
94  download_mgr_->Init(16,
97 
98  if (settings.proxy() != "") {
99  download_mgr_->SetProxyChain(settings.proxy(), "",
101  }
102 
103  try {
104  DownloadRootObjects(settings.url(), settings.fqrn(), settings.tmp_dir());
105  } catch (const EPublish& e) {
106  signature_mgr_->Fini();
107  download_mgr_->Fini();
108  delete signature_mgr_;
109  delete download_mgr_;
110  delete statistics_;
111  throw;
112  }
113 }
114 
116  if (signature_mgr_ != NULL) signature_mgr_->Fini();
117  if (download_mgr_ != NULL) download_mgr_->Fini();
118 
119  delete history_;
120  delete manifest_;
121  delete reflog_;
122  delete whitelist_;
123  delete signature_mgr_;
124  delete download_mgr_;
125  delete simple_catalog_mgr_;
126  delete statistics_;
127 }
128 
129 const history::History *Repository::history() const { return history_; }
130 
132  if (simple_catalog_mgr_ != NULL) return simple_catalog_mgr_;
133 
136  settings_.url(),
137  settings_.tmp_dir(),
139  statistics_,
140  true /* manage_catalog_files */);
142  return simple_catalog_mgr_;
143 }
144 
145 
147  const std::string &url, const std::string &fqrn, const std::string &tmp_dir)
148 {
149  delete whitelist_;
151  whitelist::Failures rv_whitelist = whitelist_->LoadUrl(url);
153  throw EPublish(std::string("cannot load whitelist [") +
154  whitelist::Code2Ascii(rv_whitelist) + "]");
155  }
156 
158  const uint64_t minimum_timestamp = 0;
159  const shash::Any *base_catalog = NULL;
160  manifest::Failures rv_manifest = manifest::Fetch(
161  url, fqrn, minimum_timestamp, base_catalog, signature_mgr_, download_mgr_,
162  &ensemble);
163  if (rv_manifest != manifest::kFailOk) throw EPublish("cannot load manifest");
164  delete manifest_;
165  manifest_ = new manifest::Manifest(*ensemble.manifest);
166 
167  std::string reflog_path;
168  FILE *reflog_fd =
169  CreateTempFile(tmp_dir + "/reflog", kPrivateFileMode, "w", &reflog_path);
170  std::string reflog_url = url + "/.cvmfsreflog";
171  // TODO(jblomer): verify reflog hash
172  // shash::Any reflog_hash(manifest_->GetHashAlgorithm());
173  download::JobInfo download_reflog(
174  &reflog_url,
175  false /* compressed */,
176  false /* probe hosts */,
177  reflog_fd,
178  NULL);
179  download::Failures rv_dl = download_mgr_->Fetch(&download_reflog);
180  fclose(reflog_fd);
181  if (rv_dl == download::kFailOk) {
182  delete reflog_;
183  reflog_ = manifest::Reflog::Open(reflog_path);
184  if (reflog_ == NULL) throw EPublish("cannot open reflog");
186  } else {
187  if (!download_reflog.IsFileNotFound()) {
188  throw EPublish(std::string("cannot load reflog [") +
189  download::Code2Ascii(rv_dl) + "]");
190  }
191  assert(reflog_ == NULL);
192  }
193 
194  std::string tags_path;
195  FILE *tags_fd =
196  CreateTempFile(tmp_dir + "/tags", kPrivateFileMode, "w", &tags_path);
197  if (!manifest_->history().IsNull()) {
198  std::string tags_url = url + "/data/" + manifest_->history().MakePath();
199  shash::Any tags_hash(manifest_->history());
200  download::JobInfo download_tags(
201  &tags_url,
202  true /* compressed */,
203  true /* probe hosts */,
204  tags_fd,
205  &tags_hash);
206  rv_dl = download_mgr_->Fetch(&download_tags);
207  fclose(tags_fd);
208  if (rv_dl != download::kFailOk) throw EPublish("cannot load tag database");
209  delete history_;
211  if (history_ == NULL) throw EPublish("cannot open tag database");
212  } else {
213  fclose(tags_fd);
214  delete history_;
215  history_ = history::SqliteHistory::Create(tags_path, fqrn);
216  if (history_ == NULL) throw EPublish("cannot create tag database");
217  }
219 
220  if (!manifest_->meta_info().IsNull()) {
221  shash::Any info_hash(manifest_->meta_info());
222  std::string info_url = url + "/data/" + info_hash.MakePath();
223  download::JobInfo download_info(
224  &info_url,
225  true /* compressed */,
226  true /* probe_hosts */,
227  &info_hash);
228  download::Failures rv_info = download_mgr_->Fetch(&download_info);
229  if (rv_info != download::kFailOk) {
230  throw EPublish(std::string("cannot load meta info [") +
231  download::Code2Ascii(rv_info) + "]");
232  }
233  meta_info_ = std::string(download_info.destination_mem.data,
234  download_info.destination_mem.pos);
235  } else {
236  meta_info_ = "n/a";
237  }
238 }
239 
240 
241 std::string Repository::GetFqrnFromUrl(const std::string &url) {
242  return GetFileName(MakeCanonicalPath(url));
243 }
244 
245 
247  std::string url = settings_.url() + "/.cvmfs_master_replica";
248  download::JobInfo head(&url, false /* probe_hosts */);
249  download::Failures retval = download_mgr_->Fetch(&head);
250  if (retval == download::kFailOk) return true;
251  if (head.IsFileNotFound()) return false;
252 
253  throw EPublish(std::string("error looking for .cvmfs_master_replica [") +
254  download::Code2Ascii(retval) + "]");
255 }
256 
257 
258 //------------------------------------------------------------------------------
259 
260 
262  if ((spooler_files_ != NULL) && (spooler_catalogs_ != NULL))
263  return;
264  assert((spooler_files_ == NULL) && (spooler_catalogs_ == NULL));
265 
267  settings_.storage().GetLocator(),
268  settings_.transaction().hash_algorithm(),
269  settings_.transaction().compression_algorithm());
270  sd.session_token_file =
271  settings_.transaction().spool_area().gw_session_token();
272  sd.key_file = settings_.keychain().gw_key_path();
273 
275  upload::Spooler::Construct(sd, statistics_publish_.weak_ref());
276  if (spooler_files_ == NULL)
277  throw EPublish("could not initialize file spooler");
278 
279  upload::SpoolerDefinition sd_catalogs(sd.Dup2DefaultCompression());
281  upload::Spooler::Construct(sd_catalogs, statistics_publish_.weak_ref());
282  if (spooler_catalogs_ == NULL) {
283  delete spooler_files_;
284  throw EPublish("could not initialize catalog spooler");
285  }
286 }
287 
288 
291  throw EPublish("dangling master key pair");
292  }
294  throw EPublish("dangling repository keys");
295  }
300 
302  std::string whitelist_str = whitelist::Whitelist::CreateString(
303  settings_.fqrn(),
304  settings_.whitelist_validity_days(),
305  settings_.transaction().hash_algorithm(),
307  whitelist::Failures rv_wl = whitelist_->LoadMem(whitelist_str);
308  if (rv_wl != whitelist::kFailOk)
309  throw EPublish("whitelist generation failed");
310 }
311 
312 
314  // Reflog
315  const std::string reflog_path = CreateTempPath(
316  settings_.transaction().spool_area().tmp_dir() + "/cvmfs_reflog", 0600);
318  if (reflog_ == NULL) throw EPublish("could not create reflog");
319  reflog_->TakeDatabaseFileOwnership();
320 
321  // Root file catalog and initial manifest
323  settings_.transaction().spool_area().tmp_dir(),
324  settings_.transaction().is_volatile(),
325  settings_.transaction().voms_authz(),
327  spooler_catalogs_->WaitForUpload();
328  if (manifest_ == NULL)
329  throw EPublish("could not create initial file catalog");
330  reflog_->AddCatalog(manifest_->catalog_hash());
331 
333  manifest_->set_ttl(settings_.transaction().ttl_second());
334  const bool needs_bootstrap_shortcuts =
335  !settings_.transaction().voms_authz().empty();
336  manifest_->set_has_alt_catalog_path(needs_bootstrap_shortcuts);
338  settings_.transaction().is_garbage_collectable());
339 
340  // Tag database
341  const std::string tags_path = CreateTempPath(
342  settings_.transaction().spool_area().tmp_dir() + "/cvmfs_tags", 0600);
344  if (history_ == NULL) throw EPublish("could not create tag database");
345  history_->TakeDatabaseFileOwnership();
346  history::History::Tag tag_trunk(
347  "trunk",
350  "empty repository", "" /* branch */);
351  history_->Insert(tag_trunk);
352 
353  // Meta information, TODO(jblomer)
354  meta_info_ = "{}";
355 }
356 
357 
360  if (!spooler_files_->Create())
361  throw EPublish("could not initialize repository storage area");
362 }
363 
364 
366  upload::Spooler::CallbackPtr callback =
367  spooler_files_->RegisterListener(&Publisher::OnProcessCertificate, this);
368  spooler_files_->ProcessCertificate(
370  spooler_files_->WaitForUpload();
371  spooler_files_->UnregisterListener(callback);
372 }
373 
374 
376  assert(history_ != NULL);
378  const string history_path = history_->filename();
380  delete history_;
381 
382  upload::Spooler::CallbackPtr callback =
383  spooler_files_->RegisterListener(&Publisher::OnProcessHistory, this);
384  spooler_files_->ProcessHistory(history_path);
385  spooler_files_->WaitForUpload();
386  spooler_files_->UnregisterListener(callback);
387 
389  assert(history_ != NULL);
391 }
392 
393 
395  upload::Spooler::CallbackPtr callback =
396  spooler_files_->RegisterListener(&Publisher::OnProcessMetainfo, this);
397  spooler_files_->ProcessMetainfo(new StringIngestionSource(meta_info_));
398  spooler_files_->WaitForUpload();
399  spooler_files_->UnregisterListener(callback);
400 }
401 
402 
404  std::string signed_manifest = manifest_->ExportString();
405  shash::Any manifest_hash(settings_.transaction().hash_algorithm());
407  reinterpret_cast<const unsigned char *>(signed_manifest.data()),
408  signed_manifest.length(), &manifest_hash);
409  signed_manifest += "--\n" + manifest_hash.ToString() + "\n";
410  unsigned char *signature;
411  unsigned sig_size;
412  bool rvb = signature_mgr_->Sign(
413  reinterpret_cast<const unsigned char *>(manifest_hash.ToString().data()),
414  manifest_hash.GetHexSize(), &signature, &sig_size);
415  if (!rvb) throw EPublish("cannot sign manifest");
416  signed_manifest += std::string(reinterpret_cast<char *>(signature), sig_size);
417  free(signature);
418 
419  // Create alternative bootstrapping symlinks for VOMS secured repos
421  rvb =
422  spooler_files_->PlaceBootstrappingShortcut(manifest_->certificate()) &&
423  spooler_files_->PlaceBootstrappingShortcut(manifest_->catalog_hash()) &&
424  (manifest_->history().IsNull() ||
425  spooler_files_->PlaceBootstrappingShortcut(manifest_->history())) &&
426  (manifest_->meta_info().IsNull() ||
427  spooler_files_->PlaceBootstrappingShortcut(manifest_->meta_info()));
428  if (!rvb) EPublish("cannot place VOMS bootstrapping symlinks");
429  }
430 
431  upload::Spooler::CallbackPtr callback =
432  spooler_files_->RegisterListener(&Publisher::OnUploadManifest, this);
433  spooler_files_->Upload(".cvmfspublished",
434  new StringIngestionSource(signed_manifest));
435  spooler_files_->WaitForUpload();
436  spooler_files_->UnregisterListener(callback);
437 }
438 
439 
441  const string reflog_path = reflog_->database_file();
443  delete reflog_;
444 
445  shash::Any hash_reflog(settings_.transaction().hash_algorithm());
446  manifest::Reflog::HashDatabase(reflog_path, &hash_reflog);
447 
448  upload::Spooler::CallbackPtr callback =
449  spooler_files_->RegisterListener(&Publisher::OnUploadReflog, this);
450  spooler_files_->UploadReflog(reflog_path);
451  spooler_files_->WaitForUpload();
452  spooler_files_->UnregisterListener(callback);
453 
454  manifest_->set_reflog_hash(hash_reflog);
455 
456  reflog_ = manifest::Reflog::Open(reflog_path);
457  assert(reflog_ != NULL);
459 }
460 
461 
463  // TODO(jblomer): PKCS7 handling
464  upload::Spooler::CallbackPtr callback =
465  spooler_files_->RegisterListener(&Publisher::OnUploadWhitelist, this);
466  spooler_files_->Upload(".cvmfswhitelist",
468  spooler_files_->WaitForUpload();
469  spooler_files_->UnregisterListener(callback);
470 }
471 
472 
473 Publisher *Publisher::Create(const SettingsPublisher &settings) {
474  UniquePtr<Publisher> publisher(new Publisher());
475 
476  publisher->settings_ = settings;
477  if (settings.is_silent())
478  publisher->llvl_ = kLogNone;
479  publisher->signature_mgr_ = new signature::SignatureManager();
480  publisher->signature_mgr_->Init();
481 
482  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout | kLogNoLinebreak,
483  "Creating Key Chain... ");
484  publisher->CreateKeychain();
485  publisher->ExportKeychain();
486  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout, "done");
487 
488  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout | kLogNoLinebreak,
489  "Creating Backend Storage... ");
490  publisher->CreateStorage();
491  publisher->PushWhitelist();
492  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout, "done");
493 
494  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout | kLogNoLinebreak,
495  "Creating Initial Repository... ");
496  publisher->InitSpoolArea();
497  publisher->CreateRootObjects();
498  publisher->PushHistory();
499  publisher->PushCertificate();
500  publisher->PushMetainfo();
501  publisher->PushReflog();
502  publisher->PushManifest();
503  // TODO(jblomer): meta-info
504 
505  // Re-create from empty repository in order to properly initialize
506  // parent Repository object
507  publisher = new Publisher(settings);
508 
509  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout, "done");
510 
511  return publisher.Release();
512 }
513 
516 
517  bool rvb;
520  if (!rvb) throw EPublish("cannot export public master key");
523  if (!rvb) throw EPublish("cannot export certificate");
524 
527  if (!rvb) throw EPublish("cannot export private certificate key");
530  if (!rvb) throw EPublish("cannot export private master key");
531 
532  int rvi;
533  rvi = chown(settings_.keychain().master_public_key_path().c_str(),
534  settings_.owner_uid(), settings_.owner_gid());
535  if (rvi != 0) throw EPublish("cannot set key file ownership");
536  rvi = chown(settings_.keychain().certificate_path().c_str(),
537  settings_.owner_uid(), settings_.owner_gid());
538  if (rvi != 0) throw EPublish("cannot set key file ownership");
539  rvi = chown(settings_.keychain().private_key_path().c_str(),
540  settings_.owner_uid(), settings_.owner_gid());
541  if (rvi != 0) throw EPublish("cannot set key file ownership");
542  rvi = chown(settings_.keychain().master_private_key_path().c_str(),
543  settings_.owner_uid(), settings_.owner_gid());
544  if (rvi != 0) throw EPublish("cannot set key file ownership");
545 }
546 
548  if (result.return_code != 0) {
549  throw EPublish("cannot write certificate to storage");
550  }
553 }
554 
556  if (result.return_code != 0) {
557  throw EPublish("cannot write tag database to storage");
558  }
561 }
562 
564  if (result.return_code != 0) {
565  throw EPublish("cannot write repository meta info to storage");
566  }
569 }
570 
572  if (result.return_code != 0) {
573  throw EPublish("cannot write manifest to storage");
574  }
575 }
576 
578  if (result.return_code != 0) {
579  throw EPublish("cannot write reflog to storage");
580  }
581 }
582 
584  if (result.return_code != 0) {
585  throw EPublish("cannot write whitelist to storage");
586  }
587 }
588 
589 void Publisher::CreateDirectoryAsOwner(const std::string &path, int mode)
590 {
591  bool rvb = MkdirDeep(path, kPrivateDirMode);
592  if (!rvb) throw EPublish("cannot create directory " + path);
593  int rvi = chown(path.c_str(), settings_.owner_uid(), settings_.owner_gid());
594  if (rvi != 0) throw EPublish("cannot set ownership on directory " + path);
595 }
596 
598  CreateDirectoryAsOwner(settings_.transaction().spool_area().workspace(),
600  CreateDirectoryAsOwner(settings_.transaction().spool_area().tmp_dir(),
602  CreateDirectoryAsOwner(settings_.transaction().spool_area().cache_dir(),
604  CreateDirectoryAsOwner(settings_.transaction().spool_area().scratch_dir(),
606  CreateDirectoryAsOwner(settings_.transaction().spool_area().ovl_work_dir(),
608 
609  // On a managed node, the mount points are already mounted
610  if (!DirectoryExists(settings_.transaction().spool_area().readonly_mnt())) {
611  CreateDirectoryAsOwner(settings_.transaction().spool_area().readonly_mnt(),
613  }
614  if (!DirectoryExists(settings_.transaction().spool_area().union_mnt())) {
616  settings_.transaction().spool_area().union_mnt(), kPrivateDirMode);
617  }
618 }
619 
621  : settings_("invalid.cvmfs.io")
622  , statistics_publish_(new perf::StatisticsTemplate("publish", statistics_))
623  , llvl_(kLogNormal)
624  , in_transaction_(false)
625  , spooler_files_(NULL)
626  , spooler_catalogs_(NULL)
627  , catalog_mgr_(NULL)
628  , sync_parameters_(NULL)
629  , sync_mediator_(NULL)
630  , sync_union_(NULL)
631 {
632 }
633 
635  : Repository(SettingsRepository(settings))
636  , settings_(settings)
637  , statistics_publish_(new perf::StatisticsTemplate("publish", statistics_))
638  , llvl_(settings.is_silent() ? kLogNone : kLogNormal)
639  , in_transaction_(false)
640  , spooler_files_(NULL)
641  , spooler_catalogs_(NULL)
642  , catalog_mgr_(NULL)
643  , sync_parameters_(NULL)
644  , sync_mediator_(NULL)
645  , sync_union_(NULL)
646 {
648  unsigned layout_revision = settings.transaction().layout_revision();
649  throw EPublish(
650  "This repository uses layout revision " + StringifyInt(layout_revision)
651  + ".\n"
652  "This version of CernVM-FS requires layout revision " + StringifyInt(
653  kRequiredLayoutRevision) + ", which is\n"
654  "incompatible to " + StringifyInt(layout_revision) + ".\n\n"
655  "Please run `cvmfs_server migrate` to update your repository before "
656  "proceeding.",
658  }
659 
660  CreateDirectoryAsOwner(settings_.transaction().spool_area().tmp_dir(),
662 
663  if (settings.storage().type() == upload::SpoolerDefinition::Gateway) {
664  if (!settings.keychain().HasGatewayKey()) {
665  throw EPublish("gateway key missing: " +
666  settings.keychain().gw_key_path());
667  }
669  if (!gw_key_.IsValid()) {
670  throw EPublish("cannot read gateway key: " +
671  settings.keychain().gw_key_path());
672  }
673  }
674 
675  if ((settings.storage().type() != upload::SpoolerDefinition::Gateway) &&
676  !settings.transaction().in_enter_session())
677  {
679  settings.keychain().certificate_path());
680  if (!rvb)
681  throw EPublish("cannot load certificate, thus cannot commit changes");
683  settings.keychain().private_key_path(), "");
684  if (!rvb)
685  throw EPublish("cannot load private key, thus cannot commit changes");
686  // The private master key might be on a key card instead
687  if (FileExists(settings.keychain().master_private_key_path())) {
689  settings.keychain().master_private_key_path());
690  if (!rvb) throw EPublish("cannot load private master key");
691  }
692  if (!signature_mgr_->KeysMatch())
693  throw EPublish("corrupted keychain");
694  }
695 
696  if (settings.is_managed())
697  managed_node_ = new ManagedNode(this);
699  if (in_transaction_)
701 }
702 
704  delete sync_union_;
705  delete sync_mediator_;
706  delete sync_parameters_;
707  delete catalog_mgr_;
708  delete spooler_catalogs_;
709  delete spooler_files_;
710 }
711 
712 
715 
716  if (catalog_mgr_ == NULL) {
718  settings_.transaction().base_hash(),
719  settings_.url(),
720  settings_.transaction().spool_area().tmp_dir(),
723  settings_.transaction().enforce_limits(),
724  settings_.transaction().limit_nested_catalog_kentries(),
725  settings_.transaction().limit_root_catalog_kentries(),
726  settings_.transaction().limit_file_size_mb(),
727  statistics_,
728  settings_.transaction().use_catalog_autobalance(),
729  settings_.transaction().autobalance_max_weight(),
730  settings_.transaction().autobalance_min_weight());
731  catalog_mgr_->Init();
732  }
733 
734  if (sync_parameters_ == NULL) {
735  SyncParameters *p = new SyncParameters();
736  p->spooler = spooler_files_;
737  p->repo_name = settings_.fqrn();
738  p->dir_union = settings_.transaction().spool_area().union_mnt();
739  p->dir_scratch = settings_.transaction().spool_area().scratch_dir();
740  p->dir_rdonly = settings_.transaction().spool_area().readonly_mnt();
741  p->dir_temp = settings_.transaction().spool_area().tmp_dir();
742  p->base_hash = settings_.transaction().base_hash();
743  p->stratum0 = settings_.url();
744  // p->manifest_path = SHOULD NOT BE NEEDED
745  // p->spooler_definition = SHOULD NOT BE NEEDED;
746  // p->union_fs_type = SHOULD NOT BE NEEDED
747  p->print_changeset = settings_.transaction().print_changeset();
748  p->dry_run = settings_.transaction().dry_run();
749  sync_parameters_ = p;
750  }
751 
752  if (sync_mediator_ == NULL) {
755  }
756 
757  if (sync_union_ == NULL) {
758  switch (settings_.transaction().union_fs()) {
759  case kUnionFsAufs:
762  settings_.transaction().spool_area().readonly_mnt(),
763  settings_.transaction().spool_area().union_mnt(),
764  settings_.transaction().spool_area().scratch_dir());
765  break;
766  case kUnionFsOverlay:
769  settings_.transaction().spool_area().readonly_mnt(),
770  settings_.transaction().spool_area().union_mnt(),
771  settings_.transaction().spool_area().scratch_dir());
772  break;
773  case kUnionFsTarball:
776  settings_.transaction().spool_area().readonly_mnt(),
777  // TODO(jblomer): get from settings
778  "tar_file",
779  "base_directory",
780  "to_delete",
781  false /* create_catalog */);
782  break;
783  default:
784  throw EPublish("unknown union file system");
785  }
786  bool rvb = sync_union_->Initialize();
787  if (!rvb) {
788  delete sync_union_;
789  sync_union_ = NULL;
790  throw EPublish("cannot initialize union file system engine");
791  }
792  }
793 }
794 
796  if (settings_.transaction().dry_run()) {
797  SyncImpl();
798  return;
799  }
800 
801  const std::string publishing_lock =
802  settings_.transaction().spool_area().publishing_lock();
803 
804  ServerLockFile::Acquire(publishing_lock, false /* ignore_stale */);
805  try {
806  SyncImpl();
807  ServerLockFile::Release(publishing_lock);
808  } catch (...) {
809  ServerLockFile::Release(publishing_lock);
810  throw;
811  }
812 }
813 
815  std::string session_dir = Env::GetEnterSessionDir();
816  std::string session_pid_tmp = session_dir + "/session_pid";
817  std::string session_pid;
818  int fd_session_pid = open(session_pid_tmp.c_str(), O_RDONLY);
819  if (fd_session_pid < 0) throw EPublish("Session pid cannot be retrieved");
820  SafeReadToString(fd_session_pid, &session_pid);
821 
822  pid_t pid_child = String2Uint64(session_pid);
823  kill(pid_child, SIGUSR1);
824 }
825 
828 
830  bool rvb = sync_mediator_->Commit(manifest_);
831  if (!rvb) throw EPublish("cannot write change set to storage");
832 
833  if (!settings_.transaction().dry_run()) {
834  spooler_files_->WaitForUpload();
835  spooler_catalogs_->WaitForUpload();
836  spooler_files_->FinalizeSession(false /* commit */);
837 
838  const std::string old_root_hash =
839  settings_.transaction().base_hash().ToString(true /* with_suffix */);
840  const std::string new_root_hash =
841  manifest_->catalog_hash().ToString(true /* with_suffix */);
842  rvb = spooler_catalogs_->FinalizeSession(true /* commit */,
843  old_root_hash, new_root_hash,
844  /* TODO(jblomer) */ sync_parameters_->repo_tag);
845  if (!rvb)
846  throw EPublish("failed to commit transaction");
847 
848  // Reset to the new catalog root hash
849  settings_.GetTransaction()->SetBaseHash(manifest_->catalog_hash());
850  // TODO(jblomer): think about how to deal with the scratch area at
851  // this point
852  // WipeScratchArea();
853  }
854 
855  delete sync_union_;
856  delete sync_mediator_;
857  delete sync_parameters_;
858  delete catalog_mgr_;
859  sync_union_ = NULL;
860  sync_mediator_ = NULL;
861  sync_parameters_ = NULL;
862  catalog_mgr_ = NULL;
863 
864  if (!settings_.transaction().dry_run()) {
865  LogCvmfs(kLogCvmfs, kLogStdout, "New revision: %d", manifest_->revision());
867  }
868 }
869 
871  if (!in_transaction_) throw EPublish("cannot publish outside transaction");
872 
873  PushReflog();
874  PushManifest();
875  in_transaction_ = false;
876 }
877 
878 
879 void Publisher::MarkReplicatible(bool value) {
881 
882  if (value) {
883  spooler_files_->Upload("/dev/null", "/.cvmfs_master_replica");
884  } else {
885  spooler_files_->RemoveAsync("/.cvmfs_master_replica");
886  }
887  spooler_files_->WaitForUpload();
888  if (spooler_files_->GetNumberOfErrors() > 0)
889  throw EPublish("cannot set replication mode");
890 }
891 
893 void Publisher::Migrate() {}
895 void Publisher::Rollback() {}
897 
898 void Publisher::Transaction() {
900  session()->SetKeepAlive(true);
901 }
902 
903 //------------------------------------------------------------------------------
904 
905 
907 }
908 
909 
911 }
912 
913 } // namespace publish
bool Commit(manifest::Manifest *manifest)
int return_code
the return value of the spooler operation
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
void CheckTransactionStatus()
void set_repository_name(const std::string &repository_name)
Definition: manifest.h:93
void TakeDatabaseFileOwnership()
Definition: reflog.cc:307
const SettingsRepository settings_
Definition: repository.h:136
std::string database_file() const
Definition: reflog.cc:337
void CreateRootObjects()
Definition: repository.cc:313
manifest::Reflog * reflog_
Definition: repository.h:146
bool IsNull() const
Definition: hash.h:382
download::DownloadManager * download_mgr_
Definition: repository.h:140
bool HasGatewayKey() const
Definition: settings.cc:272
std::string stratum0
bool AddHistory(const shash::Any &history)
Definition: reflog.cc:134
void ConstructSyncManagers()
Definition: repository.cc:713
void OnUploadReflog(const upload::SpoolerResult &result)
Definition: repository.cc:577
catalog::SimpleCatalogManager * GetSimpleCatalogManager()
Definition: repository.cc:131
UniquePtr< ManagedNode > managed_node_
Definition: repository.h:383
Session * session() const
Definition: repository.h:317
std::string GetFileName(const std::string &path)
Definition: posix.cc:162
Repository(const SettingsRepository &settings)
Definition: repository.cc:63
const int kPrivateDirMode
Definition: posix.h:35
T * weak_ref() const
Definition: pointer.h:42
void Ingest()
Definition: repository.cc:892
UniquePtr< perf::StatisticsTemplate > statistics_publish_
Definition: repository.h:367
static Publisher * Create(const SettingsPublisher &settings)
std::string keychain_dir() const
Definition: settings.h:325
std::string GetCertificate() const
Definition: signature.cc:370
static const std::string kPreviousHeadTag
Definition: repository.cc:41
Failures LoadUrl(const std::string &base_url)
Definition: whitelist.cc:216
std::string gw_key_path() const
Definition: settings.h:334
FILE * CreateTempFile(const std::string &path_prefix, const int mode, const char *open_flags, std::string *final_path)
Definition: posix.cc:1030
string JoinStrings(const vector< string > &strings, const string &joint)
Definition: string.cc:318
std::string ToString(const bool with_suffix=false) const
Definition: hash.h:248
void OnProcessCertificate(const upload::SpoolerResult &result)
Definition: repository.cc:547
virtual bool Initialize()
Definition: sync_union.cc:24
void set_certificate(const shash::Any &certificate)
Definition: manifest.h:87
const history::History * history() const
bool IsFileNotFound()
Definition: download.cc:383
void CreateDirectoryAsOwner(const std::string &path, int mode)
Definition: repository.cc:589
void PushCertificate()
Definition: repository.cc:365
unsigned layout_revision() const
Definition: settings.h:188
void SetProxyChain(const std::string &proxy_list, const std::string &fallback_proxy_list, const ProxySetModes set_mode)
Definition: download.cc:2407
void OnUploadWhitelist(const upload::SpoolerResult &result)
Definition: repository.cc:583
void CreateStorage()
Definition: repository.cc:358
bool IsMasterReplica()
Definition: repository.cc:246
std::string private_key_path() const
Definition: settings.h:332
const SettingsPublisher & settings() const
Definition: repository.h:314
perf::Statistics * statistics_
Definition: repository.h:138
Status status() const
Definition: whitelist.h:103
std::string CreateTempPath(const std::string &path_prefix, const int mode)
Definition: posix.cc:1059
void set_meta_info(const shash::Any &meta_info)
Definition: manifest.h:111
std::string GetPrivateKey()
Definition: signature.cc:386
std::string ExportString() const
Definition: manifest.cc:174
virtual ~Replica()
Definition: repository.cc:910
std::string repo_name
assert((mem||(size==0))&&"Out Of Memory")
bool has_alt_catalog_path() const
Definition: manifest.h:130
upload::Spooler * spooler_catalogs_
Definition: repository.h:386
gateway::GatewayKey gw_key_
Definition: repository.h:374
bool SafeWriteToFile(const std::string &content, const std::string &path, int mode)
Definition: posix.cc:2027
bool in_transaction_
Definition: repository.h:372
void PushMetainfo()
Definition: repository.cc:394
bool LoadPrivateKeyPath(const std::string &file_pem, const std::string &password)
Definition: signature.cc:171
class __attribute__((visibility("default"))) Publisher ManagedNode(Publisher *p)
Definition: repository.h:173
void PushReflog()
Definition: repository.cc:440
uint64_t catalog_size() const
Definition: manifest.h:125
void ExitShell()
Definition: repository.cc:814
bool AddCatalog(const shash::Any &catalog)
Definition: reflog.cc:128
static Reflog * Create(const std::string &database_path, const std::string &repo_name)
Definition: reflog.cc:32
std::string cert_bundle() const
Definition: settings.h:372
catalog::WritableCatalogManager * catalog_mgr_
Definition: repository.h:387
std::string dir_scratch
bool HasMasterKeys() const
Definition: settings.cc:253
void GenerateCertificate(const std::string &cn)
Definition: signature.cc:452
uint64_t revision() const
Definition: manifest.h:121
static Reflog * Open(const std::string &database_path)
Definition: reflog.cc:17
static SqliteHistory * OpenWritable(const std::string &file_name)
std::string master_public_key_path() const
Definition: settings.h:329
bool FileExists(const std::string &path)
Definition: posix.cc:816
std::string GetActivePubkeys() const
Definition: signature.cc:357
std::string dir_temp
const int kPrivateFileMode
Definition: posix.h:34
void OnProcessMetainfo(const upload::SpoolerResult &result)
Definition: repository.cc:563
void Init(const unsigned max_pool_handles, const perf::StatisticsTemplate &statistics)
Definition: download.cc:1600
const char * Code2Ascii(const Failures error)
Definition: download.h:88
static manifest::Manifest * CreateRepository(const std::string &dir_temp, const bool volatile_content, const std::string &voms_authz, upload::Spooler *spooler)
bool HasRepositoryKeys() const
Definition: settings.cc:267
std::string proxy() const
Definition: settings.h:370
std::string meta_info_
Definition: repository.h:150
void PushHistory()
Definition: repository.cc:375
const SettingsKeychain & keychain() const
Definition: settings.h:434
publish::SyncUnion * sync_union_
Definition: repository.h:390
uint64_t publish_timestamp() const
Definition: manifest.h:128
bool AddMetainfo(const shash::Any &metainfo)
Definition: reflog.cc:140
void Transaction()
const int kDefaultDirMode
Definition: posix.h:32
bool Sign(const unsigned char *buffer, const unsigned buffer_size, unsigned char **signature, unsigned *signature_size)
Definition: signature.cc:730
void OnUploadManifest(const upload::SpoolerResult &result)
Definition: repository.cc:571
std::string dir_rdonly
signature::SignatureManager * signature_mgr_
Definition: repository.h:139
void SetKeepAlive(bool value)
Failures Fetch(const std::string &base_url, const std::string &repository_name, const uint64_t minimum_timestamp, const shash::Any *base_catalog, signature::SignatureManager *signature_manager, download::DownloadManager *download_manager, ManifestEnsemble *ensemble)
void UpdateMetaInfo()
Definition: repository.cc:896
std::string master_private_key_path() const
Definition: settings.h:326
void DownloadRootObjects(const std::string &url, const std::string &fqrn, const std::string &tmp_dir)
Definition: repository.cc:146
shash::Any base_hash
static const std::string kHeadTag
Definition: repository.cc:40
shash::Any certificate() const
Definition: manifest.h:126
virtual ~Publisher()
upload::Spooler * spooler
bool MkdirDeep(const std::string &path, const mode_t mode, bool verify_writable)
Definition: posix.cc:871
bool LoadCertificatePath(const std::string &file_pem)
Definition: signature.cc:217
const SettingsTransaction & transaction() const
Definition: settings.h:433
bool IsValid() const
Definition: gateway_util.h:17
const char * Code2Ascii(const Failures error)
Definition: whitelist.h:49
std::string dir_union
void Sync()
Definition: repository.cc:795
void InitSpoolArea()
Definition: repository.cc:597
static void HashDatabase(const std::string &database_path, shash::Any *hash_reflog)
Definition: reflog.cc:322
void PushManifest()
Definition: repository.cc:403
shash::Any catalog_hash() const
Definition: manifest.h:124
string StringifyInt(const int64_t value)
Definition: string.cc:78
std::string certificate_path() const
Definition: settings.h:333
catalog::SimpleCatalogManager * simple_catalog_mgr_
Definition: repository.h:144
history::SqliteHistory * history_
Definition: repository.h:148
void Resign()
Definition: repository.cc:894
bool is_managed() const
Definition: settings.h:430
void Publish()
const SettingsStorage & storage() const
Definition: settings.h:432
void DropDatabaseFileOwnership()
Definition: reflog.cc:313
bool SetPreviousRevision(const shash::Any &history_hash)
bool LoadPrivateMasterKeyPath(const std::string &file_pem)
Definition: signature.cc:153
bool DirectoryExists(const std::string &path)
Definition: posix.cc:838
void set_history(const shash::Any &history_db)
Definition: manifest.h:90
void set_garbage_collectability(const bool garbage_collectable)
Definition: manifest.h:105
bool AddCertificate(const shash::Any &certificate)
Definition: reflog.cc:121
std::string filename() const
bool SafeReadToString(int fd, std::string *final_result)
Definition: posix.cc:2011
manifest::Manifest * manifest_
Definition: repository.h:147
void OnProcessHistory(const upload::SpoolerResult &result)
Definition: repository.cc:555
void set_ttl(const uint32_t ttl)
Definition: manifest.h:85
upload::Spooler * spooler_files_
Definition: repository.h:385
void HashMem(const unsigned char *buffer, const unsigned buffer_size, Any *any_digest)
Definition: hash.cc:255
void SyncImpl()
Definition: repository.cc:826
void MarkReplicatible(bool value)
Definition: repository.cc:879
Publisher(const SettingsPublisher &settings)
Definition: repository.cc:634
class __attribute__((visibility("default"))) Replica Replica(const SettingsReplica &settings)
Definition: repository.cc:906
const SettingsKeychain & keychain() const
Definition: settings.h:374
std::string ExportString() const
Definition: whitelist.cc:77
virtual void Traverse()=0
uint64_t String2Uint64(const string &value)
Definition: string.cc:228
std::string GetPrivateMasterKey()
Definition: signature.cc:402
void Migrate(InodeTracker *old_tracker, glue::InodeTracker *new_tracker)
Definition: compat.cc:83
Failures Fetch(JobInfo *info)
Definition: download.cc:1716
struct download::JobInfo::@3 destination_mem
shash::Any history() const
Definition: manifest.h:127
static std::string CreateString(const std::string &fqrn, int validity_days, shash::Algorithms hash_algorithm, signature::SignatureManager *signature_manager)
Definition: whitelist.cc:47
bool LoadPublicRsaKeys(const std::string &path_list)
Definition: signature.cc:277
whitelist::Whitelist * whitelist_
Definition: repository.h:145
GatewayKey ReadGatewayKey(const std::string &key_file_name)
Definition: gateway_util.cc:28
static SqliteHistory * Create(const std::string &file_name, const std::string &fqrn)
bool HasDanglingRepositoryKeys() const
Definition: settings.cc:259
Failures LoadMem(const std::string &whitelist)
Definition: whitelist.cc:196
RepositoryTag repo_tag
SyncMediator * sync_mediator_
Definition: repository.h:389
std::string fqrn() const
Definition: settings.h:368
void ExportKeychain()
Definition: repository.cc:514
void set_has_alt_catalog_path(const bool &has_alt_path)
Definition: manifest.h:108
int llvl_
Definition: repository.h:371
upload::SpoolerDefinition::DriverType type() const
Definition: settings.h:294
std::string MakePath() const
Definition: hash.h:315
std::string MakeCanonicalPath(const std::string &path)
Definition: posix.cc:96
bool in_enter_session() const
Definition: settings.h:189
virtual ~Repository()
Definition: repository.cc:115
void set_reflog_hash(const shash::Any &checksum)
Definition: manifest.h:117
std::string url() const
Definition: settings.h:369
void ConstructSpoolers()
Definition: repository.cc:261
void Rollback()
void CreateKeychain()
Definition: repository.cc:289
void PushWhitelist()
Definition: repository.cc:462
std::vector< std::string > FindFilesBySuffix(const std::string &dir, const std::string &suffix)
Definition: posix.cc:1149
shash::Any meta_info() const
Definition: manifest.h:131
std::string tmp_dir() const
Definition: settings.h:371
static const unsigned kRequiredLayoutRevision
Definition: repository.h:276
SyncParameters * sync_parameters_
Definition: repository.h:388
class static __attribute__((visibility("default"))) Repository std::string GetFqrnFromUrl(const std::string &url)
Definition: repository.cc:241
bool HasDanglingMasterKeys() const
Definition: settings.cc:245