CernVM-FS  2.12.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
repository.cc
Go to the documentation of this file.
1 
5 #include "cvmfs_config.h"
6 #include "publish/repository.h"
7 
8 #include <cassert>
9 #include <cstddef>
10 #include <cstdlib>
11 
12 #include "catalog_mgr_ro.h"
13 #include "catalog_mgr_rw.h"
14 #include "crypto/hash.h"
15 #include "crypto/signature.h"
16 #include "gateway_util.h"
17 #include "history_sqlite.h"
19 #include "manifest.h"
20 #include "manifest_fetch.h"
21 #include "network/download.h"
22 #include "publish/except.h"
24 #include "publish/settings.h"
25 #include "reflog.h"
26 #include "statistics.h"
27 #include "sync_mediator.h"
28 #include "sync_union_aufs.h"
29 #include "sync_union_overlayfs.h"
30 #include "sync_union_tarball.h"
31 #include "upload.h"
33 #include "util/logging.h"
34 #include "util/pointer.h"
35 #include "whitelist.h"
36 
37 // TODO(jblomer): Remove Me
38 namespace swissknife {
39 class CommandTag {
40  static const std::string kHeadTag;
41  static const std::string kPreviousHeadTag;
42 };
43 const std::string CommandTag::kHeadTag = "trunk";
44 const std::string CommandTag::kPreviousHeadTag = "trunk-previous";
45 }
46 
47 namespace publish {
48 
50  : settings_(settings)
51  , statistics_(new perf::Statistics())
52  , signature_mgr_(new signature::SignatureManager())
53  , download_mgr_(NULL)
54  , simple_catalog_mgr_(NULL)
55  , whitelist_(NULL)
56  , reflog_(NULL)
57  , manifest_(NULL)
58  , history_(NULL)
59 {
61 
62  if (exists) {
63  int rvb;
64  std::string keys = JoinStrings(FindFilesBySuffix(
65  settings.keychain().keychain_dir(), ".pub"), ":");
66  rvb = signature_mgr_->LoadPublicRsaKeys(keys);
67  if (!rvb) {
69  delete signature_mgr_;
70  delete statistics_;
71  throw EPublish("cannot load public rsa key");
72  }
73  }
74 
75  if (!settings.cert_bundle().empty()) {
76  int rvi = setenv("X509_CERT_BUNDLE", settings.cert_bundle().c_str(),
77  1 /* overwrite */);
78  if (rvi != 0)
79  throw EPublish("cannot set X509_CERT_BUNDLE environment variable");
80  }
84 
85  if (settings.proxy() != "") {
86  download_mgr_->SetProxyChain(settings.proxy(), "",
88  }
89 
90  if (exists) {
91  try {
92  DownloadRootObjects(settings.url(), settings.fqrn(), settings.tmp_dir());
93  } catch (const EPublish& e) {
95  delete signature_mgr_;
96  delete download_mgr_;
97  delete statistics_;
98  throw;
99  }
100  }
101 }
102 
104  if (signature_mgr_ != NULL) signature_mgr_->Fini();
105 
106  delete history_;
107  delete manifest_;
108  delete reflog_;
109  delete whitelist_;
110  delete signature_mgr_;
111  delete download_mgr_;
112  delete simple_catalog_mgr_;
113  delete statistics_;
114 }
115 
116 const history::History *Repository::history() const { return history_; }
117 
119  if (simple_catalog_mgr_ != NULL) return simple_catalog_mgr_;
120 
123  settings_.url(),
124  settings_.tmp_dir(),
126  statistics_,
127  true /* manage_catalog_files */);
129  return simple_catalog_mgr_;
130 }
131 
132 
134  const std::string &url, const std::string &fqrn, const std::string &tmp_dir)
135 {
136  delete whitelist_;
138  whitelist::Failures rv_whitelist = whitelist_->LoadUrl(url);
140  throw EPublish(std::string("cannot load whitelist [") +
141  whitelist::Code2Ascii(rv_whitelist) + "]");
142  }
143 
145  const uint64_t minimum_timestamp = 0;
146  const shash::Any *base_catalog = NULL;
147  manifest::Failures rv_manifest = manifest::Fetch(
148  url, fqrn, minimum_timestamp, base_catalog, signature_mgr_, download_mgr_,
149  &ensemble);
150  if (rv_manifest != manifest::kFailOk) throw EPublish("cannot load manifest");
151  delete manifest_;
152  manifest_ = new manifest::Manifest(*ensemble.manifest);
153 
154  std::string reflog_path;
155  FILE *reflog_fd =
156  CreateTempFile(tmp_dir + "/reflog", kPrivateFileMode, "w", &reflog_path);
157  std::string reflog_url = url + "/.cvmfsreflog";
158  // TODO(jblomer): verify reflog hash
159  // shash::Any reflog_hash(manifest_->GetHashAlgorithm());
160  cvmfs::FileSink filesink(reflog_fd);
161  download::JobInfo download_reflog(&reflog_url, false /* compressed */,
162  false /* probe hosts */, NULL, &filesink);
163  download::Failures rv_dl = download_mgr_->Fetch(&download_reflog);
164  fclose(reflog_fd);
165  if (rv_dl == download::kFailOk) {
166  delete reflog_;
167  reflog_ = manifest::Reflog::Open(reflog_path);
168  if (reflog_ == NULL) throw EPublish("cannot open reflog");
170  } else {
171  if (!download_reflog.IsFileNotFound()) {
172  throw EPublish(std::string("cannot load reflog [") +
173  download::Code2Ascii(rv_dl) + "]");
174  }
175  assert(reflog_ == NULL);
176  }
177 
178  std::string tags_path;
179  FILE *tags_fd =
180  CreateTempFile(tmp_dir + "/tags", kPrivateFileMode, "w", &tags_path);
181  if (!manifest_->history().IsNull()) {
182  std::string tags_url = url + "/data/" + manifest_->history().MakePath();
183  shash::Any tags_hash(manifest_->history());
184  cvmfs::FileSink filesink(tags_fd);
185  download::JobInfo download_tags(&tags_url, true /* compressed */,
186  true /* probe hosts */, &tags_hash,
187  &filesink);
188  rv_dl = download_mgr_->Fetch(&download_tags);
189  fclose(tags_fd);
190  if (rv_dl != download::kFailOk) throw EPublish("cannot load tag database");
191  delete history_;
193  if (history_ == NULL) throw EPublish("cannot open tag database");
194  } else {
195  fclose(tags_fd);
196  delete history_;
197  history_ = history::SqliteHistory::Create(tags_path, fqrn);
198  if (history_ == NULL) throw EPublish("cannot create tag database");
199  }
201 
202  if (!manifest_->meta_info().IsNull()) {
203  shash::Any info_hash(manifest_->meta_info());
204  std::string info_url = url + "/data/" + info_hash.MakePath();
205  cvmfs::MemSink metainfo_memsink;
206  download::JobInfo download_info(&info_url, true /* compressed */,
207  true /* probe_hosts */, &info_hash,
208  &metainfo_memsink);
209  download::Failures rv_info = download_mgr_->Fetch(&download_info);
210  if (rv_info != download::kFailOk) {
211  throw EPublish(std::string("cannot load meta info [") +
212  download::Code2Ascii(rv_info) + "]");
213  }
214  meta_info_ = std::string(reinterpret_cast<char*>(metainfo_memsink.data()),
215  metainfo_memsink.pos());
216  } else {
217  meta_info_ = "n/a";
218  }
219 }
220 
221 
222 std::string Repository::GetFqrnFromUrl(const std::string &url) {
223  return GetFileName(MakeCanonicalPath(url));
224 }
225 
226 
228  std::string url = settings_.url() + "/.cvmfs_master_replica";
229  download::JobInfo head(&url, false /* probe_hosts */);
230  download::Failures retval = download_mgr_->Fetch(&head);
231  if (retval == download::kFailOk) { return true; }
232  if (head.IsFileNotFound()) { return false; }
233 
234  throw EPublish(std::string("error looking for .cvmfs_master_replica [") +
235  download::Code2Ascii(retval) + "]");
236 }
237 
238 
239 //------------------------------------------------------------------------------
240 
241 
243  if ((spooler_files_ != NULL) && (spooler_catalogs_ != NULL))
244  return;
245  assert((spooler_files_ == NULL) && (spooler_catalogs_ == NULL));
246 
248  settings_.storage().GetLocator(),
249  settings_.transaction().hash_algorithm(),
250  settings_.transaction().compression_algorithm());
251  sd.session_token_file =
252  settings_.transaction().spool_area().gw_session_token();
253  sd.key_file = settings_.keychain().gw_key_path();
254 
256  upload::Spooler::Construct(sd, statistics_publish_.weak_ref());
257  if (spooler_files_ == NULL)
258  throw EPublish("could not initialize file spooler");
259 
260  upload::SpoolerDefinition sd_catalogs(sd.Dup2DefaultCompression());
262  upload::Spooler::Construct(sd_catalogs, statistics_publish_.weak_ref());
263  if (spooler_catalogs_ == NULL) {
264  delete spooler_files_;
265  throw EPublish("could not initialize catalog spooler");
266  }
267 }
268 
269 
272  throw EPublish("dangling master key pair");
273  }
275  throw EPublish("dangling repository keys");
276  }
281 
283  std::string whitelist_str = whitelist::Whitelist::CreateString(
284  settings_.fqrn(),
285  settings_.whitelist_validity_days(),
286  settings_.transaction().hash_algorithm(),
288  whitelist::Failures rv_wl = whitelist_->LoadMem(whitelist_str);
289  if (rv_wl != whitelist::kFailOk)
290  throw EPublish("whitelist generation failed");
291 }
292 
293 
295  // Reflog
296  const std::string reflog_path = CreateTempPath(
297  settings_.transaction().spool_area().tmp_dir() + "/cvmfs_reflog", 0600);
299  if (reflog_ == NULL) throw EPublish("could not create reflog");
300  reflog_->TakeDatabaseFileOwnership();
301 
302  // Root file catalog and initial manifest
304  settings_.transaction().spool_area().tmp_dir(),
305  settings_.transaction().is_volatile(),
306  settings_.transaction().voms_authz(),
308  spooler_catalogs_->WaitForUpload();
309  if (manifest_ == NULL)
310  throw EPublish("could not create initial file catalog");
311  reflog_->AddCatalog(manifest_->catalog_hash());
312 
314  manifest_->set_ttl(settings_.transaction().ttl_second());
315  const bool needs_bootstrap_shortcuts =
316  !settings_.transaction().voms_authz().empty();
317  manifest_->set_has_alt_catalog_path(needs_bootstrap_shortcuts);
319  settings_.transaction().is_garbage_collectable());
320 
321  // Tag database
322  const std::string tags_path = CreateTempPath(
323  settings_.transaction().spool_area().tmp_dir() + "/cvmfs_tags", 0600);
325  if (history_ == NULL) throw EPublish("could not create tag database");
326  history_->TakeDatabaseFileOwnership();
327  history::History::Tag tag_trunk(
328  "trunk",
330  manifest_->publish_timestamp(), "empty repository", "" /* branch */);
331  history_->Insert(tag_trunk);
332 
333  // Meta information, TODO(jblomer)
334  meta_info_ = "{}";
335 }
336 
337 
340  if (!spooler_files_->Create())
341  throw EPublish("could not initialize repository storage area");
342 }
343 
344 
346  upload::Spooler::CallbackPtr callback =
347  spooler_files_->RegisterListener(&Publisher::OnProcessCertificate, this);
348  spooler_files_->ProcessCertificate(
350  spooler_files_->WaitForUpload();
351  spooler_files_->UnregisterListener(callback);
352 }
353 
354 
356  assert(history_ != NULL);
358  const string history_path = history_->filename();
360  delete history_;
361 
362  upload::Spooler::CallbackPtr callback =
363  spooler_files_->RegisterListener(&Publisher::OnProcessHistory, this);
364  spooler_files_->ProcessHistory(history_path);
365  spooler_files_->WaitForUpload();
366  spooler_files_->UnregisterListener(callback);
367 
369  assert(history_ != NULL);
371 }
372 
373 
375  upload::Spooler::CallbackPtr callback =
376  spooler_files_->RegisterListener(&Publisher::OnProcessMetainfo, this);
377  spooler_files_->ProcessMetainfo(new StringIngestionSource(meta_info_));
378  spooler_files_->WaitForUpload();
379  spooler_files_->UnregisterListener(callback);
380 }
381 
382 
384  std::string signed_manifest = manifest_->ExportString();
385  shash::Any manifest_hash(settings_.transaction().hash_algorithm());
387  reinterpret_cast<const unsigned char *>(signed_manifest.data()),
388  signed_manifest.length(), &manifest_hash);
389  signed_manifest += "--\n" + manifest_hash.ToString() + "\n";
390  unsigned char *signature;
391  unsigned sig_size;
392  bool rvb = signature_mgr_->Sign(
393  reinterpret_cast<const unsigned char *>(manifest_hash.ToString().data()),
394  manifest_hash.GetHexSize(), &signature, &sig_size);
395  if (!rvb) throw EPublish("cannot sign manifest");
396  signed_manifest += std::string(reinterpret_cast<char *>(signature), sig_size);
397  free(signature);
398 
399  // Create alternative bootstrapping symlinks for VOMS secured repos
401  rvb =
402  spooler_files_->PlaceBootstrappingShortcut(manifest_->certificate()) &&
403  spooler_files_->PlaceBootstrappingShortcut(manifest_->catalog_hash()) &&
404  (manifest_->history().IsNull() ||
405  spooler_files_->PlaceBootstrappingShortcut(manifest_->history())) &&
406  (manifest_->meta_info().IsNull() ||
407  spooler_files_->PlaceBootstrappingShortcut(manifest_->meta_info()));
408  if (!rvb) EPublish("cannot place VOMS bootstrapping symlinks");
409  }
410 
411  upload::Spooler::CallbackPtr callback =
412  spooler_files_->RegisterListener(&Publisher::OnUploadManifest, this);
413  spooler_files_->Upload(".cvmfspublished",
414  new StringIngestionSource(signed_manifest));
415  spooler_files_->WaitForUpload();
416  spooler_files_->UnregisterListener(callback);
417 }
418 
419 
421  const string reflog_path = reflog_->database_file();
423  delete reflog_;
424 
425  shash::Any hash_reflog(settings_.transaction().hash_algorithm());
426  manifest::Reflog::HashDatabase(reflog_path, &hash_reflog);
427 
428  upload::Spooler::CallbackPtr callback =
429  spooler_files_->RegisterListener(&Publisher::OnUploadReflog, this);
430  spooler_files_->UploadReflog(reflog_path);
431  spooler_files_->WaitForUpload();
432  spooler_files_->UnregisterListener(callback);
433 
434  manifest_->set_reflog_hash(hash_reflog);
435 
436  reflog_ = manifest::Reflog::Open(reflog_path);
437  assert(reflog_ != NULL);
439 }
440 
441 
443  // TODO(jblomer): PKCS7 handling
444  upload::Spooler::CallbackPtr callback =
445  spooler_files_->RegisterListener(&Publisher::OnUploadWhitelist, this);
446  spooler_files_->Upload(".cvmfswhitelist",
448  spooler_files_->WaitForUpload();
449  spooler_files_->UnregisterListener(callback);
450 }
451 
452 
453 Publisher *Publisher::Create(const SettingsPublisher &settings) {
454  UniquePtr<Publisher> publisher(new Publisher(settings, false));
455 
456  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout | kLogNoLinebreak,
457  "Creating Key Chain... ");
458  publisher->CreateKeychain();
459  publisher->ExportKeychain();
460  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout, "done");
461 
462  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout | kLogNoLinebreak,
463  "Creating Backend Storage... ");
464  publisher->CreateStorage();
465  publisher->PushWhitelist();
466  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout, "done");
467 
468  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout | kLogNoLinebreak,
469  "Creating Initial Repository... ");
470  publisher->InitSpoolArea();
471  publisher->CreateRootObjects();
472  publisher->PushHistory();
473  publisher->PushCertificate();
474  publisher->PushMetainfo();
475  publisher->PushReflog();
476  publisher->PushManifest();
477  // TODO(jblomer): meta-info
478 
479  // Re-create from empty repository in order to properly initialize
480  // parent Repository object
481  publisher = new Publisher(settings);
482 
483  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout, "done");
484 
485  return publisher.Release();
486 }
487 
490 
491  bool rvb;
494  if (!rvb) throw EPublish("cannot export public master key");
497  if (!rvb) throw EPublish("cannot export certificate");
498 
501  if (!rvb) throw EPublish("cannot export private certificate key");
504  if (!rvb) throw EPublish("cannot export private master key");
505 
506  int rvi;
507  rvi = chown(settings_.keychain().master_public_key_path().c_str(),
508  settings_.owner_uid(), settings_.owner_gid());
509  if (rvi != 0) throw EPublish("cannot set key file ownership");
510  rvi = chown(settings_.keychain().certificate_path().c_str(),
511  settings_.owner_uid(), settings_.owner_gid());
512  if (rvi != 0) throw EPublish("cannot set key file ownership");
513  rvi = chown(settings_.keychain().private_key_path().c_str(),
514  settings_.owner_uid(), settings_.owner_gid());
515  if (rvi != 0) throw EPublish("cannot set key file ownership");
516  rvi = chown(settings_.keychain().master_private_key_path().c_str(),
517  settings_.owner_uid(), settings_.owner_gid());
518  if (rvi != 0) throw EPublish("cannot set key file ownership");
519 }
520 
522  if (result.return_code != 0) {
523  throw EPublish("cannot write certificate to storage");
524  }
527 }
528 
530  if (result.return_code != 0) {
531  throw EPublish("cannot write tag database to storage");
532  }
535 }
536 
538  if (result.return_code != 0) {
539  throw EPublish("cannot write repository meta info to storage");
540  }
543 }
544 
546  if (result.return_code != 0) {
547  throw EPublish("cannot write manifest to storage");
548  }
549 }
550 
552  if (result.return_code != 0) {
553  throw EPublish("cannot write reflog to storage");
554  }
555 }
556 
558  if (result.return_code != 0) {
559  throw EPublish("cannot write whitelist to storage");
560  }
561 }
562 
563 void Publisher::CreateDirectoryAsOwner(const std::string &path, int mode)
564 {
565  bool rvb = MkdirDeep(path, kPrivateDirMode);
566  if (!rvb) throw EPublish("cannot create directory " + path);
567  int rvi = chown(path.c_str(), settings_.owner_uid(), settings_.owner_gid());
568  if (rvi != 0) throw EPublish("cannot set ownership on directory " + path);
569 }
570 
572  CreateDirectoryAsOwner(settings_.transaction().spool_area().workspace(),
574  CreateDirectoryAsOwner(settings_.transaction().spool_area().tmp_dir(),
576  CreateDirectoryAsOwner(settings_.transaction().spool_area().cache_dir(),
578  CreateDirectoryAsOwner(settings_.transaction().spool_area().scratch_dir(),
580  CreateDirectoryAsOwner(settings_.transaction().spool_area().ovl_work_dir(),
582 
583  // On a managed node, the mount points are already mounted
584  if (!DirectoryExists(settings_.transaction().spool_area().readonly_mnt())) {
585  CreateDirectoryAsOwner(settings_.transaction().spool_area().readonly_mnt(),
587  }
588  if (!DirectoryExists(settings_.transaction().spool_area().union_mnt())) {
590  settings_.transaction().spool_area().union_mnt(), kPrivateDirMode);
591  }
592 }
593 
594 Publisher::Publisher(const SettingsPublisher &settings, const bool exists)
595  : Repository(SettingsRepository(settings), exists)
596  , settings_(settings)
597  , statistics_publish_(new perf::StatisticsTemplate("publish", statistics_))
598  , llvl_(settings.is_silent() ? kLogNone : kLogNormal)
599  , in_transaction_(settings.transaction().spool_area().transaction_lock())
600  , is_publishing_(settings.transaction().spool_area().publishing_lock())
601  , spooler_files_(NULL)
602  , spooler_catalogs_(NULL)
603  , catalog_mgr_(NULL)
604  , sync_parameters_(NULL)
605  , sync_mediator_(NULL)
606  , sync_union_(NULL)
607 {
609  unsigned layout_revision = settings.transaction().layout_revision();
610  throw EPublish(
611  "This repository uses layout revision " + StringifyInt(layout_revision)
612  + ".\n"
613  "This version of CernVM-FS requires layout revision " + StringifyInt(
614  kRequiredLayoutRevision) + ", which is\n"
615  "incompatible to " + StringifyInt(layout_revision) + ".\n\n"
616  "Please run `cvmfs_server migrate` to update your repository before "
617  "proceeding.",
619  }
620 
621  if (!exists)
622  return;
623 
624  CreateDirectoryAsOwner(settings_.transaction().spool_area().tmp_dir(),
626 
627  if (settings.storage().type() == upload::SpoolerDefinition::Gateway) {
628  if (!settings.keychain().HasGatewayKey()) {
629  throw EPublish("gateway key missing: " +
630  settings.keychain().gw_key_path());
631  }
633  if (!gw_key_.IsValid()) {
634  throw EPublish("cannot read gateway key: " +
635  settings.keychain().gw_key_path());
636  }
637  }
638 
639  if ((settings.storage().type() != upload::SpoolerDefinition::Gateway) &&
640  !settings.transaction().in_enter_session())
641  {
643  settings.keychain().certificate_path());
644  if (!rvb)
645  throw EPublish("cannot load certificate, thus cannot commit changes");
647  settings.keychain().private_key_path(), "");
648  if (!rvb)
649  throw EPublish("cannot load private key, thus cannot commit changes");
650  // The private master key might be on a key card instead
651  if (FileExists(settings.keychain().master_private_key_path())) {
653  settings.keychain().master_private_key_path());
654  if (!rvb) throw EPublish("cannot load private master key");
655  }
656  if (!signature_mgr_->KeysMatch())
657  throw EPublish("corrupted keychain");
658  }
659 
660  if (settings.is_managed())
661  managed_node_ = new ManagedNode(this);
663  if (in_transaction_.IsSet())
665 }
666 
668  delete sync_union_;
669  delete sync_mediator_;
670  delete sync_parameters_;
671  delete catalog_mgr_;
672  delete spooler_catalogs_;
673  delete spooler_files_;
674 }
675 
676 
679 
680  if (catalog_mgr_ == NULL) {
682  settings_.transaction().base_hash(),
683  settings_.url(),
684  settings_.transaction().spool_area().tmp_dir(),
687  settings_.transaction().enforce_limits(),
688  settings_.transaction().limit_nested_catalog_kentries(),
689  settings_.transaction().limit_root_catalog_kentries(),
690  settings_.transaction().limit_file_size_mb(),
691  statistics_,
692  settings_.transaction().use_catalog_autobalance(),
693  settings_.transaction().autobalance_max_weight(),
694  settings_.transaction().autobalance_min_weight());
695  catalog_mgr_->Init();
696  }
697 
698  if (sync_parameters_ == NULL) {
699  SyncParameters *p = new SyncParameters();
700  p->spooler = spooler_files_;
701  p->repo_name = settings_.fqrn();
702  p->dir_union = settings_.transaction().spool_area().union_mnt();
703  p->dir_scratch = settings_.transaction().spool_area().scratch_dir();
704  p->dir_rdonly = settings_.transaction().spool_area().readonly_mnt();
705  p->dir_temp = settings_.transaction().spool_area().tmp_dir();
706  p->base_hash = settings_.transaction().base_hash();
707  p->stratum0 = settings_.url();
708  // p->manifest_path = SHOULD NOT BE NEEDED
709  // p->spooler_definition = SHOULD NOT BE NEEDED;
710  // p->union_fs_type = SHOULD NOT BE NEEDED
711  p->print_changeset = settings_.transaction().print_changeset();
712  p->dry_run = settings_.transaction().dry_run();
713  sync_parameters_ = p;
714  }
715 
716  if (sync_mediator_ == NULL) {
719  }
720 
721  if (sync_union_ == NULL) {
722  switch (settings_.transaction().union_fs()) {
723  case kUnionFsAufs:
726  settings_.transaction().spool_area().readonly_mnt(),
727  settings_.transaction().spool_area().union_mnt(),
728  settings_.transaction().spool_area().scratch_dir());
729  break;
730  case kUnionFsOverlay:
733  settings_.transaction().spool_area().readonly_mnt(),
734  settings_.transaction().spool_area().union_mnt(),
735  settings_.transaction().spool_area().scratch_dir());
736  break;
737  case kUnionFsTarball:
740  settings_.transaction().spool_area().readonly_mnt(),
741  // TODO(jblomer): get from settings
742  "tar_file",
743  "base_directory",
744  -1u,
745  -1u,
746  "to_delete",
747  false /* create_catalog */);
748  break;
749  default:
750  throw EPublish("unknown union file system");
751  }
752  bool rvb = sync_union_->Initialize();
753  if (!rvb) {
754  delete sync_union_;
755  sync_union_ = NULL;
756  throw EPublish("cannot initialize union file system engine");
757  }
758  }
759 }
760 
762  std::string session_dir = Env::GetEnterSessionDir();
763  std::string session_pid_tmp = session_dir + "/session_pid";
764  std::string session_pid;
765  int fd_session_pid = open(session_pid_tmp.c_str(), O_RDONLY);
766  if (fd_session_pid < 0) throw EPublish("Session pid cannot be retrieved");
767  SafeReadToString(fd_session_pid, &session_pid);
768 
769  pid_t pid_child = String2Uint64(session_pid);
770  kill(pid_child, SIGUSR1);
771 }
772 
775 
777 
779  bool rvb = sync_mediator_->Commit(manifest_);
780  if (!rvb) throw EPublish("cannot write change set to storage");
781 
782  if (!settings_.transaction().dry_run()) {
783  spooler_files_->WaitForUpload();
784  spooler_catalogs_->WaitForUpload();
785  spooler_files_->FinalizeSession(false /* commit */);
786 
787  const std::string old_root_hash =
788  settings_.transaction().base_hash().ToString(true /* with_suffix */);
789  const std::string new_root_hash =
790  manifest_->catalog_hash().ToString(true /* with_suffix */);
791  rvb = spooler_catalogs_->FinalizeSession(true /* commit */,
792  old_root_hash, new_root_hash,
793  /* TODO(jblomer) */ sync_parameters_->repo_tag);
794  if (!rvb)
795  throw EPublish("failed to commit transaction");
796 
797  // Reset to the new catalog root hash
798  settings_.GetTransaction()->SetBaseHash(manifest_->catalog_hash());
799  // TODO(jblomer): think about how to deal with the scratch area at
800  // this point
801  // WipeScratchArea();
802  }
803 
804  delete sync_union_;
805  delete sync_mediator_;
806  delete sync_parameters_;
807  delete catalog_mgr_;
808  sync_union_ = NULL;
809  sync_mediator_ = NULL;
810  sync_parameters_ = NULL;
811  catalog_mgr_ = NULL;
812 
813  if (!settings_.transaction().dry_run()) {
814  LogCvmfs(kLogCvmfs, kLogStdout, "New revision: %" PRIu64,
815  manifest_->revision());
817  }
818 }
819 
821  if (!in_transaction_.IsSet())
822  throw EPublish("cannot publish outside transaction");
823 
824  PushReflog();
825  PushManifest();
827 }
828 
829 
830 void Publisher::MarkReplicatible(bool value) {
832 
833  if (value) {
834  spooler_files_->Upload("/dev/null", "/.cvmfs_master_replica");
835  } else {
836  spooler_files_->RemoveAsync("/.cvmfs_master_replica");
837  }
838  spooler_files_->WaitForUpload();
839  if (spooler_files_->GetNumberOfErrors() > 0)
840  throw EPublish("cannot set replication mode");
841 }
842 
844 void Publisher::Migrate() {}
846 void Publisher::Rollback() {}
848 
849 void Publisher::Transaction() {
851  session()->SetKeepAlive(true);
852 }
853 
854 //------------------------------------------------------------------------------
855 
856 
858  : Repository(SettingsRepository(settings))
859 {
860 }
861 
862 
864 }
865 
866 } // namespace publish
bool Commit(manifest::Manifest *manifest)
int return_code
the return value of the spooler operation
void set_repository_name(const std::string &repository_name)
Definition: manifest.h:101
void TakeDatabaseFileOwnership()
Definition: reflog.cc:307
const SettingsRepository settings_
Definition: repository.h:137
std::string database_file() const
Definition: reflog.cc:337
void CreateRootObjects()
Definition: repository.cc:294
manifest::Reflog * reflog_
Definition: repository.h:147
bool IsNull() const
Definition: hash.h:383
download::DownloadManager * download_mgr_
Definition: repository.h:141
bool HasGatewayKey() const
Definition: settings.cc:272
std::string stratum0
bool AddHistory(const shash::Any &history)
Definition: reflog.cc:134
void ConstructSyncManagers()
Definition: repository.cc:677
void OnUploadReflog(const upload::SpoolerResult &result)
Definition: repository.cc:551
catalog::SimpleCatalogManager * GetSimpleCatalogManager()
Definition: repository.cc:118
UniquePtr< ManagedNode > managed_node_
Definition: repository.h:381
Session * session() const
Definition: repository.h:320
NameString GetFileName(const PathString &path)
Definition: shortstring.cc:29
const int kPrivateDirMode
Definition: posix.h:36
T * weak_ref() const
Definition: pointer.h:42
void Ingest()
Definition: repository.cc:843
UniquePtr< perf::StatisticsTemplate > statistics_publish_
Definition: repository.h:365
static Publisher * Create(const SettingsPublisher &settings)
unsigned char * data()
Definition: sink_mem.h:122
std::string keychain_dir() const
Definition: settings.h:325
std::string GetCertificate() const
Definition: signature.cc:409
static const std::string kPreviousHeadTag
Definition: repository.cc:41
Failures LoadUrl(const std::string &base_url)
Definition: whitelist.cc:216
std::string gw_key_path() const
Definition: settings.h:334
FILE * CreateTempFile(const std::string &path_prefix, const int mode, const char *open_flags, std::string *final_path)
Definition: posix.cc:1005
string JoinStrings(const vector< string > &strings, const string &joint)
Definition: string.cc:325
std::string ToString(const bool with_suffix=false) const
Definition: hash.h:249
void OnProcessCertificate(const upload::SpoolerResult &result)
Definition: repository.cc:521
virtual bool Initialize()
Definition: sync_union.cc:24
void set_certificate(const shash::Any &certificate)
Definition: manifest.h:95
const history::History * history() const
void CreateDirectoryAsOwner(const std::string &path, int mode)
Definition: repository.cc:563
void PushCertificate()
Definition: repository.cc:345
unsigned layout_revision() const
Definition: settings.h:188
void SetProxyChain(const std::string &proxy_list, const std::string &fallback_proxy_list, const ProxySetModes set_mode)
Definition: download.cc:2557
void OnUploadWhitelist(const upload::SpoolerResult &result)
Definition: repository.cc:557
void CreateStorage()
Definition: repository.cc:338
bool IsMasterReplica()
Definition: repository.cc:227
std::string private_key_path() const
Definition: settings.h:332
const SettingsPublisher & settings() const
Definition: repository.h:317
perf::Statistics * statistics_
Definition: repository.h:139
Status status() const
Definition: whitelist.h:103
std::string CreateTempPath(const std::string &path_prefix, const int mode)
Definition: posix.cc:1034
void set_meta_info(const shash::Any &meta_info)
Definition: manifest.h:119
std::string GetPrivateKey()
Definition: signature.cc:425
std::string ExportString() const
Definition: manifest.cc:188
virtual ~Replica()
Definition: repository.cc:863
std::string repo_name
assert((mem||(size==0))&&"Out Of Memory")
bool has_alt_catalog_path() const
Definition: manifest.h:138
upload::Spooler * spooler_catalogs_
Definition: repository.h:384
gateway::GatewayKey gw_key_
Definition: repository.h:372
bool SafeWriteToFile(const std::string &content, const std::string &path, int mode)
Definition: posix.cc:2081
void PushMetainfo()
Definition: repository.cc:374
bool LoadPrivateKeyPath(const std::string &file_pem, const std::string &password)
Definition: signature.cc:185
class __attribute__((visibility("default"))) Publisher ManagedNode(Publisher *p)
Definition: repository.h:174
void PushReflog()
Definition: repository.cc:420
uint64_t catalog_size() const
Definition: manifest.h:133
void ExitShell()
Definition: repository.cc:761
bool AddCatalog(const shash::Any &catalog)
Definition: reflog.cc:128
static Reflog * Create(const std::string &database_path, const std::string &repo_name)
Definition: reflog.cc:32
std::string cert_bundle() const
Definition: settings.h:374
catalog::WritableCatalogManager * catalog_mgr_
Definition: repository.h:385
std::string dir_scratch
bool IsFileNotFound()
Definition: jobinfo.cc:32
bool HasMasterKeys() const
Definition: settings.cc:253
void GenerateCertificate(const std::string &cn)
Definition: signature.cc:491
uint64_t revision() const
Definition: manifest.h:129
Repository(const SettingsRepository &settings, const bool exists=true)
Definition: repository.cc:49
static Reflog * Open(const std::string &database_path)
Definition: reflog.cc:17
static SqliteHistory * OpenWritable(const std::string &file_name)
std::string master_public_key_path() const
Definition: settings.h:329
bool FileExists(const std::string &path)
Definition: posix.cc:791
std::string GetActivePubkeys() const
Definition: signature.cc:385
std::string dir_temp
const int kPrivateFileMode
Definition: posix.h:35
void OnProcessMetainfo(const upload::SpoolerResult &result)
Definition: repository.cc:537
const char * Code2Ascii(const Failures error)
static manifest::Manifest * CreateRepository(const std::string &dir_temp, const bool volatile_content, const std::string &voms_authz, upload::Spooler *spooler)
bool HasRepositoryKeys() const
Definition: settings.cc:267
std::string proxy() const
Definition: settings.h:372
std::string meta_info_
Definition: repository.h:151
void PushHistory()
Definition: repository.cc:355
const SettingsKeychain & keychain() const
Definition: settings.h:439
publish::SyncUnion * sync_union_
Definition: repository.h:388
uint64_t publish_timestamp() const
Definition: manifest.h:136
bool AddMetainfo(const shash::Any &metainfo)
Definition: reflog.cc:140
void Transaction()
const int kDefaultDirMode
Definition: posix.h:33
bool Sign(const unsigned char *buffer, const unsigned buffer_size, unsigned char **signature, unsigned *signature_size)
Definition: signature.cc:769
void OnUploadManifest(const upload::SpoolerResult &result)
Definition: repository.cc:545
std::string dir_rdonly
signature::SignatureManager * signature_mgr_
Definition: repository.h:140
void SetKeepAlive(bool value)
Failures Fetch(const std::string &base_url, const std::string &repository_name, const uint64_t minimum_timestamp, const shash::Any *base_catalog, signature::SignatureManager *signature_manager, download::DownloadManager *download_manager, ManifestEnsemble *ensemble)
void UpdateMetaInfo()
Definition: repository.cc:847
std::string master_private_key_path() const
Definition: settings.h:326
void DownloadRootObjects(const std::string &url, const std::string &fqrn, const std::string &tmp_dir)
Definition: repository.cc:133
shash::Any base_hash
static const std::string kHeadTag
Definition: repository.cc:40
shash::Any certificate() const
Definition: manifest.h:134
virtual ~Publisher()
upload::Spooler * spooler
bool MkdirDeep(const std::string &path, const mode_t mode, bool verify_writable)
Definition: posix.cc:846
bool LoadCertificatePath(const std::string &file_pem)
Definition: signature.cc:245
const SettingsTransaction & transaction() const
Definition: settings.h:438
bool IsValid() const
Definition: gateway_util.h:17
const char * Code2Ascii(const Failures error)
Definition: whitelist.h:49
ServerLockFile is_publishing_
Definition: repository.h:371
std::string dir_union
void Sync()
Definition: repository.cc:773
void InitSpoolArea()
Definition: repository.cc:571
static void HashDatabase(const std::string &database_path, shash::Any *hash_reflog)
Definition: reflog.cc:322
void PushManifest()
Definition: repository.cc:383
shash::Any catalog_hash() const
Definition: manifest.h:132
string StringifyInt(const int64_t value)
Definition: string.cc:78
std::string certificate_path() const
Definition: settings.h:333
catalog::SimpleCatalogManager * simple_catalog_mgr_
Definition: repository.h:145
history::SqliteHistory * history_
Definition: repository.h:149
void Resign()
Definition: repository.cc:845
bool is_managed() const
Definition: settings.h:434
void Publish()
const SettingsStorage & storage() const
Definition: settings.h:437
void DropDatabaseFileOwnership()
Definition: reflog.cc:313
bool SetPreviousRevision(const shash::Any &history_hash)
bool LoadPrivateMasterKeyPath(const std::string &file_pem)
Definition: signature.cc:153
bool DirectoryExists(const std::string &path)
Definition: posix.cc:813
UniquePtr< Session > session_
Definition: repository.h:380
void set_history(const shash::Any &history_db)
Definition: manifest.h:98
void set_garbage_collectability(const bool garbage_collectable)
Definition: manifest.h:113
bool AddCertificate(const shash::Any &certificate)
Definition: reflog.cc:121
std::string filename() const
bool SafeReadToString(int fd, std::string *final_result)
Definition: posix.cc:2065
manifest::Manifest * manifest_
Definition: repository.h:148
void OnProcessHistory(const upload::SpoolerResult &result)
Definition: repository.cc:529
void set_ttl(const uint32_t ttl)
Definition: manifest.h:93
void HashMem(const unsigned char *buffer, const unsigned buffer_size, Any *any_digest)
Definition: hash.cc:255
upload::Spooler * spooler_files_
Definition: repository.h:383
void MarkReplicatible(bool value)
Definition: repository.cc:830
size_t pos()
Definition: sink_mem.h:121
class __attribute__((visibility("default"))) Replica Replica(const SettingsReplica &settings)
Definition: repository.cc:857
const SettingsKeychain & keychain() const
Definition: settings.h:376
std::string ExportString() const
Definition: whitelist.cc:77
virtual void Traverse()=0
uint64_t String2Uint64(const string &value)
Definition: string.cc:228
std::string GetPrivateMasterKey()
Definition: signature.cc:441
void Migrate(InodeTracker *old_tracker, glue::InodeTracker *new_tracker)
Definition: compat.cc:80
Failures Fetch(JobInfo *info)
Definition: download.cc:1834
shash::Any history() const
Definition: manifest.h:135
ServerFlagFile in_transaction_
Definition: repository.h:370
static std::string CreateString(const std::string &fqrn, int validity_days, shash::Algorithms hash_algorithm, signature::SignatureManager *signature_manager)
Definition: whitelist.cc:47
bool LoadPublicRsaKeys(const std::string &path_list)
Definition: signature.cc:305
whitelist::Whitelist * whitelist_
Definition: repository.h:146
GatewayKey ReadGatewayKey(const std::string &key_file_name)
Definition: gateway_util.cc:28
static SqliteHistory * Create(const std::string &file_name, const std::string &fqrn)
bool HasDanglingRepositoryKeys() const
Definition: settings.cc:259
Failures LoadMem(const std::string &whitelist)
Definition: whitelist.cc:196
RepositoryTag repo_tag
SyncMediator * sync_mediator_
Definition: repository.h:387
std::string fqrn() const
Definition: settings.h:370
void ExportKeychain()
Definition: repository.cc:488
void set_has_alt_catalog_path(const bool &has_alt_path)
Definition: manifest.h:116
int llvl_
Definition: repository.h:369
upload::SpoolerDefinition::DriverType type() const
Definition: settings.h:294
Publisher(const SettingsPublisher &settings, const bool exists=true)
Definition: repository.cc:594
std::string MakePath() const
Definition: hash.h:316
std::string MakeCanonicalPath(const std::string &path)
Definition: posix.cc:98
bool in_enter_session() const
Definition: settings.h:189
virtual ~Repository()
Definition: repository.cc:103
void set_reflog_hash(const shash::Any &checksum)
Definition: manifest.h:125
std::string url() const
Definition: settings.h:371
void ConstructSpoolers()
Definition: repository.cc:242
void Rollback()
void CreateKeychain()
Definition: repository.cc:270
void PushWhitelist()
Definition: repository.cc:442
std::vector< std::string > FindFilesBySuffix(const std::string &dir, const std::string &suffix)
Definition: posix.cc:1124
shash::Any meta_info() const
Definition: manifest.h:139
std::string tmp_dir() const
Definition: settings.h:373
static const unsigned kRequiredLayoutRevision
Definition: repository.h:278
SyncParameters * sync_parameters_
Definition: repository.h:386
class static __attribute__((visibility("default"))) Repository std::string GetFqrnFromUrl(const std::string &url)
Definition: repository.cc:222
bool HasDanglingMasterKeys() const
Definition: settings.cc:245
CVMFS_EXPORT void LogCvmfs(const LogSource source, const int mask, const char *format,...)
Definition: logging.cc:528