CernVM-FS  2.11.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
repository.cc
Go to the documentation of this file.
1 
5 #include "cvmfs_config.h"
6 #include "publish/repository.h"
7 
8 #include <cassert>
9 #include <cstddef>
10 #include <cstdlib>
11 
12 #include "catalog_mgr_ro.h"
13 #include "catalog_mgr_rw.h"
14 #include "crypto/hash.h"
15 #include "crypto/signature.h"
16 #include "gateway_util.h"
17 #include "history_sqlite.h"
19 #include "manifest.h"
20 #include "manifest_fetch.h"
21 #include "network/download.h"
22 #include "publish/except.h"
24 #include "publish/settings.h"
25 #include "reflog.h"
26 #include "statistics.h"
27 #include "sync_mediator.h"
28 #include "sync_union_aufs.h"
29 #include "sync_union_overlayfs.h"
30 #include "sync_union_tarball.h"
31 #include "upload.h"
33 #include "util/logging.h"
34 #include "util/pointer.h"
35 #include "whitelist.h"
36 
37 // TODO(jblomer): Remove Me
38 namespace swissknife {
39 class CommandTag {
40  static const std::string kHeadTag;
41  static const std::string kPreviousHeadTag;
42 };
43 const std::string CommandTag::kHeadTag = "trunk";
44 const std::string CommandTag::kPreviousHeadTag = "trunk-previous";
45 }
46 
47 namespace publish {
48 
50  : settings_(settings)
51  , statistics_(new perf::Statistics())
52  , signature_mgr_(new signature::SignatureManager())
53  , download_mgr_(NULL)
54  , simple_catalog_mgr_(NULL)
55  , whitelist_(NULL)
56  , reflog_(NULL)
57  , manifest_(NULL)
58  , history_(NULL)
59 {
61 
62  if (exists) {
63  int rvb;
64  std::string keys = JoinStrings(FindFilesBySuffix(
65  settings.keychain().keychain_dir(), ".pub"), ":");
66  rvb = signature_mgr_->LoadPublicRsaKeys(keys);
67  if (!rvb) {
69  delete signature_mgr_;
70  delete statistics_;
71  throw EPublish("cannot load public rsa key");
72  }
73  }
74 
75  if (!settings.cert_bundle().empty()) {
76  int rvi = setenv("X509_CERT_BUNDLE", settings.cert_bundle().c_str(),
77  1 /* overwrite */);
78  if (rvi != 0)
79  throw EPublish("cannot set X509_CERT_BUNDLE environment variable");
80  }
82  download_mgr_->Init(16,
85 
86  if (settings.proxy() != "") {
87  download_mgr_->SetProxyChain(settings.proxy(), "",
89  }
90 
91  if (exists) {
92  try {
93  DownloadRootObjects(settings.url(), settings.fqrn(), settings.tmp_dir());
94  } catch (const EPublish& e) {
97  delete signature_mgr_;
98  delete download_mgr_;
99  delete statistics_;
100  throw;
101  }
102  }
103 }
104 
106  if (signature_mgr_ != NULL) signature_mgr_->Fini();
107  if (download_mgr_ != NULL) download_mgr_->Fini();
108 
109  delete history_;
110  delete manifest_;
111  delete reflog_;
112  delete whitelist_;
113  delete signature_mgr_;
114  delete download_mgr_;
115  delete simple_catalog_mgr_;
116  delete statistics_;
117 }
118 
119 const history::History *Repository::history() const { return history_; }
120 
122  if (simple_catalog_mgr_ != NULL) return simple_catalog_mgr_;
123 
126  settings_.url(),
127  settings_.tmp_dir(),
129  statistics_,
130  true /* manage_catalog_files */);
132  return simple_catalog_mgr_;
133 }
134 
135 
137  const std::string &url, const std::string &fqrn, const std::string &tmp_dir)
138 {
139  delete whitelist_;
141  whitelist::Failures rv_whitelist = whitelist_->LoadUrl(url);
143  throw EPublish(std::string("cannot load whitelist [") +
144  whitelist::Code2Ascii(rv_whitelist) + "]");
145  }
146 
148  const uint64_t minimum_timestamp = 0;
149  const shash::Any *base_catalog = NULL;
150  manifest::Failures rv_manifest = manifest::Fetch(
151  url, fqrn, minimum_timestamp, base_catalog, signature_mgr_, download_mgr_,
152  &ensemble);
153  if (rv_manifest != manifest::kFailOk) throw EPublish("cannot load manifest");
154  delete manifest_;
155  manifest_ = new manifest::Manifest(*ensemble.manifest);
156 
157  std::string reflog_path;
158  FILE *reflog_fd =
159  CreateTempFile(tmp_dir + "/reflog", kPrivateFileMode, "w", &reflog_path);
160  std::string reflog_url = url + "/.cvmfsreflog";
161  // TODO(jblomer): verify reflog hash
162  // shash::Any reflog_hash(manifest_->GetHashAlgorithm());
163  download::JobInfo download_reflog(
164  &reflog_url,
165  false /* compressed */,
166  false /* probe hosts */,
167  reflog_fd,
168  NULL);
169  download::Failures rv_dl = download_mgr_->Fetch(&download_reflog);
170  fclose(reflog_fd);
171  if (rv_dl == download::kFailOk) {
172  delete reflog_;
173  reflog_ = manifest::Reflog::Open(reflog_path);
174  if (reflog_ == NULL) throw EPublish("cannot open reflog");
176  } else {
177  if (!download_reflog.IsFileNotFound()) {
178  throw EPublish(std::string("cannot load reflog [") +
179  download::Code2Ascii(rv_dl) + "]");
180  }
181  assert(reflog_ == NULL);
182  }
183 
184  std::string tags_path;
185  FILE *tags_fd =
186  CreateTempFile(tmp_dir + "/tags", kPrivateFileMode, "w", &tags_path);
187  if (!manifest_->history().IsNull()) {
188  std::string tags_url = url + "/data/" + manifest_->history().MakePath();
189  shash::Any tags_hash(manifest_->history());
190  download::JobInfo download_tags(
191  &tags_url,
192  true /* compressed */,
193  true /* probe hosts */,
194  tags_fd,
195  &tags_hash);
196  rv_dl = download_mgr_->Fetch(&download_tags);
197  fclose(tags_fd);
198  if (rv_dl != download::kFailOk) throw EPublish("cannot load tag database");
199  delete history_;
201  if (history_ == NULL) throw EPublish("cannot open tag database");
202  } else {
203  fclose(tags_fd);
204  delete history_;
205  history_ = history::SqliteHistory::Create(tags_path, fqrn);
206  if (history_ == NULL) throw EPublish("cannot create tag database");
207  }
209 
210  if (!manifest_->meta_info().IsNull()) {
211  shash::Any info_hash(manifest_->meta_info());
212  std::string info_url = url + "/data/" + info_hash.MakePath();
213  download::JobInfo download_info(
214  &info_url,
215  true /* compressed */,
216  true /* probe_hosts */,
217  &info_hash);
218  download::Failures rv_info = download_mgr_->Fetch(&download_info);
219  if (rv_info != download::kFailOk) {
220  throw EPublish(std::string("cannot load meta info [") +
221  download::Code2Ascii(rv_info) + "]");
222  }
223  meta_info_ = std::string(download_info.destination_mem.data,
224  download_info.destination_mem.pos);
225  } else {
226  meta_info_ = "n/a";
227  }
228 }
229 
230 
231 std::string Repository::GetFqrnFromUrl(const std::string &url) {
232  return GetFileName(MakeCanonicalPath(url));
233 }
234 
235 
237  std::string url = settings_.url() + "/.cvmfs_master_replica";
238  download::JobInfo head(&url, false /* probe_hosts */);
239  download::Failures retval = download_mgr_->Fetch(&head);
240  if (retval == download::kFailOk) return true;
241  if (head.IsFileNotFound()) return false;
242 
243  throw EPublish(std::string("error looking for .cvmfs_master_replica [") +
244  download::Code2Ascii(retval) + "]");
245 }
246 
247 
248 //------------------------------------------------------------------------------
249 
250 
252  if ((spooler_files_ != NULL) && (spooler_catalogs_ != NULL))
253  return;
254  assert((spooler_files_ == NULL) && (spooler_catalogs_ == NULL));
255 
257  settings_.storage().GetLocator(),
258  settings_.transaction().hash_algorithm(),
259  settings_.transaction().compression_algorithm());
260  sd.session_token_file =
261  settings_.transaction().spool_area().gw_session_token();
262  sd.key_file = settings_.keychain().gw_key_path();
263 
265  upload::Spooler::Construct(sd, statistics_publish_.weak_ref());
266  if (spooler_files_ == NULL)
267  throw EPublish("could not initialize file spooler");
268 
269  upload::SpoolerDefinition sd_catalogs(sd.Dup2DefaultCompression());
271  upload::Spooler::Construct(sd_catalogs, statistics_publish_.weak_ref());
272  if (spooler_catalogs_ == NULL) {
273  delete spooler_files_;
274  throw EPublish("could not initialize catalog spooler");
275  }
276 }
277 
278 
281  throw EPublish("dangling master key pair");
282  }
284  throw EPublish("dangling repository keys");
285  }
290 
292  std::string whitelist_str = whitelist::Whitelist::CreateString(
293  settings_.fqrn(),
294  settings_.whitelist_validity_days(),
295  settings_.transaction().hash_algorithm(),
297  whitelist::Failures rv_wl = whitelist_->LoadMem(whitelist_str);
298  if (rv_wl != whitelist::kFailOk)
299  throw EPublish("whitelist generation failed");
300 }
301 
302 
304  // Reflog
305  const std::string reflog_path = CreateTempPath(
306  settings_.transaction().spool_area().tmp_dir() + "/cvmfs_reflog", 0600);
308  if (reflog_ == NULL) throw EPublish("could not create reflog");
309  reflog_->TakeDatabaseFileOwnership();
310 
311  // Root file catalog and initial manifest
313  settings_.transaction().spool_area().tmp_dir(),
314  settings_.transaction().is_volatile(),
315  settings_.transaction().voms_authz(),
317  spooler_catalogs_->WaitForUpload();
318  if (manifest_ == NULL)
319  throw EPublish("could not create initial file catalog");
320  reflog_->AddCatalog(manifest_->catalog_hash());
321 
323  manifest_->set_ttl(settings_.transaction().ttl_second());
324  const bool needs_bootstrap_shortcuts =
325  !settings_.transaction().voms_authz().empty();
326  manifest_->set_has_alt_catalog_path(needs_bootstrap_shortcuts);
328  settings_.transaction().is_garbage_collectable());
329 
330  // Tag database
331  const std::string tags_path = CreateTempPath(
332  settings_.transaction().spool_area().tmp_dir() + "/cvmfs_tags", 0600);
334  if (history_ == NULL) throw EPublish("could not create tag database");
335  history_->TakeDatabaseFileOwnership();
336  history::History::Tag tag_trunk(
337  "trunk",
339  manifest_->publish_timestamp(), "empty repository", "" /* branch */);
340  history_->Insert(tag_trunk);
341 
342  // Meta information, TODO(jblomer)
343  meta_info_ = "{}";
344 }
345 
346 
349  if (!spooler_files_->Create())
350  throw EPublish("could not initialize repository storage area");
351 }
352 
353 
355  upload::Spooler::CallbackPtr callback =
356  spooler_files_->RegisterListener(&Publisher::OnProcessCertificate, this);
357  spooler_files_->ProcessCertificate(
359  spooler_files_->WaitForUpload();
360  spooler_files_->UnregisterListener(callback);
361 }
362 
363 
365  assert(history_ != NULL);
367  const string history_path = history_->filename();
369  delete history_;
370 
371  upload::Spooler::CallbackPtr callback =
372  spooler_files_->RegisterListener(&Publisher::OnProcessHistory, this);
373  spooler_files_->ProcessHistory(history_path);
374  spooler_files_->WaitForUpload();
375  spooler_files_->UnregisterListener(callback);
376 
378  assert(history_ != NULL);
380 }
381 
382 
384  upload::Spooler::CallbackPtr callback =
385  spooler_files_->RegisterListener(&Publisher::OnProcessMetainfo, this);
386  spooler_files_->ProcessMetainfo(new StringIngestionSource(meta_info_));
387  spooler_files_->WaitForUpload();
388  spooler_files_->UnregisterListener(callback);
389 }
390 
391 
393  std::string signed_manifest = manifest_->ExportString();
394  shash::Any manifest_hash(settings_.transaction().hash_algorithm());
396  reinterpret_cast<const unsigned char *>(signed_manifest.data()),
397  signed_manifest.length(), &manifest_hash);
398  signed_manifest += "--\n" + manifest_hash.ToString() + "\n";
399  unsigned char *signature;
400  unsigned sig_size;
401  bool rvb = signature_mgr_->Sign(
402  reinterpret_cast<const unsigned char *>(manifest_hash.ToString().data()),
403  manifest_hash.GetHexSize(), &signature, &sig_size);
404  if (!rvb) throw EPublish("cannot sign manifest");
405  signed_manifest += std::string(reinterpret_cast<char *>(signature), sig_size);
406  free(signature);
407 
408  // Create alternative bootstrapping symlinks for VOMS secured repos
410  rvb =
411  spooler_files_->PlaceBootstrappingShortcut(manifest_->certificate()) &&
412  spooler_files_->PlaceBootstrappingShortcut(manifest_->catalog_hash()) &&
413  (manifest_->history().IsNull() ||
414  spooler_files_->PlaceBootstrappingShortcut(manifest_->history())) &&
415  (manifest_->meta_info().IsNull() ||
416  spooler_files_->PlaceBootstrappingShortcut(manifest_->meta_info()));
417  if (!rvb) EPublish("cannot place VOMS bootstrapping symlinks");
418  }
419 
420  upload::Spooler::CallbackPtr callback =
421  spooler_files_->RegisterListener(&Publisher::OnUploadManifest, this);
422  spooler_files_->Upload(".cvmfspublished",
423  new StringIngestionSource(signed_manifest));
424  spooler_files_->WaitForUpload();
425  spooler_files_->UnregisterListener(callback);
426 }
427 
428 
430  const string reflog_path = reflog_->database_file();
432  delete reflog_;
433 
434  shash::Any hash_reflog(settings_.transaction().hash_algorithm());
435  manifest::Reflog::HashDatabase(reflog_path, &hash_reflog);
436 
437  upload::Spooler::CallbackPtr callback =
438  spooler_files_->RegisterListener(&Publisher::OnUploadReflog, this);
439  spooler_files_->UploadReflog(reflog_path);
440  spooler_files_->WaitForUpload();
441  spooler_files_->UnregisterListener(callback);
442 
443  manifest_->set_reflog_hash(hash_reflog);
444 
445  reflog_ = manifest::Reflog::Open(reflog_path);
446  assert(reflog_ != NULL);
448 }
449 
450 
452  // TODO(jblomer): PKCS7 handling
453  upload::Spooler::CallbackPtr callback =
454  spooler_files_->RegisterListener(&Publisher::OnUploadWhitelist, this);
455  spooler_files_->Upload(".cvmfswhitelist",
457  spooler_files_->WaitForUpload();
458  spooler_files_->UnregisterListener(callback);
459 }
460 
461 
462 Publisher *Publisher::Create(const SettingsPublisher &settings) {
463  UniquePtr<Publisher> publisher(new Publisher(settings, false));
464 
465  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout | kLogNoLinebreak,
466  "Creating Key Chain... ");
467  publisher->CreateKeychain();
468  publisher->ExportKeychain();
469  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout, "done");
470 
471  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout | kLogNoLinebreak,
472  "Creating Backend Storage... ");
473  publisher->CreateStorage();
474  publisher->PushWhitelist();
475  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout, "done");
476 
477  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout | kLogNoLinebreak,
478  "Creating Initial Repository... ");
479  publisher->InitSpoolArea();
480  publisher->CreateRootObjects();
481  publisher->PushHistory();
482  publisher->PushCertificate();
483  publisher->PushMetainfo();
484  publisher->PushReflog();
485  publisher->PushManifest();
486  // TODO(jblomer): meta-info
487 
488  // Re-create from empty repository in order to properly initialize
489  // parent Repository object
490  publisher = new Publisher(settings);
491 
492  LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout, "done");
493 
494  return publisher.Release();
495 }
496 
499 
500  bool rvb;
503  if (!rvb) throw EPublish("cannot export public master key");
506  if (!rvb) throw EPublish("cannot export certificate");
507 
510  if (!rvb) throw EPublish("cannot export private certificate key");
513  if (!rvb) throw EPublish("cannot export private master key");
514 
515  int rvi;
516  rvi = chown(settings_.keychain().master_public_key_path().c_str(),
517  settings_.owner_uid(), settings_.owner_gid());
518  if (rvi != 0) throw EPublish("cannot set key file ownership");
519  rvi = chown(settings_.keychain().certificate_path().c_str(),
520  settings_.owner_uid(), settings_.owner_gid());
521  if (rvi != 0) throw EPublish("cannot set key file ownership");
522  rvi = chown(settings_.keychain().private_key_path().c_str(),
523  settings_.owner_uid(), settings_.owner_gid());
524  if (rvi != 0) throw EPublish("cannot set key file ownership");
525  rvi = chown(settings_.keychain().master_private_key_path().c_str(),
526  settings_.owner_uid(), settings_.owner_gid());
527  if (rvi != 0) throw EPublish("cannot set key file ownership");
528 }
529 
531  if (result.return_code != 0) {
532  throw EPublish("cannot write certificate to storage");
533  }
536 }
537 
539  if (result.return_code != 0) {
540  throw EPublish("cannot write tag database to storage");
541  }
544 }
545 
547  if (result.return_code != 0) {
548  throw EPublish("cannot write repository meta info to storage");
549  }
552 }
553 
555  if (result.return_code != 0) {
556  throw EPublish("cannot write manifest to storage");
557  }
558 }
559 
561  if (result.return_code != 0) {
562  throw EPublish("cannot write reflog to storage");
563  }
564 }
565 
567  if (result.return_code != 0) {
568  throw EPublish("cannot write whitelist to storage");
569  }
570 }
571 
572 void Publisher::CreateDirectoryAsOwner(const std::string &path, int mode)
573 {
574  bool rvb = MkdirDeep(path, kPrivateDirMode);
575  if (!rvb) throw EPublish("cannot create directory " + path);
576  int rvi = chown(path.c_str(), settings_.owner_uid(), settings_.owner_gid());
577  if (rvi != 0) throw EPublish("cannot set ownership on directory " + path);
578 }
579 
581  CreateDirectoryAsOwner(settings_.transaction().spool_area().workspace(),
583  CreateDirectoryAsOwner(settings_.transaction().spool_area().tmp_dir(),
585  CreateDirectoryAsOwner(settings_.transaction().spool_area().cache_dir(),
587  CreateDirectoryAsOwner(settings_.transaction().spool_area().scratch_dir(),
589  CreateDirectoryAsOwner(settings_.transaction().spool_area().ovl_work_dir(),
591 
592  // On a managed node, the mount points are already mounted
593  if (!DirectoryExists(settings_.transaction().spool_area().readonly_mnt())) {
594  CreateDirectoryAsOwner(settings_.transaction().spool_area().readonly_mnt(),
596  }
597  if (!DirectoryExists(settings_.transaction().spool_area().union_mnt())) {
599  settings_.transaction().spool_area().union_mnt(), kPrivateDirMode);
600  }
601 }
602 
603 Publisher::Publisher(const SettingsPublisher &settings, const bool exists)
604  : Repository(SettingsRepository(settings), exists)
605  , settings_(settings)
606  , statistics_publish_(new perf::StatisticsTemplate("publish", statistics_))
607  , llvl_(settings.is_silent() ? kLogNone : kLogNormal)
608  , in_transaction_(settings.transaction().spool_area().transaction_lock())
609  , is_publishing_(settings.transaction().spool_area().publishing_lock())
610  , spooler_files_(NULL)
611  , spooler_catalogs_(NULL)
612  , catalog_mgr_(NULL)
613  , sync_parameters_(NULL)
614  , sync_mediator_(NULL)
615  , sync_union_(NULL)
616 {
618  unsigned layout_revision = settings.transaction().layout_revision();
619  throw EPublish(
620  "This repository uses layout revision " + StringifyInt(layout_revision)
621  + ".\n"
622  "This version of CernVM-FS requires layout revision " + StringifyInt(
623  kRequiredLayoutRevision) + ", which is\n"
624  "incompatible to " + StringifyInt(layout_revision) + ".\n\n"
625  "Please run `cvmfs_server migrate` to update your repository before "
626  "proceeding.",
628  }
629 
630  if (!exists)
631  return;
632 
633  CreateDirectoryAsOwner(settings_.transaction().spool_area().tmp_dir(),
635 
636  if (settings.storage().type() == upload::SpoolerDefinition::Gateway) {
637  if (!settings.keychain().HasGatewayKey()) {
638  throw EPublish("gateway key missing: " +
639  settings.keychain().gw_key_path());
640  }
642  if (!gw_key_.IsValid()) {
643  throw EPublish("cannot read gateway key: " +
644  settings.keychain().gw_key_path());
645  }
646  }
647 
648  if ((settings.storage().type() != upload::SpoolerDefinition::Gateway) &&
649  !settings.transaction().in_enter_session())
650  {
652  settings.keychain().certificate_path());
653  if (!rvb)
654  throw EPublish("cannot load certificate, thus cannot commit changes");
656  settings.keychain().private_key_path(), "");
657  if (!rvb)
658  throw EPublish("cannot load private key, thus cannot commit changes");
659  // The private master key might be on a key card instead
660  if (FileExists(settings.keychain().master_private_key_path())) {
662  settings.keychain().master_private_key_path());
663  if (!rvb) throw EPublish("cannot load private master key");
664  }
665  if (!signature_mgr_->KeysMatch())
666  throw EPublish("corrupted keychain");
667  }
668 
669  if (settings.is_managed())
670  managed_node_ = new ManagedNode(this);
672  if (in_transaction_.IsSet())
674 }
675 
677  delete sync_union_;
678  delete sync_mediator_;
679  delete sync_parameters_;
680  delete catalog_mgr_;
681  delete spooler_catalogs_;
682  delete spooler_files_;
683 }
684 
685 
688 
689  if (catalog_mgr_ == NULL) {
691  settings_.transaction().base_hash(),
692  settings_.url(),
693  settings_.transaction().spool_area().tmp_dir(),
696  settings_.transaction().enforce_limits(),
697  settings_.transaction().limit_nested_catalog_kentries(),
698  settings_.transaction().limit_root_catalog_kentries(),
699  settings_.transaction().limit_file_size_mb(),
700  statistics_,
701  settings_.transaction().use_catalog_autobalance(),
702  settings_.transaction().autobalance_max_weight(),
703  settings_.transaction().autobalance_min_weight());
704  catalog_mgr_->Init();
705  }
706 
707  if (sync_parameters_ == NULL) {
708  SyncParameters *p = new SyncParameters();
709  p->spooler = spooler_files_;
710  p->repo_name = settings_.fqrn();
711  p->dir_union = settings_.transaction().spool_area().union_mnt();
712  p->dir_scratch = settings_.transaction().spool_area().scratch_dir();
713  p->dir_rdonly = settings_.transaction().spool_area().readonly_mnt();
714  p->dir_temp = settings_.transaction().spool_area().tmp_dir();
715  p->base_hash = settings_.transaction().base_hash();
716  p->stratum0 = settings_.url();
717  // p->manifest_path = SHOULD NOT BE NEEDED
718  // p->spooler_definition = SHOULD NOT BE NEEDED;
719  // p->union_fs_type = SHOULD NOT BE NEEDED
720  p->print_changeset = settings_.transaction().print_changeset();
721  p->dry_run = settings_.transaction().dry_run();
722  sync_parameters_ = p;
723  }
724 
725  if (sync_mediator_ == NULL) {
728  }
729 
730  if (sync_union_ == NULL) {
731  switch (settings_.transaction().union_fs()) {
732  case kUnionFsAufs:
735  settings_.transaction().spool_area().readonly_mnt(),
736  settings_.transaction().spool_area().union_mnt(),
737  settings_.transaction().spool_area().scratch_dir());
738  break;
739  case kUnionFsOverlay:
742  settings_.transaction().spool_area().readonly_mnt(),
743  settings_.transaction().spool_area().union_mnt(),
744  settings_.transaction().spool_area().scratch_dir());
745  break;
746  case kUnionFsTarball:
749  settings_.transaction().spool_area().readonly_mnt(),
750  // TODO(jblomer): get from settings
751  "tar_file",
752  "base_directory",
753  "to_delete",
754  false /* create_catalog */);
755  break;
756  default:
757  throw EPublish("unknown union file system");
758  }
759  bool rvb = sync_union_->Initialize();
760  if (!rvb) {
761  delete sync_union_;
762  sync_union_ = NULL;
763  throw EPublish("cannot initialize union file system engine");
764  }
765  }
766 }
767 
769  std::string session_dir = Env::GetEnterSessionDir();
770  std::string session_pid_tmp = session_dir + "/session_pid";
771  std::string session_pid;
772  int fd_session_pid = open(session_pid_tmp.c_str(), O_RDONLY);
773  if (fd_session_pid < 0) throw EPublish("Session pid cannot be retrieved");
774  SafeReadToString(fd_session_pid, &session_pid);
775 
776  pid_t pid_child = String2Uint64(session_pid);
777  kill(pid_child, SIGUSR1);
778 }
779 
782 
784 
786  bool rvb = sync_mediator_->Commit(manifest_);
787  if (!rvb) throw EPublish("cannot write change set to storage");
788 
789  if (!settings_.transaction().dry_run()) {
790  spooler_files_->WaitForUpload();
791  spooler_catalogs_->WaitForUpload();
792  spooler_files_->FinalizeSession(false /* commit */);
793 
794  const std::string old_root_hash =
795  settings_.transaction().base_hash().ToString(true /* with_suffix */);
796  const std::string new_root_hash =
797  manifest_->catalog_hash().ToString(true /* with_suffix */);
798  rvb = spooler_catalogs_->FinalizeSession(true /* commit */,
799  old_root_hash, new_root_hash,
800  /* TODO(jblomer) */ sync_parameters_->repo_tag);
801  if (!rvb)
802  throw EPublish("failed to commit transaction");
803 
804  // Reset to the new catalog root hash
805  settings_.GetTransaction()->SetBaseHash(manifest_->catalog_hash());
806  // TODO(jblomer): think about how to deal with the scratch area at
807  // this point
808  // WipeScratchArea();
809  }
810 
811  delete sync_union_;
812  delete sync_mediator_;
813  delete sync_parameters_;
814  delete catalog_mgr_;
815  sync_union_ = NULL;
816  sync_mediator_ = NULL;
817  sync_parameters_ = NULL;
818  catalog_mgr_ = NULL;
819 
820  if (!settings_.transaction().dry_run()) {
821  LogCvmfs(kLogCvmfs, kLogStdout, "New revision: %d", manifest_->revision());
823  }
824 }
825 
827  if (!in_transaction_.IsSet())
828  throw EPublish("cannot publish outside transaction");
829 
830  PushReflog();
831  PushManifest();
833 }
834 
835 
836 void Publisher::MarkReplicatible(bool value) {
838 
839  if (value) {
840  spooler_files_->Upload("/dev/null", "/.cvmfs_master_replica");
841  } else {
842  spooler_files_->RemoveAsync("/.cvmfs_master_replica");
843  }
844  spooler_files_->WaitForUpload();
845  if (spooler_files_->GetNumberOfErrors() > 0)
846  throw EPublish("cannot set replication mode");
847 }
848 
850 void Publisher::Migrate() {}
852 void Publisher::Rollback() {}
854 
855 void Publisher::Transaction() {
857  session()->SetKeepAlive(true);
858 }
859 
860 //------------------------------------------------------------------------------
861 
862 
864  : Repository(SettingsRepository(settings))
865 {
866 }
867 
868 
870 }
871 
872 } // namespace publish
bool Commit(manifest::Manifest *manifest)
int return_code
the return value of the spooler operation
#define LogCvmfs(source, mask,...)
Definition: logging.h:25
void set_repository_name(const std::string &repository_name)
Definition: manifest.h:93
void TakeDatabaseFileOwnership()
Definition: reflog.cc:307
const SettingsRepository settings_
Definition: repository.h:137
std::string database_file() const
Definition: reflog.cc:337
void CreateRootObjects()
Definition: repository.cc:303
manifest::Reflog * reflog_
Definition: repository.h:147
bool IsNull() const
Definition: hash.h:383
download::DownloadManager * download_mgr_
Definition: repository.h:141
bool HasGatewayKey() const
Definition: settings.cc:272
std::string stratum0
bool AddHistory(const shash::Any &history)
Definition: reflog.cc:134
void ConstructSyncManagers()
Definition: repository.cc:686
void OnUploadReflog(const upload::SpoolerResult &result)
Definition: repository.cc:560
catalog::SimpleCatalogManager * GetSimpleCatalogManager()
Definition: repository.cc:121
UniquePtr< ManagedNode > managed_node_
Definition: repository.h:381
Session * session() const
Definition: repository.h:320
NameString GetFileName(const PathString &path)
Definition: shortstring.cc:29
const int kPrivateDirMode
Definition: posix.h:36
T * weak_ref() const
Definition: pointer.h:42
void Ingest()
Definition: repository.cc:849
UniquePtr< perf::StatisticsTemplate > statistics_publish_
Definition: repository.h:365
static Publisher * Create(const SettingsPublisher &settings)
std::string keychain_dir() const
Definition: settings.h:325
std::string GetCertificate() const
Definition: signature.cc:398
static const std::string kPreviousHeadTag
Definition: repository.cc:41
Failures LoadUrl(const std::string &base_url)
Definition: whitelist.cc:216
std::string gw_key_path() const
Definition: settings.h:334
FILE * CreateTempFile(const std::string &path_prefix, const int mode, const char *open_flags, std::string *final_path)
Definition: posix.cc:1002
string JoinStrings(const vector< string > &strings, const string &joint)
Definition: string.cc:325
std::string ToString(const bool with_suffix=false) const
Definition: hash.h:249
void OnProcessCertificate(const upload::SpoolerResult &result)
Definition: repository.cc:530
virtual bool Initialize()
Definition: sync_union.cc:24
void set_certificate(const shash::Any &certificate)
Definition: manifest.h:87
const history::History * history() const
bool IsFileNotFound()
Definition: download.cc:385
void CreateDirectoryAsOwner(const std::string &path, int mode)
Definition: repository.cc:572
void PushCertificate()
Definition: repository.cc:354
unsigned layout_revision() const
Definition: settings.h:188
void SetProxyChain(const std::string &proxy_list, const std::string &fallback_proxy_list, const ProxySetModes set_mode)
Definition: download.cc:2425
void OnUploadWhitelist(const upload::SpoolerResult &result)
Definition: repository.cc:566
void CreateStorage()
Definition: repository.cc:347
bool IsMasterReplica()
Definition: repository.cc:236
std::string private_key_path() const
Definition: settings.h:332
const SettingsPublisher & settings() const
Definition: repository.h:317
perf::Statistics * statistics_
Definition: repository.h:139
Status status() const
Definition: whitelist.h:103
std::string CreateTempPath(const std::string &path_prefix, const int mode)
Definition: posix.cc:1031
void set_meta_info(const shash::Any &meta_info)
Definition: manifest.h:111
std::string GetPrivateKey()
Definition: signature.cc:414
std::string ExportString() const
Definition: manifest.cc:174
virtual ~Replica()
Definition: repository.cc:869
std::string repo_name
assert((mem||(size==0))&&"Out Of Memory")
bool has_alt_catalog_path() const
Definition: manifest.h:130
upload::Spooler * spooler_catalogs_
Definition: repository.h:384
gateway::GatewayKey gw_key_
Definition: repository.h:372
bool SafeWriteToFile(const std::string &content, const std::string &path, int mode)
Definition: posix.cc:2078
void PushMetainfo()
Definition: repository.cc:383
bool LoadPrivateKeyPath(const std::string &file_pem, const std::string &password)
Definition: signature.cc:185
class __attribute__((visibility("default"))) Publisher ManagedNode(Publisher *p)
Definition: repository.h:174
void PushReflog()
Definition: repository.cc:429
uint64_t catalog_size() const
Definition: manifest.h:125
void ExitShell()
Definition: repository.cc:768
bool AddCatalog(const shash::Any &catalog)
Definition: reflog.cc:128
static Reflog * Create(const std::string &database_path, const std::string &repo_name)
Definition: reflog.cc:32
std::string cert_bundle() const
Definition: settings.h:374
catalog::WritableCatalogManager * catalog_mgr_
Definition: repository.h:385
std::string dir_scratch
bool HasMasterKeys() const
Definition: settings.cc:253
void GenerateCertificate(const std::string &cn)
Definition: signature.cc:480
uint64_t revision() const
Definition: manifest.h:121
Repository(const SettingsRepository &settings, const bool exists=true)
Definition: repository.cc:49
static Reflog * Open(const std::string &database_path)
Definition: reflog.cc:17
static SqliteHistory * OpenWritable(const std::string &file_name)
std::string master_public_key_path() const
Definition: settings.h:329
bool FileExists(const std::string &path)
Definition: posix.cc:788
std::string GetActivePubkeys() const
Definition: signature.cc:385
std::string dir_temp
const int kPrivateFileMode
Definition: posix.h:35
void OnProcessMetainfo(const upload::SpoolerResult &result)
Definition: repository.cc:546
void Init(const unsigned max_pool_handles, const perf::StatisticsTemplate &statistics)
Definition: download.cc:1621
const char * Code2Ascii(const Failures error)
Definition: download.h:92
static manifest::Manifest * CreateRepository(const std::string &dir_temp, const bool volatile_content, const std::string &voms_authz, upload::Spooler *spooler)
bool HasRepositoryKeys() const
Definition: settings.cc:267
std::string proxy() const
Definition: settings.h:372
std::string meta_info_
Definition: repository.h:151
void PushHistory()
Definition: repository.cc:364
const SettingsKeychain & keychain() const
Definition: settings.h:439
publish::SyncUnion * sync_union_
Definition: repository.h:388
uint64_t publish_timestamp() const
Definition: manifest.h:128
bool AddMetainfo(const shash::Any &metainfo)
Definition: reflog.cc:140
void Transaction()
const int kDefaultDirMode
Definition: posix.h:33
bool Sign(const unsigned char *buffer, const unsigned buffer_size, unsigned char **signature, unsigned *signature_size)
Definition: signature.cc:758
void OnUploadManifest(const upload::SpoolerResult &result)
Definition: repository.cc:554
std::string dir_rdonly
signature::SignatureManager * signature_mgr_
Definition: repository.h:140
void SetKeepAlive(bool value)
Failures Fetch(const std::string &base_url, const std::string &repository_name, const uint64_t minimum_timestamp, const shash::Any *base_catalog, signature::SignatureManager *signature_manager, download::DownloadManager *download_manager, ManifestEnsemble *ensemble)
void UpdateMetaInfo()
Definition: repository.cc:853
std::string master_private_key_path() const
Definition: settings.h:326
void DownloadRootObjects(const std::string &url, const std::string &fqrn, const std::string &tmp_dir)
Definition: repository.cc:136
shash::Any base_hash
static const std::string kHeadTag
Definition: repository.cc:40
shash::Any certificate() const
Definition: manifest.h:126
virtual ~Publisher()
upload::Spooler * spooler
bool MkdirDeep(const std::string &path, const mode_t mode, bool verify_writable)
Definition: posix.cc:843
bool LoadCertificatePath(const std::string &file_pem)
Definition: signature.cc:245
const SettingsTransaction & transaction() const
Definition: settings.h:438
bool IsValid() const
Definition: gateway_util.h:17
const char * Code2Ascii(const Failures error)
Definition: whitelist.h:49
ServerLockFile is_publishing_
Definition: repository.h:371
std::string dir_union
void Sync()
Definition: repository.cc:780
void InitSpoolArea()
Definition: repository.cc:580
static void HashDatabase(const std::string &database_path, shash::Any *hash_reflog)
Definition: reflog.cc:322
void PushManifest()
Definition: repository.cc:392
shash::Any catalog_hash() const
Definition: manifest.h:124
string StringifyInt(const int64_t value)
Definition: string.cc:78
std::string certificate_path() const
Definition: settings.h:333
catalog::SimpleCatalogManager * simple_catalog_mgr_
Definition: repository.h:145
history::SqliteHistory * history_
Definition: repository.h:149
void Resign()
Definition: repository.cc:851
bool is_managed() const
Definition: settings.h:434
void Publish()
const SettingsStorage & storage() const
Definition: settings.h:437
void DropDatabaseFileOwnership()
Definition: reflog.cc:313
bool SetPreviousRevision(const shash::Any &history_hash)
bool LoadPrivateMasterKeyPath(const std::string &file_pem)
Definition: signature.cc:153
bool DirectoryExists(const std::string &path)
Definition: posix.cc:810
UniquePtr< Session > session_
Definition: repository.h:380
void set_history(const shash::Any &history_db)
Definition: manifest.h:90
void set_garbage_collectability(const bool garbage_collectable)
Definition: manifest.h:105
bool AddCertificate(const shash::Any &certificate)
Definition: reflog.cc:121
std::string filename() const
bool SafeReadToString(int fd, std::string *final_result)
Definition: posix.cc:2062
manifest::Manifest * manifest_
Definition: repository.h:148
void OnProcessHistory(const upload::SpoolerResult &result)
Definition: repository.cc:538
void set_ttl(const uint32_t ttl)
Definition: manifest.h:85
void HashMem(const unsigned char *buffer, const unsigned buffer_size, Any *any_digest)
Definition: hash.cc:255
upload::Spooler * spooler_files_
Definition: repository.h:383
void MarkReplicatible(bool value)
Definition: repository.cc:836
class __attribute__((visibility("default"))) Replica Replica(const SettingsReplica &settings)
Definition: repository.cc:863
const SettingsKeychain & keychain() const
Definition: settings.h:376
std::string ExportString() const
Definition: whitelist.cc:77
virtual void Traverse()=0
uint64_t String2Uint64(const string &value)
Definition: string.cc:228
std::string GetPrivateMasterKey()
Definition: signature.cc:430
void Migrate(InodeTracker *old_tracker, glue::InodeTracker *new_tracker)
Definition: compat.cc:80
Failures Fetch(JobInfo *info)
Definition: download.cc:1734
shash::Any history() const
Definition: manifest.h:127
ServerFlagFile in_transaction_
Definition: repository.h:370
static std::string CreateString(const std::string &fqrn, int validity_days, shash::Algorithms hash_algorithm, signature::SignatureManager *signature_manager)
Definition: whitelist.cc:47
bool LoadPublicRsaKeys(const std::string &path_list)
Definition: signature.cc:305
whitelist::Whitelist * whitelist_
Definition: repository.h:146
GatewayKey ReadGatewayKey(const std::string &key_file_name)
Definition: gateway_util.cc:28
static SqliteHistory * Create(const std::string &file_name, const std::string &fqrn)
bool HasDanglingRepositoryKeys() const
Definition: settings.cc:259
Failures LoadMem(const std::string &whitelist)
Definition: whitelist.cc:196
struct download::JobInfo::@4 destination_mem
RepositoryTag repo_tag
SyncMediator * sync_mediator_
Definition: repository.h:387
std::string fqrn() const
Definition: settings.h:370
void ExportKeychain()
Definition: repository.cc:497
void set_has_alt_catalog_path(const bool &has_alt_path)
Definition: manifest.h:108
int llvl_
Definition: repository.h:369
upload::SpoolerDefinition::DriverType type() const
Definition: settings.h:294
Publisher(const SettingsPublisher &settings, const bool exists=true)
Definition: repository.cc:603
std::string MakePath() const
Definition: hash.h:316
std::string MakeCanonicalPath(const std::string &path)
Definition: posix.cc:98
bool in_enter_session() const
Definition: settings.h:189
virtual ~Repository()
Definition: repository.cc:105
void set_reflog_hash(const shash::Any &checksum)
Definition: manifest.h:117
std::string url() const
Definition: settings.h:371
void ConstructSpoolers()
Definition: repository.cc:251
void Rollback()
void CreateKeychain()
Definition: repository.cc:279
void PushWhitelist()
Definition: repository.cc:451
std::vector< std::string > FindFilesBySuffix(const std::string &dir, const std::string &suffix)
Definition: posix.cc:1121
shash::Any meta_info() const
Definition: manifest.h:131
std::string tmp_dir() const
Definition: settings.h:373
static const unsigned kRequiredLayoutRevision
Definition: repository.h:278
SyncParameters * sync_parameters_
Definition: repository.h:386
class static __attribute__((visibility("default"))) Repository std::string GetFqrnFromUrl(const std::string &url)
Definition: repository.cc:231
bool HasDanglingMasterKeys() const
Definition: settings.cc:245