CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
catalog_mgr_rw.cc
Go to the documentation of this file.
1 
5 #define __STDC_FORMAT_MACROS
6 
7 #include "catalog_mgr_rw.h"
8 
9 #include <inttypes.h>
10 #include <unistd.h>
11 
12 #include <cassert>
13 #include <cstdio>
14 #include <cstdlib>
15 #include <string>
16 
17 #include "catalog_balancer.h"
18 #include "catalog_rw.h"
19 #include "logging.h"
20 #include "manifest.h"
21 #include "smalloc.h"
22 #include "statistics.h"
23 #include "upload.h"
24 #include "util/exception.h"
25 #include "util/posix.h"
26 
27 using namespace std; // NOLINT
28 
29 namespace catalog {
30 
31 WritableCatalogManager::WritableCatalogManager(
32  const shash::Any &base_hash,
33  const std::string &stratum0,
34  const string &dir_temp,
35  upload::Spooler *spooler,
36  download::DownloadManager *download_manager,
37  bool enforce_limits,
38  const unsigned nested_kcatalog_limit,
39  const unsigned root_kcatalog_limit,
40  const unsigned file_mbyte_limit,
41  perf::Statistics *statistics,
42  bool is_balanceable,
43  unsigned max_weight,
44  unsigned min_weight)
45  : SimpleCatalogManager(base_hash, stratum0, dir_temp, download_manager,
46  statistics)
47  , spooler_(spooler)
48  , enforce_limits_(enforce_limits)
49  , nested_kcatalog_limit_(nested_kcatalog_limit)
50  , root_kcatalog_limit_(root_kcatalog_limit)
51  , file_mbyte_limit_(file_mbyte_limit)
52  , is_balanceable_(is_balanceable)
53  , max_weight_(max_weight)
54  , min_weight_(min_weight)
55  , balance_weight_(max_weight / 2)
56 {
57  sync_lock_ =
58  reinterpret_cast<pthread_mutex_t *>(smalloc(sizeof(pthread_mutex_t)));
59  int retval = pthread_mutex_init(sync_lock_, NULL);
60  assert(retval == 0);
62  reinterpret_cast<pthread_mutex_t *>(smalloc(sizeof(pthread_mutex_t)));
63  retval = pthread_mutex_init(catalog_processing_lock_, NULL);
64  assert(retval == 0);
65 }
66 
67 
69  pthread_mutex_destroy(sync_lock_);
70  free(sync_lock_);
71  pthread_mutex_destroy(catalog_processing_lock_);
73 }
74 
75 
86  const PathString &mountpoint,
87  const shash::Any &catalog_hash,
88  Catalog *parent_catalog)
89 {
90  return new WritableCatalog(mountpoint.ToString(),
91  catalog_hash,
92  parent_catalog);
93 }
94 
95 
97  catalog->TakeDatabaseFileOwnership();
98 }
99 
100 
108  const string &dir_temp,
109  const bool volatile_content,
110  const std::string &voms_authz,
111  upload::Spooler *spooler)
112 {
113  // Create a new root catalog at file_path
114  string file_path = dir_temp + "/new_root_catalog";
115 
116  shash::Algorithms hash_algorithm = spooler->GetHashAlgorithm();
117 
118  // A newly created catalog always needs a root entry
119  // we create and configure this here
120  DirectoryEntry root_entry;
122  root_entry.mode_ = 16877;
123  root_entry.size_ = 4096;
124  root_entry.mtime_ = time(NULL);
125  root_entry.uid_ = getuid();
126  root_entry.gid_ = getgid();
127  root_entry.checksum_ = shash::Any(hash_algorithm);
128  root_entry.linkcount_ = 2;
129  string root_path = "";
130 
131  // Create the database schema and the inital root entry
132  {
134  if (!new_clg_db.IsValid() ||
135  !new_clg_db->InsertInitialValues(root_path,
136  volatile_content,
137  voms_authz,
138  root_entry))
139  {
140  LogCvmfs(kLogCatalog, kLogStderr, "creation of catalog '%s' failed",
141  file_path.c_str());
142  return NULL;
143  }
144  }
145 
146  // Compress root catalog;
147  int64_t catalog_size = GetFileSize(file_path);
148  if (catalog_size < 0) {
149  unlink(file_path.c_str());
150  return NULL;
151  }
152  string file_path_compressed = file_path + ".compressed";
153  shash::Any hash_catalog(hash_algorithm, shash::kSuffixCatalog);
154  bool retval = zlib::CompressPath2Path(file_path, file_path_compressed,
155  &hash_catalog);
156  if (!retval) {
157  LogCvmfs(kLogCatalog, kLogStderr, "compression of catalog '%s' failed",
158  file_path.c_str());
159  unlink(file_path.c_str());
160  return NULL;
161  }
162  unlink(file_path.c_str());
163 
164  // Create manifest
165  const string manifest_path = dir_temp + "/manifest";
167  new manifest::Manifest(hash_catalog, catalog_size, "");
168  if (!voms_authz.empty()) {
169  manifest->set_has_alt_catalog_path(true);
170  }
171 
172  // Upload catalog
173  spooler->Upload(file_path_compressed, "data/" + hash_catalog.MakePath());
174  spooler->WaitForUpload();
175  unlink(file_path_compressed.c_str());
176  if (spooler->GetNumberOfErrors() > 0) {
177  LogCvmfs(kLogCatalog, kLogStderr, "failed to commit catalog %s",
178  file_path_compressed.c_str());
179  delete manifest;
180  return NULL;
181  }
182 
183  return manifest;
184 }
185 
186 
198 bool WritableCatalogManager::FindCatalog(const string &path,
199  WritableCatalog **result,
200  DirectoryEntry *dirent) {
201  const PathString ps_path(path);
202 
203  Catalog *best_fit =
205  assert(best_fit != NULL);
206  Catalog *catalog = NULL;
207  bool retval =
208  MountSubtree(ps_path, best_fit, true /* is_listable */, &catalog);
209  if (!retval)
210  return false;
211 
213  if (NULL == dirent) {
214  dirent = &dummy;
215  }
216  bool found = catalog->LookupPath(ps_path, dirent);
217  if (!found || !catalog->IsWritable())
218  return false;
219 
220  *result = static_cast<WritableCatalog *>(catalog);
221  return true;
222 }
223 
224 
226  const std::string &path)
227 {
228  WritableCatalog *result = NULL;
229  bool retval = FindCatalog(MakeRelativePath(path), &result, NULL);
230  if (!retval) return NULL;
231  return result;
232 }
233 
234 
240 void WritableCatalogManager::RemoveFile(const std::string &path) {
241  const string file_path = MakeRelativePath(path);
242  const string parent_path = GetParentPath(file_path);
243 
244  SyncLock();
245  WritableCatalog *catalog;
246  if (!FindCatalog(parent_path, &catalog)) {
247  PANIC(kLogStderr, "catalog for file '%s' cannot be found",
248  file_path.c_str());
249  }
250 
251  catalog->RemoveEntry(file_path);
252  SyncUnlock();
253 }
254 
255 
261 void WritableCatalogManager::RemoveDirectory(const std::string &path) {
262  const string directory_path = MakeRelativePath(path);
263  const string parent_path = GetParentPath(directory_path);
264 
265  SyncLock();
266  WritableCatalog *catalog;
267  DirectoryEntry parent_entry;
268  if (!FindCatalog(parent_path, &catalog, &parent_entry)) {
269  PANIC(kLogStderr, "catalog for directory '%s' cannot be found",
270  directory_path.c_str());
271  }
272 
273  parent_entry.set_linkcount(parent_entry.linkcount() - 1);
274 
275  catalog->RemoveEntry(directory_path);
276  catalog->UpdateEntry(parent_entry, parent_path);
277  if (parent_entry.IsNestedCatalogRoot()) {
278  LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating transition point %s",
279  parent_path.c_str());
280  WritableCatalog *parent_catalog =
281  reinterpret_cast<WritableCatalog *>(catalog->parent());
282  parent_entry.set_is_nested_catalog_mountpoint(true);
283  parent_entry.set_is_nested_catalog_root(false);
284  parent_catalog->UpdateEntry(parent_entry, parent_path);
285  }
286  SyncUnlock();
287 }
288 
297 void WritableCatalogManager::Clone(const std::string destination,
298  const std::string source) {
299  const std::string relative_source = MakeRelativePath(source);
300 
301  DirectoryEntry source_dirent;
302  if (!LookupPath(relative_source, kLookupSole, &source_dirent)) {
303  PANIC(kLogStderr, "catalog for file '%s' cannot be found, aborting",
304  source.c_str());
305  }
306  if (source_dirent.IsDirectory()) {
307  PANIC(kLogStderr, "Trying to clone a directory: '%s', aborting",
308  source.c_str());
309  }
310 
311  // if the file is already there we remove it and we add it back
312  DirectoryEntry check_dirent;
313  bool destination_already_present =
314  LookupPath(MakeRelativePath(destination), kLookupSole, &check_dirent);
315  if (destination_already_present) {
316  this->RemoveFile(destination);
317  }
318 
319  DirectoryEntry destination_dirent(source_dirent);
320  std::string destination_dirname;
321  std::string destination_filename;
322  SplitPath(destination, &destination_dirname, &destination_filename);
323 
324  destination_dirent.name_.Assign(
325  NameString(destination_filename.c_str(), destination_filename.length()));
326 
327  // TODO(jblomer): clone is used by tarball engine and should eventually
328  // support extended attributes
329  this->AddFile(destination_dirent, empty_xattrs, destination_dirname);
330 }
331 
332 
339 void WritableCatalogManager::CloneTree(const std::string &from_dir,
340  const std::string &to_dir)
341 {
342  // Sanitize input paths
343  if (from_dir.empty() || to_dir.empty())
344  PANIC(kLogStderr, "clone tree from or to root impossible");
345 
346  const std::string relative_source = MakeRelativePath(from_dir);
347  const std::string relative_dest = MakeRelativePath(to_dir);
348 
349  if (relative_source == relative_dest) {
350  PANIC(kLogStderr, "cannot clone tree into itself ('%s')", to_dir.c_str());
351  }
352  if (HasPrefix(relative_dest, relative_source + "/", false /*ignore_case*/)) {
354  "cannot clone tree into sub directory of source '%s' --> '%s'",
355  from_dir.c_str(), to_dir.c_str());
356  }
357 
358  DirectoryEntry source_dirent;
359  if (!LookupPath(relative_source, kLookupSole, &source_dirent)) {
360  PANIC(kLogStderr, "path '%s' cannot be found, aborting", from_dir.c_str());
361  }
362  if (!source_dirent.IsDirectory()) {
363  PANIC(kLogStderr, "CloneTree: source '%s' not a directory, aborting",
364  from_dir.c_str());
365  }
366 
367  DirectoryEntry dest_dirent;
368  if (LookupPath(relative_dest, kLookupSole, &dest_dirent)) {
369  PANIC(kLogStderr, "destination '%s' exists, aborting", to_dir.c_str());
370  }
371 
372  const std::string dest_parent = GetParentPath(relative_dest);
373  DirectoryEntry dest_parent_dirent;
374  if (!LookupPath(dest_parent, kLookupSole, &dest_parent_dirent)) {
375  PANIC(kLogStderr, "destination '%s' not on a known path, aborting",
376  to_dir.c_str());
377  }
378 
379  CloneTreeImpl(PathString(from_dir),
380  GetParentPath(to_dir),
381  NameString(GetFileName(to_dir)));
382 }
383 
384 
390  const PathString &source_dir,
391  const std::string &dest_parent_dir,
392  const NameString &dest_name)
393 {
394  LogCvmfs(kLogCatalog, kLogDebug, "cloning %s --> %s/%s", source_dir.c_str(),
395  dest_parent_dir.c_str(), dest_name.ToString().c_str());
396  PathString relative_source(MakeRelativePath(source_dir.ToString()));
397 
398  DirectoryEntry source_dirent;
399  bool retval = LookupPath(relative_source, kLookupSole, &source_dirent);
400  assert(retval);
401  assert(!source_dirent.IsBindMountpoint());
402 
403  DirectoryEntry dest_dirent(source_dirent);
404  dest_dirent.name_.Assign(dest_name);
405  // Just in case, reset the nested catalog markers
406  dest_dirent.set_is_nested_catalog_mountpoint(false);
407  dest_dirent.set_is_nested_catalog_root(false);
408 
409  XattrList xattrs;
410  if (source_dirent.HasXattrs()) {
411  retval = LookupXattrs(relative_source, &xattrs);
412  assert(retval);
413  }
414  AddDirectory(dest_dirent, xattrs, dest_parent_dir);
415 
416  std::string dest_dir = dest_parent_dir;
417  if (!dest_dir.empty())
418  dest_dir.push_back('/');
419  dest_dir += dest_name.ToString();
420  if (source_dirent.IsNestedCatalogRoot() ||
421  source_dirent.IsNestedCatalogMountpoint())
422  {
423  CreateNestedCatalog(dest_dir);
424  }
425 
427  retval = Listing(relative_source, &ls, false /* expand_symlink */);
428  assert(retval);
429  for (unsigned i = 0; i < ls.size(); ++i) {
430  PathString sub_path(source_dir);
431  assert(!sub_path.IsEmpty());
432  sub_path.Append("/", 1);
433  sub_path.Append(ls[i].name().GetChars(), ls[i].name().GetLength());
434 
435  if (ls[i].IsDirectory()) {
436  CloneTreeImpl(sub_path, dest_dir, ls[i].name());
437  continue;
438  }
439 
440  // We break hard-links during cloning
441  ls[i].set_hardlink_group(0);
442  ls[i].set_linkcount(1);
443 
444  xattrs.Clear();
445  if (ls[i].HasXattrs()) {
446  retval = LookupXattrs(sub_path, &xattrs);
447  assert(retval);
448  }
449 
450  if (ls[i].IsChunkedFile()) {
451  FileChunkList chunks;
452  std::string relative_sub_path = MakeRelativePath(sub_path.ToString());
453  retval = ListFileChunks(
454  PathString(relative_sub_path), ls[i].hash_algorithm(), &chunks);
455  assert(retval);
456  AddChunkedFile(ls[i], xattrs, dest_dir, chunks);
457  } else {
458  AddFile(ls[i], xattrs, dest_dir);
459  }
460  }
461 }
462 
463 
472  const XattrList &xattrs,
473  const std::string &parent_directory)
474 {
475  const string parent_path = MakeRelativePath(parent_directory);
476  string directory_path = parent_path + "/";
477  directory_path.append(entry.name().GetChars(), entry.name().GetLength());
478 
479  SyncLock();
480  WritableCatalog *catalog;
481  DirectoryEntry parent_entry;
482  if (!FindCatalog(parent_path, &catalog, &parent_entry)) {
483  PANIC(kLogStderr, "catalog for directory '%s' cannot be found",
484  directory_path.c_str());
485  }
486 
487  DirectoryEntry fixed_hardlink_count(entry);
488  fixed_hardlink_count.set_linkcount(2);
489  catalog->AddEntry(fixed_hardlink_count, xattrs,
490  directory_path, parent_path);
491 
492  parent_entry.set_linkcount(parent_entry.linkcount() + 1);
493  catalog->UpdateEntry(parent_entry, parent_path);
494  if (parent_entry.IsNestedCatalogRoot()) {
495  LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating transition point %s",
496  parent_path.c_str());
497  WritableCatalog *parent_catalog =
498  reinterpret_cast<WritableCatalog *>(catalog->parent());
499  parent_entry.set_is_nested_catalog_mountpoint(true);
500  parent_entry.set_is_nested_catalog_root(false);
501  parent_catalog->UpdateEntry(parent_entry, parent_path);
502  }
503  SyncUnlock();
504 }
505 
514  const DirectoryEntry &entry,
515  const XattrList &xattrs,
516  const std::string &parent_directory)
517 {
518  const string parent_path = MakeRelativePath(parent_directory);
519  const string file_path = entry.GetFullPath(parent_path);
520 
521  SyncLock();
522  WritableCatalog *catalog;
523  if (!FindCatalog(parent_path, &catalog)) {
524  PANIC(kLogStderr, "catalog for file '%s' cannot be found",
525  file_path.c_str());
526  }
527 
528  assert(!entry.IsRegular() || entry.IsChunkedFile() ||
529  !entry.checksum().IsNull());
530  assert(entry.IsRegular() || !entry.IsExternalFile());
531 
532  // check if file is too big
533  unsigned mbytes = entry.size() / (1024 * 1024);
534  if ((file_mbyte_limit_ > 0) && (mbytes > file_mbyte_limit_)) {
536  "%s: file at %s is larger than %u megabytes (%u). "
537  "CernVM-FS works best with small files. "
538  "Please remove the file or increase the limit.",
539  enforce_limits_ ? "FATAL" : "WARNING", file_path.c_str(),
540  file_mbyte_limit_, mbytes);
541  if (enforce_limits_)
542  PANIC(kLogStderr, "file at %s is larger than %u megabytes (%u).",
543  file_path.c_str(), file_mbyte_limit_, mbytes);
544  }
545 
546  catalog->AddEntry(entry, xattrs, file_path, parent_path);
547  SyncUnlock();
548 }
549 
550 
552  const DirectoryEntryBase &entry,
553  const XattrList &xattrs,
554  const std::string &parent_directory,
555  const FileChunkList &file_chunks)
556 {
557  assert(file_chunks.size() > 0);
558 
559  DirectoryEntry full_entry(entry);
560  full_entry.set_is_chunked_file(true);
561 
562  AddFile(full_entry, xattrs, parent_directory);
563 
564  const string parent_path = MakeRelativePath(parent_directory);
565  const string file_path = entry.GetFullPath(parent_path);
566 
567  SyncLock();
568  WritableCatalog *catalog;
569  if (!FindCatalog(parent_path, &catalog)) {
570  PANIC(kLogStderr, "catalog for file '%s' cannot be found",
571  file_path.c_str());
572  }
573 
574  for (unsigned i = 0; i < file_chunks.size(); ++i) {
575  catalog->AddFileChunk(file_path, *file_chunks.AtPtr(i));
576  }
577  SyncUnlock();
578 }
579 
580 
589  const DirectoryEntryBaseList &entries,
590  const XattrList &xattrs,
591  const std::string &parent_directory,
592  const FileChunkList &file_chunks)
593 {
594  assert(entries.size() >= 1);
595  assert(file_chunks.IsEmpty() || entries[0].IsRegular());
596  if (entries.size() == 1) {
597  DirectoryEntry fix_linkcount(entries[0]);
598  fix_linkcount.set_linkcount(1);
599  if (file_chunks.IsEmpty())
600  return AddFile(fix_linkcount, xattrs, parent_directory);
601  return AddChunkedFile(fix_linkcount, xattrs, parent_directory, file_chunks);
602  }
603 
604  LogCvmfs(kLogCatalog, kLogVerboseMsg, "adding hardlink group %s/%s",
605  parent_directory.c_str(), entries[0].name().c_str());
606 
607  // Hardlink groups have to reside in the same directory.
608  // Therefore we only have one parent directory here
609  const string parent_path = MakeRelativePath(parent_directory);
610 
611  // check if hard link is too big
612  unsigned mbytes = entries[0].size() / (1024 * 1024);
613  if ((file_mbyte_limit_ > 0) && (mbytes > file_mbyte_limit_)) {
615  "%s: hard link at %s is larger than %u megabytes (%u). "
616  "CernVM-FS works best with small files. "
617  "Please remove the file or increase the limit.",
618  enforce_limits_ ? "FATAL" : "WARNING",
619  (parent_path + entries[0].name().ToString()).c_str(),
621  mbytes);
622  if (enforce_limits_)
623  PANIC(kLogStderr, "hard link at %s is larger than %u megabytes (%u)",
624  (parent_path + entries[0].name().ToString()).c_str(),
625  file_mbyte_limit_, mbytes);
626  }
627 
628  SyncLock();
629  WritableCatalog *catalog;
630  if (!FindCatalog(parent_path, &catalog)) {
632  "catalog for hardlink group containing '%s' cannot be found",
633  parent_path.c_str());
634  }
635 
636  // Get a valid hardlink group id for the catalog the group will end up in
637  // TODO(unkown): Compaction
638  uint32_t new_group_id = catalog->GetMaxLinkId() + 1;
639  LogCvmfs(kLogCatalog, kLogVerboseMsg, "hardlink group id %u issued",
640  new_group_id);
641  assert(new_group_id > 0);
642 
643  // Add the file entries to the catalog
644  for (DirectoryEntryBaseList::const_iterator i = entries.begin(),
645  iEnd = entries.end(); i != iEnd; ++i)
646  {
647  string file_path = parent_path + "/";
648  file_path.append(i->name().GetChars(), i->name().GetLength());
649 
650  // create a fully fledged DirectoryEntry to add the hardlink group to it
651  // which is CVMFS specific meta data.
652  DirectoryEntry hardlink(*i);
653  hardlink.set_hardlink_group(new_group_id);
654  hardlink.set_linkcount(entries.size());
655  hardlink.set_is_chunked_file(!file_chunks.IsEmpty());
656 
657  catalog->AddEntry(hardlink, xattrs, file_path, parent_path);
658  if (hardlink.IsChunkedFile()) {
659  for (unsigned i = 0; i < file_chunks.size(); ++i) {
660  catalog->AddFileChunk(file_path, *file_chunks.AtPtr(i));
661  }
662  }
663  }
664  SyncUnlock();
665 }
666 
667 
668 void WritableCatalogManager::ShrinkHardlinkGroup(const string &remove_path) {
669  const string relative_path = MakeRelativePath(remove_path);
670 
671  SyncLock();
672  WritableCatalog *catalog;
673  if (!FindCatalog(relative_path, &catalog)) {
675  "catalog for hardlink group containing '%s' cannot be found",
676  remove_path.c_str());
677  }
678 
679  catalog->IncLinkcount(relative_path, -1);
680  SyncUnlock();
681 }
682 
683 
693  const XattrList &xattrs,
694  const std::string &directory_path)
695 {
696  assert(entry.IsDirectory());
697 
698  const string entry_path = MakeRelativePath(directory_path);
699  const string parent_path = GetParentPath(entry_path);
700 
701  SyncLock();
702  // find the catalog to be updated
703  WritableCatalog *catalog;
704  if (!FindCatalog(parent_path, &catalog)) {
705  PANIC(kLogStderr, "catalog for entry '%s' cannot be found",
706  entry_path.c_str());
707  }
708 
709  catalog->TouchEntry(entry, xattrs, entry_path);
710 
711  // since we deal with a directory here, we might just touch a
712  // nested catalog transition point. If this is the case we would need to
713  // update two catalog entries:
714  // * the nested catalog MOUNTPOINT in the parent catalog
715  // * the nested catalog ROOT in the nested catalog
716 
717  // first check if we really have a nested catalog transition point
718  catalog::DirectoryEntry potential_transition_point;
719  PathString transition_path(entry_path.data(), entry_path.length());
720  bool retval = catalog->LookupPath(transition_path,
721  &potential_transition_point);
722  assert(retval);
723  if (potential_transition_point.IsNestedCatalogMountpoint()) {
725  "updating transition point at %s", entry_path.c_str());
726 
727  // find and mount nested catalog assciated to this transition point
728  shash::Any nested_hash;
729  uint64_t nested_size;
730  retval = catalog->FindNested(transition_path, &nested_hash, &nested_size);
731  assert(retval);
732  Catalog *nested_catalog;
733  nested_catalog = MountCatalog(transition_path, nested_hash, catalog);
734  assert(nested_catalog != NULL);
735 
736  // update nested catalog root in the child catalog
737  reinterpret_cast<WritableCatalog *>(nested_catalog)->
738  TouchEntry(entry, xattrs, entry_path);
739  }
740 
741  SyncUnlock();
742 }
743 
744 
751 void WritableCatalogManager::CreateNestedCatalog(const std::string &mountpoint)
752 {
753  const string nested_root_path = MakeRelativePath(mountpoint);
754  const PathString ps_nested_root_path(nested_root_path);
755 
756  SyncLock();
757  // Find the catalog currently containing the directory structure, which
758  // will be represented as a new nested catalog and its root-entry/mountpoint
759  // along the way
760  WritableCatalog *old_catalog = NULL;
761  DirectoryEntry new_root_entry;
762  if (!FindCatalog(nested_root_path, &old_catalog, &new_root_entry)) {
764  "failed to create nested catalog '%s': "
765  "mountpoint was not found in current catalog structure",
766  nested_root_path.c_str());
767  }
768 
769  // Create the database schema and the inital root entry
770  // for the new nested catalog
771  const string database_file_path = CreateTempPath(dir_temp() + "/catalog",
772  0666);
773  const bool volatile_content = false;
774  CatalogDatabase *new_catalog_db = CatalogDatabase::Create(database_file_path);
775  assert(NULL != new_catalog_db);
776  // Note we do not set the external_data bit for nested catalogs
777  bool retval =
778  new_catalog_db->InsertInitialValues(nested_root_path,
779  volatile_content,
780  "", // At this point, only root
781  // catalog gets VOMS authz
782  new_root_entry);
783  assert(retval);
784  // TODO(rmeusel): we need a way to attach a catalog directy from an open
785  // database to remove this indirection
786  delete new_catalog_db;
787  new_catalog_db = NULL;
788 
789  // Attach the just created nested catalog
790  Catalog *new_catalog =
791  CreateCatalog(ps_nested_root_path, shash::Any(), old_catalog);
792  retval = AttachCatalog(database_file_path, new_catalog);
793  assert(retval);
794 
795  assert(new_catalog->IsWritable());
796  WritableCatalog *wr_new_catalog = static_cast<WritableCatalog *>(new_catalog);
797 
798  if (new_root_entry.HasXattrs()) {
799  XattrList xattrs;
800  retval = old_catalog->LookupXattrsPath(ps_nested_root_path, &xattrs);
801  assert(retval);
802  wr_new_catalog->TouchEntry(new_root_entry, xattrs, nested_root_path);
803  }
804 
805  // From now on, there are two catalogs, spanning the same directory structure
806  // we have to split the overlapping directory entries from the old catalog
807  // to the new catalog to re-gain a valid catalog structure
808  old_catalog->Partition(wr_new_catalog);
809 
810  // Add the newly created nested catalog to the references of the containing
811  // catalog
812  old_catalog->InsertNestedCatalog(new_catalog->mountpoint().ToString(), NULL,
813  shash::Any(spooler_->GetHashAlgorithm()), 0);
814 
815  // Fix subtree counters in new nested catalogs: subtree is the sum of all
816  // entries of all "grand-nested" catalogs
817  // Note: taking a copy of the nested catalog list here
818  const Catalog::NestedCatalogList &grand_nested =
819  wr_new_catalog->ListOwnNestedCatalogs();
820  DeltaCounters fix_subtree_counters;
821  for (Catalog::NestedCatalogList::const_iterator i = grand_nested.begin(),
822  iEnd = grand_nested.end(); i != iEnd; ++i)
823  {
824  WritableCatalog *grand_catalog;
825  retval = FindCatalog(i->mountpoint.ToString(), &grand_catalog);
826  assert(retval);
827  const Counters &grand_counters = grand_catalog->GetCounters();
828  grand_counters.AddAsSubtree(&fix_subtree_counters);
829  }
830  DeltaCounters save_counters = wr_new_catalog->delta_counters_;
831  wr_new_catalog->delta_counters_ = fix_subtree_counters;
832  wr_new_catalog->UpdateCounters();
833  wr_new_catalog->delta_counters_ = save_counters;
834 
835  SyncUnlock();
836 }
837 
838 
850 void WritableCatalogManager::RemoveNestedCatalog(const string &mountpoint,
851  const bool merge) {
852  const string nested_root_path = MakeRelativePath(mountpoint);
853 
854  SyncLock();
855  // Find the catalog which should be removed
856  WritableCatalog *nested_catalog = NULL;
857  if (!FindCatalog(nested_root_path, &nested_catalog)) {
859  "failed to remove nested catalog '%s': "
860  "mountpoint was not found in current catalog structure",
861  nested_root_path.c_str());
862  }
863 
864  // Check if the found catalog is really the nested catalog to be deleted
865  assert(!nested_catalog->IsRoot() &&
866  (nested_catalog->mountpoint().ToString() == nested_root_path));
867 
868  if (merge) {
869  // Merge all data from the nested catalog into it's parent
870  nested_catalog->MergeIntoParent();
871  } else {
872  nested_catalog->RemoveFromParent();
873  }
874 
875  // Delete the catalog database file from the working copy
876  if (unlink(nested_catalog->database_path().c_str()) != 0) {
878  "unable to delete the removed nested catalog database file '%s'",
879  nested_catalog->database_path().c_str());
880  }
881 
882  // Remove the catalog from internal data structures
883  DetachCatalog(nested_catalog);
884  SyncUnlock();
885 }
886 
887 
899 void WritableCatalogManager::SwapNestedCatalog(const string &mountpoint,
900  const shash::Any &new_hash,
901  const uint64_t new_size) {
902  const string nested_root_path = MakeRelativePath(mountpoint);
903  const string parent_path = GetParentPath(nested_root_path);
904  const PathString nested_root_ps = PathString(nested_root_path);
905 
906  SyncLock();
907 
908  // Find the immediate parent catalog
909  WritableCatalog *parent = NULL;
910  if (!FindCatalog(parent_path, &parent)) {
912  "failed to swap nested catalog '%s': could not find parent '%s'",
913  nested_root_path.c_str(), parent_path.c_str());
914  }
915 
916  // Get old nested catalog counters
917  Catalog *old_attached_catalog = parent->FindChild(nested_root_ps);
918  Counters old_counters;
919  if (old_attached_catalog) {
920 
921  // Old catalog was already attached (e.g. as a child catalog
922  // attached by a prior call to CreateNestedCatalog()). Ensure
923  // that it has not been modified, get counters, and detach it.
924  WritableCatalogList list;
925  if (GetModifiedCatalogLeafsRecursively(old_attached_catalog, &list)) {
927  "failed to swap nested catalog '%s': already modified",
928  nested_root_path.c_str());
929  }
930  old_counters = old_attached_catalog->GetCounters();
931  DetachSubtree(old_attached_catalog);
932 
933  } else {
934 
935  // Old catalog was not attached. Download a freely attached
936  // version and get counters.
937  shash::Any old_hash;
938  uint64_t old_size;
939  const bool old_found = parent->FindNested(nested_root_ps, &old_hash,
940  &old_size);
941  if (!old_found) {
943  "failed to swap nested catalog '%s': not found in parent",
944  nested_root_path.c_str());
945  }
946  UniquePtr<Catalog> old_free_catalog(LoadFreeCatalog(nested_root_ps, old_hash));
947  if (!old_free_catalog) {
949  "failed to swap nested catalog '%s': failed to load old catalog",
950  nested_root_path.c_str());
951  }
952  old_counters = old_free_catalog->GetCounters();
953  }
954 
955  // Load freely attached new catalog
956  UniquePtr<Catalog> new_catalog(LoadFreeCatalog(nested_root_ps, new_hash));
957  if (!new_catalog) {
959  "failed to swap nested catalog '%s': failed to load new catalog",
960  nested_root_path.c_str());
961  }
962 
963  // Get new catalog root directory entry
964  DirectoryEntry dirent;
965  XattrList xattrs;
966  const bool dirent_found = new_catalog->LookupPath(nested_root_ps, &dirent);
967  if (!dirent_found) {
969  "failed to swap nested catalog '%s': missing dirent in new catalog",
970  nested_root_path.c_str());
971  }
972  if (dirent.HasXattrs()) {
973  const bool xattrs_found = new_catalog->LookupXattrsPath(nested_root_ps,
974  &xattrs);
975  if (!xattrs_found) {
977  "failed to swap nested catalog '%s': missing xattrs in new catalog",
978  nested_root_path.c_str());
979  }
980  }
981 
982  // Swap catalogs
983  parent->RemoveNestedCatalog(nested_root_path, NULL);
984  parent->InsertNestedCatalog(nested_root_path, NULL, new_hash, new_size);
985 
986  // Update parent directory entry
988  dirent.set_is_nested_catalog_root(false);
989  parent->UpdateEntry(dirent, nested_root_path);
990  parent->TouchEntry(dirent, xattrs, nested_root_path);
991 
992  // Update counters
993  DeltaCounters delta = Counters::Diff(old_counters,
994  new_catalog->GetCounters());
995  delta.PopulateToParent(&parent->delta_counters_);
996 
997  SyncUnlock();
998 }
999 
1000 
1004 bool WritableCatalogManager::IsTransitionPoint(const string &mountpoint) {
1005  const string path = MakeRelativePath(mountpoint);
1006 
1007  SyncLock();
1008  WritableCatalog *catalog;
1009  DirectoryEntry entry;
1010  if (!FindCatalog(path, &catalog, &entry)) {
1011  PANIC(kLogStderr, "catalog for directory '%s' cannot be found",
1012  path.c_str());
1013  }
1014  const bool result = entry.IsNestedCatalogRoot();
1015  SyncUnlock();
1016  return result;
1017 }
1018 
1019 
1021  // TODO(jblomer): meant for micro catalogs
1022 }
1023 
1024 
1025 void WritableCatalogManager::SetTTL(const uint64_t new_ttl) {
1026  SyncLock();
1027  reinterpret_cast<WritableCatalog *>(GetRootCatalog())->SetTTL(new_ttl);
1028  SyncUnlock();
1029 }
1030 
1031 
1032 bool WritableCatalogManager::SetVOMSAuthz(const std::string &voms_authz) {
1033  bool result;
1034  SyncLock();
1035  result = reinterpret_cast<WritableCatalog *>(
1036  GetRootCatalog())->SetVOMSAuthz(voms_authz);
1037  SyncUnlock();
1038  return result;
1039 }
1040 
1041 
1042 bool WritableCatalogManager::Commit(const bool stop_for_tweaks,
1043  const uint64_t manual_revision,
1045  WritableCatalog *root_catalog =
1046  reinterpret_cast<WritableCatalog *>(GetRootCatalog());
1047  root_catalog->SetDirty();
1048 
1049  // set root catalog revision to manually provided number if available
1050  if (manual_revision > 0) {
1051  const uint64_t revision = root_catalog->GetRevision();
1052  if (revision >= manual_revision) {
1053  LogCvmfs(kLogCatalog, kLogStderr, "Manual revision (%d) must not be "
1054  "smaller than the current root "
1055  "catalog's (%d). Skipped!",
1056  manual_revision, revision);
1057  } else {
1058  // Gets incremented by FinalizeCatalog() afterwards!
1059  root_catalog->SetRevision(manual_revision - 1);
1060  }
1061  }
1062 
1063  // do the actual catalog snapshotting and upload
1064  CatalogInfo root_catalog_info;
1065  if (getenv("_CVMFS_SERIALIZED_CATALOG_PROCESSING_") == NULL)
1066  root_catalog_info = SnapshotCatalogs(stop_for_tweaks);
1067  else
1068  root_catalog_info = SnapshotCatalogsSerialized(stop_for_tweaks);
1069  if (spooler_->GetNumberOfErrors() > 0) {
1070  LogCvmfs(kLogCatalog, kLogStderr, "failed to commit catalogs");
1071  return false;
1072  }
1073 
1074  // .cvmfspublished export
1075  LogCvmfs(kLogCatalog, kLogVerboseMsg, "Committing repository manifest");
1076  set_base_hash(root_catalog_info.content_hash);
1077 
1078  manifest->set_catalog_hash(root_catalog_info.content_hash);
1079  manifest->set_catalog_size(root_catalog_info.size);
1080  manifest->set_root_path("");
1081  manifest->set_ttl(root_catalog_info.ttl);
1082  manifest->set_revision(root_catalog_info.revision);
1083 
1084  return true;
1085 }
1086 
1087 
1115  const bool stop_for_tweaks) {
1116  // prepare environment for parallel processing
1117  Future<CatalogInfo> root_catalog_info_future;
1118  CatalogUploadContext upload_context;
1119  upload_context.root_catalog_info = &root_catalog_info_future;
1120  upload_context.stop_for_tweaks = stop_for_tweaks;
1121 
1122  spooler_->RegisterListener(
1123  &WritableCatalogManager::CatalogUploadCallback, this, upload_context);
1124 
1125  // find dirty leaf catalogs and annotate non-leaf catalogs (dirty child count)
1126  // post-condition: the entire catalog tree is ready for concurrent processing
1127  WritableCatalogList leafs_to_snapshot;
1128  GetModifiedCatalogLeafs(&leafs_to_snapshot);
1129 
1130  // finalize and schedule the catalog processing
1131  WritableCatalogList::const_iterator i = leafs_to_snapshot.begin();
1132  const WritableCatalogList::const_iterator iend = leafs_to_snapshot.end();
1133  for (; i != iend; ++i) {
1134  FinalizeCatalog(*i, stop_for_tweaks);
1136  }
1137 
1138  LogCvmfs(kLogCatalog, kLogVerboseMsg, "waiting for upload of catalogs");
1139  CatalogInfo& root_catalog_info = root_catalog_info_future.Get();
1140  spooler_->WaitForUpload();
1141 
1142  spooler_->UnregisterListeners();
1143  return root_catalog_info;
1144 }
1145 
1146 
1148  const bool stop_for_tweaks) {
1149  // update meta information of this catalog
1150  LogCvmfs(kLogCatalog, kLogVerboseMsg, "creating snapshot of catalog '%s'",
1151  catalog->mountpoint().c_str());
1152 
1153  catalog->UpdateCounters();
1154  catalog->UpdateLastModified();
1155  catalog->IncrementRevision();
1156 
1157  // update the previous catalog revision pointer
1158  if (catalog->IsRoot()) {
1159  LogCvmfs(kLogCatalog, kLogVerboseMsg, "setting '%s' as previous revision "
1160  "for root catalog",
1161  base_hash().ToStringWithSuffix().c_str());
1162  catalog->SetPreviousRevision(base_hash());
1163  } else {
1164  // Multiple catalogs might query the parent concurrently
1165  SyncLock();
1166  shash::Any hash_previous;
1167  uint64_t size_previous;
1168  const bool retval =
1169  catalog->parent()->FindNested(catalog->mountpoint(),
1170  &hash_previous, &size_previous);
1171  assert(retval);
1172  SyncUnlock();
1173 
1174  LogCvmfs(kLogCatalog, kLogVerboseMsg, "found '%s' as previous revision "
1175  "for nested catalog '%s'",
1176  hash_previous.ToStringWithSuffix().c_str(),
1177  catalog->mountpoint().c_str());
1178  catalog->SetPreviousRevision(hash_previous);
1179  }
1180  catalog->Commit();
1181 
1182  // check if catalog has too many entries
1183  uint64_t catalog_limit = uint64_t(1000) *
1184  uint64_t((catalog->IsRoot()
1187  if ((catalog_limit > 0) &&
1188  (catalog->GetCounters().GetSelfEntries() > catalog_limit)) {
1190  "%s: catalog at %s has more than %u entries (%u). "
1191  "Large catalogs stress the CernVM-FS transport infrastructure. "
1192  "Please split it into nested catalogs or increase the limit.",
1193  enforce_limits_ ? "FATAL" : "WARNING",
1194  (catalog->IsRoot() ? "/" : catalog->mountpoint().c_str()),
1195  catalog_limit, catalog->GetCounters().GetSelfEntries());
1196  if (enforce_limits_)
1197  PANIC(kLogStderr, "catalog at %s has more than %u entries (%u). ",
1198  (catalog->IsRoot() ? "/" : catalog->mountpoint().c_str()),
1199  catalog_limit, catalog->GetCounters().GetSelfEntries());
1200  }
1201 
1202  // allow for manual adjustments in the catalog
1203  if (stop_for_tweaks) {
1204  LogCvmfs(kLogCatalog, kLogStdout, "Allowing for tweaks in %s at %s "
1205  "(hit return to continue)",
1206  catalog->database_path().c_str(), catalog->mountpoint().c_str());
1207  int read_char = getchar();
1208  assert(read_char != EOF);
1209  }
1210 
1211  // compaction of bloated catalogs (usually after high database churn)
1212  catalog->VacuumDatabaseIfNecessary();
1213 }
1214 
1215 
1217  WritableCatalog *catalog) {
1218  {
1220  // register catalog object for WritableCatalogManager::CatalogUploadCallback
1221  catalog_processing_map_[catalog->database_path()] = catalog;
1222  }
1223  spooler_->ProcessCatalog(catalog->database_path());
1224 }
1225 
1226 
1228  const upload::SpoolerResult &result,
1229  const CatalogUploadContext catalog_upload_context) {
1230  if (result.return_code != 0) {
1231  PANIC(kLogStderr, "failed to upload '%s' (retval: %d)",
1232  result.local_path.c_str(), result.return_code);
1233  }
1234 
1235  // retrieve the catalog object based on the callback information
1236  // see WritableCatalogManager::ScheduleCatalogProcessing()
1237  WritableCatalog *catalog = NULL;
1238  {
1240  std::map<std::string, WritableCatalog*>::iterator c =
1241  catalog_processing_map_.find(result.local_path);
1242  assert(c != catalog_processing_map_.end());
1243  catalog = c->second;
1244  }
1245 
1246  uint64_t catalog_size = GetFileSize(result.local_path);
1247  assert(catalog_size > 0);
1248 
1249  SyncLock();
1250  if (catalog->HasParent()) {
1251  // finalized nested catalogs will update their parent's pointer and schedule
1252  // them for processing (continuation) if the 'dirty children count' == 0
1253  LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating nested catalog link");
1254  WritableCatalog *parent = catalog->GetWritableParent();
1255 
1256  parent->UpdateNestedCatalog(catalog->mountpoint().ToString(),
1257  result.content_hash,
1258  catalog_size,
1259  catalog->delta_counters_);
1260  catalog->delta_counters_.SetZero();
1261 
1262  const int remaining_dirty_children =
1264 
1265  SyncUnlock();
1266 
1267  // continuation of the dirty catalog tree traversal
1268  // see WritableCatalogManager::SnapshotCatalogs()
1269  if (remaining_dirty_children == 0) {
1270  FinalizeCatalog(parent, catalog_upload_context.stop_for_tweaks);
1271  ScheduleCatalogProcessing(parent);
1272  }
1273 
1274  } else if (catalog->IsRoot()) {
1275  // once the root catalog is reached, we are done with processing and report
1276  // back to the main via a Future<> and provide the necessary information
1277  CatalogInfo root_catalog_info;
1278  root_catalog_info.size = catalog_size;
1279  root_catalog_info.ttl = catalog->GetTTL();
1280  root_catalog_info.content_hash = result.content_hash;
1281  root_catalog_info.revision = catalog->GetRevision();
1282  catalog_upload_context.root_catalog_info->Set(root_catalog_info);
1283  SyncUnlock();
1284  } else {
1285  PANIC(kLogStderr, "inconsistent state detected");
1286  }
1287 }
1288 
1289 
1301  Catalog *catalog,
1302  WritableCatalogList *result) const {
1303  WritableCatalog *wr_catalog = static_cast<WritableCatalog *>(catalog);
1304 
1305  // Look for dirty catalogs in the descendants of *catalog
1306  int dirty_children = 0;
1307  CatalogList children = wr_catalog->GetChildren();
1308  CatalogList::const_iterator i = children.begin();
1309  const CatalogList::const_iterator iend = children.end();
1310  for (; i != iend; ++i) {
1311  if (GetModifiedCatalogLeafsRecursively(*i, result)) {
1312  ++dirty_children;
1313  }
1314  }
1315 
1316  // a catalog is dirty if itself or one of its children has changed
1317  // a leaf catalog doesn't have any dirty children
1318  wr_catalog->set_dirty_children(dirty_children);
1319  const bool is_dirty = wr_catalog->IsDirty() || dirty_children > 0;
1320  const bool is_leaf = dirty_children == 0;
1321  if (is_dirty && is_leaf) {
1322  result->push_back(const_cast<WritableCatalog *>(wr_catalog));
1323  }
1324 
1325  return is_dirty;
1326 }
1327 
1328 
1330  CatalogList catalog_list = GetCatalogs();
1331  reverse(catalog_list.begin(), catalog_list.end());
1332  for (unsigned i = 0; i < catalog_list.size(); ++i) {
1333  FixWeight(static_cast<WritableCatalog*>(catalog_list[i]));
1334  }
1335 }
1336 
1338  // firstly check underflow because they can provoke overflows
1339  if (catalog->GetNumEntries() < min_weight_ &&
1340  !catalog->IsRoot() &&
1341  catalog->IsAutogenerated()) {
1343  "Deleting an autogenerated catalog in '%s'",
1344  catalog->mountpoint().c_str());
1345  // Remove the .cvmfscatalog and .cvmfsautocatalog files first
1346  string path = catalog->mountpoint().ToString();
1347  catalog->RemoveEntry(path + "/.cvmfscatalog");
1348  catalog->RemoveEntry(path + "/.cvmfsautocatalog");
1349  // Remove the actual catalog
1350  string catalog_path = catalog->mountpoint().ToString().substr(1);
1351  RemoveNestedCatalog(catalog_path);
1352  } else if (catalog->GetNumEntries() > max_weight_) {
1353  CatalogBalancer<WritableCatalogManager> catalog_balancer(this);
1354  catalog_balancer.Balance(catalog);
1355  }
1356 }
1357 
1358 
1359 //****************************************************************************
1360 // Workaround -- Serialized Catalog Committing
1361 
1363  const Catalog *catalog,
1364  WritableCatalogList *result) const
1365 {
1366  // A catalog must be snapshot, if itself or one of it's descendants is dirty.
1367  // So we traverse the catalog tree recursively and look for dirty catalogs
1368  // on the way.
1369  const WritableCatalog *wr_catalog =
1370  static_cast<const WritableCatalog *>(catalog);
1371  // This variable will contain the number of dirty catalogs in the sub tree
1372  // with *catalog as it's root.
1373  int dirty_catalogs = (wr_catalog->IsDirty()) ? 1 : 0;
1374 
1375  // Look for dirty catalogs in the descendants of *catalog
1376  CatalogList children = wr_catalog->GetChildren();
1377  for (CatalogList::const_iterator i = children.begin(), iEnd = children.end();
1378  i != iEnd; ++i)
1379  {
1380  dirty_catalogs += GetModifiedCatalogsRecursively(*i, result);
1381  }
1382 
1383  // If we found a dirty catalog in the checked sub tree, the root (*catalog)
1384  // must be snapshot and ends up in the result list
1385  if (dirty_catalogs > 0)
1386  result->push_back(const_cast<WritableCatalog *>(wr_catalog));
1387 
1388  // tell the upper layer about number of catalogs
1389  return dirty_catalogs;
1390 }
1391 
1392 
1394  const upload::SpoolerResult &result,
1395  const CatalogUploadContext unused)
1396 {
1397  if (result.return_code != 0) {
1398  PANIC(kLogStderr, "failed to upload '%s' (retval: %d)",
1399  result.local_path.c_str(), result.return_code);
1400  }
1401  unlink(result.local_path.c_str());
1402 }
1403 
1404 
1407  const bool stop_for_tweaks)
1408 {
1409  LogCvmfs(kLogCvmfs, kLogStdout, "Serialized committing of file catalogs...");
1410  reinterpret_cast<WritableCatalog *>(GetRootCatalog())->SetDirty();
1411  WritableCatalogList catalogs_to_snapshot;
1412  GetModifiedCatalogs(&catalogs_to_snapshot);
1413  CatalogUploadContext unused;
1414  unused.root_catalog_info = NULL;
1415  unused.stop_for_tweaks = false;
1416  spooler_->RegisterListener(
1418 
1419  CatalogInfo root_catalog_info;
1420  WritableCatalogList::const_iterator i = catalogs_to_snapshot.begin();
1421  const WritableCatalogList::const_iterator iend = catalogs_to_snapshot.end();
1422  for (; i != iend; ++i) {
1423  FinalizeCatalog(*i, stop_for_tweaks);
1424 
1425  // Compress and upload catalog
1426  shash::Any hash_catalog(spooler_->GetHashAlgorithm(),
1428  if (!zlib::CompressPath2Null((*i)->database_path(),
1429  &hash_catalog))
1430  {
1431  PANIC(kLogStderr, "could not compress catalog %s",
1432  (*i)->mountpoint().ToString().c_str());
1433  }
1434 
1435  int64_t catalog_size = GetFileSize((*i)->database_path());
1436  assert(catalog_size > 0);
1437 
1438  if ((*i)->HasParent()) {
1439  LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating nested catalog link");
1440  WritableCatalog *parent = (*i)->GetWritableParent();
1441  parent->UpdateNestedCatalog((*i)->mountpoint().ToString(), hash_catalog,
1442  catalog_size, (*i)->delta_counters_);
1443  (*i)->delta_counters_.SetZero();
1444  } else if ((*i)->IsRoot()) {
1445  root_catalog_info.size = catalog_size;
1446  root_catalog_info.ttl = (*i)->GetTTL();
1447  root_catalog_info.content_hash = hash_catalog;
1448  root_catalog_info.revision = (*i)->GetRevision();
1449  } else {
1450  PANIC(kLogStderr, "inconsistent state detected");
1451  }
1452 
1453  spooler_->ProcessCatalog((*i)->database_path());
1454  }
1455  spooler_->WaitForUpload();
1456 
1457  spooler_->UnregisterListeners();
1458  return root_catalog_info;
1459 }
1460 
1461 } // namespace catalog
bool CompressPath2Null(const string &src, shash::Any *compressed_hash)
Definition: compression.cc:515
uint32_t linkcount() const
int return_code
the return value of the spooler operation
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
bool IsExternalFile() const
const Counters & GetCounters() const
Definition: catalog.h:175
void AddChunkedFile(const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &parent_directory, const FileChunkList &file_chunks)
bool IsNull() const
Definition: hash.h:379
const manifest::Manifest * manifest() const
Definition: repository.h:123
void set_base_hash(const shash::Any &hash)
bool IsRoot() const
Definition: catalog.h:193
ShortString< kDefaultMaxName, 1 > NameString
Definition: shortstring.h:190
void CloneTree(const std::string &from_dir, const std::string &to_dir)
void AddHardlinkGroup(const DirectoryEntryBaseList &entries, const XattrList &xattrs, const std::string &parent_directory, const FileChunkList &file_chunks)
const std::vector< Catalog * > & GetCatalogs() const
Definition: catalog_mgr.h:218
void set_dirty_children(const int count)
Definition: catalog_rw.h:143
void set_is_chunked_file(const bool val)
void AddDirectory(const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &parent_directory)
std::string GetFileName(const std::string &path)
Definition: posix.cc:159
std::string database_path() const
Definition: catalog.h:184
bool IsDirectory() const
void set_catalog_hash(const shash::Any &catalog_hash)
Definition: manifest.h:102
Catalog * LoadFreeCatalog(const PathString &mountpoint, const shash::Any &hash)
bool IsChunkedFile() const
inode_t inode_
void Clear()
Definition: xattr.h:41
bool HasParent() const
Definition: catalog.h:200
bool MountSubtree(const PathString &path, const Catalog *entry_point, bool can_listing, Catalog **leaf_catalog)
void RemoveDirectory(const std::string &directory_path)
uint32_t GetMaxLinkId() const
Definition: catalog_rw.cc:124
void UpdateNestedCatalog(const std::string &path, const shash::Any &hash, const uint64_t size, const DeltaCounters &child_counters)
Definition: catalog_rw.cc:620
#define PANIC(...)
Definition: exception.h:26
uint64_t size() const
WritableCatalog * GetHostingCatalog(const std::string &path)
void Assign(const char *chars, const unsigned length)
Definition: shortstring.h:53
void set_linkcount(const uint32_t linkcount)
void set_is_nested_catalog_root(const bool val)
bool IsDirty() const
Definition: catalog_rw.h:57
void FixWeight(WritableCatalog *catalog)
std::string ToStringWithSuffix() const
Definition: hash.h:300
gid_t gid_
void Balance(catalog_t *catalog)
bool LookupPath(const PathString &path, DirectoryEntry *dirent) const
Definition: catalog.h:124
void ScheduleCatalogProcessing(WritableCatalog *catalog)
std::map< std::string, WritableCatalog * > catalog_processing_map_
std::string CreateTempPath(const std::string &path_prefix, const int mode)
Definition: posix.cc:1054
void SetTTL(const uint64_t new_ttl)
static DeltaCounters Diff(const Counters &from, const Counters &to)
int GetModifiedCatalogsRecursively(const Catalog *catalog, WritableCatalogList *result) const
Counters_t GetSelfEntries() const
void set_revision(const uint64_t revision)
Definition: manifest.h:86
void InsertNestedCatalog(const std::string &mountpoint, Catalog *attached_reference, const shash::Any content_hash, const uint64_t size)
Definition: catalog_rw.cc:502
void TouchEntry(const DirectoryEntryBase &entry, const XattrList &xattrs, const shash::Md5 &path_hash)
Definition: catalog_rw.cc:224
uint64_t GetTTL() const
Definition: catalog.cc:495
assert((mem||(size==0))&&"Out Of Memory")
bool FindCatalog(const std::string &path, WritableCatalog **result, DirectoryEntry *dirent=NULL)
bool Commit(const bool stop_for_tweaks, const uint64_t manual_revision, manifest::Manifest *manifest)
CatalogList GetChildren() const
Definition: catalog.cc:752
bool LookupPath(const PathString &path, const LookupOptions options, DirectoryEntry *entry)
std::string GetParentPath(const std::string &path)
Definition: posix.cc:129
void SetPreviousRevision(const shash::Any &hash)
Definition: catalog_rw.cc:352
Catalog * parent() const
Definition: catalog.h:180
pthread_mutex_t * catalog_processing_lock_
Catalog * FindChild(const PathString &mountpoint) const
Definition: catalog.cc:808
std::vector< WritableCatalog * > WritableCatalogList
Definition: catalog_rw.h:203
shash::Any checksum() const
void CloneTreeImpl(const PathString &source_dir, const std::string &dest_parent_dir, const NameString &dest_name)
uint64_t size_
void AddAsSubtree(DeltaCounters *delta) const
bool IsNestedCatalogMountpoint() const
bool IsTransitionPoint(const std::string &mountpoint)
Algorithms
Definition: hash.h:39
bool Listing(const PathString &path, DirectoryEntryList *listing, const bool expand_symlink)
void CatalogUploadCallback(const upload::SpoolerResult &result, const CatalogUploadContext clg_upload_context)
bool IsNestedCatalogRoot() const
NameString name_
void TouchDirectory(const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &directory_path)
uint64_t GetNumEntries() const
Definition: catalog.cc:546
uint64_t GetRevision() const
Definition: catalog.cc:528
void AddFileChunk(const std::string &entry_path, const FileChunk &chunk)
Definition: catalog_rw.cc:264
bool IsAutogenerated() const
Definition: catalog.h:194
std::vector< DirectoryEntry > DirectoryEntryList
NameString name() const
static manifest::Manifest * CreateRepository(const std::string &dir_temp, const bool volatile_content, const std::string &voms_authz, upload::Spooler *spooler)
bool AttachCatalog(const std::string &db_path, Catalog *new_catalog)
void SplitPath(const std::string &path, std::string *dirname, std::string *filename)
Definition: posix.cc:110
bool HasXattrs() const
void ActivateCatalog(Catalog *catalog)
std::string local_path
the local_path previously given as input
bool LookupXattrsPath(const PathString &path, XattrList *xattrs) const
Definition: catalog.h:128
void ShrinkHardlinkGroup(const std::string &remove_path)
bool IsRegular() const
bool GetModifiedCatalogLeafsRecursively(Catalog *catalog, WritableCatalogList *result) const
const char kSuffixCatalog
Definition: hash.h:52
uint32_t linkcount_
const std::string & dir_temp() const
void PopulateToParent(DeltaCounters *parent) const
bool InsertInitialValues(const std::string &root_path, const bool volatile_content, const std::string &voms_authz, const DirectoryEntry &root_entry=DirectoryEntry(kDirentNegative))
Definition: catalog_sql.cc:263
CatalogInfo SnapshotCatalogsSerialized(const bool stop_for_tweaks)
CatalogInfo SnapshotCatalogs(const bool stop_for_tweaks)
PathString mountpoint() const
Definition: catalog.h:179
time_t mtime_
static const inode_t kInvalidInode
void Append(const char *chars, const unsigned length)
Definition: shortstring.h:70
void TakeDatabaseFileOwnership()
Definition: catalog.cc:479
bool SetVOMSAuthz(const std::string &voms_authz)
bool HasPrefix(const string &str, const string &prefix, const bool ignore_case)
Definition: string.cc:264
void Clone(const std::string from, const std::string to)
std::vector< Catalog * > CatalogList
Definition: catalog_mgr.h:119
void RemoveFile(const std::string &file_path)
bool IsBindMountpoint() const
std::string MakeRelativePath(const std::string &relative_path) const
void CatalogUploadSerializedCallback(const upload::SpoolerResult &result, const CatalogUploadContext unused)
void RemoveNestedCatalog(const std::string &mountpoint, Catalog **attached_reference)
Definition: catalog_rw.cc:561
bool IsEmpty() const
Definition: bigvector.h:67
void AddEntry(const DirectoryEntry &entry, const XattrList &xattr, const std::string &entry_path, const std::string &parent_path)
void RemoveNestedCatalog(const std::string &mountpoint, const bool merge=true)
void set_ttl(const uint32_t ttl)
Definition: manifest.h:85
shash::Algorithms GetHashAlgorithm() const
Definition: manifest.h:83
std::string ToString() const
Definition: shortstring.h:113
std::vector< NestedCatalog > NestedCatalogList
Definition: catalog.h:208
bool IsEmpty() const
Definition: shortstring.h:109
unsigned int mode_
bool ListFileChunks(const PathString &path, const shash::Algorithms interpret_hashes_as, FileChunkList *chunks)
void Partition(WritableCatalog *new_nested_catalog)
Definition: catalog_rw.cc:360
ShortString< kDefaultMaxPath, 0 > PathString
Definition: shortstring.h:189
void SetRevision(const uint64_t new_revision)
Definition: catalog_rw.cc:329
void set_hardlink_group(const uint32_t group)
WritableCatalog * GetWritableParent() const
Definition: catalog_rw.h:136
void set_catalog_size(const uint64_t catalog_size)
Definition: manifest.h:99
DeltaCounters delta_counters_
Definition: catalog_rw.h:163
bool FindNested(const PathString &mountpoint, shash::Any *hash, uint64_t *size) const
Definition: catalog.cc:690
std::string GetFullPath(const std::string &parent_directory) const
bool LookupXattrs(const PathString &path, XattrList *xattrs)
Catalog * MountCatalog(const PathString &mountpoint, const shash::Any &hash, Catalog *parent_catalog)
int64_t GetFileSize(const std::string &path)
Definition: posix.cc:821
std::vector< DirectoryEntryBase > DirectoryEntryBaseList
const int kLogVerboseMsg
void set_has_alt_catalog_path(const bool &has_alt_path)
Definition: manifest.h:108
unsigned GetLength() const
Definition: shortstring.h:103
shash::Any checksum_
CatalogT * FindCatalog(const PathString &path) const
std::string MakePath() const
Definition: hash.h:312
void AddFile(const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &parent_directory)
const char * c_str() const
Definition: shortstring.h:117
void SwapNestedCatalog(const string &mountpoint, const shash::Any &new_hash, const uint64_t new_size)
void RemoveEntry(const std::string &entry_path)
Definition: catalog_rw.cc:184
const char * GetChars() const
Definition: shortstring.h:96
virtual bool IsWritable() const
Definition: catalog.h:201
void GetModifiedCatalogLeafs(WritableCatalogList *result) const
Catalog * CreateCatalog(const PathString &mountpoint, const shash::Any &catalog_hash, Catalog *parent_catalog)
void FinalizeCatalog(WritableCatalog *catalog, const bool stop_for_tweaks)
void IncLinkcount(const std::string &path_within_group, const int delta)
Definition: catalog_rw.cc:208
const Item * AtPtr(const size_t index) const
Definition: bigvector.h:55
void set_is_nested_catalog_mountpoint(const bool val)
bool CompressPath2Path(const string &src, const string &dest)
Definition: compression.cc:318
void CreateNestedCatalog(const std::string &mountpoint)
void UpdateEntry(const DirectoryEntry &entry, const shash::Md5 &path_hash)
Definition: catalog_rw.cc:252
static DerivedT * Create(const std::string &filename)
Definition: sql_impl.h:30
void GetModifiedCatalogs(WritableCatalogList *result) const
size_t size() const
Definition: bigvector.h:100
uid_t uid_
void set_root_path(const std::string &root_path)
Definition: manifest.h:114
const shash::Any & base_hash() const