CernVM-FS  2.10.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
catalog_mgr_rw.cc
Go to the documentation of this file.
1 
5 #define __STDC_FORMAT_MACROS
6 
7 #include "catalog_mgr_rw.h"
8 
9 #include <inttypes.h>
10 #include <unistd.h>
11 
12 #include <cassert>
13 #include <cstdio>
14 #include <cstdlib>
15 #include <string>
16 
17 #include "catalog_balancer.h"
18 #include "catalog_rw.h"
19 #include "logging.h"
20 #include "manifest.h"
21 #include "smalloc.h"
22 #include "statistics.h"
23 #include "upload.h"
24 #include "util/exception.h"
25 #include "util/posix.h"
26 
27 using namespace std; // NOLINT
28 
29 namespace catalog {
30 
31 WritableCatalogManager::WritableCatalogManager(
32  const shash::Any &base_hash,
33  const std::string &stratum0,
34  const string &dir_temp,
35  upload::Spooler *spooler,
36  download::DownloadManager *download_manager,
37  bool enforce_limits,
38  const unsigned nested_kcatalog_limit,
39  const unsigned root_kcatalog_limit,
40  const unsigned file_mbyte_limit,
41  perf::Statistics *statistics,
42  bool is_balanceable,
43  unsigned max_weight,
44  unsigned min_weight)
45  : SimpleCatalogManager(base_hash, stratum0, dir_temp, download_manager,
46  statistics)
47  , spooler_(spooler)
48  , enforce_limits_(enforce_limits)
49  , nested_kcatalog_limit_(nested_kcatalog_limit)
50  , root_kcatalog_limit_(root_kcatalog_limit)
51  , file_mbyte_limit_(file_mbyte_limit)
52  , is_balanceable_(is_balanceable)
53  , max_weight_(max_weight)
54  , min_weight_(min_weight)
55  , balance_weight_(max_weight / 2)
56 {
57  sync_lock_ =
58  reinterpret_cast<pthread_mutex_t *>(smalloc(sizeof(pthread_mutex_t)));
59  int retval = pthread_mutex_init(sync_lock_, NULL);
60  assert(retval == 0);
62  reinterpret_cast<pthread_mutex_t *>(smalloc(sizeof(pthread_mutex_t)));
63  retval = pthread_mutex_init(catalog_processing_lock_, NULL);
64  assert(retval == 0);
65 }
66 
67 
69  pthread_mutex_destroy(sync_lock_);
70  free(sync_lock_);
71  pthread_mutex_destroy(catalog_processing_lock_);
73 }
74 
75 
86  const PathString &mountpoint,
87  const shash::Any &catalog_hash,
88  Catalog *parent_catalog)
89 {
90  return new WritableCatalog(mountpoint.ToString(),
91  catalog_hash,
92  parent_catalog);
93 }
94 
95 
97  catalog->TakeDatabaseFileOwnership();
98 }
99 
100 
108  const string &dir_temp,
109  const bool volatile_content,
110  const std::string &voms_authz,
111  upload::Spooler *spooler)
112 {
113  // Create a new root catalog at file_path
114  string file_path = dir_temp + "/new_root_catalog";
115 
116  shash::Algorithms hash_algorithm = spooler->GetHashAlgorithm();
117 
118  // A newly created catalog always needs a root entry
119  // we create and configure this here
120  DirectoryEntry root_entry;
122  root_entry.mode_ = 16877;
123  root_entry.size_ = 4096;
124  root_entry.mtime_ = time(NULL);
125  root_entry.uid_ = getuid();
126  root_entry.gid_ = getgid();
127  root_entry.checksum_ = shash::Any(hash_algorithm);
128  root_entry.linkcount_ = 2;
129  string root_path = "";
130 
131  // Create the database schema and the inital root entry
132  {
134  if (!new_clg_db.IsValid() ||
135  !new_clg_db->InsertInitialValues(root_path,
136  volatile_content,
137  voms_authz,
138  root_entry))
139  {
140  LogCvmfs(kLogCatalog, kLogStderr, "creation of catalog '%s' failed",
141  file_path.c_str());
142  return NULL;
143  }
144  }
145 
146  // Compress root catalog;
147  int64_t catalog_size = GetFileSize(file_path);
148  if (catalog_size < 0) {
149  unlink(file_path.c_str());
150  return NULL;
151  }
152  string file_path_compressed = file_path + ".compressed";
153  shash::Any hash_catalog(hash_algorithm, shash::kSuffixCatalog);
154  bool retval = zlib::CompressPath2Path(file_path, file_path_compressed,
155  &hash_catalog);
156  if (!retval) {
157  LogCvmfs(kLogCatalog, kLogStderr, "compression of catalog '%s' failed",
158  file_path.c_str());
159  unlink(file_path.c_str());
160  return NULL;
161  }
162  unlink(file_path.c_str());
163 
164  // Create manifest
165  const string manifest_path = dir_temp + "/manifest";
167  new manifest::Manifest(hash_catalog, catalog_size, "");
168  if (!voms_authz.empty()) {
169  manifest->set_has_alt_catalog_path(true);
170  }
171 
172  // Upload catalog
173  spooler->Upload(file_path_compressed, "data/" + hash_catalog.MakePath());
174  spooler->WaitForUpload();
175  unlink(file_path_compressed.c_str());
176  if (spooler->GetNumberOfErrors() > 0) {
177  LogCvmfs(kLogCatalog, kLogStderr, "failed to commit catalog %s",
178  file_path_compressed.c_str());
179  delete manifest;
180  return NULL;
181  }
182 
183  return manifest;
184 }
185 
186 
198 bool WritableCatalogManager::FindCatalog(const string &path,
199  WritableCatalog **result,
200  DirectoryEntry *dirent) {
201  const PathString ps_path(path);
202 
203  Catalog *best_fit =
205  assert(best_fit != NULL);
206  Catalog *catalog = NULL;
207  bool retval =
208  MountSubtree(ps_path, best_fit, true /* is_listable */, &catalog);
209  if (!retval)
210  return false;
211 
213  if (NULL == dirent) {
214  dirent = &dummy;
215  }
216  bool found = catalog->LookupPath(ps_path, dirent);
217  if (!found || !catalog->IsWritable())
218  return false;
219 
220  *result = static_cast<WritableCatalog *>(catalog);
221  return true;
222 }
223 
224 
226  const std::string &path)
227 {
228  WritableCatalog *result = NULL;
229  bool retval = FindCatalog(MakeRelativePath(path), &result, NULL);
230  if (!retval) return NULL;
231  return result;
232 }
233 
234 
240 void WritableCatalogManager::RemoveFile(const std::string &path) {
241  const string file_path = MakeRelativePath(path);
242  const string parent_path = GetParentPath(file_path);
243 
244  SyncLock();
245  WritableCatalog *catalog;
246  if (!FindCatalog(parent_path, &catalog)) {
247  PANIC(kLogStderr, "catalog for file '%s' cannot be found",
248  file_path.c_str());
249  }
250 
251  catalog->RemoveEntry(file_path);
252  SyncUnlock();
253 }
254 
255 
261 void WritableCatalogManager::RemoveDirectory(const std::string &path) {
262  const string directory_path = MakeRelativePath(path);
263  const string parent_path = GetParentPath(directory_path);
264 
265  SyncLock();
266  WritableCatalog *catalog;
267  DirectoryEntry parent_entry;
268  if (!FindCatalog(parent_path, &catalog, &parent_entry)) {
269  PANIC(kLogStderr, "catalog for directory '%s' cannot be found",
270  directory_path.c_str());
271  }
272 
273  parent_entry.set_linkcount(parent_entry.linkcount() - 1);
274 
275  catalog->RemoveEntry(directory_path);
276  catalog->UpdateEntry(parent_entry, parent_path);
277  if (parent_entry.IsNestedCatalogRoot()) {
278  LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating transition point %s",
279  parent_path.c_str());
280  WritableCatalog *parent_catalog =
281  reinterpret_cast<WritableCatalog *>(catalog->parent());
282  parent_entry.set_is_nested_catalog_mountpoint(true);
283  parent_entry.set_is_nested_catalog_root(false);
284  parent_catalog->UpdateEntry(parent_entry, parent_path);
285  }
286  SyncUnlock();
287 }
288 
297 void WritableCatalogManager::Clone(const std::string destination,
298  const std::string source) {
299  const std::string relative_source = MakeRelativePath(source);
300 
301  DirectoryEntry source_dirent;
302  if (!LookupPath(relative_source, kLookupSole, &source_dirent)) {
303  PANIC(kLogStderr, "catalog for file '%s' cannot be found, aborting",
304  source.c_str());
305  }
306  if (source_dirent.IsDirectory()) {
307  PANIC(kLogStderr, "Trying to clone a directory: '%s', aborting",
308  source.c_str());
309  }
310 
311  // if the file is already there we remove it and we add it back
312  DirectoryEntry check_dirent;
313  bool destination_already_present =
314  LookupPath(MakeRelativePath(destination), kLookupSole, &check_dirent);
315  if (destination_already_present) {
316  this->RemoveFile(destination);
317  }
318 
319  DirectoryEntry destination_dirent(source_dirent);
320  std::string destination_dirname;
321  std::string destination_filename;
322  SplitPath(destination, &destination_dirname, &destination_filename);
323 
324  destination_dirent.name_.Assign(
325  NameString(destination_filename.c_str(), destination_filename.length()));
326 
327  // TODO(jblomer): clone is used by tarball engine and should eventually
328  // support extended attributes
329  this->AddFile(destination_dirent, empty_xattrs, destination_dirname);
330 }
331 
332 
339 void WritableCatalogManager::CloneTree(const std::string &from_dir,
340  const std::string &to_dir)
341 {
342  // Sanitize input paths
343  if (from_dir.empty() || to_dir.empty())
344  PANIC(kLogStderr, "clone tree from or to root impossible");
345 
346  const std::string relative_source = MakeRelativePath(from_dir);
347  const std::string relative_dest = MakeRelativePath(to_dir);
348 
349  if (relative_source == relative_dest) {
350  PANIC(kLogStderr, "cannot clone tree into itself ('%s')", to_dir.c_str());
351  }
352  if (HasPrefix(relative_dest, relative_source + "/", false /*ignore_case*/)) {
354  "cannot clone tree into sub directory of source '%s' --> '%s'",
355  from_dir.c_str(), to_dir.c_str());
356  }
357 
358  DirectoryEntry source_dirent;
359  if (!LookupPath(relative_source, kLookupSole, &source_dirent)) {
360  PANIC(kLogStderr, "path '%s' cannot be found, aborting", from_dir.c_str());
361  }
362  if (!source_dirent.IsDirectory()) {
363  PANIC(kLogStderr, "CloneTree: source '%s' not a directory, aborting",
364  from_dir.c_str());
365  }
366 
367  DirectoryEntry dest_dirent;
368  if (LookupPath(relative_dest, kLookupSole, &dest_dirent)) {
369  PANIC(kLogStderr, "destination '%s' exists, aborting", to_dir.c_str());
370  }
371 
372  const std::string dest_parent = GetParentPath(relative_dest);
373  DirectoryEntry dest_parent_dirent;
374  if (!LookupPath(dest_parent, kLookupSole, &dest_parent_dirent)) {
375  PANIC(kLogStderr, "destination '%s' not on a known path, aborting",
376  to_dir.c_str());
377  }
378 
379  CloneTreeImpl(PathString(from_dir),
380  GetParentPath(to_dir),
381  NameString(GetFileName(to_dir)));
382 }
383 
384 
390  const PathString &source_dir,
391  const std::string &dest_parent_dir,
392  const NameString &dest_name)
393 {
394  LogCvmfs(kLogCatalog, kLogDebug, "cloning %s --> %s/%s", source_dir.c_str(),
395  dest_parent_dir.c_str(), dest_name.ToString().c_str());
396  PathString relative_source(MakeRelativePath(source_dir.ToString()));
397 
398  DirectoryEntry source_dirent;
399  bool retval = LookupPath(relative_source, kLookupSole, &source_dirent);
400  assert(retval);
401  assert(!source_dirent.IsBindMountpoint());
402 
403  DirectoryEntry dest_dirent(source_dirent);
404  dest_dirent.name_.Assign(dest_name);
405  // Just in case, reset the nested catalog markers
406  dest_dirent.set_is_nested_catalog_mountpoint(false);
407  dest_dirent.set_is_nested_catalog_root(false);
408 
409  XattrList xattrs;
410  if (source_dirent.HasXattrs()) {
411  retval = LookupXattrs(relative_source, &xattrs);
412  assert(retval);
413  }
414  AddDirectory(dest_dirent, xattrs, dest_parent_dir);
415 
416  std::string dest_dir = dest_parent_dir;
417  if (!dest_dir.empty())
418  dest_dir.push_back('/');
419  dest_dir += dest_name.ToString();
420  if (source_dirent.IsNestedCatalogRoot() ||
421  source_dirent.IsNestedCatalogMountpoint())
422  {
423  CreateNestedCatalog(dest_dir);
424  }
425 
427  retval = Listing(relative_source, &ls, false /* expand_symlink */);
428  assert(retval);
429  for (unsigned i = 0; i < ls.size(); ++i) {
430  PathString sub_path(source_dir);
431  assert(!sub_path.IsEmpty());
432  sub_path.Append("/", 1);
433  sub_path.Append(ls[i].name().GetChars(), ls[i].name().GetLength());
434 
435  if (ls[i].IsDirectory()) {
436  CloneTreeImpl(sub_path, dest_dir, ls[i].name());
437  continue;
438  }
439 
440  // We break hard-links during cloning
441  ls[i].set_hardlink_group(0);
442  ls[i].set_linkcount(1);
443 
444  xattrs.Clear();
445  if (ls[i].HasXattrs()) {
446  retval = LookupXattrs(sub_path, &xattrs);
447  assert(retval);
448  }
449 
450  if (ls[i].IsChunkedFile()) {
451  FileChunkList chunks;
452  std::string relative_sub_path = MakeRelativePath(sub_path.ToString());
453  retval = ListFileChunks(
454  PathString(relative_sub_path), ls[i].hash_algorithm(), &chunks);
455  assert(retval);
456  AddChunkedFile(ls[i], xattrs, dest_dir, chunks);
457  } else {
458  AddFile(ls[i], xattrs, dest_dir);
459  }
460  }
461 }
462 
463 
472  const XattrList &xattrs,
473  const std::string &parent_directory)
474 {
475  const string parent_path = MakeRelativePath(parent_directory);
476  string directory_path = parent_path + "/";
477  directory_path.append(entry.name().GetChars(), entry.name().GetLength());
478 
479  SyncLock();
480  WritableCatalog *catalog;
481  DirectoryEntry parent_entry;
482  if (!FindCatalog(parent_path, &catalog, &parent_entry)) {
483  PANIC(kLogStderr, "catalog for directory '%s' cannot be found",
484  directory_path.c_str());
485  }
486 
487  DirectoryEntry fixed_hardlink_count(entry);
488  fixed_hardlink_count.set_linkcount(2);
489  catalog->AddEntry(fixed_hardlink_count, xattrs,
490  directory_path, parent_path);
491 
492  parent_entry.set_linkcount(parent_entry.linkcount() + 1);
493  catalog->UpdateEntry(parent_entry, parent_path);
494  if (parent_entry.IsNestedCatalogRoot()) {
495  LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating transition point %s",
496  parent_path.c_str());
497  WritableCatalog *parent_catalog =
498  reinterpret_cast<WritableCatalog *>(catalog->parent());
499  parent_entry.set_is_nested_catalog_mountpoint(true);
500  parent_entry.set_is_nested_catalog_root(false);
501  parent_catalog->UpdateEntry(parent_entry, parent_path);
502  }
503  SyncUnlock();
504 }
505 
514  const DirectoryEntry &entry,
515  const XattrList &xattrs,
516  const std::string &parent_directory)
517 {
518  const string parent_path = MakeRelativePath(parent_directory);
519  const string file_path = entry.GetFullPath(parent_path);
520 
521  SyncLock();
522  WritableCatalog *catalog;
523  if (!FindCatalog(parent_path, &catalog)) {
524  PANIC(kLogStderr, "catalog for file '%s' cannot be found",
525  file_path.c_str());
526  }
527 
528  assert(!entry.IsRegular() || entry.IsChunkedFile() ||
529  !entry.checksum().IsNull());
530  assert(entry.IsRegular() || !entry.IsExternalFile());
531 
532  // check if file is too big
533  unsigned mbytes = entry.size() / (1024 * 1024);
534  if ((file_mbyte_limit_ > 0) && (mbytes > file_mbyte_limit_)) {
536  "%s: file at %s is larger than %u megabytes (%u). "
537  "CernVM-FS works best with small files. "
538  "Please remove the file or increase the limit.",
539  enforce_limits_ ? "FATAL" : "WARNING", file_path.c_str(),
540  file_mbyte_limit_, mbytes);
541  if (enforce_limits_)
542  PANIC(kLogStderr, "file at %s is larger than %u megabytes (%u).",
543  file_path.c_str(), file_mbyte_limit_, mbytes);
544  }
545 
546  catalog->AddEntry(entry, xattrs, file_path, parent_path);
547  SyncUnlock();
548 }
549 
550 
552  const DirectoryEntryBase &entry,
553  const XattrList &xattrs,
554  const std::string &parent_directory,
555  const FileChunkList &file_chunks)
556 {
557  assert(file_chunks.size() > 0);
558 
559  DirectoryEntry full_entry(entry);
560  full_entry.set_is_chunked_file(true);
561 
562  AddFile(full_entry, xattrs, parent_directory);
563 
564  const string parent_path = MakeRelativePath(parent_directory);
565  const string file_path = entry.GetFullPath(parent_path);
566 
567  SyncLock();
568  WritableCatalog *catalog;
569  if (!FindCatalog(parent_path, &catalog)) {
570  PANIC(kLogStderr, "catalog for file '%s' cannot be found",
571  file_path.c_str());
572  }
573 
574  for (unsigned i = 0; i < file_chunks.size(); ++i) {
575  catalog->AddFileChunk(file_path, *file_chunks.AtPtr(i));
576  }
577  SyncUnlock();
578 }
579 
580 
589  const DirectoryEntryBaseList &entries,
590  const XattrList &xattrs,
591  const std::string &parent_directory,
592  const FileChunkList &file_chunks)
593 {
594  assert(entries.size() >= 1);
595  assert(file_chunks.IsEmpty() || entries[0].IsRegular());
596  if (entries.size() == 1) {
597  DirectoryEntry fix_linkcount(entries[0]);
598  fix_linkcount.set_linkcount(1);
599  if (file_chunks.IsEmpty())
600  return AddFile(fix_linkcount, xattrs, parent_directory);
601  return AddChunkedFile(fix_linkcount, xattrs, parent_directory, file_chunks);
602  }
603 
604  LogCvmfs(kLogCatalog, kLogVerboseMsg, "adding hardlink group %s/%s",
605  parent_directory.c_str(), entries[0].name().c_str());
606 
607  // Hardlink groups have to reside in the same directory.
608  // Therefore we only have one parent directory here
609  const string parent_path = MakeRelativePath(parent_directory);
610 
611  // check if hard link is too big
612  unsigned mbytes = entries[0].size() / (1024 * 1024);
613  if ((file_mbyte_limit_ > 0) && (mbytes > file_mbyte_limit_)) {
615  "%s: hard link at %s is larger than %u megabytes (%u). "
616  "CernVM-FS works best with small files. "
617  "Please remove the file or increase the limit.",
618  enforce_limits_ ? "FATAL" : "WARNING",
619  (parent_path + entries[0].name().ToString()).c_str(),
621  mbytes);
622  if (enforce_limits_)
623  PANIC(kLogStderr, "hard link at %s is larger than %u megabytes (%u)",
624  (parent_path + entries[0].name().ToString()).c_str(),
625  file_mbyte_limit_, mbytes);
626  }
627 
628  SyncLock();
629  WritableCatalog *catalog;
630  if (!FindCatalog(parent_path, &catalog)) {
632  "catalog for hardlink group containing '%s' cannot be found",
633  parent_path.c_str());
634  }
635 
636  // Get a valid hardlink group id for the catalog the group will end up in
637  // TODO(unkown): Compaction
638  uint32_t new_group_id = catalog->GetMaxLinkId() + 1;
639  LogCvmfs(kLogCatalog, kLogVerboseMsg, "hardlink group id %u issued",
640  new_group_id);
641  assert(new_group_id > 0);
642 
643  // Add the file entries to the catalog
644  for (DirectoryEntryBaseList::const_iterator i = entries.begin(),
645  iEnd = entries.end(); i != iEnd; ++i)
646  {
647  string file_path = parent_path + "/";
648  file_path.append(i->name().GetChars(), i->name().GetLength());
649 
650  // create a fully fledged DirectoryEntry to add the hardlink group to it
651  // which is CVMFS specific meta data.
652  DirectoryEntry hardlink(*i);
653  hardlink.set_hardlink_group(new_group_id);
654  hardlink.set_linkcount(entries.size());
655  hardlink.set_is_chunked_file(!file_chunks.IsEmpty());
656 
657  catalog->AddEntry(hardlink, xattrs, file_path, parent_path);
658  if (hardlink.IsChunkedFile()) {
659  for (unsigned i = 0; i < file_chunks.size(); ++i) {
660  catalog->AddFileChunk(file_path, *file_chunks.AtPtr(i));
661  }
662  }
663  }
664  SyncUnlock();
665 }
666 
667 
668 void WritableCatalogManager::ShrinkHardlinkGroup(const string &remove_path) {
669  const string relative_path = MakeRelativePath(remove_path);
670 
671  SyncLock();
672  WritableCatalog *catalog;
673  if (!FindCatalog(relative_path, &catalog)) {
675  "catalog for hardlink group containing '%s' cannot be found",
676  remove_path.c_str());
677  }
678 
679  catalog->IncLinkcount(relative_path, -1);
680  SyncUnlock();
681 }
682 
683 
693  const XattrList &xattrs,
694  const std::string &directory_path)
695 {
696  assert(entry.IsDirectory());
697 
698  const string entry_path = MakeRelativePath(directory_path);
699  const string parent_path = GetParentPath(entry_path);
700 
701  SyncLock();
702  // find the catalog to be updated
703  WritableCatalog *catalog;
704  if (!FindCatalog(parent_path, &catalog)) {
705  PANIC(kLogStderr, "catalog for entry '%s' cannot be found",
706  entry_path.c_str());
707  }
708 
709  catalog->TouchEntry(entry, xattrs, entry_path);
710 
711  // since we deal with a directory here, we might just touch a
712  // nested catalog transition point. If this is the case we would need to
713  // update two catalog entries:
714  // * the nested catalog MOUNTPOINT in the parent catalog
715  // * the nested catalog ROOT in the nested catalog
716 
717  // first check if we really have a nested catalog transition point
718  catalog::DirectoryEntry potential_transition_point;
719  PathString transition_path(entry_path.data(), entry_path.length());
720  bool retval = catalog->LookupPath(transition_path,
721  &potential_transition_point);
722  assert(retval);
723  if (potential_transition_point.IsNestedCatalogMountpoint()) {
725  "updating transition point at %s", entry_path.c_str());
726 
727  // find and mount nested catalog assciated to this transition point
728  shash::Any nested_hash;
729  uint64_t nested_size;
730  retval = catalog->FindNested(transition_path, &nested_hash, &nested_size);
731  assert(retval);
732  Catalog *nested_catalog;
733  nested_catalog = MountCatalog(transition_path, nested_hash, catalog);
734  assert(nested_catalog != NULL);
735 
736  // update nested catalog root in the child catalog
737  reinterpret_cast<WritableCatalog *>(nested_catalog)->
738  TouchEntry(entry, xattrs, entry_path);
739  }
740 
741  SyncUnlock();
742 }
743 
744 
751 void WritableCatalogManager::CreateNestedCatalog(const std::string &mountpoint)
752 {
753  const string nested_root_path = MakeRelativePath(mountpoint);
754  const PathString ps_nested_root_path(nested_root_path);
755 
756  SyncLock();
757  // Find the catalog currently containing the directory structure, which
758  // will be represented as a new nested catalog and its root-entry/mountpoint
759  // along the way
760  WritableCatalog *old_catalog = NULL;
761  DirectoryEntry new_root_entry;
762  if (!FindCatalog(nested_root_path, &old_catalog, &new_root_entry)) {
764  "failed to create nested catalog '%s': "
765  "mountpoint was not found in current catalog structure",
766  nested_root_path.c_str());
767  }
768 
769  // Create the database schema and the inital root entry
770  // for the new nested catalog
771  const string database_file_path = CreateTempPath(dir_temp() + "/catalog",
772  0666);
773  const bool volatile_content = false;
774  CatalogDatabase *new_catalog_db = CatalogDatabase::Create(database_file_path);
775  assert(NULL != new_catalog_db);
776  // Note we do not set the external_data bit for nested catalogs
777  bool retval =
778  new_catalog_db->InsertInitialValues(nested_root_path,
779  volatile_content,
780  "", // At this point, only root
781  // catalog gets VOMS authz
782  new_root_entry);
783  assert(retval);
784  // TODO(rmeusel): we need a way to attach a catalog directy from an open
785  // database to remove this indirection
786  delete new_catalog_db;
787  new_catalog_db = NULL;
788 
789  // Attach the just created nested catalog
790  Catalog *new_catalog =
791  CreateCatalog(ps_nested_root_path, shash::Any(), old_catalog);
792  retval = AttachCatalog(database_file_path, new_catalog);
793  assert(retval);
794 
795  assert(new_catalog->IsWritable());
796  WritableCatalog *wr_new_catalog = static_cast<WritableCatalog *>(new_catalog);
797 
798  if (new_root_entry.HasXattrs()) {
799  XattrList xattrs;
800  retval = old_catalog->LookupXattrsPath(ps_nested_root_path, &xattrs);
801  assert(retval);
802  wr_new_catalog->TouchEntry(new_root_entry, xattrs, nested_root_path);
803  }
804 
805  // From now on, there are two catalogs, spanning the same directory structure
806  // we have to split the overlapping directory entries from the old catalog
807  // to the new catalog to re-gain a valid catalog structure
808  old_catalog->Partition(wr_new_catalog);
809 
810  // Add the newly created nested catalog to the references of the containing
811  // catalog
812  old_catalog->InsertNestedCatalog(new_catalog->mountpoint().ToString(), NULL,
813  shash::Any(spooler_->GetHashAlgorithm()), 0);
814 
815  // Fix subtree counters in new nested catalogs: subtree is the sum of all
816  // entries of all "grand-nested" catalogs
817  // Note: taking a copy of the nested catalog list here
818  const Catalog::NestedCatalogList &grand_nested =
819  wr_new_catalog->ListOwnNestedCatalogs();
820  DeltaCounters fix_subtree_counters;
821  for (Catalog::NestedCatalogList::const_iterator i = grand_nested.begin(),
822  iEnd = grand_nested.end(); i != iEnd; ++i)
823  {
824  WritableCatalog *grand_catalog;
825  retval = FindCatalog(i->mountpoint.ToString(), &grand_catalog);
826  assert(retval);
827  const Counters &grand_counters = grand_catalog->GetCounters();
828  grand_counters.AddAsSubtree(&fix_subtree_counters);
829  }
830  DeltaCounters save_counters = wr_new_catalog->delta_counters_;
831  wr_new_catalog->delta_counters_ = fix_subtree_counters;
832  wr_new_catalog->UpdateCounters();
833  wr_new_catalog->delta_counters_ = save_counters;
834 
835  SyncUnlock();
836 }
837 
838 
850 void WritableCatalogManager::RemoveNestedCatalog(const string &mountpoint,
851  const bool merge) {
852  const string nested_root_path = MakeRelativePath(mountpoint);
853 
854  SyncLock();
855  // Find the catalog which should be removed
856  WritableCatalog *nested_catalog = NULL;
857  if (!FindCatalog(nested_root_path, &nested_catalog)) {
859  "failed to remove nested catalog '%s': "
860  "mountpoint was not found in current catalog structure",
861  nested_root_path.c_str());
862  }
863 
864  // Check if the found catalog is really the nested catalog to be deleted
865  assert(!nested_catalog->IsRoot() &&
866  (nested_catalog->mountpoint().ToString() == nested_root_path));
867 
868  if (merge) {
869  // Merge all data from the nested catalog into it's parent
870  nested_catalog->MergeIntoParent();
871  } else {
872  nested_catalog->RemoveFromParent();
873  }
874 
875  // Delete the catalog database file from the working copy
876  if (unlink(nested_catalog->database_path().c_str()) != 0) {
878  "unable to delete the removed nested catalog database file '%s'",
879  nested_catalog->database_path().c_str());
880  }
881 
882  // Remove the catalog from internal data structures
883  DetachCatalog(nested_catalog);
884  SyncUnlock();
885 }
886 
887 
899 void WritableCatalogManager::SwapNestedCatalog(const string &mountpoint,
900  const shash::Any &new_hash,
901  const uint64_t new_size) {
902  const string nested_root_path = MakeRelativePath(mountpoint);
903  const string parent_path = GetParentPath(nested_root_path);
904  const PathString nested_root_ps = PathString(nested_root_path);
905 
906  SyncLock();
907 
908  // Find the immediate parent catalog
909  WritableCatalog *parent = NULL;
910  if (!FindCatalog(parent_path, &parent)) {
912  "failed to swap nested catalog '%s': could not find parent '%s'",
913  nested_root_path.c_str(), parent_path.c_str());
914  }
915 
916  // Get old nested catalog counters
917  Catalog *old_attached_catalog = parent->FindChild(nested_root_ps);
918  Counters old_counters;
919  if (old_attached_catalog) {
920  // Old catalog was already attached (e.g. as a child catalog
921  // attached by a prior call to CreateNestedCatalog()). Ensure
922  // that it has not been modified, get counters, and detach it.
923  WritableCatalogList list;
924  if (GetModifiedCatalogLeafsRecursively(old_attached_catalog, &list)) {
926  "failed to swap nested catalog '%s': already modified",
927  nested_root_path.c_str());
928  }
929  old_counters = old_attached_catalog->GetCounters();
930  DetachSubtree(old_attached_catalog);
931 
932  } else {
933  // Old catalog was not attached. Download a freely attached
934  // version and get counters.
935  shash::Any old_hash;
936  uint64_t old_size;
937  const bool old_found = parent->FindNested(nested_root_ps, &old_hash,
938  &old_size);
939  if (!old_found) {
941  "failed to swap nested catalog '%s': not found in parent",
942  nested_root_path.c_str());
943  }
944  UniquePtr<Catalog> old_free_catalog(
945  LoadFreeCatalog(nested_root_ps, old_hash));
946  if (!old_free_catalog.IsValid()) {
948  "failed to swap nested catalog '%s': failed to load old catalog",
949  nested_root_path.c_str());
950  }
951  old_counters = old_free_catalog->GetCounters();
952  }
953 
954  // Load freely attached new catalog
955  UniquePtr<Catalog> new_catalog(LoadFreeCatalog(nested_root_ps, new_hash));
956  if (!new_catalog.IsValid()) {
958  "failed to swap nested catalog '%s': failed to load new catalog",
959  nested_root_path.c_str());
960  }
961 
962  // Get new catalog root directory entry
963  DirectoryEntry dirent;
964  XattrList xattrs;
965  const bool dirent_found = new_catalog->LookupPath(nested_root_ps, &dirent);
966  if (!dirent_found) {
968  "failed to swap nested catalog '%s': missing dirent in new catalog",
969  nested_root_path.c_str());
970  }
971  if (dirent.HasXattrs()) {
972  const bool xattrs_found = new_catalog->LookupXattrsPath(nested_root_ps,
973  &xattrs);
974  if (!xattrs_found) {
976  "failed to swap nested catalog '%s': missing xattrs in new catalog",
977  nested_root_path.c_str());
978  }
979  }
980 
981  // Swap catalogs
982  parent->RemoveNestedCatalog(nested_root_path, NULL);
983  parent->InsertNestedCatalog(nested_root_path, NULL, new_hash, new_size);
984 
985  // Update parent directory entry
987  dirent.set_is_nested_catalog_root(false);
988  parent->UpdateEntry(dirent, nested_root_path);
989  parent->TouchEntry(dirent, xattrs, nested_root_path);
990 
991  // Update counters
992  DeltaCounters delta = Counters::Diff(old_counters,
993  new_catalog->GetCounters());
994  delta.PopulateToParent(&parent->delta_counters_);
995 
996  SyncUnlock();
997 }
998 
999 
1003 bool WritableCatalogManager::IsTransitionPoint(const string &mountpoint) {
1004  const string path = MakeRelativePath(mountpoint);
1005 
1006  SyncLock();
1007  WritableCatalog *catalog;
1008  DirectoryEntry entry;
1009  if (!FindCatalog(path, &catalog, &entry)) {
1010  PANIC(kLogStderr, "catalog for directory '%s' cannot be found",
1011  path.c_str());
1012  }
1013  const bool result = entry.IsNestedCatalogRoot();
1014  SyncUnlock();
1015  return result;
1016 }
1017 
1018 
1020  // TODO(jblomer): meant for micro catalogs
1021 }
1022 
1023 
1024 void WritableCatalogManager::SetTTL(const uint64_t new_ttl) {
1025  SyncLock();
1026  reinterpret_cast<WritableCatalog *>(GetRootCatalog())->SetTTL(new_ttl);
1027  SyncUnlock();
1028 }
1029 
1030 
1031 bool WritableCatalogManager::SetVOMSAuthz(const std::string &voms_authz) {
1032  bool result;
1033  SyncLock();
1034  result = reinterpret_cast<WritableCatalog *>(
1035  GetRootCatalog())->SetVOMSAuthz(voms_authz);
1036  SyncUnlock();
1037  return result;
1038 }
1039 
1040 
1041 bool WritableCatalogManager::Commit(const bool stop_for_tweaks,
1042  const uint64_t manual_revision,
1044  WritableCatalog *root_catalog =
1045  reinterpret_cast<WritableCatalog *>(GetRootCatalog());
1046  root_catalog->SetDirty();
1047 
1048  // set root catalog revision to manually provided number if available
1049  if (manual_revision > 0) {
1050  const uint64_t revision = root_catalog->GetRevision();
1051  if (revision >= manual_revision) {
1052  LogCvmfs(kLogCatalog, kLogStderr, "Manual revision (%d) must not be "
1053  "smaller than the current root "
1054  "catalog's (%d). Skipped!",
1055  manual_revision, revision);
1056  } else {
1057  // Gets incremented by FinalizeCatalog() afterwards!
1058  root_catalog->SetRevision(manual_revision - 1);
1059  }
1060  }
1061 
1062  // do the actual catalog snapshotting and upload
1063  CatalogInfo root_catalog_info;
1064  if (getenv("_CVMFS_SERIALIZED_CATALOG_PROCESSING_") == NULL)
1065  root_catalog_info = SnapshotCatalogs(stop_for_tweaks);
1066  else
1067  root_catalog_info = SnapshotCatalogsSerialized(stop_for_tweaks);
1068  if (spooler_->GetNumberOfErrors() > 0) {
1069  LogCvmfs(kLogCatalog, kLogStderr, "failed to commit catalogs");
1070  return false;
1071  }
1072 
1073  // .cvmfspublished export
1074  LogCvmfs(kLogCatalog, kLogVerboseMsg, "Committing repository manifest");
1075  set_base_hash(root_catalog_info.content_hash);
1076 
1077  manifest->set_catalog_hash(root_catalog_info.content_hash);
1078  manifest->set_catalog_size(root_catalog_info.size);
1079  manifest->set_root_path("");
1080  manifest->set_ttl(root_catalog_info.ttl);
1081  manifest->set_revision(root_catalog_info.revision);
1082 
1083  return true;
1084 }
1085 
1086 
1114  const bool stop_for_tweaks) {
1115  // prepare environment for parallel processing
1116  Future<CatalogInfo> root_catalog_info_future;
1117  CatalogUploadContext upload_context;
1118  upload_context.root_catalog_info = &root_catalog_info_future;
1119  upload_context.stop_for_tweaks = stop_for_tweaks;
1120 
1121  spooler_->RegisterListener(
1122  &WritableCatalogManager::CatalogUploadCallback, this, upload_context);
1123 
1124  // find dirty leaf catalogs and annotate non-leaf catalogs (dirty child count)
1125  // post-condition: the entire catalog tree is ready for concurrent processing
1126  WritableCatalogList leafs_to_snapshot;
1127  GetModifiedCatalogLeafs(&leafs_to_snapshot);
1128 
1129  // finalize and schedule the catalog processing
1130  WritableCatalogList::const_iterator i = leafs_to_snapshot.begin();
1131  const WritableCatalogList::const_iterator iend = leafs_to_snapshot.end();
1132  for (; i != iend; ++i) {
1133  FinalizeCatalog(*i, stop_for_tweaks);
1135  }
1136 
1137  LogCvmfs(kLogCatalog, kLogVerboseMsg, "waiting for upload of catalogs");
1138  CatalogInfo& root_catalog_info = root_catalog_info_future.Get();
1139  spooler_->WaitForUpload();
1140 
1141  spooler_->UnregisterListeners();
1142  return root_catalog_info;
1143 }
1144 
1145 
1147  const bool stop_for_tweaks) {
1148  // update meta information of this catalog
1149  LogCvmfs(kLogCatalog, kLogVerboseMsg, "creating snapshot of catalog '%s'",
1150  catalog->mountpoint().c_str());
1151 
1152  catalog->UpdateCounters();
1153  catalog->UpdateLastModified();
1154  catalog->IncrementRevision();
1155 
1156  // update the previous catalog revision pointer
1157  if (catalog->IsRoot()) {
1158  LogCvmfs(kLogCatalog, kLogVerboseMsg, "setting '%s' as previous revision "
1159  "for root catalog",
1160  base_hash().ToStringWithSuffix().c_str());
1161  catalog->SetPreviousRevision(base_hash());
1162  } else {
1163  // Multiple catalogs might query the parent concurrently
1164  SyncLock();
1165  shash::Any hash_previous;
1166  uint64_t size_previous;
1167  const bool retval =
1168  catalog->parent()->FindNested(catalog->mountpoint(),
1169  &hash_previous, &size_previous);
1170  assert(retval);
1171  SyncUnlock();
1172 
1173  LogCvmfs(kLogCatalog, kLogVerboseMsg, "found '%s' as previous revision "
1174  "for nested catalog '%s'",
1175  hash_previous.ToStringWithSuffix().c_str(),
1176  catalog->mountpoint().c_str());
1177  catalog->SetPreviousRevision(hash_previous);
1178  }
1179  catalog->Commit();
1180 
1181  // check if catalog has too many entries
1182  uint64_t catalog_limit = uint64_t(1000) *
1183  uint64_t((catalog->IsRoot()
1186  if ((catalog_limit > 0) &&
1187  (catalog->GetCounters().GetSelfEntries() > catalog_limit)) {
1189  "%s: catalog at %s has more than %u entries (%u). "
1190  "Large catalogs stress the CernVM-FS transport infrastructure. "
1191  "Please split it into nested catalogs or increase the limit.",
1192  enforce_limits_ ? "FATAL" : "WARNING",
1193  (catalog->IsRoot() ? "/" : catalog->mountpoint().c_str()),
1194  catalog_limit, catalog->GetCounters().GetSelfEntries());
1195  if (enforce_limits_)
1196  PANIC(kLogStderr, "catalog at %s has more than %u entries (%u). ",
1197  (catalog->IsRoot() ? "/" : catalog->mountpoint().c_str()),
1198  catalog_limit, catalog->GetCounters().GetSelfEntries());
1199  }
1200 
1201  // allow for manual adjustments in the catalog
1202  if (stop_for_tweaks) {
1203  LogCvmfs(kLogCatalog, kLogStdout, "Allowing for tweaks in %s at %s "
1204  "(hit return to continue)",
1205  catalog->database_path().c_str(), catalog->mountpoint().c_str());
1206  int read_char = getchar();
1207  assert(read_char != EOF);
1208  }
1209 
1210  // compaction of bloated catalogs (usually after high database churn)
1211  catalog->VacuumDatabaseIfNecessary();
1212 }
1213 
1214 
1216  WritableCatalog *catalog) {
1217  {
1219  // register catalog object for WritableCatalogManager::CatalogUploadCallback
1220  catalog_processing_map_[catalog->database_path()] = catalog;
1221  }
1222  spooler_->ProcessCatalog(catalog->database_path());
1223 }
1224 
1225 
1227  const upload::SpoolerResult &result,
1228  const CatalogUploadContext catalog_upload_context) {
1229  if (result.return_code != 0) {
1230  PANIC(kLogStderr, "failed to upload '%s' (retval: %d)",
1231  result.local_path.c_str(), result.return_code);
1232  }
1233 
1234  // retrieve the catalog object based on the callback information
1235  // see WritableCatalogManager::ScheduleCatalogProcessing()
1236  WritableCatalog *catalog = NULL;
1237  {
1239  std::map<std::string, WritableCatalog*>::iterator c =
1240  catalog_processing_map_.find(result.local_path);
1241  assert(c != catalog_processing_map_.end());
1242  catalog = c->second;
1243  }
1244 
1245  uint64_t catalog_size = GetFileSize(result.local_path);
1246  assert(catalog_size > 0);
1247 
1248  SyncLock();
1249  if (catalog->HasParent()) {
1250  // finalized nested catalogs will update their parent's pointer and schedule
1251  // them for processing (continuation) if the 'dirty children count' == 0
1252  LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating nested catalog link");
1253  WritableCatalog *parent = catalog->GetWritableParent();
1254 
1255  parent->UpdateNestedCatalog(catalog->mountpoint().ToString(),
1256  result.content_hash,
1257  catalog_size,
1258  catalog->delta_counters_);
1259  catalog->delta_counters_.SetZero();
1260 
1261  const int remaining_dirty_children =
1263 
1264  SyncUnlock();
1265 
1266  // continuation of the dirty catalog tree traversal
1267  // see WritableCatalogManager::SnapshotCatalogs()
1268  if (remaining_dirty_children == 0) {
1269  FinalizeCatalog(parent, catalog_upload_context.stop_for_tweaks);
1270  ScheduleCatalogProcessing(parent);
1271  }
1272 
1273  } else if (catalog->IsRoot()) {
1274  // once the root catalog is reached, we are done with processing and report
1275  // back to the main via a Future<> and provide the necessary information
1276  CatalogInfo root_catalog_info;
1277  root_catalog_info.size = catalog_size;
1278  root_catalog_info.ttl = catalog->GetTTL();
1279  root_catalog_info.content_hash = result.content_hash;
1280  root_catalog_info.revision = catalog->GetRevision();
1281  catalog_upload_context.root_catalog_info->Set(root_catalog_info);
1282  SyncUnlock();
1283  } else {
1284  PANIC(kLogStderr, "inconsistent state detected");
1285  }
1286 }
1287 
1288 
1300  Catalog *catalog,
1301  WritableCatalogList *result) const {
1302  WritableCatalog *wr_catalog = static_cast<WritableCatalog *>(catalog);
1303 
1304  // Look for dirty catalogs in the descendants of *catalog
1305  int dirty_children = 0;
1306  CatalogList children = wr_catalog->GetChildren();
1307  CatalogList::const_iterator i = children.begin();
1308  const CatalogList::const_iterator iend = children.end();
1309  for (; i != iend; ++i) {
1310  if (GetModifiedCatalogLeafsRecursively(*i, result)) {
1311  ++dirty_children;
1312  }
1313  }
1314 
1315  // a catalog is dirty if itself or one of its children has changed
1316  // a leaf catalog doesn't have any dirty children
1317  wr_catalog->set_dirty_children(dirty_children);
1318  const bool is_dirty = wr_catalog->IsDirty() || dirty_children > 0;
1319  const bool is_leaf = dirty_children == 0;
1320  if (is_dirty && is_leaf) {
1321  result->push_back(const_cast<WritableCatalog *>(wr_catalog));
1322  }
1323 
1324  return is_dirty;
1325 }
1326 
1327 
1329  CatalogList catalog_list = GetCatalogs();
1330  reverse(catalog_list.begin(), catalog_list.end());
1331  for (unsigned i = 0; i < catalog_list.size(); ++i) {
1332  FixWeight(static_cast<WritableCatalog*>(catalog_list[i]));
1333  }
1334 }
1335 
1337  // firstly check underflow because they can provoke overflows
1338  if (catalog->GetNumEntries() < min_weight_ &&
1339  !catalog->IsRoot() &&
1340  catalog->IsAutogenerated()) {
1342  "Deleting an autogenerated catalog in '%s'",
1343  catalog->mountpoint().c_str());
1344  // Remove the .cvmfscatalog and .cvmfsautocatalog files first
1345  string path = catalog->mountpoint().ToString();
1346  catalog->RemoveEntry(path + "/.cvmfscatalog");
1347  catalog->RemoveEntry(path + "/.cvmfsautocatalog");
1348  // Remove the actual catalog
1349  string catalog_path = catalog->mountpoint().ToString().substr(1);
1350  RemoveNestedCatalog(catalog_path);
1351  } else if (catalog->GetNumEntries() > max_weight_) {
1352  CatalogBalancer<WritableCatalogManager> catalog_balancer(this);
1353  catalog_balancer.Balance(catalog);
1354  }
1355 }
1356 
1357 
1358 //****************************************************************************
1359 // Workaround -- Serialized Catalog Committing
1360 
1362  const Catalog *catalog,
1363  WritableCatalogList *result) const
1364 {
1365  // A catalog must be snapshot, if itself or one of it's descendants is dirty.
1366  // So we traverse the catalog tree recursively and look for dirty catalogs
1367  // on the way.
1368  const WritableCatalog *wr_catalog =
1369  static_cast<const WritableCatalog *>(catalog);
1370  // This variable will contain the number of dirty catalogs in the sub tree
1371  // with *catalog as it's root.
1372  int dirty_catalogs = (wr_catalog->IsDirty()) ? 1 : 0;
1373 
1374  // Look for dirty catalogs in the descendants of *catalog
1375  CatalogList children = wr_catalog->GetChildren();
1376  for (CatalogList::const_iterator i = children.begin(), iEnd = children.end();
1377  i != iEnd; ++i)
1378  {
1379  dirty_catalogs += GetModifiedCatalogsRecursively(*i, result);
1380  }
1381 
1382  // If we found a dirty catalog in the checked sub tree, the root (*catalog)
1383  // must be snapshot and ends up in the result list
1384  if (dirty_catalogs > 0)
1385  result->push_back(const_cast<WritableCatalog *>(wr_catalog));
1386 
1387  // tell the upper layer about number of catalogs
1388  return dirty_catalogs;
1389 }
1390 
1391 
1393  const upload::SpoolerResult &result,
1394  const CatalogUploadContext unused)
1395 {
1396  if (result.return_code != 0) {
1397  PANIC(kLogStderr, "failed to upload '%s' (retval: %d)",
1398  result.local_path.c_str(), result.return_code);
1399  }
1400  unlink(result.local_path.c_str());
1401 }
1402 
1403 
1406  const bool stop_for_tweaks)
1407 {
1408  LogCvmfs(kLogCvmfs, kLogStdout, "Serialized committing of file catalogs...");
1409  reinterpret_cast<WritableCatalog *>(GetRootCatalog())->SetDirty();
1410  WritableCatalogList catalogs_to_snapshot;
1411  GetModifiedCatalogs(&catalogs_to_snapshot);
1412  CatalogUploadContext unused;
1413  unused.root_catalog_info = NULL;
1414  unused.stop_for_tweaks = false;
1415  spooler_->RegisterListener(
1417 
1418  CatalogInfo root_catalog_info;
1419  WritableCatalogList::const_iterator i = catalogs_to_snapshot.begin();
1420  const WritableCatalogList::const_iterator iend = catalogs_to_snapshot.end();
1421  for (; i != iend; ++i) {
1422  FinalizeCatalog(*i, stop_for_tweaks);
1423 
1424  // Compress and upload catalog
1425  shash::Any hash_catalog(spooler_->GetHashAlgorithm(),
1427  if (!zlib::CompressPath2Null((*i)->database_path(),
1428  &hash_catalog))
1429  {
1430  PANIC(kLogStderr, "could not compress catalog %s",
1431  (*i)->mountpoint().ToString().c_str());
1432  }
1433 
1434  int64_t catalog_size = GetFileSize((*i)->database_path());
1435  assert(catalog_size > 0);
1436 
1437  if ((*i)->HasParent()) {
1438  LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating nested catalog link");
1439  WritableCatalog *parent = (*i)->GetWritableParent();
1440  parent->UpdateNestedCatalog((*i)->mountpoint().ToString(), hash_catalog,
1441  catalog_size, (*i)->delta_counters_);
1442  (*i)->delta_counters_.SetZero();
1443  } else if ((*i)->IsRoot()) {
1444  root_catalog_info.size = catalog_size;
1445  root_catalog_info.ttl = (*i)->GetTTL();
1446  root_catalog_info.content_hash = hash_catalog;
1447  root_catalog_info.revision = (*i)->GetRevision();
1448  } else {
1449  PANIC(kLogStderr, "inconsistent state detected");
1450  }
1451 
1452  spooler_->ProcessCatalog((*i)->database_path());
1453  }
1454  spooler_->WaitForUpload();
1455 
1456  spooler_->UnregisterListeners();
1457  return root_catalog_info;
1458 }
1459 
1460 } // namespace catalog
bool CompressPath2Null(const string &src, shash::Any *compressed_hash)
Definition: compression.cc:515
uint32_t linkcount() const
int return_code
the return value of the spooler operation
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
bool IsExternalFile() const
const Counters & GetCounters() const
Definition: catalog.h:175
void AddChunkedFile(const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &parent_directory, const FileChunkList &file_chunks)
bool IsNull() const
Definition: hash.h:382
const manifest::Manifest * manifest() const
Definition: repository.h:123
void set_base_hash(const shash::Any &hash)
bool IsRoot() const
Definition: catalog.h:193
ShortString< kDefaultMaxName, 1 > NameString
Definition: shortstring.h:191
void CloneTree(const std::string &from_dir, const std::string &to_dir)
void AddHardlinkGroup(const DirectoryEntryBaseList &entries, const XattrList &xattrs, const std::string &parent_directory, const FileChunkList &file_chunks)
const std::vector< Catalog * > & GetCatalogs() const
Definition: catalog_mgr.h:218
void set_dirty_children(const int count)
Definition: catalog_rw.h:143
void set_is_chunked_file(const bool val)
void AddDirectory(const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &parent_directory)
std::string GetFileName(const std::string &path)
Definition: posix.cc:162
std::string database_path() const
Definition: catalog.h:184
bool IsDirectory() const
void set_catalog_hash(const shash::Any &catalog_hash)
Definition: manifest.h:102
Catalog * LoadFreeCatalog(const PathString &mountpoint, const shash::Any &hash)
bool IsChunkedFile() const
inode_t inode_
void Clear()
Definition: xattr.h:41
bool HasParent() const
Definition: catalog.h:200
bool MountSubtree(const PathString &path, const Catalog *entry_point, bool can_listing, Catalog **leaf_catalog)
void RemoveDirectory(const std::string &directory_path)
uint32_t GetMaxLinkId() const
Definition: catalog_rw.cc:124
void UpdateNestedCatalog(const std::string &path, const shash::Any &hash, const uint64_t size, const DeltaCounters &child_counters)
Definition: catalog_rw.cc:620
#define PANIC(...)
Definition: exception.h:26
uint64_t size() const
WritableCatalog * GetHostingCatalog(const std::string &path)
void Assign(const char *chars, const unsigned length)
Definition: shortstring.h:53
void set_linkcount(const uint32_t linkcount)
void set_is_nested_catalog_root(const bool val)
bool IsDirty() const
Definition: catalog_rw.h:57
void FixWeight(WritableCatalog *catalog)
std::string ToStringWithSuffix() const
Definition: hash.h:303
gid_t gid_
void Balance(catalog_t *catalog)
bool LookupPath(const PathString &path, DirectoryEntry *dirent) const
Definition: catalog.h:124
void ScheduleCatalogProcessing(WritableCatalog *catalog)
std::map< std::string, WritableCatalog * > catalog_processing_map_
std::string CreateTempPath(const std::string &path_prefix, const int mode)
Definition: posix.cc:1059
void SetTTL(const uint64_t new_ttl)
static DeltaCounters Diff(const Counters &from, const Counters &to)
int GetModifiedCatalogsRecursively(const Catalog *catalog, WritableCatalogList *result) const
Counters_t GetSelfEntries() const
void set_revision(const uint64_t revision)
Definition: manifest.h:86
void InsertNestedCatalog(const std::string &mountpoint, Catalog *attached_reference, const shash::Any content_hash, const uint64_t size)
Definition: catalog_rw.cc:502
void TouchEntry(const DirectoryEntryBase &entry, const XattrList &xattrs, const shash::Md5 &path_hash)
Definition: catalog_rw.cc:224
uint64_t GetTTL() const
Definition: catalog.cc:495
assert((mem||(size==0))&&"Out Of Memory")
bool FindCatalog(const std::string &path, WritableCatalog **result, DirectoryEntry *dirent=NULL)
bool Commit(const bool stop_for_tweaks, const uint64_t manual_revision, manifest::Manifest *manifest)
CatalogList GetChildren() const
Definition: catalog.cc:752
bool LookupPath(const PathString &path, const LookupOptions options, DirectoryEntry *entry)
std::string GetParentPath(const std::string &path)
Definition: posix.cc:131
void SetPreviousRevision(const shash::Any &hash)
Definition: catalog_rw.cc:352
Catalog * parent() const
Definition: catalog.h:180
pthread_mutex_t * catalog_processing_lock_
Catalog * FindChild(const PathString &mountpoint) const
Definition: catalog.cc:808
std::vector< WritableCatalog * > WritableCatalogList
Definition: catalog_rw.h:203
shash::Any checksum() const
void CloneTreeImpl(const PathString &source_dir, const std::string &dest_parent_dir, const NameString &dest_name)
uint64_t size_
void AddAsSubtree(DeltaCounters *delta) const
bool IsNestedCatalogMountpoint() const
bool IsTransitionPoint(const std::string &mountpoint)
Algorithms
Definition: hash.h:40
bool Listing(const PathString &path, DirectoryEntryList *listing, const bool expand_symlink)
void CatalogUploadCallback(const upload::SpoolerResult &result, const CatalogUploadContext clg_upload_context)
bool IsNestedCatalogRoot() const
NameString name_
void TouchDirectory(const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &directory_path)
uint64_t GetNumEntries() const
Definition: catalog.cc:546
uint64_t GetRevision() const
Definition: catalog.cc:528
void AddFileChunk(const std::string &entry_path, const FileChunk &chunk)
Definition: catalog_rw.cc:264
bool IsAutogenerated() const
Definition: catalog.h:194
std::vector< DirectoryEntry > DirectoryEntryList
NameString name() const
static manifest::Manifest * CreateRepository(const std::string &dir_temp, const bool volatile_content, const std::string &voms_authz, upload::Spooler *spooler)
bool AttachCatalog(const std::string &db_path, Catalog *new_catalog)
void SplitPath(const std::string &path, std::string *dirname, std::string *filename)
Definition: posix.cc:112
bool HasXattrs() const
void ActivateCatalog(Catalog *catalog)
std::string local_path
the local_path previously given as input
bool LookupXattrsPath(const PathString &path, XattrList *xattrs) const
Definition: catalog.h:128
void ShrinkHardlinkGroup(const std::string &remove_path)
bool IsRegular() const
bool GetModifiedCatalogLeafsRecursively(Catalog *catalog, WritableCatalogList *result) const
const char kSuffixCatalog
Definition: hash.h:53
uint32_t linkcount_
const std::string & dir_temp() const
void PopulateToParent(DeltaCounters *parent) const
bool InsertInitialValues(const std::string &root_path, const bool volatile_content, const std::string &voms_authz, const DirectoryEntry &root_entry=DirectoryEntry(kDirentNegative))
Definition: catalog_sql.cc:263
CatalogInfo SnapshotCatalogsSerialized(const bool stop_for_tweaks)
CatalogInfo SnapshotCatalogs(const bool stop_for_tweaks)
PathString mountpoint() const
Definition: catalog.h:179
time_t mtime_
static const inode_t kInvalidInode
void Append(const char *chars, const unsigned length)
Definition: shortstring.h:70
void TakeDatabaseFileOwnership()
Definition: catalog.cc:479
bool SetVOMSAuthz(const std::string &voms_authz)
bool HasPrefix(const string &str, const string &prefix, const bool ignore_case)
Definition: string.cc:265
void Clone(const std::string from, const std::string to)
std::vector< Catalog * > CatalogList
Definition: catalog_mgr.h:119
void RemoveFile(const std::string &file_path)
bool IsBindMountpoint() const
std::string MakeRelativePath(const std::string &relative_path) const
void CatalogUploadSerializedCallback(const upload::SpoolerResult &result, const CatalogUploadContext unused)
void RemoveNestedCatalog(const std::string &mountpoint, Catalog **attached_reference)
Definition: catalog_rw.cc:561
bool IsEmpty() const
Definition: bigvector.h:67
void AddEntry(const DirectoryEntry &entry, const XattrList &xattr, const std::string &entry_path, const std::string &parent_path)
void RemoveNestedCatalog(const std::string &mountpoint, const bool merge=true)
void set_ttl(const uint32_t ttl)
Definition: manifest.h:85
shash::Algorithms GetHashAlgorithm() const
Definition: manifest.h:83
std::string ToString() const
Definition: shortstring.h:114
std::vector< NestedCatalog > NestedCatalogList
Definition: catalog.h:208
bool IsEmpty() const
Definition: shortstring.h:110
unsigned int mode_
bool ListFileChunks(const PathString &path, const shash::Algorithms interpret_hashes_as, FileChunkList *chunks)
void Partition(WritableCatalog *new_nested_catalog)
Definition: catalog_rw.cc:360
ShortString< kDefaultMaxPath, 0 > PathString
Definition: shortstring.h:190
void SetRevision(const uint64_t new_revision)
Definition: catalog_rw.cc:329
Definition: mutex.h:42
void set_hardlink_group(const uint32_t group)
WritableCatalog * GetWritableParent() const
Definition: catalog_rw.h:136
void set_catalog_size(const uint64_t catalog_size)
Definition: manifest.h:99
DeltaCounters delta_counters_
Definition: catalog_rw.h:163
bool FindNested(const PathString &mountpoint, shash::Any *hash, uint64_t *size) const
Definition: catalog.cc:690
std::string GetFullPath(const std::string &parent_directory) const
bool LookupXattrs(const PathString &path, XattrList *xattrs)
Catalog * MountCatalog(const PathString &mountpoint, const shash::Any &hash, Catalog *parent_catalog)
int64_t GetFileSize(const std::string &path)
Definition: posix.cc:826
std::vector< DirectoryEntryBase > DirectoryEntryBaseList
const int kLogVerboseMsg
void set_has_alt_catalog_path(const bool &has_alt_path)
Definition: manifest.h:108
unsigned GetLength() const
Definition: shortstring.h:104
shash::Any checksum_
CatalogT * FindCatalog(const PathString &path) const
std::string MakePath() const
Definition: hash.h:315
void AddFile(const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &parent_directory)
const char * c_str() const
Definition: shortstring.h:118
void SwapNestedCatalog(const string &mountpoint, const shash::Any &new_hash, const uint64_t new_size)
void RemoveEntry(const std::string &entry_path)
Definition: catalog_rw.cc:184
const char * GetChars() const
Definition: shortstring.h:96
virtual bool IsWritable() const
Definition: catalog.h:201
void GetModifiedCatalogLeafs(WritableCatalogList *result) const
Catalog * CreateCatalog(const PathString &mountpoint, const shash::Any &catalog_hash, Catalog *parent_catalog)
void FinalizeCatalog(WritableCatalog *catalog, const bool stop_for_tweaks)
void IncLinkcount(const std::string &path_within_group, const int delta)
Definition: catalog_rw.cc:208
const Item * AtPtr(const size_t index) const
Definition: bigvector.h:55
void set_is_nested_catalog_mountpoint(const bool val)
bool CompressPath2Path(const string &src, const string &dest)
Definition: compression.cc:318
void CreateNestedCatalog(const std::string &mountpoint)
void UpdateEntry(const DirectoryEntry &entry, const shash::Md5 &path_hash)
Definition: catalog_rw.cc:252
static DerivedT * Create(const std::string &filename)
Definition: sql_impl.h:30
void GetModifiedCatalogs(WritableCatalogList *result) const
size_t size() const
Definition: bigvector.h:100
uid_t uid_
void set_root_path(const std::string &root_path)
Definition: manifest.h:114
const shash::Any & base_hash() const