CernVM-FS  2.12.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
catalog_mgr_impl.h
Go to the documentation of this file.
1 
5 // avoid clang-tidy false positives (at least starting with clang14)
6 //NOLINTBEGIN
7 
8 #ifndef CVMFS_CATALOG_MGR_IMPL_H_
9 #define CVMFS_CATALOG_MGR_IMPL_H_
10 
11 #ifndef __STDC_FORMAT_MACROS
12 #define __STDC_FORMAT_MACROS
13 #endif
14 
15 
16 
17 #include <cassert>
18 #include <string>
19 #include <vector>
20 
21 #include "shortstring.h"
22 #include "statistics.h"
23 #include "util/logging.h"
24 #include "xattr.h"
25 
26 using namespace std; // NOLINT
27 
28 namespace catalog {
29 
30 template <class CatalogT>
32  perf::Statistics *statistics) :
33  statistics_(statistics) {
36  revision_cache_ = 0;
37  timestamp_cache_ = 0;
39  volatile_flag_ = false;
40  has_authz_cache_ = false;
41  inode_annotation_ = NULL;
42  incarnation_ = 0;
43  rwlock_ =
44  reinterpret_cast<pthread_rwlock_t *>(smalloc(sizeof(pthread_rwlock_t)));
45  int retval = pthread_rwlock_init(rwlock_, NULL);
46  assert(retval == 0);
47  retval = pthread_key_create(&pkey_sqlitemem_, NULL);
48  assert(retval == 0);
49 }
50 
51 template <class CatalogT>
53  DetachAll();
54  pthread_key_delete(pkey_sqlitemem_);
55  pthread_rwlock_destroy(rwlock_);
56  free(rwlock_);
57 }
58 
59 template <class CatalogT>
61  InodeAnnotation *new_annotation)
62 {
63  assert(catalogs_.empty() || (new_annotation == inode_annotation_));
64  inode_annotation_ = new_annotation;
65 }
66 
67 template <class CatalogT>
69  const OwnerMap &gid_map)
70 {
71  uid_map_ = uid_map;
72  gid_map_ = gid_map;
73 }
74 
75 template <class CatalogT>
77  catalog_watermark_ = limit;
78 }
79 
80 template <class CatalogT>
82  if (inode_watermark_status_ > 0)
83  return;
84 
85  uint64_t highest_inode = inode_gauge_;
86  if (inode_annotation_)
87  highest_inode += inode_annotation_->GetGeneration();
88  uint64_t uint32_border = 1;
89  uint32_border = uint32_border << 32;
90  if (highest_inode >= uint32_border) {
91  LogCvmfs(kLogCatalog, kLogDebug | kLogSyslogWarn, "inodes exceed 32bit");
92  inode_watermark_status_++;
93  }
94 }
95 
96 
101 template <class CatalogT>
103  LogCvmfs(kLogCatalog, kLogDebug, "Initialize catalog");
104  WriteLock();
105  bool attached = MountCatalog(PathString("", 0), shash::Any(), NULL);
106  Unlock();
107 
108  if (!attached) {
109  LogCvmfs(kLogCatalog, kLogDebug, "failed to initialize root catalog");
110  }
111 
112  return attached;
113 }
114 
115 
121 template <class CatalogT>
124  "dryrun remounting repositories");
125  CatalogContext ctlg_context;
126  return GetNewRootCatalogContext(&ctlg_context);
127 }
128 
129 template <class CatalogT>
131  LogCvmfs(kLogCatalog, kLogDebug, "remounting repositories");
132  CatalogContext ctlg_context;
133 
134  if (GetNewRootCatalogContext(&ctlg_context) != kLoadNew
135  && GetNewRootCatalogContext(&ctlg_context) != kLoadUp2Date) {
136  LogCvmfs(kLogCatalog, kLogDebug, "remounting repositories: "
137  "Did not find any valid root catalog to mount");
138  return kLoadFail;
139  }
140 
141  WriteLock();
142 
143  const LoadReturn load_error = LoadCatalogByHash(&ctlg_context);
144 
145  if (load_error == kLoadNew) {
146  inode_t old_inode_gauge = inode_gauge_;
147  DetachAll();
149 
150  CatalogT *new_root = CreateCatalog(ctlg_context.mountpoint(),
151  ctlg_context.hash(), NULL);
152  assert(new_root);
153  bool retval = AttachCatalog(ctlg_context.sqlite_path(), new_root);
154  assert(retval);
155 
156  if (inode_annotation_) {
157  inode_annotation_->IncGeneration(old_inode_gauge);
158  }
159  }
160  CheckInodeWatermark();
161  Unlock();
162 
163  return load_error;
164 }
165 
169 template <class CatalogT>
171  const shash::Any &root_hash)
172 {
173  assert(!root_hash.IsNull());
175  "switching to root hash %s", root_hash.ToString().c_str());
176 
177  WriteLock();
178 
179  CatalogContext ctlg_context(root_hash, PathString("", 0),
181  // we do not need to set revision as LoadCatalogByHash
182  // needs only mountpoint, hash
183 
184  const LoadReturn load_error = LoadCatalogByHash(&ctlg_context);
185 
186  if (load_error == kLoadNew) {
187  inode_t old_inode_gauge = inode_gauge_;
188  DetachAll();
190 
191  CatalogT *new_root =
192  CreateCatalog(PathString("", 0), ctlg_context.hash(), NULL);
193  assert(new_root);
194  bool retval = AttachCatalog(ctlg_context.sqlite_path(), new_root);
195  assert(retval);
196 
197  if (inode_annotation_) {
198  inode_annotation_->IncGeneration(old_inode_gauge);
199  }
200  }
201  CheckInodeWatermark();
202  Unlock();
203 
204  return load_error;
205 }
206 
207 
211 template <class CatalogT>
213  WriteLock();
214  if (catalogs_.empty()) {
215  Unlock();
216  return;
217  }
218 
219  typename CatalogList::const_iterator i;
220  typename CatalogList::const_iterator iend;
221  CatalogList catalogs_to_detach = GetRootCatalog()->GetChildren();
222  for (i = catalogs_to_detach.begin(), iend = catalogs_to_detach.end();
223  i != iend; ++i)
224  {
225  DetachSubtree(*i);
226  }
227 
228  Unlock();
229 }
230 
231 
235 template <class CatalogT>
237  const PathString &mountpoint)
238 {
239  assert(!mountpoint.IsEmpty());
240  CatalogT *catalog = FindCatalog(mountpoint);
241  assert(catalog != NULL);
242  if (catalog->mountpoint() == mountpoint) {
243  catalog = catalog->parent();
244  assert(catalog != NULL);
245  }
246  shash::Any result;
247  uint64_t size;
248  catalog->FindNested(mountpoint, &result, &size);
249  return result;
250 }
251 
252 
262 template <class CatalogT>
264  const LookupOptions options,
265  DirectoryEntry *dirent)
266 {
267  // initialize as non-negative
268  assert(dirent);
269  *dirent = DirectoryEntry();
270 
271  // create a dummy negative directory entry
272  const DirectoryEntry dirent_negative =
274 
275  EnforceSqliteMemLimit();
276  ReadLock();
277 
278  CatalogT *best_fit = FindCatalog(path);
279  assert(best_fit != NULL);
280 
281  perf::Inc(statistics_.n_lookup_path);
282  LogCvmfs(kLogCatalog, kLogDebug, "looking up '%s' in catalog: '%s'",
283  path.c_str(), best_fit->mountpoint().c_str());
284  bool found = best_fit->LookupPath(path, dirent);
285 
286  // Possibly in a nested catalog
287  if (!found && MountSubtree(path, best_fit, false /* is_listable */, NULL)) {
288  LogCvmfs(kLogCatalog, kLogDebug, "looking up '%s' in a nested catalog",
289  path.c_str());
290  StageNestedCatalogAndUnlock(path, best_fit, false /* is_listable */);
291  WriteLock();
292  // Check again to avoid race
293  best_fit = FindCatalog(path);
294  assert(best_fit != NULL);
295  perf::Inc(statistics_.n_lookup_path);
296  found = best_fit->LookupPath(path, dirent);
297 
298  if (!found) {
300  "entry not found, we may have to load nested catalogs");
301 
302  CatalogT *nested_catalog;
303  found =
304  MountSubtree(path, best_fit, false /* is_listable */, &nested_catalog);
305 
306  if (!found) {
308  "failed to load nested catalog for '%s'", path.c_str());
309  goto lookup_path_notfound;
310  }
311 
312  if (nested_catalog != best_fit) {
313  perf::Inc(statistics_.n_lookup_path);
314  found = nested_catalog->LookupPath(path, dirent);
315  if (!found) {
317  "nested catalogs loaded but entry '%s' was still not found",
318  path.c_str());
319  if (dirent != NULL) *dirent = dirent_negative;
320  goto lookup_path_notfound;
321  } else {
322  best_fit = nested_catalog;
323  }
324  } else {
325  LogCvmfs(kLogCatalog, kLogDebug, "no nested catalog fits");
326  if (dirent != NULL) *dirent = dirent_negative;
327  goto lookup_path_notfound;
328  }
329  }
330  assert(found);
331  }
332  // Not in a nested catalog (because no nested cataog fits), ENOENT
333  if (!found) {
334  LogCvmfs(kLogCatalog, kLogDebug, "ENOENT: '%s'", path.c_str());
335  if (dirent != NULL) *dirent = dirent_negative;
336  goto lookup_path_notfound;
337  }
338 
339  LogCvmfs(kLogCatalog, kLogDebug, "found entry '%s' in catalog '%s'",
340  path.c_str(), best_fit->mountpoint().c_str());
341 
342  if ((options & kLookupRawSymlink) == kLookupRawSymlink) {
343  LinkString raw_symlink;
344  bool retval = best_fit->LookupRawSymlink(path, &raw_symlink);
345  assert(retval); // Must be true, we have just found the entry
346  dirent->set_symlink(raw_symlink);
347  }
348 
349  Unlock();
350  return true;
351 
352  lookup_path_notfound:
353  Unlock();
354  // Includes both: ENOENT and not found due to I/O error
355  perf::Inc(statistics_.n_lookup_path_negative);
356  return false;
357 }
358 
359 
372 template <class CatalogT>
374  const PathString &path,
375  PathString *mountpoint,
376  shash::Any *hash,
377  uint64_t *size
378 ) {
379  EnforceSqliteMemLimit();
380  bool result = false;
381  ReadLock();
382 
383  // Look past current path to mount up to intended location
384  PathString catalog_path(path);
385  catalog_path.Append("/.cvmfscatalog", 14);
386 
387  // Find catalog, possibly load nested
388  CatalogT *best_fit = FindCatalog(catalog_path);
389  CatalogT *catalog = best_fit;
390  if (MountSubtree(catalog_path, best_fit, false /* is_listable */, NULL)) {
391  StageNestedCatalogAndUnlock(path, best_fit, false);
392  WriteLock();
393  // Check again to avoid race
394  best_fit = FindCatalog(catalog_path);
395  result =
396  MountSubtree(catalog_path, best_fit, false /* is_listable */, &catalog);
397  // Result is false if an available catalog failed to load (error happened)
398  if (!result) {
399  Unlock();
400  return false;
401  }
402  }
403 
404  // If the found catalog is the Root there is no parent to lookup
405  if (catalog->HasParent()) {
406  result = catalog->parent()->FindNested(catalog->root_prefix(), hash, size);
407  }
408 
409  // Mountpoint now points to the found catalog
410  mountpoint->Assign(catalog->root_prefix());
411 
412  // If the result is false, it means that no nested catalog was found for
413  // this path. As the root catalog does not have a Nested Catalog of
414  // itself, we manually set the values and leave the size as 0.
415  // TODO(nhazekam) Allow for Root Catalog to be returned
416  if (!result) {
417  *hash = GetRootCatalog()->hash();
418  *size = 0;
419  result = true;
420  }
421 
422  Unlock();
423  return result;
424 }
425 
426 
436 template <class CatalogT>
438  const PathString &path,
439  std::vector<PathString> *result_list
440 ) {
441  EnforceSqliteMemLimit();
442  bool result;
443  ReadLock();
444 
445  // Look past current path to mount up to intended location
446  PathString test(path);
447  test.Append("/.cvmfscatalog", 14);
448 
449  // Find catalog, possibly load nested
450  CatalogT *best_fit = FindCatalog(test);
451  CatalogT *catalog = best_fit;
452  // True if there is an available nested catalog
453  if (MountSubtree(test, best_fit, false /* is_listable */, NULL)) {
454  StageNestedCatalogAndUnlock(path, best_fit, false);
455  WriteLock();
456  // Check again to avoid race
457  best_fit = FindCatalog(test);
458  result = MountSubtree(test, best_fit, false /* is_listable */, &catalog);
459  // result is false if an available catalog failed to load
460  if (!result) {
461  Unlock();
462  return false;
463  }
464  }
465 
466  // Build listing
467  CatalogT *cur_parent = catalog->parent();
468  if (cur_parent) {
469  // Walk up parent tree to find base
470  std::vector<catalog::Catalog*> parents;
471  while (cur_parent->HasParent()) {
472  parents.push_back(cur_parent);
473  cur_parent = cur_parent->parent();
474  }
475  parents.push_back(cur_parent);
476  while (!parents.empty()) {
477  // Add to list in order starting at root
478  result_list->push_back(parents.back()->root_prefix());
479  parents.pop_back();
480  }
481  }
482  // Add the current catalog
483  result_list->push_back(catalog->root_prefix());
484 
485  Catalog::NestedCatalogList children = catalog->ListOwnNestedCatalogs();
486 
487  // Add all children nested catalogs
488  for (unsigned i = 0; i < children.size(); i++) {
489  result_list->push_back(children.at(i).mountpoint);
490  }
491 
492  Unlock();
493  return true;
494 }
495 
496 
497 template <class CatalogT>
499  const PathString &path,
500  XattrList *xattrs)
501 {
502  EnforceSqliteMemLimit();
503  bool result;
504  ReadLock();
505 
506  // Find catalog, possibly load nested
507  CatalogT *best_fit = FindCatalog(path);
508  CatalogT *catalog = best_fit;
509  if (MountSubtree(path, best_fit, false /* is_listable */, NULL)) {
510  StageNestedCatalogAndUnlock(path, best_fit, false);
511  WriteLock();
512  // Check again to avoid race
513  best_fit = FindCatalog(path);
514  result = MountSubtree(path, best_fit, false /* is_listable */, &catalog);
515  if (!result) {
516  Unlock();
517  return false;
518  }
519  }
520 
521  perf::Inc(statistics_.n_lookup_xattrs);
522  result = catalog->LookupXattrsPath(path, xattrs);
523 
524  Unlock();
525  return result;
526 }
527 
528 
535 template <class CatalogT>
537  DirectoryEntryList *listing,
538  const bool expand_symlink)
539 {
540  EnforceSqliteMemLimit();
541  bool result;
542  ReadLock();
543 
544  // Find catalog, possibly load nested
545  CatalogT *best_fit = FindCatalog(path);
546  CatalogT *catalog = best_fit;
547  if (MountSubtree(path, best_fit, true /* is_listable */, NULL)) {
548  StageNestedCatalogAndUnlock(path, best_fit, true /* is_listable */);
549  WriteLock();
550  // Check again to avoid race
551  best_fit = FindCatalog(path);
552  result = MountSubtree(path, best_fit, true /* is_listable */, &catalog);
553  if (!result) {
554  Unlock();
555  return false;
556  }
557  }
558 
559  perf::Inc(statistics_.n_listing);
560  result = catalog->ListingPath(path, listing, expand_symlink);
561 
562  Unlock();
563  return result;
564 }
565 
566 
573 template <class CatalogT>
575  StatEntryList *listing)
576 {
577  EnforceSqliteMemLimit();
578  bool result;
579  ReadLock();
580 
581  // Find catalog, possibly load nested
582  CatalogT *best_fit = FindCatalog(path);
583  CatalogT *catalog = best_fit;
584  if (MountSubtree(path, best_fit, true /* is_listable */, NULL)) {
585  StageNestedCatalogAndUnlock(path, best_fit, true /* is_listable */);
586  WriteLock();
587  // Check again to avoid race
588  best_fit = FindCatalog(path);
589  result = MountSubtree(path, best_fit, true /* is_listable */, &catalog);
590  if (!result) {
591  Unlock();
592  return false;
593  }
594  }
595 
596  perf::Inc(statistics_.n_listing);
597  result = catalog->ListingPathStat(path, listing);
598 
599  Unlock();
600  return result;
601 }
602 
603 
611 template <class CatalogT>
613  const PathString &path,
614  const shash::Algorithms interpret_hashes_as,
615  FileChunkList *chunks)
616 {
617  EnforceSqliteMemLimit();
618  bool result;
619  ReadLock();
620 
621  // Find catalog, possibly load nested
622  CatalogT *best_fit = FindCatalog(path);
623  CatalogT *catalog = best_fit;
624  if (MountSubtree(path, best_fit, false /* is_listable */, NULL)) {
625  StageNestedCatalogAndUnlock(path, best_fit, false);
626  WriteLock();
627  // Check again to avoid race
628  best_fit = FindCatalog(path);
629  result = MountSubtree(path, best_fit, false /* is_listable */, &catalog);
630  if (!result) {
631  Unlock();
632  return false;
633  }
634  }
635 
636  result = catalog->ListPathChunks(path, interpret_hashes_as, chunks);
637 
638  Unlock();
639  return result;
640 }
641 
642 template <class CatalogT>
644  const PathString &path,
645  std::string *subcatalog_path,
646  shash::Any *hash
647  )
648 {
649  EnforceSqliteMemLimit();
650  bool result;
651  ReadLock();
652 
653  // Look past current path to mount up to intended location
654  PathString catalog_path(path);
655  catalog_path.Append("/.cvmfscatalog", 14);
656 
657  // Find catalog, possibly load nested
658  CatalogT *best_fit = FindCatalog(catalog_path);
659  CatalogT *catalog = best_fit;
660  if (MountSubtree(catalog_path, best_fit, false /* is_listable */, NULL)) {
661  StageNestedCatalogAndUnlock(path, best_fit, false /* is_listable */);
662  WriteLock();
663  // Check again to avoid race
664  best_fit = FindCatalog(catalog_path);
665  result =
666  MountSubtree(catalog_path, best_fit, false /* is_listable */, &catalog);
667  // Result is false if an available catalog failed to load (error happened)
668  if (!result) {
669  Unlock();
670  *subcatalog_path = "error: failed to load catalog!";
671  *hash = shash::Any();
672  return catalog::Counters();
673  }
674  }
675 
676  *hash = catalog->hash();
677  *subcatalog_path = catalog->mountpoint().ToString();
678  catalog::Counters counters = catalog->GetCounters();
679  Unlock();
680  return counters;
681 }
682 
683 
684 template <class CatalogT>
686  ReadLock();
687  const uint64_t revision = GetRevisionNoLock();
688  Unlock();
689 
690  return revision;
691 }
692 
698 template <class CatalogT>
700  return revision_cache_;
701 }
702 
703 template <class CatalogT>
705  ReadLock();
706  const uint64_t timestamp = GetTimestampNoLock();
707  Unlock();
708 
709  return timestamp;
710 }
711 
717 template <class CatalogT>
719  return timestamp_cache_;
720 }
721 
722 template <class CatalogT>
723 bool AbstractCatalogManager<CatalogT>::GetVOMSAuthz(std::string *authz) const {
724  ReadLock();
725  const bool has_authz = has_authz_cache_;
726  if (has_authz && authz)
727  *authz = authz_cache_;
728  Unlock();
729  return has_authz;
730 }
731 
732 
733 template <class CatalogT>
735  ReadLock();
736  const bool result = GetRootCatalog()->HasExplicitTTL();
737  Unlock();
738  return result;
739 }
740 
741 
742 template <class CatalogT>
744  ReadLock();
745  const uint64_t ttl = GetRootCatalog()->GetTTL();
746  Unlock();
747  return ttl;
748 }
749 
750 
751 template <class CatalogT>
753  ReadLock();
754  int result = catalogs_.size();
755  Unlock();
756  return result;
757 }
758 
759 
763 template <class CatalogT>
765  ReadLock();
766  string output = PrintHierarchyRecursively(GetRootCatalog(), 0);
767  Unlock();
768  return output;
769 }
770 
771 
775 template <class CatalogT>
777  InodeRange result;
778  result.offset = inode_gauge_;
779  result.size = size;
780 
781  inode_gauge_ += size;
782  LogCvmfs(kLogCatalog, kLogDebug, "allocating inodes from %lu to %lu.",
783  result.offset + 1, inode_gauge_);
784 
785  return result;
786 }
787 
788 
794 template <class CatalogT>
796  // TODO(jblomer) currently inodes are only released on remount
797 }
798 
799 
806 template <class CatalogT>
808  const PathString &path) const {
809  assert(catalogs_.size() > 0);
810 
811  // Start at the root catalog and successively go down the catalog tree
812  CatalogT *best_fit = GetRootCatalog();
813  CatalogT *next_fit = NULL;
814  while (best_fit->mountpoint() != path) {
815  next_fit = best_fit->FindSubtree(path);
816  if (next_fit == NULL)
817  break;
818  best_fit = next_fit;
819  }
820 
821  return best_fit;
822 }
823 
824 
831 template <class CatalogT>
833  CatalogT **attached_catalog) const
834 {
835  if (catalogs_.size() == 0)
836  return false;
837 
838  CatalogT *best_fit = FindCatalog(root_path);
839  if (best_fit->mountpoint() != root_path)
840  return false;
841 
842  if (attached_catalog != NULL) *attached_catalog = best_fit;
843  return true;
844 }
845 
846 
847 template <class CatalogT>
849  const PathString &path,
850  const CatalogT *parent,
851  bool is_listable)
852 {
853  assert(parent);
854  const unsigned path_len = path.GetLength();
855 
856  perf::Inc(statistics_.n_nested_listing);
857  typedef typename CatalogT::NestedCatalogList NestedCatalogList;
858  const NestedCatalogList& nested_catalogs = parent->ListNestedCatalogs();
859 
860  for (typename NestedCatalogList::const_iterator i = nested_catalogs.begin(),
861  iEnd = nested_catalogs.end(); i != iEnd; ++i)
862  {
863  if (!path.StartsWith(i->mountpoint))
864  continue;
865 
866  // in this case the path doesn't start with
867  // the mountpoint in a file path sense
868  // (e.g. path is /a/bc and mountpoint is /a/b), and will be ignored
869  const unsigned mountpoint_len = i->mountpoint.GetLength();
870  if (path_len > mountpoint_len && path.GetChars()[mountpoint_len] != '/')
871  continue;
872 
873  // Found a nested catalog transition point
874  if (!is_listable && (path_len == mountpoint_len))
875  break;
876 
877  Unlock();
878  LogCvmfs(kLogCatalog, kLogDebug, "staging nested catalog at %s (%s)",
879  i->mountpoint.c_str(), i->hash.ToString().c_str());
880  StageNestedCatalogByHash(i->hash, i->mountpoint);
881  return;
882  }
883  Unlock();
884 }
885 
895 template <class CatalogT>
897  const PathString &path,
898  const CatalogT *entry_point,
899  bool is_listable,
900  CatalogT **leaf_catalog)
901 {
902  bool result = true;
903  CatalogT *parent = (entry_point == NULL) ?
904  GetRootCatalog() : const_cast<CatalogT *>(entry_point);
905  assert(path.StartsWith(parent->mountpoint()));
906 
907  unsigned path_len = path.GetLength();
908 
909  // Try to find path as a super string of nested catalog mount points
910  perf::Inc(statistics_.n_nested_listing);
911  typedef typename CatalogT::NestedCatalogList NestedCatalogList;
912  const NestedCatalogList& nested_catalogs =
913  parent->ListNestedCatalogs();
914  for (typename NestedCatalogList::const_iterator i = nested_catalogs.begin(),
915  iEnd = nested_catalogs.end(); i != iEnd; ++i)
916  {
917  // Next nesting level
918  if (path.StartsWith(i->mountpoint)) {
919  // in this case the path doesn't start with
920  // the mountpoint in a file path sense
921  // (e.g. path is /a/bc and mountpoint is /a/b), and will be ignored
922  unsigned mountpoint_len = i->mountpoint.GetLength();
923  if (path_len > mountpoint_len && path.GetChars()[mountpoint_len] != '/')
924  continue;
925 
926  // Found a nested catalog transition point
927  if (!is_listable && (path_len == mountpoint_len))
928  break;
929 
930  if (leaf_catalog == NULL)
931  return true;
932  CatalogT *new_nested;
933  LogCvmfs(kLogCatalog, kLogDebug, "load nested catalog at %s",
934  i->mountpoint.c_str());
935  // prevent endless recursion with corrupted catalogs
936  // (due to reloading root)
937  if (i->hash.IsNull())
938  return false;
939  new_nested = MountCatalog(i->mountpoint, i->hash, parent);
940  if (!new_nested)
941  return false;
942 
943  result = MountSubtree(path, new_nested, is_listable, &parent);
944  break;
945  }
946  }
947 
948  if (leaf_catalog == NULL)
949  return false;
950  *leaf_catalog = parent;
951  return result;
952 }
953 
954 
959 template <class CatalogT>
961  const PathString &mountpoint,
962  const shash::Any &hash,
963  CatalogT *parent_catalog)
964 {
965  CatalogT *attached_catalog = NULL;
966  if (IsAttached(mountpoint, &attached_catalog)) {
967  return attached_catalog;
968  }
969 
970  CatalogContext ctlg_context(hash, mountpoint, kCtlgLocationMounted);
971 
972  if (ctlg_context.IsRootCatalog() && hash.IsNull()) {
973  if (GetNewRootCatalogContext(&ctlg_context) == kLoadFail) {
975  "failed to retrieve valid root catalog '%s'",
976  mountpoint.c_str());
977  return NULL;
978  }
979  }
980 
981  const LoadReturn retval = LoadCatalogByHash(&ctlg_context);
982  if ((retval == kLoadFail) || (retval == kLoadNoSpace)) {
983  LogCvmfs(kLogCatalog, kLogDebug, "failed to load catalog '%s' (%d - %s)",
984  mountpoint.c_str(), retval, Code2Ascii(retval));
985  return NULL;
986  }
987 
988  attached_catalog = CreateCatalog(ctlg_context.mountpoint(),
989  ctlg_context.hash(),
990  parent_catalog);
991 
992  // Attach loaded catalog
993  if (!AttachCatalog(ctlg_context.sqlite_path(), attached_catalog)) {
994  LogCvmfs(kLogCatalog, kLogDebug, "failed to attach catalog '%s'",
995  mountpoint.c_str());
996  UnloadCatalog(attached_catalog);
997  return NULL;
998  }
999 
1000  if ((catalog_watermark_ > 0) && (catalogs_.size() >= catalog_watermark_)) {
1001  DetachSiblings(mountpoint);
1002  }
1003 
1004  return attached_catalog;
1005 }
1006 
1007 
1012 template <class CatalogT>
1014  const PathString &mountpoint,
1015  const shash::Any &hash)
1016 {
1017  assert(!hash.IsNull());
1018  CatalogContext ctlg_context(hash, mountpoint, kCtlgNoLocationNeeded);
1019 
1020  const LoadReturn load_ret = LoadCatalogByHash(&ctlg_context);
1021 
1022  if (load_ret != kLoadNew) {
1023  return NULL;
1024  }
1025 
1026  CatalogT *catalog = CatalogT::AttachFreely(mountpoint.ToString(),
1027  ctlg_context.sqlite_path(),
1028  ctlg_context.hash());
1029  catalog->TakeDatabaseFileOwnership();
1030  return catalog;
1031 }
1032 
1033 
1040 template <class CatalogT>
1042  CatalogT *new_catalog)
1043 {
1044  LogCvmfs(kLogCatalog, kLogDebug, "attaching catalog file %s",
1045  db_path.c_str());
1046 
1047  // Initialize the new catalog
1048  if (!new_catalog->OpenDatabase(db_path)) {
1049  LogCvmfs(kLogCatalog, kLogDebug, "initialization of catalog %s failed",
1050  db_path.c_str());
1051  return false;
1052  }
1053 
1054  // Determine the inode offset of this catalog
1055  uint64_t inode_chunk_size = new_catalog->max_row_id();
1056  InodeRange range = AcquireInodes(inode_chunk_size);
1057  new_catalog->set_inode_range(range);
1058  new_catalog->SetInodeAnnotation(inode_annotation_);
1059  new_catalog->SetOwnerMaps(&uid_map_, &gid_map_);
1060 
1061  // Add catalog to the manager
1062  if (!new_catalog->IsInitialized()) {
1064  "catalog initialization failed (obscure data)");
1065  inode_gauge_ -= inode_chunk_size;
1066  return false;
1067  }
1068  CheckInodeWatermark();
1069 
1070  // The revision of the catalog tree is given by the root catalog revision
1071  if (catalogs_.empty()) {
1072  revision_cache_ = new_catalog->GetRevision();
1073  timestamp_cache_ = new_catalog->GetLastModified();
1074  statistics_.catalog_revision->Set(revision_cache_);
1075  has_authz_cache_ = new_catalog->GetVOMSAuthz(&authz_cache_);
1076  volatile_flag_ = new_catalog->volatile_flag();
1077  }
1078 
1079  catalogs_.push_back(new_catalog);
1080  ActivateCatalog(new_catalog);
1081  return true;
1082 }
1083 
1084 
1093 template <class CatalogT>
1095  if (catalog->HasParent())
1096  catalog->parent()->RemoveChild(catalog);
1097 
1098  ReleaseInodes(catalog->inode_range());
1099  UnloadCatalog(catalog);
1100 
1101  // Delete catalog from internal lists
1102  typename CatalogList::iterator i;
1103  typename CatalogList::const_iterator iend;
1104  for (i = catalogs_.begin(), iend = catalogs_.end(); i != iend; ++i) {
1105  if (*i == catalog) {
1106  catalogs_.erase(i);
1107  delete catalog;
1108  return;
1109  }
1110  }
1111 
1112  assert(false);
1113 }
1114 
1115 
1122 template <class CatalogT>
1124  // Detach all child catalogs recursively
1125  typename CatalogList::const_iterator i;
1126  typename CatalogList::const_iterator iend;
1127  CatalogList catalogs_to_detach = catalog->GetChildren();
1128  for (i = catalogs_to_detach.begin(), iend = catalogs_to_detach.end();
1129  i != iend; ++i)
1130  {
1131  DetachSubtree(*i);
1132  }
1133 
1134  DetachCatalog(catalog);
1135 }
1136 
1137 
1142 template <class CatalogT>
1144  const PathString &current_tree)
1145 {
1146  bool again;
1147  do {
1148  again = false;
1149  unsigned N = catalogs_.size();
1150  for (unsigned i = 0; i < N; ++i) {
1151  if (!HasPrefix(current_tree.ToString(),
1152  catalogs_[i]->mountpoint().ToString(),
1153  false /* ignore_case */))
1154  {
1155  DetachSubtree(catalogs_[i]);
1156  again = true;
1157  break;
1158  }
1159  }
1160  } while (again);
1161  perf::Inc(statistics_.n_detach_siblings);
1162 }
1163 
1164 
1168 template <class CatalogT>
1170  const CatalogT *catalog,
1171  const int level) const
1172 {
1173  string output;
1174 
1175  // Indent according to level
1176  for (int i = 0; i < level; ++i)
1177  output += " ";
1178 
1179  output += "-> " + string(catalog->mountpoint().GetChars(),
1180  catalog->mountpoint().GetLength())
1181  + "\n";
1182 
1183  CatalogList children = catalog->GetChildren();
1184  typename CatalogList::const_iterator i = children.begin();
1185  typename CatalogList::const_iterator iend = children.end();
1186  for (; i != iend; ++i) {
1187  output += PrintHierarchyRecursively(*i, level + 1);
1188  }
1189 
1190  return output;
1191 }
1192 
1193 
1194 template <class CatalogT>
1196  const CatalogT *catalog) const
1197 {
1198  string result = catalog->PrintMemStatistics() + "\n";
1199 
1200  CatalogList children = catalog->GetChildren();
1201  typename CatalogList::const_iterator i = children.begin();
1202  typename CatalogList::const_iterator iend = children.end();
1203  for (; i != iend; ++i) {
1204  result += PrintMemStatsRecursively(*i);
1205  }
1206  return result;
1207 }
1208 
1209 
1213 template <class CatalogT>
1215  string result;
1216  ReadLock();
1217  result = PrintMemStatsRecursively(GetRootCatalog());
1218  Unlock();
1219  return result;
1220 }
1221 
1222 
1223 template <class CatalogT>
1225  char *mem_enforced =
1226  static_cast<char *>(pthread_getspecific(pkey_sqlitemem_));
1227  if (mem_enforced == NULL) {
1228  sqlite3_soft_heap_limit(kSqliteMemPerThread);
1229  pthread_setspecific(pkey_sqlitemem_, this);
1230  }
1231 }
1232 
1233 } // namespace catalog
1234 
1235 
1236 
1237 
1238 #endif // CVMFS_CATALOG_MGR_IMPL_H_
1239 //NOLINTEND
shash::Any hash() const
Definition: catalog_mgr.h:125
void DetachSubtree(CatalogT *catalog)
bool GetVOMSAuthz(std::string *authz) const
catalog::Counters LookupCounters(const PathString &path, std::string *subcatalog_path, shash::Any *hash)
bool IsNull() const
Definition: hash.h:383
CatalogT * LoadFreeCatalog(const PathString &mountpoint, const shash::Any &hash)
void StageNestedCatalogAndUnlock(const PathString &path, const CatalogT *parent, bool is_listable)
bool MountSubtree(const PathString &path, const CatalogT *entry_point, bool can_listing, CatalogT **leaf_catalog)
void DetachSiblings(const PathString &current_tree)
InodeAnnotation * inode_annotation_
Definition: catalog_mgr.h:452
std::string ToString(const bool with_suffix=false) const
Definition: hash.h:249
void Assign(const char *chars, const unsigned length)
Definition: shortstring.h:61
std::string PrintHierarchyRecursively(const CatalogT *catalog, const int level) const
unsigned LookupOptions
Definition: catalog_mgr.h:42
void set_symlink(const LinkString &symlink)
perf::Statistics * statistics_
Definition: repository.h:139
void DetachCatalog(CatalogT *catalog)
bool IsAttached(const PathString &root_path, CatalogT **attached_catalog) const
uint64_t inode_t
bool ListingStat(const PathString &path, StatEntryList *listing)
assert((mem||(size==0))&&"Out Of Memory")
bool LookupPath(const PathString &path, const LookupOptions options, DirectoryEntry *entry)
IntegerMap< uint64_t > OwnerMap
Definition: catalog.h:41
std::string sqlite_path() const
Definition: catalog_mgr.h:127
uint64_t size
Definition: catalog.h:51
Algorithms
Definition: hash.h:41
bool Listing(const PathString &path, DirectoryEntryList *listing, const bool expand_symlink)
std::vector< DirectoryEntry > DirectoryEntryList
bool AttachCatalog(const std::string &db_path, CatalogT *new_catalog)
LoadReturn ChangeRoot(const shash::Any &root_hash)
const unsigned kSqliteMemPerThread
Definition: catalog_mgr.h:34
shash::Any GetNestedCatalogHash(const PathString &mountpoint)
void Append(const char *chars, const unsigned length)
Definition: shortstring.h:80
void Inc(class Counter *counter)
Definition: statistics.h:50
bool HasPrefix(const string &str, const string &prefix, const bool ignore_case)
Definition: string.cc:285
std::vector< CatalogT * > CatalogList
Definition: catalog_mgr.h:237
const unsigned kLookupRawSymlink
Definition: catalog_mgr.h:44
bool ListCatalogSkein(const PathString &path, std::vector< PathString > *result_list)
InodeRange AcquireInodes(uint64_t size)
pthread_rwlock_t * rwlock_
Definition: catalog_mgr.h:453
std::string ToString() const
Definition: shortstring.h:141
std::vector< NestedCatalog > NestedCatalogList
Definition: catalog.h:208
bool IsEmpty() const
Definition: shortstring.h:137
bool ListFileChunks(const PathString &path, const shash::Algorithms interpret_hashes_as, FileChunkList *chunks)
ShortString< kDefaultMaxPath, 0 > PathString
Definition: shortstring.h:217
const char * Code2Ascii(const LoadReturn error)
Definition: catalog_mgr.h:164
bool LookupXattrs(const PathString &path, XattrList *xattrs)
PathString mountpoint() const
Definition: catalog_mgr.h:126
bool LookupNested(const PathString &path, PathString *mountpoint, shash::Any *hash, uint64_t *size)
CatalogT * MountCatalog(const PathString &mountpoint, const shash::Any &hash, CatalogT *parent_catalog)
bool StartsWith(const ShortString &other) const
Definition: shortstring.h:189
void SetCatalogWatermark(unsigned limit)
void SetOwnerMaps(const OwnerMap &uid_map, const OwnerMap &gid_map)
unsigned GetLength() const
Definition: shortstring.h:131
void ReleaseInodes(const InodeRange chunk)
CatalogT * FindCatalog(const PathString &path) const
uint64_t offset
Definition: catalog.h:50
const char * c_str() const
Definition: shortstring.h:145
const char * GetChars() const
Definition: shortstring.h:123
std::string PrintHierarchy() const
std::string PrintMemStatsRecursively(const CatalogT *catalog) const
std::string PrintAllMemStatistics() const
static void size_t size
Definition: smalloc.h:54
void SetInodeAnnotation(InodeAnnotation *new_annotation)
CVMFS_EXPORT void LogCvmfs(const LogSource source, const int mask, const char *format,...)
Definition: logging.cc:528