8 #ifndef CVMFS_CATALOG_MGR_IMPL_H_
9 #define CVMFS_CATALOG_MGR_IMPL_H_
11 #ifndef __STDC_FORMAT_MACROS
12 #define __STDC_FORMAT_MACROS
30 template <
class CatalogT>
44 reinterpret_cast<pthread_rwlock_t *
>(smalloc(
sizeof(pthread_rwlock_t)));
45 int retval = pthread_rwlock_init(
rwlock_, NULL);
51 template <
class CatalogT>
54 pthread_key_delete(pkey_sqlitemem_);
55 pthread_rwlock_destroy(rwlock_);
59 template <
class CatalogT>
63 assert(catalogs_.empty() || (new_annotation == inode_annotation_));
64 inode_annotation_ = new_annotation;
67 template <
class CatalogT>
75 template <
class CatalogT>
77 catalog_watermark_ = limit;
80 template <
class CatalogT>
82 if (inode_watermark_status_ > 0)
85 uint64_t highest_inode = inode_gauge_;
86 if (inode_annotation_)
87 highest_inode += inode_annotation_->GetGeneration();
88 uint64_t uint32_border = 1;
89 uint32_border = uint32_border << 32;
90 if (highest_inode >= uint32_border) {
92 inode_watermark_status_++;
101 template <
class CatalogT>
121 template <
class CatalogT>
124 "dryrun remounting repositories");
126 return GetNewRootCatalogContext(&ctlg_context);
129 template <
class CatalogT>
134 if (GetNewRootCatalogContext(&ctlg_context) !=
kLoadNew
135 && GetNewRootCatalogContext(&ctlg_context) !=
kLoadUp2Date) {
137 "Did not find any valid root catalog to mount");
143 const LoadReturn load_error = LoadCatalogByHash(&ctlg_context);
146 inode_t old_inode_gauge = inode_gauge_;
150 CatalogT *new_root = CreateCatalog(ctlg_context.
mountpoint(),
151 ctlg_context.
hash(), NULL);
153 bool retval = AttachCatalog(ctlg_context.
sqlite_path(), new_root);
156 if (inode_annotation_) {
157 inode_annotation_->IncGeneration(old_inode_gauge);
160 CheckInodeWatermark();
169 template <
class CatalogT>
175 "switching to root hash %s", root_hash.
ToString().c_str());
184 const LoadReturn load_error = LoadCatalogByHash(&ctlg_context);
187 inode_t old_inode_gauge = inode_gauge_;
194 bool retval = AttachCatalog(ctlg_context.
sqlite_path(), new_root);
197 if (inode_annotation_) {
198 inode_annotation_->IncGeneration(old_inode_gauge);
201 CheckInodeWatermark();
211 template <
class CatalogT>
214 if (catalogs_.empty()) {
219 typename CatalogList::const_iterator i;
220 typename CatalogList::const_iterator iend;
221 CatalogList catalogs_to_detach = GetRootCatalog()->GetChildren();
222 for (i = catalogs_to_detach.begin(), iend = catalogs_to_detach.end();
235 template <
class CatalogT>
240 CatalogT *catalog = FindCatalog(mountpoint);
242 if (catalog->mountpoint() == mountpoint) {
243 catalog = catalog->parent();
248 catalog->FindNested(mountpoint, &result, &size);
262 template <
class CatalogT>
275 EnforceSqliteMemLimit();
278 CatalogT *best_fit = FindCatalog(path);
283 path.
c_str(), best_fit->mountpoint().c_str());
284 bool found = best_fit->LookupPath(path, dirent);
287 if (!found && MountSubtree(path, best_fit,
false , NULL)) {
290 StageNestedCatalogAndUnlock(path, best_fit,
false );
293 best_fit = FindCatalog(path);
296 found = best_fit->LookupPath(path, dirent);
300 "entry not found, we may have to load nested catalogs");
302 CatalogT *nested_catalog;
304 MountSubtree(path, best_fit,
false , &nested_catalog);
308 "failed to load nested catalog for '%s'", path.
c_str());
309 goto lookup_path_notfound;
312 if (nested_catalog != best_fit) {
314 found = nested_catalog->LookupPath(path, dirent);
317 "nested catalogs loaded but entry '%s' was still not found",
319 if (dirent != NULL) *dirent = dirent_negative;
320 goto lookup_path_notfound;
322 best_fit = nested_catalog;
326 if (dirent != NULL) *dirent = dirent_negative;
327 goto lookup_path_notfound;
335 if (dirent != NULL) *dirent = dirent_negative;
336 goto lookup_path_notfound;
340 path.
c_str(), best_fit->mountpoint().c_str());
344 bool retval = best_fit->LookupRawSymlink(path, &raw_symlink);
352 lookup_path_notfound:
372 template <
class CatalogT>
379 EnforceSqliteMemLimit();
385 catalog_path.
Append(
"/.cvmfscatalog", 14);
388 CatalogT *best_fit = FindCatalog(catalog_path);
389 CatalogT *catalog = best_fit;
390 if (MountSubtree(catalog_path, best_fit,
false , NULL)) {
391 StageNestedCatalogAndUnlock(path, best_fit,
false);
394 best_fit = FindCatalog(catalog_path);
396 MountSubtree(catalog_path, best_fit,
false , &catalog);
405 if (catalog->HasParent()) {
406 result = catalog->parent()->FindNested(catalog->root_prefix(), hash,
size);
410 mountpoint->
Assign(catalog->root_prefix());
417 *hash = GetRootCatalog()->hash();
436 template <
class CatalogT>
439 std::vector<PathString> *result_list
441 EnforceSqliteMemLimit();
447 test.
Append(
"/.cvmfscatalog", 14);
450 CatalogT *best_fit = FindCatalog(test);
451 CatalogT *catalog = best_fit;
453 if (MountSubtree(test, best_fit,
false , NULL)) {
454 StageNestedCatalogAndUnlock(path, best_fit,
false);
457 best_fit = FindCatalog(test);
458 result = MountSubtree(test, best_fit,
false , &catalog);
467 CatalogT *cur_parent = catalog->parent();
470 std::vector<catalog::Catalog*> parents;
471 while (cur_parent->HasParent()) {
472 parents.push_back(cur_parent);
473 cur_parent = cur_parent->parent();
475 parents.push_back(cur_parent);
476 while (!parents.empty()) {
478 result_list->push_back(parents.back()->root_prefix());
483 result_list->push_back(catalog->root_prefix());
488 for (
unsigned i = 0; i < children.size(); i++) {
489 result_list->push_back(children.at(i).mountpoint);
497 template <
class CatalogT>
502 EnforceSqliteMemLimit();
507 CatalogT *best_fit = FindCatalog(path);
508 CatalogT *catalog = best_fit;
509 if (MountSubtree(path, best_fit,
false , NULL)) {
510 StageNestedCatalogAndUnlock(path, best_fit,
false);
513 best_fit = FindCatalog(path);
514 result = MountSubtree(path, best_fit,
false , &catalog);
522 result = catalog->LookupXattrsPath(path, xattrs);
535 template <
class CatalogT>
538 const bool expand_symlink)
540 EnforceSqliteMemLimit();
545 CatalogT *best_fit = FindCatalog(path);
546 CatalogT *catalog = best_fit;
547 if (MountSubtree(path, best_fit,
true , NULL)) {
548 StageNestedCatalogAndUnlock(path, best_fit,
true );
551 best_fit = FindCatalog(path);
552 result = MountSubtree(path, best_fit,
true , &catalog);
560 result = catalog->ListingPath(path, listing, expand_symlink);
573 template <
class CatalogT>
577 EnforceSqliteMemLimit();
582 CatalogT *best_fit = FindCatalog(path);
583 CatalogT *catalog = best_fit;
584 if (MountSubtree(path, best_fit,
true , NULL)) {
585 StageNestedCatalogAndUnlock(path, best_fit,
true );
588 best_fit = FindCatalog(path);
589 result = MountSubtree(path, best_fit,
true , &catalog);
597 result = catalog->ListingPathStat(path, listing);
611 template <
class CatalogT>
617 EnforceSqliteMemLimit();
622 CatalogT *best_fit = FindCatalog(path);
623 CatalogT *catalog = best_fit;
624 if (MountSubtree(path, best_fit,
false , NULL)) {
625 StageNestedCatalogAndUnlock(path, best_fit,
false);
628 best_fit = FindCatalog(path);
629 result = MountSubtree(path, best_fit,
false , &catalog);
636 result = catalog->ListPathChunks(path, interpret_hashes_as, chunks);
642 template <
class CatalogT>
645 std::string *subcatalog_path,
649 EnforceSqliteMemLimit();
655 catalog_path.
Append(
"/.cvmfscatalog", 14);
658 CatalogT *best_fit = FindCatalog(catalog_path);
659 CatalogT *catalog = best_fit;
660 if (MountSubtree(catalog_path, best_fit,
false , NULL)) {
661 StageNestedCatalogAndUnlock(path, best_fit,
false );
664 best_fit = FindCatalog(catalog_path);
666 MountSubtree(catalog_path, best_fit,
false , &catalog);
670 *subcatalog_path =
"error: failed to load catalog!";
676 *hash = catalog->hash();
677 *subcatalog_path = catalog->mountpoint().
ToString();
684 template <
class CatalogT>
687 const uint64_t revision = GetRevisionNoLock();
698 template <
class CatalogT>
700 return revision_cache_;
703 template <
class CatalogT>
706 const uint64_t timestamp = GetTimestampNoLock();
717 template <
class CatalogT>
719 return timestamp_cache_;
722 template <
class CatalogT>
725 const bool has_authz = has_authz_cache_;
726 if (has_authz && authz)
727 *authz = authz_cache_;
733 template <
class CatalogT>
736 const bool result = GetRootCatalog()->HasExplicitTTL();
742 template <
class CatalogT>
745 const uint64_t ttl = GetRootCatalog()->GetTTL();
751 template <
class CatalogT>
754 int result = catalogs_.size();
763 template <
class CatalogT>
766 string output = PrintHierarchyRecursively(GetRootCatalog(), 0);
775 template <
class CatalogT>
778 result.
offset = inode_gauge_;
781 inode_gauge_ +=
size;
783 result.
offset + 1, inode_gauge_);
794 template <
class CatalogT>
806 template <
class CatalogT>
809 assert(catalogs_.size() > 0);
812 CatalogT *best_fit = GetRootCatalog();
813 CatalogT *next_fit = NULL;
814 while (best_fit->mountpoint() != path) {
815 next_fit = best_fit->FindSubtree(path);
816 if (next_fit == NULL)
831 template <
class CatalogT>
833 CatalogT **attached_catalog)
const
835 if (catalogs_.size() == 0)
838 CatalogT *best_fit = FindCatalog(root_path);
839 if (best_fit->mountpoint() != root_path)
842 if (attached_catalog != NULL) *attached_catalog = best_fit;
847 template <
class CatalogT>
850 const CatalogT *parent,
854 const unsigned path_len = path.
GetLength();
857 typedef typename CatalogT::NestedCatalogList NestedCatalogList;
858 const NestedCatalogList& nested_catalogs = parent->ListNestedCatalogs();
860 for (
typename NestedCatalogList::const_iterator i = nested_catalogs.begin(),
861 iEnd = nested_catalogs.end(); i != iEnd; ++i)
869 const unsigned mountpoint_len = i->mountpoint.GetLength();
870 if (path_len > mountpoint_len && path.
GetChars()[mountpoint_len] !=
'/')
874 if (!is_listable && (path_len == mountpoint_len))
879 i->mountpoint.c_str(), i->hash.ToString().c_str());
880 StageNestedCatalogByHash(i->hash, i->mountpoint);
895 template <
class CatalogT>
898 const CatalogT *entry_point,
900 CatalogT **leaf_catalog)
903 CatalogT *parent = (entry_point == NULL) ?
904 GetRootCatalog() :
const_cast<CatalogT *
>(entry_point);
911 typedef typename CatalogT::NestedCatalogList NestedCatalogList;
912 const NestedCatalogList& nested_catalogs =
913 parent->ListNestedCatalogs();
914 for (
typename NestedCatalogList::const_iterator i = nested_catalogs.begin(),
915 iEnd = nested_catalogs.end(); i != iEnd; ++i)
922 unsigned mountpoint_len = i->mountpoint.GetLength();
923 if (path_len > mountpoint_len && path.
GetChars()[mountpoint_len] !=
'/')
927 if (!is_listable && (path_len == mountpoint_len))
930 if (leaf_catalog == NULL)
932 CatalogT *new_nested;
934 i->mountpoint.c_str());
937 if (i->hash.IsNull())
939 new_nested = MountCatalog(i->mountpoint, i->hash, parent);
943 result = MountSubtree(path, new_nested, is_listable, &parent);
948 if (leaf_catalog == NULL)
950 *leaf_catalog = parent;
959 template <
class CatalogT>
963 CatalogT *parent_catalog)
965 CatalogT *attached_catalog = NULL;
966 if (IsAttached(mountpoint, &attached_catalog)) {
967 return attached_catalog;
973 if (GetNewRootCatalogContext(&ctlg_context) ==
kLoadFail) {
975 "failed to retrieve valid root catalog '%s'",
981 const LoadReturn retval = LoadCatalogByHash(&ctlg_context);
988 attached_catalog = CreateCatalog(ctlg_context.
mountpoint(),
993 if (!AttachCatalog(ctlg_context.
sqlite_path(), attached_catalog)) {
996 UnloadCatalog(attached_catalog);
1000 if ((catalog_watermark_ > 0) && (catalogs_.size() >= catalog_watermark_)) {
1001 DetachSiblings(mountpoint);
1004 return attached_catalog;
1012 template <
class CatalogT>
1020 const LoadReturn load_ret = LoadCatalogByHash(&ctlg_context);
1026 CatalogT *catalog = CatalogT::AttachFreely(mountpoint.
ToString(),
1027 ctlg_context.sqlite_path(),
1028 ctlg_context.hash());
1029 catalog->TakeDatabaseFileOwnership();
1040 template <
class CatalogT>
1042 CatalogT *new_catalog)
1048 if (!new_catalog->OpenDatabase(db_path)) {
1055 uint64_t inode_chunk_size = new_catalog->max_row_id();
1056 InodeRange range = AcquireInodes(inode_chunk_size);
1057 new_catalog->set_inode_range(range);
1058 new_catalog->SetInodeAnnotation(inode_annotation_);
1059 new_catalog->SetOwnerMaps(&uid_map_, &gid_map_);
1062 if (!new_catalog->IsInitialized()) {
1064 "catalog initialization failed (obscure data)");
1065 inode_gauge_ -= inode_chunk_size;
1068 CheckInodeWatermark();
1071 if (catalogs_.empty()) {
1072 revision_cache_ = new_catalog->GetRevision();
1073 timestamp_cache_ = new_catalog->GetLastModified();
1074 statistics_.catalog_revision->Set(revision_cache_);
1075 has_authz_cache_ = new_catalog->GetVOMSAuthz(&authz_cache_);
1076 volatile_flag_ = new_catalog->volatile_flag();
1079 catalogs_.push_back(new_catalog);
1080 ActivateCatalog(new_catalog);
1093 template <
class CatalogT>
1095 if (catalog->HasParent())
1096 catalog->parent()->RemoveChild(catalog);
1098 ReleaseInodes(catalog->inode_range());
1099 UnloadCatalog(catalog);
1102 typename CatalogList::iterator i;
1103 typename CatalogList::const_iterator iend;
1104 for (i = catalogs_.begin(), iend = catalogs_.end(); i != iend; ++i) {
1105 if (*i == catalog) {
1122 template <
class CatalogT>
1125 typename CatalogList::const_iterator i;
1126 typename CatalogList::const_iterator iend;
1127 CatalogList catalogs_to_detach = catalog->GetChildren();
1128 for (i = catalogs_to_detach.begin(), iend = catalogs_to_detach.end();
1134 DetachCatalog(catalog);
1142 template <
class CatalogT>
1149 unsigned N = catalogs_.size();
1150 for (
unsigned i = 0; i < N; ++i) {
1152 catalogs_[i]->mountpoint().ToString(),
1155 DetachSubtree(catalogs_[i]);
1168 template <
class CatalogT>
1170 const CatalogT *catalog,
1171 const int level)
const
1176 for (
int i = 0; i < level; ++i)
1179 output +=
"-> " + string(catalog->mountpoint().GetChars(),
1180 catalog->mountpoint().GetLength())
1184 typename CatalogList::const_iterator i = children.begin();
1185 typename CatalogList::const_iterator iend = children.end();
1186 for (; i != iend; ++i) {
1187 output += PrintHierarchyRecursively(*i, level + 1);
1194 template <
class CatalogT>
1196 const CatalogT *catalog)
const
1198 string result = catalog->PrintMemStatistics() +
"\n";
1201 typename CatalogList::const_iterator i = children.begin();
1202 typename CatalogList::const_iterator iend = children.end();
1203 for (; i != iend; ++i) {
1204 result += PrintMemStatsRecursively(*i);
1213 template <
class CatalogT>
1217 result = PrintMemStatsRecursively(GetRootCatalog());
1223 template <
class CatalogT>
1225 char *mem_enforced =
1226 static_cast<char *
>(pthread_getspecific(pkey_sqlitemem_));
1227 if (mem_enforced == NULL) {
1229 pthread_setspecific(pkey_sqlitemem_,
this);
1238 #endif // CVMFS_CATALOG_MGR_IMPL_H_
void DetachSubtree(CatalogT *catalog)
bool GetVOMSAuthz(std::string *authz) const
catalog::Counters LookupCounters(const PathString &path, std::string *subcatalog_path, shash::Any *hash)
pthread_key_t pkey_sqlitemem_
virtual ~AbstractCatalogManager()
virtual void EnforceSqliteMemLimit()
CatalogT * LoadFreeCatalog(const PathString &mountpoint, const shash::Any &hash)
bool HasExplicitTTL() const
void StageNestedCatalogAndUnlock(const PathString &path, const CatalogT *parent, bool is_listable)
bool MountSubtree(const PathString &path, const CatalogT *entry_point, bool can_listing, CatalogT **leaf_catalog)
void DetachSiblings(const PathString ¤t_tree)
InodeAnnotation * inode_annotation_
std::string ToString(const bool with_suffix=false) const
void Assign(const char *chars, const unsigned length)
std::string PrintHierarchyRecursively(const CatalogT *catalog, const int level) const
void set_symlink(const LinkString &symlink)
uint64_t GetTimestamp() const
perf::Statistics * statistics_
void DetachCatalog(CatalogT *catalog)
bool IsAttached(const PathString &root_path, CatalogT **attached_catalog) const
bool ListingStat(const PathString &path, StatEntryList *listing)
assert((mem||(size==0))&&"Out Of Memory")
bool LookupPath(const PathString &path, const LookupOptions options, DirectoryEntry *entry)
IntegerMap< uint64_t > OwnerMap
unsigned catalog_watermark_
std::string sqlite_path() const
bool Listing(const PathString &path, DirectoryEntryList *listing, const bool expand_symlink)
std::vector< DirectoryEntry > DirectoryEntryList
bool AttachCatalog(const std::string &db_path, CatalogT *new_catalog)
LoadReturn ChangeRoot(const shash::Any &root_hash)
const unsigned kSqliteMemPerThread
uint64_t GetTimestampNoLock() const
LoadReturn RemountDryrun()
shash::Any GetNestedCatalogHash(const PathString &mountpoint)
void Append(const char *chars, const unsigned length)
void CheckInodeWatermark()
void Inc(class Counter *counter)
bool HasPrefix(const string &str, const string &prefix, const bool ignore_case)
std::vector< CatalogT * > CatalogList
const unsigned kLookupRawSymlink
bool ListCatalogSkein(const PathString &path, std::vector< PathString > *result_list)
InodeRange AcquireInodes(uint64_t size)
pthread_rwlock_t * rwlock_
std::string ToString() const
uint64_t GetRevision() const
std::vector< NestedCatalog > NestedCatalogList
uint64_t GetRevisionNoLock() const
int GetNumCatalogs() const
bool ListFileChunks(const PathString &path, const shash::Algorithms interpret_hashes_as, FileChunkList *chunks)
ShortString< kDefaultMaxPath, 0 > PathString
const char * Code2Ascii(const LoadReturn error)
uint64_t timestamp_cache_
bool LookupXattrs(const PathString &path, XattrList *xattrs)
PathString mountpoint() const
bool LookupNested(const PathString &path, PathString *mountpoint, shash::Any *hash, uint64_t *size)
CatalogT * MountCatalog(const PathString &mountpoint, const shash::Any &hash, CatalogT *parent_catalog)
bool StartsWith(const ShortString &other) const
void SetCatalogWatermark(unsigned limit)
void SetOwnerMaps(const OwnerMap &uid_map, const OwnerMap &gid_map)
unsigned GetLength() const
void ReleaseInodes(const InodeRange chunk)
CatalogT * FindCatalog(const PathString &path) const
const char * c_str() const
int inode_watermark_status_
const char * GetChars() const
std::string PrintHierarchy() const
std::string PrintMemStatsRecursively(const CatalogT *catalog) const
std::string PrintAllMemStatistics() const
void SetInodeAnnotation(InodeAnnotation *new_annotation)
CVMFS_EXPORT void LogCvmfs(const LogSource source, const int mask, const char *format,...)