8 #ifndef CVMFS_CATALOG_MGR_IMPL_H_
9 #define CVMFS_CATALOG_MGR_IMPL_H_
11 #ifndef __STDC_FORMAT_MACROS
12 #define __STDC_FORMAT_MACROS
29 template<
class CatalogT>
42 rwlock_ =
reinterpret_cast<pthread_rwlock_t *
>(
43 smalloc(
sizeof(pthread_rwlock_t)));
44 int retval = pthread_rwlock_init(
rwlock_, NULL);
50 template<
class CatalogT>
53 pthread_key_delete(pkey_sqlitemem_);
54 pthread_rwlock_destroy(rwlock_);
58 template<
class CatalogT>
61 assert(catalogs_.empty() || (new_annotation == inode_annotation_));
62 inode_annotation_ = new_annotation;
65 template<
class CatalogT>
72 template<
class CatalogT>
74 catalog_watermark_ = limit;
77 template<
class CatalogT>
79 if (inode_watermark_status_ > 0)
82 uint64_t highest_inode = inode_gauge_;
83 if (inode_annotation_)
84 highest_inode += inode_annotation_->GetGeneration();
85 uint64_t uint32_border = 1;
86 uint32_border = uint32_border << 32;
87 if (highest_inode >= uint32_border) {
89 inode_watermark_status_++;
98 template<
class CatalogT>
118 template<
class CatalogT>
122 return GetNewRootCatalogContext(&ctlg_context);
125 template<
class CatalogT>
130 if (GetNewRootCatalogContext(&ctlg_context) !=
kLoadNew
131 && GetNewRootCatalogContext(&ctlg_context) !=
kLoadUp2Date) {
133 "remounting repositories: "
134 "Did not find any valid root catalog to mount");
140 const LoadReturn load_error = LoadCatalogByHash(&ctlg_context);
143 inode_t old_inode_gauge = inode_gauge_;
147 CatalogT *new_root = CreateCatalog(ctlg_context.
mountpoint(),
148 ctlg_context.
hash(), NULL);
150 bool retval = AttachCatalog(ctlg_context.
sqlite_path(), new_root);
153 if (inode_annotation_) {
154 inode_annotation_->IncGeneration(old_inode_gauge);
157 CheckInodeWatermark();
166 template<
class CatalogT>
180 const LoadReturn load_error = LoadCatalogByHash(&ctlg_context);
183 inode_t old_inode_gauge = inode_gauge_;
187 CatalogT *new_root = CreateCatalog(
PathString(
"", 0), ctlg_context.
hash(),
190 bool retval = AttachCatalog(ctlg_context.
sqlite_path(), new_root);
193 if (inode_annotation_) {
194 inode_annotation_->IncGeneration(old_inode_gauge);
197 CheckInodeWatermark();
207 template<
class CatalogT>
210 if (catalogs_.empty()) {
215 typename CatalogList::const_iterator i;
216 typename CatalogList::const_iterator iend;
217 CatalogList catalogs_to_detach = GetRootCatalog()->GetChildren();
218 for (i = catalogs_to_detach.begin(), iend = catalogs_to_detach.end();
230 template<
class CatalogT>
234 CatalogT *catalog = FindCatalog(mountpoint);
236 if (catalog->mountpoint() == mountpoint) {
237 catalog = catalog->parent();
242 catalog->FindNested(mountpoint, &result, &size);
256 template<
class CatalogT>
268 EnforceSqliteMemLimit();
271 CatalogT *best_fit = FindCatalog(path);
276 path.
c_str(), best_fit->mountpoint().c_str());
277 bool found = best_fit->LookupPath(path, dirent);
280 if (!found && MountSubtree(path, best_fit,
false , NULL)) {
283 StageNestedCatalogAndUnlock(path, best_fit,
false );
286 best_fit = FindCatalog(path);
289 found = best_fit->LookupPath(path, dirent);
293 "entry not found, we may have to load nested catalogs");
295 CatalogT *nested_catalog;
296 found = MountSubtree(path, best_fit,
false ,
301 "failed to load nested catalog for '%s'", path.
c_str());
302 goto lookup_path_notfound;
305 if (nested_catalog != best_fit) {
307 found = nested_catalog->LookupPath(path, dirent);
310 "nested catalogs loaded but entry '%s' was still not found",
313 *dirent = dirent_negative;
314 goto lookup_path_notfound;
316 best_fit = nested_catalog;
321 *dirent = dirent_negative;
322 goto lookup_path_notfound;
331 *dirent = dirent_negative;
332 goto lookup_path_notfound;
336 path.
c_str(), best_fit->mountpoint().c_str());
340 bool retval = best_fit->LookupRawSymlink(path, &raw_symlink);
348 lookup_path_notfound:
368 template<
class CatalogT>
373 EnforceSqliteMemLimit();
379 catalog_path.
Append(
"/.cvmfscatalog", 14);
382 CatalogT *best_fit = FindCatalog(catalog_path);
383 CatalogT *catalog = best_fit;
384 if (MountSubtree(catalog_path, best_fit,
false , NULL)) {
385 StageNestedCatalogAndUnlock(path, best_fit,
false);
388 best_fit = FindCatalog(catalog_path);
389 result = MountSubtree(catalog_path, best_fit,
false ,
399 if (catalog->HasParent()) {
400 result = catalog->parent()->FindNested(catalog->root_prefix(), hash,
size);
404 mountpoint->
Assign(catalog->root_prefix());
411 *hash = GetRootCatalog()->hash();
430 template<
class CatalogT>
432 const PathString &path, std::vector<PathString> *result_list) {
433 EnforceSqliteMemLimit();
439 test.
Append(
"/.cvmfscatalog", 14);
442 CatalogT *best_fit = FindCatalog(test);
443 CatalogT *catalog = best_fit;
445 if (MountSubtree(test, best_fit,
false , NULL)) {
446 StageNestedCatalogAndUnlock(path, best_fit,
false);
449 best_fit = FindCatalog(test);
450 result = MountSubtree(test, best_fit,
false , &catalog);
459 CatalogT *cur_parent = catalog->parent();
462 std::vector<catalog::Catalog *> parents;
463 while (cur_parent->HasParent()) {
464 parents.push_back(cur_parent);
465 cur_parent = cur_parent->parent();
467 parents.push_back(cur_parent);
468 while (!parents.empty()) {
470 result_list->push_back(parents.back()->root_prefix());
475 result_list->push_back(catalog->root_prefix());
480 for (
unsigned i = 0; i < children.size(); i++) {
481 result_list->push_back(children.at(i).mountpoint);
489 template<
class CatalogT>
492 EnforceSqliteMemLimit();
497 CatalogT *best_fit = FindCatalog(path);
498 CatalogT *catalog = best_fit;
499 if (MountSubtree(path, best_fit,
false , NULL)) {
500 StageNestedCatalogAndUnlock(path, best_fit,
false);
503 best_fit = FindCatalog(path);
504 result = MountSubtree(path, best_fit,
false , &catalog);
512 result = catalog->LookupXattrsPath(path, xattrs);
525 template<
class CatalogT>
528 const bool expand_symlink) {
529 EnforceSqliteMemLimit();
534 CatalogT *best_fit = FindCatalog(path);
535 CatalogT *catalog = best_fit;
536 if (MountSubtree(path, best_fit,
true , NULL)) {
537 StageNestedCatalogAndUnlock(path, best_fit,
true );
540 best_fit = FindCatalog(path);
541 result = MountSubtree(path, best_fit,
true , &catalog);
549 result = catalog->ListingPath(path, listing, expand_symlink);
562 template<
class CatalogT>
565 EnforceSqliteMemLimit();
570 CatalogT *best_fit = FindCatalog(path);
571 CatalogT *catalog = best_fit;
572 if (MountSubtree(path, best_fit,
true , NULL)) {
573 StageNestedCatalogAndUnlock(path, best_fit,
true );
576 best_fit = FindCatalog(path);
577 result = MountSubtree(path, best_fit,
true , &catalog);
585 result = catalog->ListingPathStat(path, listing);
599 template<
class CatalogT>
604 EnforceSqliteMemLimit();
609 CatalogT *best_fit = FindCatalog(path);
610 CatalogT *catalog = best_fit;
611 if (MountSubtree(path, best_fit,
false , NULL)) {
612 StageNestedCatalogAndUnlock(path, best_fit,
false);
615 best_fit = FindCatalog(path);
616 result = MountSubtree(path, best_fit,
false , &catalog);
623 result = catalog->ListPathChunks(path, interpret_hashes_as, chunks);
629 template<
class CatalogT>
632 EnforceSqliteMemLimit();
638 catalog_path.
Append(
"/.cvmfscatalog", 14);
641 CatalogT *best_fit = FindCatalog(catalog_path);
642 CatalogT *catalog = best_fit;
643 if (MountSubtree(catalog_path, best_fit,
false , NULL)) {
644 StageNestedCatalogAndUnlock(path, best_fit,
false );
647 best_fit = FindCatalog(catalog_path);
648 result = MountSubtree(catalog_path, best_fit,
false ,
653 *subcatalog_path =
"error: failed to load catalog!";
659 *hash = catalog->hash();
660 *subcatalog_path = catalog->mountpoint().
ToString();
667 template<
class CatalogT>
670 const uint64_t revision = GetRevisionNoLock();
681 template<
class CatalogT>
683 return revision_cache_;
686 template<
class CatalogT>
689 const uint64_t timestamp = GetTimestampNoLock();
700 template<
class CatalogT>
702 return timestamp_cache_;
705 template<
class CatalogT>
708 const bool has_authz = has_authz_cache_;
709 if (has_authz && authz)
710 *authz = authz_cache_;
716 template<
class CatalogT>
719 const bool result = GetRootCatalog()->HasExplicitTTL();
725 template<
class CatalogT>
728 const uint64_t ttl = GetRootCatalog()->GetTTL();
734 template<
class CatalogT>
737 int result = catalogs_.size();
746 template<
class CatalogT>
749 string output = PrintHierarchyRecursively(GetRootCatalog(), 0);
758 template<
class CatalogT>
761 result.
offset = inode_gauge_;
764 inode_gauge_ +=
size;
766 result.
offset + 1, inode_gauge_);
777 template<
class CatalogT>
789 template<
class CatalogT>
792 assert(catalogs_.size() > 0);
795 CatalogT *best_fit = GetRootCatalog();
796 CatalogT *next_fit = NULL;
797 while (best_fit->mountpoint() != path) {
798 next_fit = best_fit->FindSubtree(path);
799 if (next_fit == NULL)
814 template<
class CatalogT>
816 const PathString &root_path, CatalogT **attached_catalog)
const {
817 if (catalogs_.size() == 0)
820 CatalogT *best_fit = FindCatalog(root_path);
821 if (best_fit->mountpoint() != root_path)
824 if (attached_catalog != NULL)
825 *attached_catalog = best_fit;
830 template<
class CatalogT>
832 const PathString &path,
const CatalogT *parent,
bool is_listable) {
834 const unsigned path_len = path.
GetLength();
837 typedef typename CatalogT::NestedCatalogList NestedCatalogList;
838 const NestedCatalogList &nested_catalogs = parent->ListNestedCatalogs();
840 for (
typename NestedCatalogList::const_iterator i = nested_catalogs.begin(),
841 iEnd = nested_catalogs.end();
850 const unsigned mountpoint_len = i->mountpoint.GetLength();
851 if (path_len > mountpoint_len && path.
GetChars()[mountpoint_len] !=
'/')
855 if (!is_listable && (path_len == mountpoint_len))
860 i->mountpoint.c_str(), i->hash.ToString().c_str());
861 StageNestedCatalogByHash(i->hash, i->mountpoint);
876 template<
class CatalogT>
878 const CatalogT *entry_point,
880 CatalogT **leaf_catalog) {
882 CatalogT *parent = (entry_point == NULL)
884 :
const_cast<CatalogT *
>(entry_point);
891 typedef typename CatalogT::NestedCatalogList NestedCatalogList;
892 const NestedCatalogList &nested_catalogs = parent->ListNestedCatalogs();
893 for (
typename NestedCatalogList::const_iterator i = nested_catalogs.begin(),
894 iEnd = nested_catalogs.end();
902 unsigned mountpoint_len = i->mountpoint.GetLength();
903 if (path_len > mountpoint_len && path.
GetChars()[mountpoint_len] !=
'/')
907 if (!is_listable && (path_len == mountpoint_len))
910 if (leaf_catalog == NULL)
912 CatalogT *new_nested;
914 i->mountpoint.c_str());
917 if (i->hash.IsNull())
919 new_nested = MountCatalog(i->mountpoint, i->hash, parent);
923 result = MountSubtree(path, new_nested, is_listable, &parent);
928 if (leaf_catalog == NULL)
930 *leaf_catalog = parent;
939 template<
class CatalogT>
943 CatalogT *parent_catalog) {
944 CatalogT *attached_catalog = NULL;
945 if (IsAttached(mountpoint, &attached_catalog)) {
946 return attached_catalog;
952 if (GetNewRootCatalogContext(&ctlg_context) ==
kLoadFail) {
954 "failed to retrieve valid root catalog '%s'",
960 const LoadReturn retval = LoadCatalogByHash(&ctlg_context);
967 attached_catalog = CreateCatalog(
971 if (!AttachCatalog(ctlg_context.
sqlite_path(), attached_catalog)) {
974 UnloadCatalog(attached_catalog);
978 if ((catalog_watermark_ > 0) && (catalogs_.size() >= catalog_watermark_)) {
979 DetachSiblings(mountpoint);
982 return attached_catalog;
990 template<
class CatalogT>
996 const LoadReturn load_ret = LoadCatalogByHash(&ctlg_context);
1002 CatalogT *catalog = CatalogT::AttachFreely(
1003 mountpoint.
ToString(), ctlg_context.sqlite_path(), ctlg_context.hash());
1004 catalog->TakeDatabaseFileOwnership();
1015 template<
class CatalogT>
1017 CatalogT *new_catalog) {
1022 if (!new_catalog->OpenDatabase(db_path)) {
1029 uint64_t inode_chunk_size = new_catalog->max_row_id();
1030 InodeRange range = AcquireInodes(inode_chunk_size);
1031 new_catalog->set_inode_range(range);
1032 new_catalog->SetInodeAnnotation(inode_annotation_);
1033 new_catalog->SetOwnerMaps(&uid_map_, &gid_map_);
1036 if (!new_catalog->IsInitialized()) {
1038 "catalog initialization failed (obscure data)");
1039 inode_gauge_ -= inode_chunk_size;
1042 CheckInodeWatermark();
1045 if (catalogs_.empty()) {
1046 revision_cache_ = new_catalog->GetRevision();
1047 timestamp_cache_ = new_catalog->GetLastModified();
1048 statistics_.catalog_revision->Set(revision_cache_);
1049 has_authz_cache_ = new_catalog->GetVOMSAuthz(&authz_cache_);
1050 volatile_flag_ = new_catalog->volatile_flag();
1053 catalogs_.push_back(new_catalog);
1054 ActivateCatalog(new_catalog);
1067 template<
class CatalogT>
1069 if (catalog->HasParent())
1070 catalog->parent()->RemoveChild(catalog);
1072 ReleaseInodes(catalog->inode_range());
1073 UnloadCatalog(catalog);
1076 typename CatalogList::iterator i;
1077 typename CatalogList::const_iterator iend;
1078 for (i = catalogs_.begin(), iend = catalogs_.end(); i != iend; ++i) {
1079 if (*i == catalog) {
1096 template<
class CatalogT>
1099 typename CatalogList::const_iterator i;
1100 typename CatalogList::const_iterator iend;
1101 CatalogList catalogs_to_detach = catalog->GetChildren();
1102 for (i = catalogs_to_detach.begin(), iend = catalogs_to_detach.end();
1107 DetachCatalog(catalog);
1115 template<
class CatalogT>
1121 unsigned N = catalogs_.size();
1122 for (
unsigned i = 0; i < N; ++i) {
1124 catalogs_[i]->mountpoint().ToString(),
1126 DetachSubtree(catalogs_[i]);
1139 template<
class CatalogT>
1141 const CatalogT *catalog,
const int level)
const {
1145 for (
int i = 0; i < level; ++i)
1149 + string(catalog->mountpoint().GetChars(),
1150 catalog->mountpoint().GetLength())
1154 typename CatalogList::const_iterator i = children.begin();
1155 typename CatalogList::const_iterator iend = children.end();
1156 for (; i != iend; ++i) {
1157 output += PrintHierarchyRecursively(*i, level + 1);
1164 template<
class CatalogT>
1166 const CatalogT *catalog)
const {
1167 string result = catalog->PrintMemStatistics() +
"\n";
1170 typename CatalogList::const_iterator i = children.begin();
1171 typename CatalogList::const_iterator iend = children.end();
1172 for (; i != iend; ++i) {
1173 result += PrintMemStatsRecursively(*i);
1182 template<
class CatalogT>
1186 result = PrintMemStatsRecursively(GetRootCatalog());
1192 template<
class CatalogT>
1194 char *mem_enforced =
static_cast<char *
>(
1195 pthread_getspecific(pkey_sqlitemem_));
1196 if (mem_enforced == NULL) {
1198 pthread_setspecific(pkey_sqlitemem_,
this);
1205 #endif // CVMFS_CATALOG_MGR_IMPL_H_
void DetachSubtree(CatalogT *catalog)
bool GetVOMSAuthz(std::string *authz) const
catalog::Counters LookupCounters(const PathString &path, std::string *subcatalog_path, shash::Any *hash)
pthread_key_t pkey_sqlitemem_
virtual ~AbstractCatalogManager()
virtual void EnforceSqliteMemLimit()
CatalogT * LoadFreeCatalog(const PathString &mountpoint, const shash::Any &hash)
bool HasExplicitTTL() const
void StageNestedCatalogAndUnlock(const PathString &path, const CatalogT *parent, bool is_listable)
bool MountSubtree(const PathString &path, const CatalogT *entry_point, bool can_listing, CatalogT **leaf_catalog)
void DetachSiblings(const PathString ¤t_tree)
InodeAnnotation * inode_annotation_
std::string ToString(const bool with_suffix=false) const
void Assign(const char *chars, const unsigned length)
std::string PrintHierarchyRecursively(const CatalogT *catalog, const int level) const
void set_symlink(const LinkString &symlink)
uint64_t GetTimestamp() const
perf::Statistics * statistics_
std::vector< CatalogT * > CatalogList
void DetachCatalog(CatalogT *catalog)
bool IsAttached(const PathString &root_path, CatalogT **attached_catalog) const
bool ListingStat(const PathString &path, StatEntryList *listing)
assert((mem||(size==0))&&"Out Of Memory")
bool LookupPath(const PathString &path, const LookupOptions options, DirectoryEntry *entry)
IntegerMap< uint64_t > OwnerMap
unsigned catalog_watermark_
std::string sqlite_path() const
bool Listing(const PathString &path, DirectoryEntryList *listing, const bool expand_symlink)
std::vector< DirectoryEntry > DirectoryEntryList
bool AttachCatalog(const std::string &db_path, CatalogT *new_catalog)
LoadReturn ChangeRoot(const shash::Any &root_hash)
const unsigned kSqliteMemPerThread
uint64_t GetTimestampNoLock() const
LoadReturn RemountDryrun()
shash::Any GetNestedCatalogHash(const PathString &mountpoint)
void Append(const char *chars, const unsigned length)
void CheckInodeWatermark()
void Inc(class Counter *counter)
bool HasPrefix(const string &str, const string &prefix, const bool ignore_case)
const unsigned kLookupRawSymlink
bool ListCatalogSkein(const PathString &path, std::vector< PathString > *result_list)
InodeRange AcquireInodes(uint64_t size)
pthread_rwlock_t * rwlock_
std::string ToString() const
uint64_t GetRevision() const
std::vector< NestedCatalog > NestedCatalogList
uint64_t GetRevisionNoLock() const
int GetNumCatalogs() const
bool ListFileChunks(const PathString &path, const shash::Algorithms interpret_hashes_as, FileChunkList *chunks)
ShortString< kDefaultMaxPath, 0 > PathString
const char * Code2Ascii(const LoadReturn error)
uint64_t timestamp_cache_
bool LookupXattrs(const PathString &path, XattrList *xattrs)
PathString mountpoint() const
bool LookupNested(const PathString &path, PathString *mountpoint, shash::Any *hash, uint64_t *size)
CatalogT * MountCatalog(const PathString &mountpoint, const shash::Any &hash, CatalogT *parent_catalog)
bool StartsWith(const ShortString &other) const
void SetCatalogWatermark(unsigned limit)
void SetOwnerMaps(const OwnerMap &uid_map, const OwnerMap &gid_map)
unsigned GetLength() const
void ReleaseInodes(const InodeRange chunk)
CatalogT * FindCatalog(const PathString &path) const
const char * c_str() const
int inode_watermark_status_
const char * GetChars() const
std::string PrintHierarchy() const
std::string PrintMemStatsRecursively(const CatalogT *catalog) const
std::string PrintAllMemStatistics() const
void SetInodeAnnotation(InodeAnnotation *new_annotation)
CVMFS_EXPORT void LogCvmfs(const LogSource source, const int mask, const char *format,...)