5 #ifndef CVMFS_CATALOG_MGR_IMPL_H_
6 #define CVMFS_CATALOG_MGR_IMPL_H_
8 #ifndef __STDC_FORMAT_MACROS
9 #define __STDC_FORMAT_MACROS
12 #include "cvmfs_config.h"
27 template <
class CatalogT>
40 reinterpret_cast<pthread_rwlock_t *
>(smalloc(
sizeof(pthread_rwlock_t)));
41 int retval = pthread_rwlock_init(
rwlock_, NULL);
47 template <
class CatalogT>
50 pthread_key_delete(pkey_sqlitemem_);
51 pthread_rwlock_destroy(rwlock_);
55 template <
class CatalogT>
59 assert(catalogs_.empty() || (new_annotation == inode_annotation_));
60 inode_annotation_ = new_annotation;
63 template <
class CatalogT>
71 template <
class CatalogT>
73 catalog_watermark_ = limit;
76 template <
class CatalogT>
78 if (inode_watermark_status_ > 0)
81 uint64_t highest_inode = inode_gauge_;
82 if (inode_annotation_)
83 highest_inode += inode_annotation_->GetGeneration();
84 uint64_t uint32_border = 1;
85 uint32_border = uint32_border << 32;
86 if (highest_inode >= uint32_border) {
88 inode_watermark_status_++;
97 template <
class CatalogT>
117 template <
class CatalogT>
120 "remounting repositories (dry run %d)", dry_run);
133 inode_t old_inode_gauge = inode_gauge_;
137 CatalogT *new_root = CreateCatalog(
PathString(
"", 0), catalog_hash, NULL);
139 bool retval = AttachCatalog(catalog_path, new_root);
142 if (inode_annotation_) {
143 inode_annotation_->IncGeneration(old_inode_gauge);
146 CheckInodeWatermark();
155 template <
class CatalogT>
160 "switching to root hash %s", root_hash.
ToString().c_str());
171 inode_t old_inode_gauge = inode_gauge_;
175 CatalogT *new_root = CreateCatalog(
PathString(
"", 0), catalog_hash, NULL);
177 bool retval = AttachCatalog(catalog_path, new_root);
180 if (inode_annotation_) {
181 inode_annotation_->IncGeneration(old_inode_gauge);
184 CheckInodeWatermark();
194 template <
class CatalogT>
197 if (catalogs_.empty()) {
202 typename CatalogList::const_iterator i;
203 typename CatalogList::const_iterator iend;
204 CatalogList catalogs_to_detach = GetRootCatalog()->GetChildren();
205 for (i = catalogs_to_detach.begin(), iend = catalogs_to_detach.end();
218 template <
class CatalogT>
223 CatalogT *catalog = FindCatalog(mountpoint);
225 if (catalog->mountpoint() == mountpoint) {
226 catalog = catalog->parent();
231 catalog->FindNested(mountpoint, &result, &size);
245 template <
class CatalogT>
258 EnforceSqliteMemLimit();
261 CatalogT *best_fit = FindCatalog(path);
266 path.
c_str(), best_fit->mountpoint().c_str());
267 bool found = best_fit->LookupPath(path, dirent);
270 if (!found && MountSubtree(path, best_fit,
false , NULL)) {
276 best_fit = FindCatalog(path);
279 found = best_fit->LookupPath(path, dirent);
283 "entry not found, we may have to load nested catalogs");
285 CatalogT *nested_catalog;
287 MountSubtree(path, best_fit,
false , &nested_catalog);
291 "failed to load nested catalog for '%s'", path.
c_str());
292 goto lookup_path_notfound;
295 if (nested_catalog != best_fit) {
297 found = nested_catalog->LookupPath(path, dirent);
300 "nested catalogs loaded but entry '%s' was still not found",
302 if (dirent != NULL) *dirent = dirent_negative;
303 goto lookup_path_notfound;
305 best_fit = nested_catalog;
309 if (dirent != NULL) *dirent = dirent_negative;
310 goto lookup_path_notfound;
318 if (dirent != NULL) *dirent = dirent_negative;
319 goto lookup_path_notfound;
323 path.
c_str(), best_fit->mountpoint().c_str());
327 bool retval = best_fit->LookupRawSymlink(path, &raw_symlink);
335 lookup_path_notfound:
355 template <
class CatalogT>
362 EnforceSqliteMemLimit();
368 catalog_path.
Append(
"/.cvmfscatalog", 14);
371 CatalogT *best_fit = FindCatalog(catalog_path);
372 CatalogT *catalog = best_fit;
373 if (MountSubtree(catalog_path, best_fit,
false , NULL)) {
377 best_fit = FindCatalog(catalog_path);
379 MountSubtree(catalog_path, best_fit,
false , &catalog);
388 if (catalog->HasParent()) {
389 result = catalog->parent()->FindNested(catalog->root_prefix(), hash,
size);
393 mountpoint->
Assign(catalog->root_prefix());
400 *hash = GetRootCatalog()->hash();
419 template <
class CatalogT>
422 std::vector<PathString> *result_list
424 EnforceSqliteMemLimit();
430 test.
Append(
"/.cvmfscatalog", 14);
433 CatalogT *best_fit = FindCatalog(test);
434 CatalogT *catalog = best_fit;
436 if (MountSubtree(test, best_fit,
false , NULL)) {
440 best_fit = FindCatalog(test);
441 result = MountSubtree(test, best_fit,
false , &catalog);
450 CatalogT *cur_parent = catalog->parent();
453 std::vector<catalog::Catalog*> parents;
454 while (cur_parent->HasParent()) {
455 parents.push_back(cur_parent);
456 cur_parent = cur_parent->parent();
458 parents.push_back(cur_parent);
459 while (!parents.empty()) {
461 result_list->push_back(parents.back()->root_prefix());
466 result_list->push_back(catalog->root_prefix());
471 for (
unsigned i = 0; i < children.size(); i++) {
472 result_list->push_back(children.at(i).mountpoint);
480 template <
class CatalogT>
485 EnforceSqliteMemLimit();
490 CatalogT *best_fit = FindCatalog(path);
491 CatalogT *catalog = best_fit;
492 if (MountSubtree(path, best_fit,
false , NULL)) {
496 best_fit = FindCatalog(path);
497 result = MountSubtree(path, best_fit,
false , &catalog);
505 result = catalog->LookupXattrsPath(path, xattrs);
518 template <
class CatalogT>
521 const bool expand_symlink)
523 EnforceSqliteMemLimit();
528 CatalogT *best_fit = FindCatalog(path);
529 CatalogT *catalog = best_fit;
530 if (MountSubtree(path, best_fit,
true , NULL)) {
534 best_fit = FindCatalog(path);
535 result = MountSubtree(path, best_fit,
true , &catalog);
543 result = catalog->ListingPath(path, listing, expand_symlink);
556 template <
class CatalogT>
560 EnforceSqliteMemLimit();
565 CatalogT *best_fit = FindCatalog(path);
566 CatalogT *catalog = best_fit;
567 if (MountSubtree(path, best_fit,
true , NULL)) {
571 best_fit = FindCatalog(path);
572 result = MountSubtree(path, best_fit,
true , &catalog);
580 result = catalog->ListingPathStat(path, listing);
594 template <
class CatalogT>
600 EnforceSqliteMemLimit();
605 CatalogT *best_fit = FindCatalog(path);
606 CatalogT *catalog = best_fit;
607 if (MountSubtree(path, best_fit,
false , NULL)) {
611 best_fit = FindCatalog(path);
612 result = MountSubtree(path, best_fit,
false , &catalog);
619 result = catalog->ListPathChunks(path, interpret_hashes_as, chunks);
625 template <
class CatalogT>
628 std::string *subcatalog_path,
632 EnforceSqliteMemLimit();
638 catalog_path.
Append(
"/.cvmfscatalog", 14);
641 CatalogT *best_fit = FindCatalog(catalog_path);
642 CatalogT *catalog = best_fit;
643 if (MountSubtree(catalog_path, best_fit,
false , NULL)) {
647 best_fit = FindCatalog(catalog_path);
649 MountSubtree(catalog_path, best_fit,
false , &catalog);
653 *subcatalog_path =
"error: failed to load catalog!";
659 *hash = catalog->hash();
660 *subcatalog_path = catalog->mountpoint().
ToString();
667 template <
class CatalogT>
670 const uint64_t revision = revision_cache_;
677 template <
class CatalogT>
680 const bool has_authz = has_authz_cache_;
681 if (has_authz && authz)
682 *authz = authz_cache_;
688 template <
class CatalogT>
691 const bool result = GetRootCatalog()->HasExplicitTTL();
697 template <
class CatalogT>
700 const uint64_t ttl = GetRootCatalog()->GetTTL();
706 template <
class CatalogT>
709 int result = catalogs_.size();
718 template <
class CatalogT>
721 string output = PrintHierarchyRecursively(GetRootCatalog(), 0);
730 template <
class CatalogT>
733 result.
offset = inode_gauge_;
736 inode_gauge_ +=
size;
738 result.
offset + 1, inode_gauge_);
749 template <
class CatalogT>
761 template <
class CatalogT>
764 assert(catalogs_.size() > 0);
767 CatalogT *best_fit = GetRootCatalog();
768 CatalogT *next_fit = NULL;
769 while (best_fit->mountpoint() != path) {
770 next_fit = best_fit->FindSubtree(path);
771 if (next_fit == NULL)
786 template <
class CatalogT>
788 CatalogT **attached_catalog)
const
790 if (catalogs_.size() == 0)
793 CatalogT *best_fit = FindCatalog(root_path);
794 if (best_fit->mountpoint() != root_path)
797 if (attached_catalog != NULL) *attached_catalog = best_fit;
811 template <
class CatalogT>
814 const CatalogT *entry_point,
816 CatalogT **leaf_catalog)
819 CatalogT *parent = (entry_point == NULL) ?
820 GetRootCatalog() :
const_cast<CatalogT *
>(entry_point);
827 typedef typename CatalogT::NestedCatalogList NestedCatalogList;
828 const NestedCatalogList& nested_catalogs =
829 parent->ListNestedCatalogs();
830 for (
typename NestedCatalogList::const_iterator i = nested_catalogs.begin(),
831 iEnd = nested_catalogs.end(); i != iEnd; ++i)
838 unsigned mountpoint_len = i->mountpoint.GetLength();
839 if (path_len > mountpoint_len && path.
GetChars()[mountpoint_len] !=
'/')
843 if (!is_listable && (path_len == mountpoint_len))
846 if (leaf_catalog == NULL)
848 CatalogT *new_nested;
850 i->mountpoint.c_str());
853 if (i->hash.IsNull())
855 new_nested = MountCatalog(i->mountpoint, i->hash, parent);
859 result = MountSubtree(path, new_nested, is_listable, &parent);
864 if (leaf_catalog == NULL)
866 *leaf_catalog = parent;
875 template <
class CatalogT>
879 CatalogT *parent_catalog)
881 CatalogT *attached_catalog = NULL;
882 if (IsAttached(mountpoint, &attached_catalog))
883 return attached_catalog;
888 LoadCatalog(mountpoint, hash, &catalog_path, &catalog_hash);
895 attached_catalog = CreateCatalog(mountpoint, catalog_hash, parent_catalog);
898 if (!AttachCatalog(catalog_path, attached_catalog)) {
901 UnloadCatalog(attached_catalog);
905 if ((catalog_watermark_ > 0) && (catalogs_.size() >= catalog_watermark_)) {
906 DetachSiblings(mountpoint);
909 return attached_catalog;
917 template <
class CatalogT>
924 const LoadError load_error = LoadCatalog(mountpoint, hash, &new_path,
928 assert(hash == check_hash);
929 CatalogT *catalog = CatalogT::AttachFreely(mountpoint.
ToString(),
931 catalog->TakeDatabaseFileOwnership();
942 template <
class CatalogT>
944 CatalogT *new_catalog)
950 if (!new_catalog->OpenDatabase(db_path)) {
957 uint64_t inode_chunk_size = new_catalog->max_row_id();
958 InodeRange range = AcquireInodes(inode_chunk_size);
959 new_catalog->set_inode_range(range);
960 new_catalog->SetInodeAnnotation(inode_annotation_);
961 new_catalog->SetOwnerMaps(&uid_map_, &gid_map_);
964 if (!new_catalog->IsInitialized()) {
966 "catalog initialization failed (obscure data)");
967 inode_gauge_ -= inode_chunk_size;
970 CheckInodeWatermark();
973 if (catalogs_.empty()) {
974 revision_cache_ = new_catalog->GetRevision();
975 statistics_.catalog_revision->Set(revision_cache_);
976 has_authz_cache_ = new_catalog->GetVOMSAuthz(&authz_cache_);
977 volatile_flag_ = new_catalog->volatile_flag();
980 catalogs_.push_back(new_catalog);
981 ActivateCatalog(new_catalog);
994 template <
class CatalogT>
996 if (catalog->HasParent())
997 catalog->parent()->RemoveChild(catalog);
999 ReleaseInodes(catalog->inode_range());
1000 UnloadCatalog(catalog);
1003 typename CatalogList::iterator i;
1004 typename CatalogList::const_iterator iend;
1005 for (i = catalogs_.begin(), iend = catalogs_.end(); i != iend; ++i) {
1006 if (*i == catalog) {
1023 template <
class CatalogT>
1026 typename CatalogList::const_iterator i;
1027 typename CatalogList::const_iterator iend;
1028 CatalogList catalogs_to_detach = catalog->GetChildren();
1029 for (i = catalogs_to_detach.begin(), iend = catalogs_to_detach.end();
1035 DetachCatalog(catalog);
1043 template <
class CatalogT>
1050 unsigned N = catalogs_.size();
1051 for (
unsigned i = 0; i < N; ++i) {
1053 catalogs_[i]->mountpoint().ToString(),
1056 DetachSubtree(catalogs_[i]);
1069 template <
class CatalogT>
1071 const CatalogT *catalog,
1072 const int level)
const
1077 for (
int i = 0; i < level; ++i)
1080 output +=
"-> " + string(catalog->mountpoint().GetChars(),
1081 catalog->mountpoint().GetLength())
1085 typename CatalogList::const_iterator i = children.begin();
1086 typename CatalogList::const_iterator iend = children.end();
1087 for (; i != iend; ++i) {
1088 output += PrintHierarchyRecursively(*i, level + 1);
1095 template <
class CatalogT>
1097 const CatalogT *catalog)
const
1099 string result = catalog->PrintMemStatistics() +
"\n";
1102 typename CatalogList::const_iterator i = children.begin();
1103 typename CatalogList::const_iterator iend = children.end();
1104 for (; i != iend; ++i) {
1105 result += PrintMemStatsRecursively(*i);
1114 template <
class CatalogT>
1118 result = PrintMemStatsRecursively(GetRootCatalog());
1124 template <
class CatalogT>
1126 char *mem_enforced =
1127 static_cast<char *
>(pthread_getspecific(pkey_sqlitemem_));
1128 if (mem_enforced == NULL) {
1130 pthread_setspecific(pkey_sqlitemem_,
this);
1139 #endif // CVMFS_CATALOG_MGR_IMPL_H_
LoadError ChangeRoot(const shash::Any &root_hash)
void DetachSubtree(CatalogT *catalog)
#define LogCvmfs(source, mask,...)
bool GetVOMSAuthz(std::string *authz) const
catalog::Counters LookupCounters(const PathString &path, std::string *subcatalog_path, shash::Any *hash)
pthread_key_t pkey_sqlitemem_
const char * Code2Ascii(const LoadError error)
virtual ~AbstractCatalogManager()
virtual void EnforceSqliteMemLimit()
CatalogT * LoadFreeCatalog(const PathString &mountpoint, const shash::Any &hash)
bool HasExplicitTTL() const
bool MountSubtree(const PathString &path, const CatalogT *entry_point, bool can_listing, CatalogT **leaf_catalog)
void DetachSiblings(const PathString ¤t_tree)
InodeAnnotation * inode_annotation_
std::string ToString(const bool with_suffix=false) const
void Assign(const char *chars, const unsigned length)
std::string PrintHierarchyRecursively(const CatalogT *catalog, const int level) const
void set_symlink(const LinkString &symlink)
perf::Statistics * statistics_
void DetachCatalog(CatalogT *catalog)
bool IsAttached(const PathString &root_path, CatalogT **attached_catalog) const
bool ListingStat(const PathString &path, StatEntryList *listing)
assert((mem||(size==0))&&"Out Of Memory")
bool LookupPath(const PathString &path, const LookupOptions options, DirectoryEntry *entry)
IntegerMap< uint64_t > OwnerMap
unsigned catalog_watermark_
LoadError Remount(const bool dry_run)
bool Listing(const PathString &path, DirectoryEntryList *listing, const bool expand_symlink)
std::vector< DirectoryEntry > DirectoryEntryList
bool AttachCatalog(const std::string &db_path, CatalogT *new_catalog)
const unsigned kSqliteMemPerThread
shash::Any GetNestedCatalogHash(const PathString &mountpoint)
void Append(const char *chars, const unsigned length)
void CheckInodeWatermark()
void Inc(class Counter *counter)
bool HasPrefix(const string &str, const string &prefix, const bool ignore_case)
std::vector< CatalogT * > CatalogList
const unsigned kLookupRawSymlink
bool ListCatalogSkein(const PathString &path, std::vector< PathString > *result_list)
InodeRange AcquireInodes(uint64_t size)
pthread_rwlock_t * rwlock_
std::string ToString() const
uint64_t GetRevision() const
std::vector< NestedCatalog > NestedCatalogList
int GetNumCatalogs() const
bool ListFileChunks(const PathString &path, const shash::Algorithms interpret_hashes_as, FileChunkList *chunks)
ShortString< kDefaultMaxPath, 0 > PathString
bool LookupXattrs(const PathString &path, XattrList *xattrs)
bool LookupNested(const PathString &path, PathString *mountpoint, shash::Any *hash, uint64_t *size)
CatalogT * MountCatalog(const PathString &mountpoint, const shash::Any &hash, CatalogT *parent_catalog)
bool StartsWith(const ShortString &other) const
void SetCatalogWatermark(unsigned limit)
void SetOwnerMaps(const OwnerMap &uid_map, const OwnerMap &gid_map)
unsigned GetLength() const
void ReleaseInodes(const InodeRange chunk)
CatalogT * FindCatalog(const PathString &path) const
const char * c_str() const
int inode_watermark_status_
const char * GetChars() const
std::string PrintHierarchy() const
std::string PrintMemStatsRecursively(const CatalogT *catalog) const
std::string PrintAllMemStatistics() const
void SetInodeAnnotation(InodeAnnotation *new_annotation)