CernVM-FS  2.11.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
catalog_mgr.h
Go to the documentation of this file.
1 
5 #ifndef CVMFS_CATALOG_MGR_H_
6 #define CVMFS_CATALOG_MGR_H_
7 
8 #ifndef __STDC_FORMAT_MACROS
9 #define __STDC_FORMAT_MACROS
10 #endif
11 
12 #include <inttypes.h>
13 #include <pthread.h>
14 
15 #include <cassert>
16 #include <map>
17 #include <string>
18 #include <vector>
19 
20 #include "catalog.h"
21 #include "crypto/hash.h"
22 #include "directory_entry.h"
23 #include "file_chunk.h"
24 #include "statistics.h"
25 #include "util/atomic.h"
26 #include "util/logging.h"
27 
28 class XattrList;
29 
30 namespace catalog {
31 
32 const unsigned kSqliteMemPerThread = 1*1024*1024;
33 
34 
40 typedef unsigned LookupOptions;
41 const unsigned kLookupDefault = 0b1;
42 const unsigned kLookupRawSymlink = 0b10;
43 
47 enum LoadError {
48  kLoadNew = 0,
52 
54 };
55 
56 inline const char *Code2Ascii(const LoadError error) {
57  const char *texts[kLoadNumEntries + 1];
58  texts[0] = "loaded new catalog";
59  texts[1] = "catalog was up to date";
60  texts[2] = "not enough space to load catalog";
61  texts[3] = "failed to load catalog";
62  texts[4] = "no text";
63  return texts[error];
64 }
65 
66 
67 struct Statistics {
75 
76  explicit Statistics(perf::Statistics *statistics) {
77  n_lookup_inode = statistics->Register("catalog_mgr.n_lookup_inode",
78  "Number of inode lookups");
79  n_lookup_path = statistics->Register("catalog_mgr.n_lookup_path",
80  "Number of path lookups");
81  n_lookup_path_negative = statistics->Register(
82  "catalog_mgr.n_lookup_path_negative",
83  "Number of negative path lookups");
84  n_lookup_xattrs = statistics->Register("catalog_mgr.n_lookup_xattrs",
85  "Number of xattrs lookups");
86  n_listing = statistics->Register("catalog_mgr.n_listing",
87  "Number of listings");
88  n_nested_listing = statistics->Register("catalog_mgr.n_nested_listing",
89  "Number of listings of nested catalogs");
90  n_detach_siblings = statistics->Register("catalog_mgr.n_detach_siblings",
91  "Number of times the CVMFS_CATALOG_WATERMARK was hit");
92  }
93 };
94 
95 
96 template <class CatalogT>
97 class AbstractCatalogManager;
98 
99 
116 template <class CatalogT>
117 class AbstractCatalogManager : public SingleCopy {
118  public:
119  typedef std::vector<CatalogT*> CatalogList;
120  typedef CatalogT catalog_t;
121 
122  static const inode_t kInodeOffset = 255;
124  virtual ~AbstractCatalogManager();
125 
126  void SetInodeAnnotation(InodeAnnotation *new_annotation);
127  virtual bool Init();
128  LoadError Remount(const bool dry_run);
129  LoadError ChangeRoot(const shash::Any &root_hash);
130  void DetachNested();
131 
132  bool LookupPath(const PathString &path, const LookupOptions options,
133  DirectoryEntry *entry);
134  bool LookupPath(const std::string &path, const LookupOptions options,
135  DirectoryEntry *entry)
136  {
137  PathString p;
138  p.Assign(&path[0], path.length());
139  return LookupPath(p, options, entry);
140  }
141  bool LookupXattrs(const PathString &path, XattrList *xattrs);
142 
143  bool LookupNested(const PathString &path,
144  PathString *mountpoint,
145  shash::Any *hash,
146  uint64_t *size);
147  bool ListCatalogSkein(const PathString &path,
148  std::vector<PathString> *result_list);
149 
150  bool Listing(const PathString &path, DirectoryEntryList *listing,
151  const bool expand_symlink);
152  bool Listing(const PathString &path, DirectoryEntryList *listing) {
153  return Listing(path, listing, true);
154  }
155  bool Listing(const std::string &path, DirectoryEntryList *listing) {
156  PathString p;
157  p.Assign(&path[0], path.length());
158  return Listing(p, listing);
159  }
160  bool ListingStat(const PathString &path, StatEntryList *listing);
161 
162  bool ListFileChunks(const PathString &path,
163  const shash::Algorithms interpret_hashes_as,
164  FileChunkList *chunks);
165  void SetOwnerMaps(const OwnerMap &uid_map, const OwnerMap &gid_map);
166  void SetCatalogWatermark(unsigned limit);
167 
168  shash::Any GetNestedCatalogHash(const PathString &mountpoint);
169 
170  Statistics statistics() const { return statistics_; }
171  uint64_t inode_gauge() {
172  ReadLock(); uint64_t r = inode_gauge_; Unlock(); return r;
173  }
174  bool volatile_flag() const { return volatile_flag_; }
175  uint64_t GetRevision() const;
176  uint64_t GetTTL() const;
177  bool HasExplicitTTL() const;
178  bool GetVOMSAuthz(std::string *authz) const;
179  int GetNumCatalogs() const;
180  std::string PrintHierarchy() const;
181  std::string PrintAllMemStatistics() const;
182 
188  inline inode_t GetRootInode() const {
189  return inode_annotation_ ?
191  }
192  inline CatalogT* GetRootCatalog() const { return catalogs_.front(); }
199  inline inode_t MangleInode(const inode_t inode) const {
200  return (inode <= kInodeOffset) ? GetRootInode() : inode;
201  }
202 
204  std::string *subcatalog_path,
205  shash::Any *hash);
206 
207  protected:
213  virtual LoadError LoadCatalog(const PathString &mountpoint,
214  const shash::Any &hash,
215  std::string *catalog_path,
216  shash::Any *catalog_hash) = 0;
217  virtual void UnloadCatalog(const CatalogT *catalog) { }
218  virtual void ActivateCatalog(CatalogT *catalog) { }
219  const std::vector<CatalogT*>& GetCatalogs() const { return catalogs_; }
220 
230  virtual CatalogT* CreateCatalog(const PathString &mountpoint,
231  const shash::Any &catalog_hash,
232  CatalogT *parent_catalog) = 0;
233 
234  CatalogT *MountCatalog(const PathString &mountpoint, const shash::Any &hash,
235  CatalogT *parent_catalog);
236  bool MountSubtree(const PathString &path,
237  const CatalogT *entry_point,
238  bool can_listing,
239  CatalogT **leaf_catalog);
240 
241  CatalogT *LoadFreeCatalog(const PathString &mountpoint,
242  const shash::Any &hash);
243 
244  bool AttachCatalog(const std::string &db_path, CatalogT *new_catalog);
245  void DetachCatalog(CatalogT *catalog);
246  void DetachSubtree(CatalogT *catalog);
247  void DetachSiblings(const PathString &current_tree);
248  void DetachAll() { if (!catalogs_.empty()) DetachSubtree(GetRootCatalog()); }
249  bool IsAttached(const PathString &root_path,
250  CatalogT **attached_catalog) const;
251 
252  CatalogT *FindCatalog(const PathString &path) const;
253 
254  inline void ReadLock() const {
255  int retval = pthread_rwlock_rdlock(rwlock_);
256  assert(retval == 0);
257  }
258  inline void WriteLock() const {
259  int retval = pthread_rwlock_wrlock(rwlock_);
260  assert(retval == 0);
261  }
262  inline void Unlock() const {
263  int retval = pthread_rwlock_unlock(rwlock_);
264  assert(retval == 0);
265  }
266  virtual void EnforceSqliteMemLimit();
267 
268  private:
269  void CheckInodeWatermark();
270 
276  uint64_t inode_gauge_;
277  uint64_t revision_cache_;
296  std::string authz_cache_;
300  uint64_t incarnation_;
301  // TODO(molina) we could just add an atomic global counter instead
303  pthread_rwlock_t *rwlock_;
305  pthread_key_t pkey_sqlitemem_;
308 
309  // Not needed anymore since there are the glue buffers
310  // Catalog *Inode2Catalog(const inode_t inode);
311  std::string PrintHierarchyRecursively(const CatalogT *catalog,
312  const int level) const;
313  std::string PrintMemStatsRecursively(const CatalogT *catalog) const;
314 
315  InodeRange AcquireInodes(uint64_t size);
316  void ReleaseInodes(const InodeRange chunk);
317 }; // class CatalogManager
318 
320  public:
323  virtual bool ValidInode(const uint64_t inode) {
324  return inode >= inode_offset_;
325  }
326  virtual inode_t Annotate(const inode_t raw_inode) {
327  return raw_inode + inode_offset_;
328  }
329  virtual inode_t Strip(const inode_t annotated_inode) {
330  return annotated_inode - inode_offset_;
331  }
332  virtual void IncGeneration(const uint64_t by) {
333  inode_offset_ += by;
334  LogCvmfs(kLogCatalog, kLogDebug, "set inode generation to %lu",
335  inode_offset_);
336  }
337  virtual inode_t GetGeneration() { return inode_offset_; }
338 
339  private:
340  uint64_t inode_offset_;
341 };
342 
349  public:
352  virtual bool ValidInode(const uint64_t inode) {
353  return (inode >= inode_offset_) || (inode == kRootInode);
354  }
355  virtual inode_t Annotate(const inode_t raw_inode) {
356  if (raw_inode <= kRootInode)
357  return kRootInode;
358  return raw_inode + inode_offset_;
359  }
360  virtual inode_t Strip(const inode_t annotated_inode) {
361  if (annotated_inode == kRootInode)
362  return annotated_inode;
363  return annotated_inode - inode_offset_;
364  }
365  virtual void IncGeneration(const uint64_t by) {
366  inode_offset_ += by;
367  LogCvmfs(kLogCatalog, kLogDebug, "set inode generation to %lu",
368  inode_offset_);
369  }
370  virtual inode_t GetGeneration() { return inode_offset_; }
371 
372  private:
373  static const uint64_t kRootInode =
375  uint64_t inode_offset_;
376 };
377 
378 } // namespace catalog
379 
380 #include "catalog_mgr_impl.h"
381 
382 #endif // CVMFS_CATALOG_MGR_H_
LoadError ChangeRoot(const shash::Any &root_hash)
void DetachSubtree(CatalogT *catalog)
#define LogCvmfs(source, mask,...)
Definition: logging.h:22
bool Listing(const std::string &path, DirectoryEntryList *listing)
Definition: catalog_mgr.h:155
bool GetVOMSAuthz(std::string *authz) const
Counter * Register(const std::string &name, const std::string &desc)
Definition: statistics.cc:139
catalog::Counters LookupCounters(const PathString &path, std::string *subcatalog_path, shash::Any *hash)
Statistics(perf::Statistics *statistics)
Definition: catalog_mgr.h:76
const char * Code2Ascii(const LoadError error)
Definition: catalog_mgr.h:56
const std::vector< CatalogT * > & GetCatalogs() const
Definition: catalog_mgr.h:219
virtual inode_t Annotate(const inode_t raw_inode)
Definition: catalog_mgr.h:355
CatalogT * LoadFreeCatalog(const PathString &mountpoint, const shash::Any &hash)
perf::Counter * n_lookup_inode
Definition: catalog_mgr.h:68
bool MountSubtree(const PathString &path, const CatalogT *entry_point, bool can_listing, CatalogT **leaf_catalog)
void DetachSiblings(const PathString &current_tree)
virtual inode_t Annotate(const inode_t raw_inode)
Definition: catalog_mgr.h:326
InodeAnnotation * inode_annotation_
Definition: catalog_mgr.h:302
void Assign(const char *chars, const unsigned length)
Definition: shortstring.h:61
std::string PrintHierarchyRecursively(const CatalogT *catalog, const int level) const
unsigned LookupOptions
Definition: catalog_mgr.h:40
perf::Counter * n_nested_listing
Definition: catalog_mgr.h:73
void DetachCatalog(CatalogT *catalog)
bool IsAttached(const PathString &root_path, CatalogT **attached_catalog) const
virtual inode_t Strip(const inode_t annotated_inode)
Definition: catalog_mgr.h:360
uint64_t inode_t
assert((mem||(size==0))&&"Out Of Memory")
bool ListingStat(const PathString &path, StatEntryList *listing)
bool LookupPath(const PathString &path, const LookupOptions options, DirectoryEntry *entry)
virtual inode_t Strip(const inode_t annotated_inode)
Definition: catalog_mgr.h:329
const unsigned kLookupDefault
Definition: catalog_mgr.h:41
bool Listing(const PathString &path, DirectoryEntryList *listing)
Definition: catalog_mgr.h:152
IntegerMap< uint64_t > OwnerMap
Definition: catalog.h:41
perf::Counter * n_lookup_path_negative
Definition: catalog_mgr.h:70
LoadError Remount(const bool dry_run)
Algorithms
Definition: hash.h:41
bool Listing(const PathString &path, DirectoryEntryList *listing, const bool expand_symlink)
perf::Counter * n_detach_siblings
Definition: catalog_mgr.h:74
virtual void UnloadCatalog(const CatalogT *catalog)
Definition: catalog_mgr.h:217
std::vector< DirectoryEntry > DirectoryEntryList
bool AttachCatalog(const std::string &db_path, CatalogT *new_catalog)
const unsigned kSqliteMemPerThread
Definition: catalog_mgr.h:32
static const uint64_t kRootInode
Definition: catalog_mgr.h:373
virtual bool ValidInode(const uint64_t inode)
Definition: catalog_mgr.h:323
shash::Any GetNestedCatalogHash(const PathString &mountpoint)
Statistics statistics() const
Definition: catalog_mgr.h:170
virtual CatalogT * CreateCatalog(const PathString &mountpoint, const shash::Any &catalog_hash, CatalogT *parent_catalog)=0
perf::Counter * n_listing
Definition: catalog_mgr.h:72
std::vector< CatalogT * > CatalogList
Definition: catalog_mgr.h:119
const unsigned kLookupRawSymlink
Definition: catalog_mgr.h:42
bool ListCatalogSkein(const PathString &path, std::vector< PathString > *result_list)
InodeRange AcquireInodes(uint64_t size)
CatalogT * GetRootCatalog() const
Definition: catalog_mgr.h:192
virtual LoadError LoadCatalog(const PathString &mountpoint, const shash::Any &hash, std::string *catalog_path, shash::Any *catalog_hash)=0
pthread_rwlock_t * rwlock_
Definition: catalog_mgr.h:303
bool LookupPath(const std::string &path, const LookupOptions options, DirectoryEntry *entry)
Definition: catalog_mgr.h:134
bool ListFileChunks(const PathString &path, const shash::Algorithms interpret_hashes_as, FileChunkList *chunks)
perf::Counter * n_lookup_xattrs
Definition: catalog_mgr.h:71
virtual bool ValidInode(const uint64_t inode)
Definition: catalog_mgr.h:352
virtual inode_t Annotate(const inode_t raw_inode)=0
virtual void IncGeneration(const uint64_t by)
Definition: catalog_mgr.h:332
perf::Counter * n_lookup_path
Definition: catalog_mgr.h:69
virtual void IncGeneration(const uint64_t by)
Definition: catalog_mgr.h:365
bool LookupXattrs(const PathString &path, XattrList *xattrs)
bool LookupNested(const PathString &path, PathString *mountpoint, shash::Any *hash, uint64_t *size)
CatalogT * MountCatalog(const PathString &mountpoint, const shash::Any &hash, CatalogT *parent_catalog)
void SetCatalogWatermark(unsigned limit)
virtual void ActivateCatalog(CatalogT *catalog)
Definition: catalog_mgr.h:218
void SetOwnerMaps(const OwnerMap &uid_map, const OwnerMap &gid_map)
void ReleaseInodes(const InodeRange chunk)
CatalogT * FindCatalog(const PathString &path) const
inode_t MangleInode(const inode_t inode) const
Definition: catalog_mgr.h:199
AbstractCatalogManager(perf::Statistics *statistics)
std::string PrintHierarchy() const
std::string PrintMemStatsRecursively(const CatalogT *catalog) const
std::string PrintAllMemStatistics() const
static void size_t size
Definition: smalloc.h:47
static const inode_t kInodeOffset
Definition: catalog_mgr.h:122
void SetInodeAnnotation(InodeAnnotation *new_annotation)