CernVM-FS  2.12.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
catalog_mgr.h
Go to the documentation of this file.
1 
5 #ifndef CVMFS_CATALOG_MGR_H_
6 #define CVMFS_CATALOG_MGR_H_
7 
8 #ifndef __STDC_FORMAT_MACROS
9 #define __STDC_FORMAT_MACROS
10 #endif
11 
12 #include <inttypes.h>
13 #include <pthread.h>
14 
15 #include <cassert>
16 #include <map>
17 #include <string>
18 #include <vector>
19 
20 #include "catalog.h"
21 #include "crypto/hash.h"
22 #include "directory_entry.h"
23 #include "file_chunk.h"
24 #include "manifest_fetch.h"
25 #include "statistics.h"
26 #include "util/algorithm.h"
27 #include "util/atomic.h"
28 #include "util/logging.h"
29 #include "util/platform.h"
30 
31 class XattrList;
32 namespace catalog {
33 
34 const unsigned kSqliteMemPerThread = 1*1024*1024;
35 
36 
42 typedef unsigned LookupOptions;
43 const unsigned kLookupDefault = 0b1;
44 const unsigned kLookupRawSymlink = 0b10;
45 
49 enum LoadReturn {
50  kLoadNew = 0,
54 
56 };
57 
67  kCtlgNoLocationNeeded = 0, // hash known, no location needed
68  kCtlgLocationMounted, // already loaded in mounted_catalogs_
71 };
72 
93  public:
95  hash_(shash::Any()),
96  mountpoint_(PathString("invalid", 7)), // empty str is root ctlg
97  sqlite_path_(""),
98  root_ctlg_revision_(-1ul),
100  manifest_ensemble_(NULL) { }
102  hash_(hash),
103  mountpoint_(mountpoint),
104  sqlite_path_(""),
105  root_ctlg_revision_(-1ul),
107  manifest_ensemble_(NULL) { }
108 
110  const RootCatalogLocation location) :
111  hash_(hash),
112  mountpoint_(mountpoint),
113  sqlite_path_(""),
114  root_ctlg_revision_(-1ul),
115  root_ctlg_location_(location),
116  manifest_ensemble_(NULL) { }
117 
118  bool IsRootCatalog() {
119  return mountpoint_.IsEmpty();
120  }
121 
122  std::string *GetSqlitePathPtr() { return &sqlite_path_; }
123  shash::Any *GetHashPtr() { return &hash_; }
124 
125  shash::Any hash() const { return hash_; }
126  PathString mountpoint() const { return mountpoint_; }
127  std::string sqlite_path() const { return sqlite_path_; }
128  uint64_t root_ctlg_revision() const { return root_ctlg_revision_; }
130  { return root_ctlg_location_; }
132  { return manifest_ensemble_.weak_ref(); }
133 
136  void SetSqlitePath(const std::string &sqlite_path)
147 
148 
149  private:
150  // mandatory for LoadCatalogByHash()
152  // mandatory for LoadCatalogByHash()
154  // out parameter, path name of the sqlite catalog
155  std::string sqlite_path_;
156  // root catalog: revision is needed for GetNewRootCatalogContext()
158  // root catalog: location is mandatory for LoadCatalogByHash()
160  // root catalog: if location = server mandatory for LoadCatalogByHash()
162 };
163 
164 inline const char *Code2Ascii(const LoadReturn error) {
165  const char *texts[kLoadNumEntries + 1];
166  texts[0] = "loaded new catalog";
167  texts[1] = "catalog was up to date";
168  texts[2] = "not enough space to load catalog";
169  texts[3] = "failed to load catalog";
170  texts[4] = "no text";
171  return texts[error];
172 }
173 
174 
175 struct Statistics {
185 
187 
188  explicit Statistics(perf::Statistics *statistics) {
189  n_lookup_inode = statistics->Register("catalog_mgr.n_lookup_inode",
190  "Number of inode lookups");
191  n_lookup_path = statistics->Register("catalog_mgr.n_lookup_path",
192  "Number of path lookups");
193  n_lookup_path_negative = statistics->Register(
194  "catalog_mgr.n_lookup_path_negative",
195  "Number of negative path lookups");
196  n_lookup_xattrs = statistics->Register("catalog_mgr.n_lookup_xattrs",
197  "Number of xattrs lookups");
198  n_listing = statistics->Register("catalog_mgr.n_listing",
199  "Number of listings");
200  n_nested_listing = statistics->Register("catalog_mgr.n_nested_listing",
201  "Number of listings of nested catalogs");
202  n_detach_siblings = statistics->Register("catalog_mgr.n_detach_siblings",
203  "Number of times the CVMFS_CATALOG_WATERMARK was hit");
204  n_write_lock = statistics->Register("catalog_mgr.n_write_lock",
205  "number of write lock calls");
206  ns_write_lock = statistics->Register("catalog_mgr.ns_write_lock",
207  "time spent in WriteLock() [ns]");
208  catalog_revision = statistics->Register("catalog_revision",
209  "Revision number of the root file catalog");
210  }
211 };
212 
213 
214 template <class CatalogT>
215 class AbstractCatalogManager;
216 
217 
234 template <class CatalogT>
235 class AbstractCatalogManager : public SingleCopy {
236  public:
237  typedef std::vector<CatalogT*> CatalogList;
238  typedef CatalogT catalog_t;
239 
240  static const inode_t kInodeOffset = 255;
242  virtual ~AbstractCatalogManager();
243 
244  void SetInodeAnnotation(InodeAnnotation *new_annotation);
245  virtual bool Init();
248  LoadReturn ChangeRoot(const shash::Any &root_hash);
249  void DetachNested();
250 
251  bool LookupPath(const PathString &path, const LookupOptions options,
252  DirectoryEntry *entry);
253  bool LookupPath(const std::string &path, const LookupOptions options,
254  DirectoryEntry *entry)
255  {
256  PathString p;
257  p.Assign(&path[0], path.length());
258  return LookupPath(p, options, entry);
259  }
260  bool LookupXattrs(const PathString &path, XattrList *xattrs);
261 
262  bool LookupNested(const PathString &path,
263  PathString *mountpoint,
264  shash::Any *hash,
265  uint64_t *size);
266  bool ListCatalogSkein(const PathString &path,
267  std::vector<PathString> *result_list);
268 
269  bool Listing(const PathString &path, DirectoryEntryList *listing,
270  const bool expand_symlink);
271  bool Listing(const PathString &path, DirectoryEntryList *listing) {
272  return Listing(path, listing, true);
273  }
274  bool Listing(const std::string &path, DirectoryEntryList *listing) {
275  PathString p;
276  p.Assign(&path[0], path.length());
277  return Listing(p, listing);
278  }
279  bool ListingStat(const PathString &path, StatEntryList *listing);
280 
281  bool ListFileChunks(const PathString &path,
282  const shash::Algorithms interpret_hashes_as,
283  FileChunkList *chunks);
284  void SetOwnerMaps(const OwnerMap &uid_map, const OwnerMap &gid_map);
285  void SetCatalogWatermark(unsigned limit);
286 
287  shash::Any GetNestedCatalogHash(const PathString &mountpoint);
288 
289  Statistics statistics() const { return statistics_; }
290  uint64_t inode_gauge() {
291  ReadLock(); uint64_t r = inode_gauge_; Unlock(); return r;
292  }
293  bool volatile_flag() const { return volatile_flag_; }
294  uint64_t GetRevision() const;
295  uint64_t GetTimestamp() const;
296  uint64_t GetTTL() const;
297  bool HasExplicitTTL() const;
298  bool GetVOMSAuthz(std::string *authz) const;
299  int GetNumCatalogs() const;
300  std::string PrintHierarchy() const;
301  std::string PrintAllMemStatistics() const;
302 
308  inline inode_t GetRootInode() const {
309  return inode_annotation_ ?
311  }
312  inline CatalogT* GetRootCatalog() const { return catalogs_.front(); }
319  inline inode_t MangleInode(const inode_t inode) const {
320  return (inode <= kInodeOffset) ? GetRootInode() : inode;
321  }
322 
324  std::string *subcatalog_path,
325  shash::Any *hash);
326 
327  protected:
340  virtual LoadReturn LoadCatalogByHash(CatalogContext *ctlg_context) = 0;
341  virtual void UnloadCatalog(const CatalogT *catalog) { }
342  virtual void ActivateCatalog(CatalogT *catalog) { }
343  const std::vector<CatalogT*>& GetCatalogs() const { return catalogs_; }
344 
353  virtual void StageNestedCatalogByHash(const shash::Any & /*hash*/,
354  const PathString & /*mountpoint*/)
355  { }
360  void StageNestedCatalogAndUnlock(const PathString &path,
361  const CatalogT *parent,
362  bool is_listable);
363 
373  virtual CatalogT* CreateCatalog(const PathString &mountpoint,
374  const shash::Any &catalog_hash,
375  CatalogT *parent_catalog) = 0;
376 
377  CatalogT *MountCatalog(const PathString &mountpoint, const shash::Any &hash,
378  CatalogT *parent_catalog);
379  bool MountSubtree(const PathString &path,
380  const CatalogT *entry_point,
381  bool can_listing,
382  CatalogT **leaf_catalog);
383 
384  CatalogT *LoadFreeCatalog(const PathString &mountpoint,
385  const shash::Any &hash);
386 
387  bool AttachCatalog(const std::string &db_path, CatalogT *new_catalog);
388  void DetachCatalog(CatalogT *catalog);
389  void DetachSubtree(CatalogT *catalog);
390  void DetachSiblings(const PathString &current_tree);
391  void DetachAll() { if (!catalogs_.empty()) DetachSubtree(GetRootCatalog()); }
392  bool IsAttached(const PathString &root_path,
393  CatalogT **attached_catalog) const;
394 
395  CatalogT *FindCatalog(const PathString &path) const;
396 
397  uint64_t GetRevisionNoLock() const;
398  uint64_t GetTimestampNoLock() const;
399  inline void ReadLock() const {
400  int retval = pthread_rwlock_rdlock(rwlock_);
401  assert(retval == 0);
402  }
403  inline void WriteLock() const {
404  uint64_t timestamp = platform_monotonic_time_ns();
405  int retval = pthread_rwlock_wrlock(rwlock_);
406  assert(retval == 0);
408  uint64_t duration = platform_monotonic_time_ns() - timestamp;
410  }
411  inline void Unlock() const {
412  int retval = pthread_rwlock_unlock(rwlock_);
413  assert(retval == 0);
414  }
415  virtual void EnforceSqliteMemLimit();
416 
417  private:
418  void CheckInodeWatermark();
419 
425  uint64_t inode_gauge_;
426  uint64_t revision_cache_;
446  std::string authz_cache_;
450  uint64_t incarnation_;
451  // TODO(molina) we could just add an atomic global counter instead
453  pthread_rwlock_t *rwlock_;
455  pthread_key_t pkey_sqlitemem_;
458 
459  // Not needed anymore since there are the glue buffers
460  // Catalog *Inode2Catalog(const inode_t inode);
461  std::string PrintHierarchyRecursively(const CatalogT *catalog,
462  const int level) const;
463  std::string PrintMemStatsRecursively(const CatalogT *catalog) const;
464 
465  InodeRange AcquireInodes(uint64_t size);
466  void ReleaseInodes(const InodeRange chunk);
467 }; // class CatalogManager
468 
470  public:
473  virtual bool ValidInode(const uint64_t inode) {
474  return inode >= inode_offset_;
475  }
476  virtual inode_t Annotate(const inode_t raw_inode) {
477  return raw_inode + inode_offset_;
478  }
479  virtual inode_t Strip(const inode_t annotated_inode) {
480  return annotated_inode - inode_offset_;
481  }
482  virtual void IncGeneration(const uint64_t by) {
483  inode_offset_ += by;
484  LogCvmfs(kLogCatalog, kLogDebug, "set inode generation to %lu",
485  inode_offset_);
486  }
487  virtual inode_t GetGeneration() { return inode_offset_; }
488 
489  private:
490  uint64_t inode_offset_;
491 };
492 
499  public:
502  virtual bool ValidInode(const uint64_t inode) {
503  return (inode >= inode_offset_) || (inode == kRootInode);
504  }
505  virtual inode_t Annotate(const inode_t raw_inode) {
506  if (raw_inode <= kRootInode)
507  return kRootInode;
508  return raw_inode + inode_offset_;
509  }
510  virtual inode_t Strip(const inode_t annotated_inode) {
511  if (annotated_inode == kRootInode)
512  return annotated_inode;
513  return annotated_inode - inode_offset_;
514  }
515  virtual void IncGeneration(const uint64_t by) {
516  inode_offset_ += by;
517  LogCvmfs(kLogCatalog, kLogDebug, "set inode generation to %lu",
518  inode_offset_);
519  }
520  virtual inode_t GetGeneration() { return inode_offset_; }
521 
522  private:
523  static const uint64_t kRootInode =
525  uint64_t inode_offset_;
526 };
527 
528 } // namespace catalog
529 
530 #include "catalog_mgr_impl.h"
531 
532 #endif // CVMFS_CATALOG_MGR_H_
shash::Any hash() const
Definition: catalog_mgr.h:125
void DetachSubtree(CatalogT *catalog)
bool Listing(const std::string &path, DirectoryEntryList *listing)
Definition: catalog_mgr.h:274
bool GetVOMSAuthz(std::string *authz) const
Counter * Register(const std::string &name, const std::string &desc)
Definition: statistics.cc:160
catalog::Counters LookupCounters(const PathString &path, std::string *subcatalog_path, shash::Any *hash)
int64_t Xadd(class Counter *counter, const int64_t delta)
Definition: statistics.h:51
Statistics(perf::Statistics *statistics)
Definition: catalog_mgr.h:188
const std::vector< CatalogT * > & GetCatalogs() const
Definition: catalog_mgr.h:343
virtual inode_t Annotate(const inode_t raw_inode)
Definition: catalog_mgr.h:505
CatalogT * LoadFreeCatalog(const PathString &mountpoint, const shash::Any &hash)
T * weak_ref() const
Definition: pointer.h:42
perf::Counter * n_lookup_inode
Definition: catalog_mgr.h:176
void StageNestedCatalogAndUnlock(const PathString &path, const CatalogT *parent, bool is_listable)
perf::Counter * catalog_revision
Definition: catalog_mgr.h:186
bool MountSubtree(const PathString &path, const CatalogT *entry_point, bool can_listing, CatalogT **leaf_catalog)
void DetachSiblings(const PathString &current_tree)
virtual inode_t Annotate(const inode_t raw_inode)
Definition: catalog_mgr.h:476
InodeAnnotation * inode_annotation_
Definition: catalog_mgr.h:452
RootCatalogLocation root_ctlg_location_
Definition: catalog_mgr.h:159
void Assign(const char *chars, const unsigned length)
Definition: shortstring.h:61
std::string PrintHierarchyRecursively(const CatalogT *catalog, const int level) const
unsigned LookupOptions
Definition: catalog_mgr.h:42
shash::Any * GetHashPtr()
Definition: catalog_mgr.h:123
RootCatalogLocation root_ctlg_location() const
Definition: catalog_mgr.h:129
perf::Counter * n_nested_listing
Definition: catalog_mgr.h:181
void DetachCatalog(CatalogT *catalog)
bool IsAttached(const PathString &root_path, CatalogT **attached_catalog) const
virtual inode_t Strip(const inode_t annotated_inode)
Definition: catalog_mgr.h:510
uint64_t inode_t
assert((mem||(size==0))&&"Out Of Memory")
bool ListingStat(const PathString &path, StatEntryList *listing)
uint64_t platform_monotonic_time_ns()
bool LookupPath(const PathString &path, const LookupOptions options, DirectoryEntry *entry)
virtual inode_t Strip(const inode_t annotated_inode)
Definition: catalog_mgr.h:479
const unsigned kLookupDefault
Definition: catalog_mgr.h:43
bool Listing(const PathString &path, DirectoryEntryList *listing)
Definition: catalog_mgr.h:271
IntegerMap< uint64_t > OwnerMap
Definition: catalog.h:41
std::string sqlite_path() const
Definition: catalog_mgr.h:127
perf::Counter * n_lookup_path_negative
Definition: catalog_mgr.h:178
manifest::ManifestEnsemble * manifest_ensemble() const
Definition: catalog_mgr.h:131
virtual LoadReturn GetNewRootCatalogContext(CatalogContext *result)=0
Algorithms
Definition: hash.h:41
bool Listing(const PathString &path, DirectoryEntryList *listing, const bool expand_symlink)
void SetMountpoint(const PathString &mountpoint)
Definition: catalog_mgr.h:135
perf::Counter * n_detach_siblings
Definition: catalog_mgr.h:182
virtual void UnloadCatalog(const CatalogT *catalog)
Definition: catalog_mgr.h:341
std::vector< DirectoryEntry > DirectoryEntryList
void TakeManifestEnsemble(manifest::ManifestEnsemble *manifest_ensemble)
Definition: catalog_mgr.h:145
bool AttachCatalog(const std::string &db_path, CatalogT *new_catalog)
RootCatalogLocation
Definition: catalog_mgr.h:66
LoadReturn ChangeRoot(const shash::Any &root_hash)
const unsigned kSqliteMemPerThread
Definition: catalog_mgr.h:34
static const uint64_t kRootInode
Definition: catalog_mgr.h:523
CatalogContext(const shash::Any &hash, const PathString &mountpoint)
Definition: catalog_mgr.h:101
virtual bool ValidInode(const uint64_t inode)
Definition: catalog_mgr.h:473
shash::Any GetNestedCatalogHash(const PathString &mountpoint)
Statistics statistics() const
Definition: catalog_mgr.h:289
virtual CatalogT * CreateCatalog(const PathString &mountpoint, const shash::Any &catalog_hash, CatalogT *parent_catalog)=0
perf::Counter * n_listing
Definition: catalog_mgr.h:180
UniquePtr< manifest::ManifestEnsemble > manifest_ensemble_
Definition: catalog_mgr.h:161
perf::Counter * n_write_lock
Definition: catalog_mgr.h:183
virtual void StageNestedCatalogByHash(const shash::Any &, const PathString &)
Definition: catalog_mgr.h:353
void Inc(class Counter *counter)
Definition: statistics.h:50
std::vector< CatalogT * > CatalogList
Definition: catalog_mgr.h:237
const unsigned kLookupRawSymlink
Definition: catalog_mgr.h:44
uint64_t root_ctlg_revision() const
Definition: catalog_mgr.h:128
void SetRootCtlgLocation(RootCatalogLocation root_ctlg_location)
Definition: catalog_mgr.h:140
bool ListCatalogSkein(const PathString &path, std::vector< PathString > *result_list)
InodeRange AcquireInodes(uint64_t size)
CatalogT * GetRootCatalog() const
Definition: catalog_mgr.h:312
pthread_rwlock_t * rwlock_
Definition: catalog_mgr.h:453
std::string * GetSqlitePathPtr()
Definition: catalog_mgr.h:122
bool IsEmpty() const
Definition: shortstring.h:137
bool LookupPath(const std::string &path, const LookupOptions options, DirectoryEntry *entry)
Definition: catalog_mgr.h:253
bool ListFileChunks(const PathString &path, const shash::Algorithms interpret_hashes_as, FileChunkList *chunks)
perf::Counter * ns_write_lock
Definition: catalog_mgr.h:184
perf::Counter * n_lookup_xattrs
Definition: catalog_mgr.h:179
virtual bool ValidInode(const uint64_t inode)
Definition: catalog_mgr.h:502
virtual inode_t Annotate(const inode_t raw_inode)=0
virtual void IncGeneration(const uint64_t by)
Definition: catalog_mgr.h:482
perf::Counter * n_lookup_path
Definition: catalog_mgr.h:177
const char * Code2Ascii(const LoadReturn error)
Definition: catalog_mgr.h:164
virtual void IncGeneration(const uint64_t by)
Definition: catalog_mgr.h:515
bool LookupXattrs(const PathString &path, XattrList *xattrs)
PathString mountpoint() const
Definition: catalog_mgr.h:126
std::string sqlite_path_
Definition: catalog_mgr.h:155
bool LookupNested(const PathString &path, PathString *mountpoint, shash::Any *hash, uint64_t *size)
CatalogT * MountCatalog(const PathString &mountpoint, const shash::Any &hash, CatalogT *parent_catalog)
void SetCatalogWatermark(unsigned limit)
CatalogContext(const shash::Any &hash, const PathString &mountpoint, const RootCatalogLocation location)
Definition: catalog_mgr.h:109
virtual void ActivateCatalog(CatalogT *catalog)
Definition: catalog_mgr.h:342
void SetHash(shash::Any hash)
Definition: catalog_mgr.h:134
void SetOwnerMaps(const OwnerMap &uid_map, const OwnerMap &gid_map)
void ReleaseInodes(const InodeRange chunk)
CatalogT * FindCatalog(const PathString &path) const
inode_t MangleInode(const inode_t inode) const
Definition: catalog_mgr.h:319
AbstractCatalogManager(perf::Statistics *statistics)
std::string PrintHierarchy() const
std::string PrintMemStatsRecursively(const CatalogT *catalog) const
std::string PrintAllMemStatistics() const
static void size_t size
Definition: smalloc.h:54
void SetSqlitePath(const std::string &sqlite_path)
Definition: catalog_mgr.h:136
static const inode_t kInodeOffset
Definition: catalog_mgr.h:240
virtual LoadReturn LoadCatalogByHash(CatalogContext *ctlg_context)=0
void SetInodeAnnotation(InodeAnnotation *new_annotation)
void SetRootCtlgRevision(uint64_t root_ctlg_revision)
Definition: catalog_mgr.h:138
CVMFS_EXPORT void LogCvmfs(const LogSource source, const int mask, const char *format,...)
Definition: logging.cc:528