CernVM-FS  2.11.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
catalog_mgr.h
Go to the documentation of this file.
1 
5 #ifndef CVMFS_CATALOG_MGR_H_
6 #define CVMFS_CATALOG_MGR_H_
7 
8 #ifndef __STDC_FORMAT_MACROS
9 #define __STDC_FORMAT_MACROS
10 #endif
11 
12 #include <inttypes.h>
13 #include <pthread.h>
14 
15 #include <cassert>
16 #include <map>
17 #include <string>
18 #include <vector>
19 
20 #include "catalog.h"
21 #include "crypto/hash.h"
22 #include "directory_entry.h"
23 #include "file_chunk.h"
24 #include "statistics.h"
25 #include "util/atomic.h"
26 #include "util/logging.h"
27 
28 class XattrList;
29 
30 namespace catalog {
31 
32 const unsigned kSqliteMemPerThread = 1*1024*1024;
33 
34 
40 typedef unsigned LookupOptions;
41 const unsigned kLookupDefault = 0b1;
42 const unsigned kLookupRawSymlink = 0b10;
43 
47 enum LoadError {
48  kLoadNew = 0,
52 
54 };
55 
56 inline const char *Code2Ascii(const LoadError error) {
57  const char *texts[kLoadNumEntries + 1];
58  texts[0] = "loaded new catalog";
59  texts[1] = "catalog was up to date";
60  texts[2] = "not enough space to load catalog";
61  texts[3] = "failed to load catalog";
62  texts[4] = "no text";
63  return texts[error];
64 }
65 
66 
67 struct Statistics {
76 
77  explicit Statistics(perf::Statistics *statistics) {
78  n_lookup_inode = statistics->Register("catalog_mgr.n_lookup_inode",
79  "Number of inode lookups");
80  n_lookup_path = statistics->Register("catalog_mgr.n_lookup_path",
81  "Number of path lookups");
82  n_lookup_path_negative = statistics->Register(
83  "catalog_mgr.n_lookup_path_negative",
84  "Number of negative path lookups");
85  n_lookup_xattrs = statistics->Register("catalog_mgr.n_lookup_xattrs",
86  "Number of xattrs lookups");
87  n_listing = statistics->Register("catalog_mgr.n_listing",
88  "Number of listings");
89  n_nested_listing = statistics->Register("catalog_mgr.n_nested_listing",
90  "Number of listings of nested catalogs");
91  n_detach_siblings = statistics->Register("catalog_mgr.n_detach_siblings",
92  "Number of times the CVMFS_CATALOG_WATERMARK was hit");
93  catalog_revision = statistics->Register("catalog_revision",
94  "Revision number of the root file catalog");
95  }
96 };
97 
98 
99 template <class CatalogT>
100 class AbstractCatalogManager;
101 
102 
119 template <class CatalogT>
120 class AbstractCatalogManager : public SingleCopy {
121  public:
122  typedef std::vector<CatalogT*> CatalogList;
123  typedef CatalogT catalog_t;
124 
125  static const inode_t kInodeOffset = 255;
127  virtual ~AbstractCatalogManager();
128 
129  void SetInodeAnnotation(InodeAnnotation *new_annotation);
130  virtual bool Init();
131  LoadError Remount(const bool dry_run);
132  LoadError ChangeRoot(const shash::Any &root_hash);
133  void DetachNested();
134 
135  bool LookupPath(const PathString &path, const LookupOptions options,
136  DirectoryEntry *entry);
137  bool LookupPath(const std::string &path, const LookupOptions options,
138  DirectoryEntry *entry)
139  {
140  PathString p;
141  p.Assign(&path[0], path.length());
142  return LookupPath(p, options, entry);
143  }
144  bool LookupXattrs(const PathString &path, XattrList *xattrs);
145 
146  bool LookupNested(const PathString &path,
147  PathString *mountpoint,
148  shash::Any *hash,
149  uint64_t *size);
150  bool ListCatalogSkein(const PathString &path,
151  std::vector<PathString> *result_list);
152 
153  bool Listing(const PathString &path, DirectoryEntryList *listing,
154  const bool expand_symlink);
155  bool Listing(const PathString &path, DirectoryEntryList *listing) {
156  return Listing(path, listing, true);
157  }
158  bool Listing(const std::string &path, DirectoryEntryList *listing) {
159  PathString p;
160  p.Assign(&path[0], path.length());
161  return Listing(p, listing);
162  }
163  bool ListingStat(const PathString &path, StatEntryList *listing);
164 
165  bool ListFileChunks(const PathString &path,
166  const shash::Algorithms interpret_hashes_as,
167  FileChunkList *chunks);
168  void SetOwnerMaps(const OwnerMap &uid_map, const OwnerMap &gid_map);
169  void SetCatalogWatermark(unsigned limit);
170 
171  shash::Any GetNestedCatalogHash(const PathString &mountpoint);
172 
173  Statistics statistics() const { return statistics_; }
174  uint64_t inode_gauge() {
175  ReadLock(); uint64_t r = inode_gauge_; Unlock(); return r;
176  }
177  bool volatile_flag() const { return volatile_flag_; }
178  uint64_t GetRevision() const;
179  uint64_t GetTTL() const;
180  bool HasExplicitTTL() const;
181  bool GetVOMSAuthz(std::string *authz) const;
182  int GetNumCatalogs() const;
183  std::string PrintHierarchy() const;
184  std::string PrintAllMemStatistics() const;
185 
191  inline inode_t GetRootInode() const {
192  return inode_annotation_ ?
194  }
195  inline CatalogT* GetRootCatalog() const { return catalogs_.front(); }
202  inline inode_t MangleInode(const inode_t inode) const {
203  return (inode <= kInodeOffset) ? GetRootInode() : inode;
204  }
205 
207  std::string *subcatalog_path,
208  shash::Any *hash);
209 
210  protected:
216  virtual LoadError LoadCatalog(const PathString &mountpoint,
217  const shash::Any &hash,
218  std::string *catalog_path,
219  shash::Any *catalog_hash) = 0;
220  virtual void UnloadCatalog(const CatalogT *catalog) { }
221  virtual void ActivateCatalog(CatalogT *catalog) { }
222  const std::vector<CatalogT*>& GetCatalogs() const { return catalogs_; }
223 
233  virtual CatalogT* CreateCatalog(const PathString &mountpoint,
234  const shash::Any &catalog_hash,
235  CatalogT *parent_catalog) = 0;
236 
237  CatalogT *MountCatalog(const PathString &mountpoint, const shash::Any &hash,
238  CatalogT *parent_catalog);
239  bool MountSubtree(const PathString &path,
240  const CatalogT *entry_point,
241  bool can_listing,
242  CatalogT **leaf_catalog);
243 
244  CatalogT *LoadFreeCatalog(const PathString &mountpoint,
245  const shash::Any &hash);
246 
247  bool AttachCatalog(const std::string &db_path, CatalogT *new_catalog);
248  void DetachCatalog(CatalogT *catalog);
249  void DetachSubtree(CatalogT *catalog);
250  void DetachSiblings(const PathString &current_tree);
251  void DetachAll() { if (!catalogs_.empty()) DetachSubtree(GetRootCatalog()); }
252  bool IsAttached(const PathString &root_path,
253  CatalogT **attached_catalog) const;
254 
255  CatalogT *FindCatalog(const PathString &path) const;
256 
257  inline void ReadLock() const {
258  int retval = pthread_rwlock_rdlock(rwlock_);
259  assert(retval == 0);
260  }
261  inline void WriteLock() const {
262  int retval = pthread_rwlock_wrlock(rwlock_);
263  assert(retval == 0);
264  }
265  inline void Unlock() const {
266  int retval = pthread_rwlock_unlock(rwlock_);
267  assert(retval == 0);
268  }
269  virtual void EnforceSqliteMemLimit();
270 
271  private:
272  void CheckInodeWatermark();
273 
279  uint64_t inode_gauge_;
280  uint64_t revision_cache_;
299  std::string authz_cache_;
303  uint64_t incarnation_;
304  // TODO(molina) we could just add an atomic global counter instead
306  pthread_rwlock_t *rwlock_;
308  pthread_key_t pkey_sqlitemem_;
311 
312  // Not needed anymore since there are the glue buffers
313  // Catalog *Inode2Catalog(const inode_t inode);
314  std::string PrintHierarchyRecursively(const CatalogT *catalog,
315  const int level) const;
316  std::string PrintMemStatsRecursively(const CatalogT *catalog) const;
317 
318  InodeRange AcquireInodes(uint64_t size);
319  void ReleaseInodes(const InodeRange chunk);
320 }; // class CatalogManager
321 
323  public:
326  virtual bool ValidInode(const uint64_t inode) {
327  return inode >= inode_offset_;
328  }
329  virtual inode_t Annotate(const inode_t raw_inode) {
330  return raw_inode + inode_offset_;
331  }
332  virtual inode_t Strip(const inode_t annotated_inode) {
333  return annotated_inode - inode_offset_;
334  }
335  virtual void IncGeneration(const uint64_t by) {
336  inode_offset_ += by;
337  LogCvmfs(kLogCatalog, kLogDebug, "set inode generation to %lu",
338  inode_offset_);
339  }
340  virtual inode_t GetGeneration() { return inode_offset_; }
341 
342  private:
343  uint64_t inode_offset_;
344 };
345 
352  public:
355  virtual bool ValidInode(const uint64_t inode) {
356  return (inode >= inode_offset_) || (inode == kRootInode);
357  }
358  virtual inode_t Annotate(const inode_t raw_inode) {
359  if (raw_inode <= kRootInode)
360  return kRootInode;
361  return raw_inode + inode_offset_;
362  }
363  virtual inode_t Strip(const inode_t annotated_inode) {
364  if (annotated_inode == kRootInode)
365  return annotated_inode;
366  return annotated_inode - inode_offset_;
367  }
368  virtual void IncGeneration(const uint64_t by) {
369  inode_offset_ += by;
370  LogCvmfs(kLogCatalog, kLogDebug, "set inode generation to %lu",
371  inode_offset_);
372  }
373  virtual inode_t GetGeneration() { return inode_offset_; }
374 
375  private:
376  static const uint64_t kRootInode =
378  uint64_t inode_offset_;
379 };
380 
381 } // namespace catalog
382 
383 #include "catalog_mgr_impl.h"
384 
385 #endif // CVMFS_CATALOG_MGR_H_
LoadError ChangeRoot(const shash::Any &root_hash)
void DetachSubtree(CatalogT *catalog)
#define LogCvmfs(source, mask,...)
Definition: logging.h:25
bool Listing(const std::string &path, DirectoryEntryList *listing)
Definition: catalog_mgr.h:158
bool GetVOMSAuthz(std::string *authz) const
Counter * Register(const std::string &name, const std::string &desc)
Definition: statistics.cc:160
catalog::Counters LookupCounters(const PathString &path, std::string *subcatalog_path, shash::Any *hash)
Statistics(perf::Statistics *statistics)
Definition: catalog_mgr.h:77
const char * Code2Ascii(const LoadError error)
Definition: catalog_mgr.h:56
const std::vector< CatalogT * > & GetCatalogs() const
Definition: catalog_mgr.h:222
virtual inode_t Annotate(const inode_t raw_inode)
Definition: catalog_mgr.h:358
CatalogT * LoadFreeCatalog(const PathString &mountpoint, const shash::Any &hash)
perf::Counter * n_lookup_inode
Definition: catalog_mgr.h:68
perf::Counter * catalog_revision
Definition: catalog_mgr.h:75
bool MountSubtree(const PathString &path, const CatalogT *entry_point, bool can_listing, CatalogT **leaf_catalog)
void DetachSiblings(const PathString &current_tree)
virtual inode_t Annotate(const inode_t raw_inode)
Definition: catalog_mgr.h:329
InodeAnnotation * inode_annotation_
Definition: catalog_mgr.h:305
void Assign(const char *chars, const unsigned length)
Definition: shortstring.h:61
std::string PrintHierarchyRecursively(const CatalogT *catalog, const int level) const
unsigned LookupOptions
Definition: catalog_mgr.h:40
perf::Counter * n_nested_listing
Definition: catalog_mgr.h:73
void DetachCatalog(CatalogT *catalog)
bool IsAttached(const PathString &root_path, CatalogT **attached_catalog) const
virtual inode_t Strip(const inode_t annotated_inode)
Definition: catalog_mgr.h:363
uint64_t inode_t
assert((mem||(size==0))&&"Out Of Memory")
bool ListingStat(const PathString &path, StatEntryList *listing)
bool LookupPath(const PathString &path, const LookupOptions options, DirectoryEntry *entry)
virtual inode_t Strip(const inode_t annotated_inode)
Definition: catalog_mgr.h:332
const unsigned kLookupDefault
Definition: catalog_mgr.h:41
bool Listing(const PathString &path, DirectoryEntryList *listing)
Definition: catalog_mgr.h:155
IntegerMap< uint64_t > OwnerMap
Definition: catalog.h:41
perf::Counter * n_lookup_path_negative
Definition: catalog_mgr.h:70
LoadError Remount(const bool dry_run)
Algorithms
Definition: hash.h:41
bool Listing(const PathString &path, DirectoryEntryList *listing, const bool expand_symlink)
perf::Counter * n_detach_siblings
Definition: catalog_mgr.h:74
virtual void UnloadCatalog(const CatalogT *catalog)
Definition: catalog_mgr.h:220
std::vector< DirectoryEntry > DirectoryEntryList
bool AttachCatalog(const std::string &db_path, CatalogT *new_catalog)
const unsigned kSqliteMemPerThread
Definition: catalog_mgr.h:32
static const uint64_t kRootInode
Definition: catalog_mgr.h:376
virtual bool ValidInode(const uint64_t inode)
Definition: catalog_mgr.h:326
shash::Any GetNestedCatalogHash(const PathString &mountpoint)
Statistics statistics() const
Definition: catalog_mgr.h:173
virtual CatalogT * CreateCatalog(const PathString &mountpoint, const shash::Any &catalog_hash, CatalogT *parent_catalog)=0
perf::Counter * n_listing
Definition: catalog_mgr.h:72
std::vector< CatalogT * > CatalogList
Definition: catalog_mgr.h:122
const unsigned kLookupRawSymlink
Definition: catalog_mgr.h:42
bool ListCatalogSkein(const PathString &path, std::vector< PathString > *result_list)
InodeRange AcquireInodes(uint64_t size)
CatalogT * GetRootCatalog() const
Definition: catalog_mgr.h:195
virtual LoadError LoadCatalog(const PathString &mountpoint, const shash::Any &hash, std::string *catalog_path, shash::Any *catalog_hash)=0
pthread_rwlock_t * rwlock_
Definition: catalog_mgr.h:306
bool LookupPath(const std::string &path, const LookupOptions options, DirectoryEntry *entry)
Definition: catalog_mgr.h:137
bool ListFileChunks(const PathString &path, const shash::Algorithms interpret_hashes_as, FileChunkList *chunks)
perf::Counter * n_lookup_xattrs
Definition: catalog_mgr.h:71
virtual bool ValidInode(const uint64_t inode)
Definition: catalog_mgr.h:355
virtual inode_t Annotate(const inode_t raw_inode)=0
virtual void IncGeneration(const uint64_t by)
Definition: catalog_mgr.h:335
perf::Counter * n_lookup_path
Definition: catalog_mgr.h:69
virtual void IncGeneration(const uint64_t by)
Definition: catalog_mgr.h:368
bool LookupXattrs(const PathString &path, XattrList *xattrs)
bool LookupNested(const PathString &path, PathString *mountpoint, shash::Any *hash, uint64_t *size)
CatalogT * MountCatalog(const PathString &mountpoint, const shash::Any &hash, CatalogT *parent_catalog)
void SetCatalogWatermark(unsigned limit)
virtual void ActivateCatalog(CatalogT *catalog)
Definition: catalog_mgr.h:221
void SetOwnerMaps(const OwnerMap &uid_map, const OwnerMap &gid_map)
void ReleaseInodes(const InodeRange chunk)
CatalogT * FindCatalog(const PathString &path) const
inode_t MangleInode(const inode_t inode) const
Definition: catalog_mgr.h:202
AbstractCatalogManager(perf::Statistics *statistics)
std::string PrintHierarchy() const
std::string PrintMemStatsRecursively(const CatalogT *catalog) const
std::string PrintAllMemStatistics() const
static void size_t size
Definition: smalloc.h:54
static const inode_t kInodeOffset
Definition: catalog_mgr.h:125
void SetInodeAnnotation(InodeAnnotation *new_annotation)