CernVM-FS  2.10.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
catalog_mgr.h
Go to the documentation of this file.
1 
5 #ifndef CVMFS_CATALOG_MGR_H_
6 #define CVMFS_CATALOG_MGR_H_
7 
8 #ifndef __STDC_FORMAT_MACROS
9 #define __STDC_FORMAT_MACROS
10 #endif
11 
12 #include <inttypes.h>
13 #include <pthread.h>
14 
15 #include <cassert>
16 #include <map>
17 #include <string>
18 #include <vector>
19 
20 #include "atomic.h"
21 #include "catalog.h"
22 #include "directory_entry.h"
23 #include "file_chunk.h"
24 #include "hash.h"
25 #include "logging.h"
26 #include "statistics.h"
27 
28 class XattrList;
29 
30 namespace catalog {
31 
32 const unsigned kSqliteMemPerThread = 1*1024*1024;
33 
38  kLookupSole = 0x01,
39  // kLookupFull = 0x02 not used anymore
41 };
42 
43 
47 enum LoadError {
48  kLoadNew = 0,
52 
54 };
55 
56 inline const char *Code2Ascii(const LoadError error) {
57  const char *texts[kLoadNumEntries + 1];
58  texts[0] = "loaded new catalog";
59  texts[1] = "catalog was up to date";
60  texts[2] = "not enough space to load catalog";
61  texts[3] = "failed to load catalog";
62  texts[4] = "no text";
63  return texts[error];
64 }
65 
66 
67 struct Statistics {
75 
76  explicit Statistics(perf::Statistics *statistics) {
77  n_lookup_inode = statistics->Register("catalog_mgr.n_lookup_inode",
78  "Number of inode lookups");
79  n_lookup_path = statistics->Register("catalog_mgr.n_lookup_path",
80  "Number of path lookups");
81  n_lookup_path_negative = statistics->Register(
82  "catalog_mgr.n_lookup_path_negative",
83  "Number of negative path lookups");
84  n_lookup_xattrs = statistics->Register("catalog_mgr.n_lookup_xattrs",
85  "Number of xattrs lookups");
86  n_listing = statistics->Register("catalog_mgr.n_listing",
87  "Number of listings");
88  n_nested_listing = statistics->Register("catalog_mgr.n_nested_listing",
89  "Number of listings of nested catalogs");
90  n_detach_siblings = statistics->Register("catalog_mgr.n_detach_siblings",
91  "Number of times the CVMFS_CATALOG_WATERMARK was hit");
92  }
93 };
94 
95 
96 template <class CatalogT>
97 class AbstractCatalogManager;
98 
99 
116 template <class CatalogT>
117 class AbstractCatalogManager : public SingleCopy {
118  public:
119  typedef std::vector<CatalogT*> CatalogList;
120  typedef CatalogT catalog_t;
121 
122  static const inode_t kInodeOffset = 255;
124  virtual ~AbstractCatalogManager();
125 
126  void SetInodeAnnotation(InodeAnnotation *new_annotation);
127  virtual bool Init();
128  LoadError Remount(const bool dry_run);
129  LoadError ChangeRoot(const shash::Any &root_hash);
130  void DetachNested();
131 
132  bool LookupPath(const PathString &path, const LookupOptions options,
133  DirectoryEntry *entry);
134  bool LookupPath(const std::string &path, const LookupOptions options,
135  DirectoryEntry *entry)
136  {
137  PathString p;
138  p.Assign(&path[0], path.length());
139  return LookupPath(p, options, entry);
140  }
141  bool LookupXattrs(const PathString &path, XattrList *xattrs);
142 
143  bool LookupNested(const PathString &path,
144  PathString *mountpoint,
145  shash::Any *hash,
146  uint64_t *size);
147  bool ListCatalogSkein(const PathString &path,
148  std::vector<PathString> *result_list);
149 
150  bool Listing(const PathString &path, DirectoryEntryList *listing,
151  const bool expand_symlink);
152  bool Listing(const PathString &path, DirectoryEntryList *listing) {
153  return Listing(path, listing, true);
154  }
155  bool Listing(const std::string &path, DirectoryEntryList *listing) {
156  PathString p;
157  p.Assign(&path[0], path.length());
158  return Listing(p, listing);
159  }
160  bool ListingStat(const PathString &path, StatEntryList *listing);
161 
162  bool ListFileChunks(const PathString &path,
163  const shash::Algorithms interpret_hashes_as,
164  FileChunkList *chunks);
165  void SetOwnerMaps(const OwnerMap &uid_map, const OwnerMap &gid_map);
166  void SetCatalogWatermark(unsigned limit);
167 
168  shash::Any GetNestedCatalogHash(const PathString &mountpoint);
169 
170  Statistics statistics() const { return statistics_; }
171  uint64_t inode_gauge() {
172  ReadLock(); uint64_t r = inode_gauge_; Unlock(); return r;
173  }
174  bool volatile_flag() const { return volatile_flag_; }
175  uint64_t GetRevision() const;
176  uint64_t GetTTL() const;
177  bool HasExplicitTTL() const;
178  bool GetVOMSAuthz(std::string *authz) const;
179  int GetNumCatalogs() const;
180  std::string PrintHierarchy() const;
181  std::string PrintAllMemStatistics() const;
182 
188  inline inode_t GetRootInode() const {
189  return inode_annotation_ ?
191  }
192  inline CatalogT* GetRootCatalog() const { return catalogs_.front(); }
199  inline inode_t MangleInode(const inode_t inode) const {
200  return (inode <= kInodeOffset) ? GetRootInode() : inode;
201  }
202 
204  std::string *subcatalog_path);
205 
206  protected:
212  virtual LoadError LoadCatalog(const PathString &mountpoint,
213  const shash::Any &hash,
214  std::string *catalog_path,
215  shash::Any *catalog_hash) = 0;
216  virtual void UnloadCatalog(const CatalogT *catalog) { }
217  virtual void ActivateCatalog(CatalogT *catalog) { }
218  const std::vector<CatalogT*>& GetCatalogs() const { return catalogs_; }
219 
229  virtual CatalogT* CreateCatalog(const PathString &mountpoint,
230  const shash::Any &catalog_hash,
231  CatalogT *parent_catalog) = 0;
232 
233  CatalogT *MountCatalog(const PathString &mountpoint, const shash::Any &hash,
234  CatalogT *parent_catalog);
235  bool MountSubtree(const PathString &path,
236  const CatalogT *entry_point,
237  bool can_listing,
238  CatalogT **leaf_catalog);
239 
240  CatalogT *LoadFreeCatalog(const PathString &mountpoint,
241  const shash::Any &hash);
242 
243  bool AttachCatalog(const std::string &db_path, CatalogT *new_catalog);
244  void DetachCatalog(CatalogT *catalog);
245  void DetachSubtree(CatalogT *catalog);
246  void DetachSiblings(const PathString &current_tree);
247  void DetachAll() { if (!catalogs_.empty()) DetachSubtree(GetRootCatalog()); }
248  bool IsAttached(const PathString &root_path,
249  CatalogT **attached_catalog) const;
250 
251  CatalogT *FindCatalog(const PathString &path) const;
252 
253  inline void ReadLock() const {
254  int retval = pthread_rwlock_rdlock(rwlock_);
255  assert(retval == 0);
256  }
257  inline void WriteLock() const {
258  int retval = pthread_rwlock_wrlock(rwlock_);
259  assert(retval == 0);
260  }
261  inline void Unlock() const {
262  int retval = pthread_rwlock_unlock(rwlock_);
263  assert(retval == 0);
264  }
265  virtual void EnforceSqliteMemLimit();
266 
267  private:
268  void CheckInodeWatermark();
269 
275  uint64_t inode_gauge_;
276  uint64_t revision_cache_;
295  std::string authz_cache_;
299  uint64_t incarnation_;
300  // TODO(molina) we could just add an atomic global counter instead
302  pthread_rwlock_t *rwlock_;
304  pthread_key_t pkey_sqlitemem_;
307 
308  // Not needed anymore since there are the glue buffers
309  // Catalog *Inode2Catalog(const inode_t inode);
310  std::string PrintHierarchyRecursively(const CatalogT *catalog,
311  const int level) const;
312  std::string PrintMemStatsRecursively(const CatalogT *catalog) const;
313 
314  InodeRange AcquireInodes(uint64_t size);
315  void ReleaseInodes(const InodeRange chunk);
316 }; // class CatalogManager
317 
319  public:
322  virtual bool ValidInode(const uint64_t inode) {
323  return inode >= inode_offset_;
324  }
325  virtual inode_t Annotate(const inode_t raw_inode) {
326  return raw_inode + inode_offset_;
327  }
328  virtual inode_t Strip(const inode_t annotated_inode) {
329  return annotated_inode - inode_offset_;
330  }
331  virtual void IncGeneration(const uint64_t by) {
332  inode_offset_ += by;
333  LogCvmfs(kLogCatalog, kLogDebug, "set inode generation to %lu",
334  inode_offset_);
335  }
336  virtual inode_t GetGeneration() { return inode_offset_; }
337 
338  private:
339  uint64_t inode_offset_;
340 };
341 
348  public:
351  virtual bool ValidInode(const uint64_t inode) {
352  return (inode >= inode_offset_) || (inode == kRootInode);
353  }
354  virtual inode_t Annotate(const inode_t raw_inode) {
355  if (raw_inode <= kRootInode)
356  return kRootInode;
357  return raw_inode + inode_offset_;
358  }
359  virtual inode_t Strip(const inode_t annotated_inode) {
360  if (annotated_inode == kRootInode)
361  return annotated_inode;
362  return annotated_inode - inode_offset_;
363  }
364  virtual void IncGeneration(const uint64_t by) {
365  inode_offset_ += by;
366  LogCvmfs(kLogCatalog, kLogDebug, "set inode generation to %lu",
367  inode_offset_);
368  }
369  virtual inode_t GetGeneration() { return inode_offset_; }
370 
371  private:
372  static const uint64_t kRootInode =
374  uint64_t inode_offset_;
375 };
376 
377 } // namespace catalog
378 
379 #include "catalog_mgr_impl.h"
380 
381 #endif // CVMFS_CATALOG_MGR_H_
LoadError ChangeRoot(const shash::Any &root_hash)
void DetachSubtree(CatalogT *catalog)
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
bool Listing(const std::string &path, DirectoryEntryList *listing)
Definition: catalog_mgr.h:155
bool GetVOMSAuthz(std::string *authz) const
Counter * Register(const std::string &name, const std::string &desc)
Definition: statistics.cc:139
Statistics(perf::Statistics *statistics)
Definition: catalog_mgr.h:76
const char * Code2Ascii(const LoadError error)
Definition: catalog_mgr.h:56
const std::vector< CatalogT * > & GetCatalogs() const
Definition: catalog_mgr.h:218
virtual inode_t Annotate(const inode_t raw_inode)
Definition: catalog_mgr.h:354
CatalogT * LoadFreeCatalog(const PathString &mountpoint, const shash::Any &hash)
perf::Counter * n_lookup_inode
Definition: catalog_mgr.h:68
bool MountSubtree(const PathString &path, const CatalogT *entry_point, bool can_listing, CatalogT **leaf_catalog)
void DetachSiblings(const PathString &current_tree)
virtual inode_t Annotate(const inode_t raw_inode)
Definition: catalog_mgr.h:325
InodeAnnotation * inode_annotation_
Definition: catalog_mgr.h:301
void Assign(const char *chars, const unsigned length)
Definition: shortstring.h:53
std::string PrintHierarchyRecursively(const CatalogT *catalog, const int level) const
perf::Counter * n_nested_listing
Definition: catalog_mgr.h:73
void DetachCatalog(CatalogT *catalog)
bool IsAttached(const PathString &root_path, CatalogT **attached_catalog) const
virtual inode_t Strip(const inode_t annotated_inode)
Definition: catalog_mgr.h:359
uint64_t inode_t
assert((mem||(size==0))&&"Out Of Memory")
bool ListingStat(const PathString &path, StatEntryList *listing)
bool LookupPath(const PathString &path, const LookupOptions options, DirectoryEntry *entry)
virtual inode_t Strip(const inode_t annotated_inode)
Definition: catalog_mgr.h:328
bool Listing(const PathString &path, DirectoryEntryList *listing)
Definition: catalog_mgr.h:152
IntegerMap< uint64_t > OwnerMap
Definition: catalog.h:41
perf::Counter * n_lookup_path_negative
Definition: catalog_mgr.h:70
LoadError Remount(const bool dry_run)
Algorithms
Definition: hash.h:40
bool Listing(const PathString &path, DirectoryEntryList *listing, const bool expand_symlink)
perf::Counter * n_detach_siblings
Definition: catalog_mgr.h:74
virtual void UnloadCatalog(const CatalogT *catalog)
Definition: catalog_mgr.h:216
std::vector< DirectoryEntry > DirectoryEntryList
bool AttachCatalog(const std::string &db_path, CatalogT *new_catalog)
const unsigned kSqliteMemPerThread
Definition: catalog_mgr.h:32
static const uint64_t kRootInode
Definition: catalog_mgr.h:372
virtual bool ValidInode(const uint64_t inode)
Definition: catalog_mgr.h:322
shash::Any GetNestedCatalogHash(const PathString &mountpoint)
Statistics statistics() const
Definition: catalog_mgr.h:170
virtual CatalogT * CreateCatalog(const PathString &mountpoint, const shash::Any &catalog_hash, CatalogT *parent_catalog)=0
perf::Counter * n_listing
Definition: catalog_mgr.h:72
std::vector< CatalogT * > CatalogList
Definition: catalog_mgr.h:119
bool ListCatalogSkein(const PathString &path, std::vector< PathString > *result_list)
InodeRange AcquireInodes(uint64_t size)
CatalogT * GetRootCatalog() const
Definition: catalog_mgr.h:192
virtual LoadError LoadCatalog(const PathString &mountpoint, const shash::Any &hash, std::string *catalog_path, shash::Any *catalog_hash)=0
pthread_rwlock_t * rwlock_
Definition: catalog_mgr.h:302
bool LookupPath(const std::string &path, const LookupOptions options, DirectoryEntry *entry)
Definition: catalog_mgr.h:134
bool ListFileChunks(const PathString &path, const shash::Algorithms interpret_hashes_as, FileChunkList *chunks)
perf::Counter * n_lookup_xattrs
Definition: catalog_mgr.h:71
virtual bool ValidInode(const uint64_t inode)
Definition: catalog_mgr.h:351
virtual inode_t Annotate(const inode_t raw_inode)=0
virtual void IncGeneration(const uint64_t by)
Definition: catalog_mgr.h:331
perf::Counter * n_lookup_path
Definition: catalog_mgr.h:69
catalog::Counters LookupCounters(const PathString &path, std::string *subcatalog_path)
virtual void IncGeneration(const uint64_t by)
Definition: catalog_mgr.h:364
bool LookupXattrs(const PathString &path, XattrList *xattrs)
bool LookupNested(const PathString &path, PathString *mountpoint, shash::Any *hash, uint64_t *size)
CatalogT * MountCatalog(const PathString &mountpoint, const shash::Any &hash, CatalogT *parent_catalog)
void SetCatalogWatermark(unsigned limit)
virtual void ActivateCatalog(CatalogT *catalog)
Definition: catalog_mgr.h:217
void SetOwnerMaps(const OwnerMap &uid_map, const OwnerMap &gid_map)
void ReleaseInodes(const InodeRange chunk)
CatalogT * FindCatalog(const PathString &path) const
inode_t MangleInode(const inode_t inode) const
Definition: catalog_mgr.h:199
AbstractCatalogManager(perf::Statistics *statistics)
std::string PrintHierarchy() const
std::string PrintMemStatsRecursively(const CatalogT *catalog) const
std::string PrintAllMemStatistics() const
static void size_t size
Definition: smalloc.h:47
static const inode_t kInodeOffset
Definition: catalog_mgr.h:122
void SetInodeAnnotation(InodeAnnotation *new_annotation)