Head

GCC Code Coverage Report

Directory:	cvmfs/		Exec	Total	Coverage
File:	cvmfs/catalog_mgr.h	Lines:	53	61	86.9 %
Date:	2019-02-03 02:48:13	Branches:	8	14	57.1 %


/**
 * This file is part of the CernVM File System.
 */

#ifndef CVMFS_CATALOG_MGR_H_
#define CVMFS_CATALOG_MGR_H_

#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS
#endif

#include <inttypes.h>
#include <pthread.h>

#include <cassert>
#include <map>
#include <string>
#include <vector>

#include "atomic.h"
#include "catalog.h"
#include "directory_entry.h"
#include "file_chunk.h"
#include "hash.h"
#include "logging.h"
#include "statistics.h"

class XattrList;

namespace catalog {

const unsigned kSqliteMemPerThread = 1*1024*1024;

/**
 * Lookup a directory entry including its parent entry or not.
 */
enum LookupOptions {
  kLookupSole        = 0x01,
  // kLookupFull        = 0x02  not used anymore
  kLookupRawSymlink  = 0x10,
};


/**
 * Results upon loading a catalog file.
 */
enum LoadError {
  kLoadNew = 0,
  kLoadUp2Date,
  kLoadNoSpace,
  kLoadFail,

  kLoadNumEntries
};

inline const char *Code2Ascii(const LoadError error) {
  const char *texts[kLoadNumEntries + 1];
  texts[0] = "loaded new catalog";
  texts[1] = "catalog was up to date";
  texts[2] = "not enough space to load catalog";
  texts[3] = "failed to load catalog";
  texts[4] = "no text";
  return texts[error];
}


struct Statistics {
  perf::Counter *n_lookup_inode;
  perf::Counter *n_lookup_path;
  perf::Counter *n_lookup_path_negative;
  perf::Counter *n_lookup_xattrs;
  perf::Counter *n_listing;
  perf::Counter *n_nested_listing;

  explicit Statistics(perf::Statistics *statistics) {
    n_lookup_inode = statistics->Register("catalog_mgr.n_lookup_inode",
        "Number of inode lookups");
    n_lookup_path = statistics->Register("catalog_mgr.n_lookup_path",
        "Number of path lookups");
    n_lookup_path_negative = statistics->Register(
        "catalog_mgr.n_lookup_path_negative",
        "Number of negative path lookups");
    n_lookup_xattrs = statistics->Register("catalog_mgr.n_lookup_xattrs",
        "Number of xattrs lookups");
    n_listing = statistics->Register("catalog_mgr.n_listing",
        "Number of listings");
    n_nested_listing = statistics->Register("catalog_mgr.n_nested_listing",
        "Number of listings of nested catalogs");
  }
};


class InodeGenerationAnnotation : public InodeAnnotation {
 public:
  InodeGenerationAnnotation() { inode_offset_ = 0; }
  ~InodeGenerationAnnotation() { }
  bool ValidInode(const uint64_t inode) {
    return inode >= inode_offset_;
  }
  inode_t Annotate(const inode_t raw_inode) {
    return raw_inode + inode_offset_;
  }
  inode_t Strip(const inode_t annotated_inode) {
    return annotated_inode - inode_offset_;
  }
  void IncGeneration(const uint64_t by) {
    inode_offset_ += by;
    LogCvmfs(kLogCatalog, kLogDebug, "set inode generation to %lu",
             inode_offset_);
  }
  inode_t GetGeneration() { return inode_offset_; }

 private:
  uint64_t inode_offset_;
};

template <class CatalogT>
class AbstractCatalogManager;


/**
 * This class provides the read-only interface to a tree of catalogs
 * representing a (subtree of a) repository.
 * Mostly lookup functions filling DirectoryEntry objects.
 * Reloading of expired catalogs, attaching of nested catalogs and delegating
 * of lookups to the appropriate catalog is done transparently.
 *
 * The loading / creating of catalogs is up to derived classes.
 *
 * CatalogT is either Catalog or MockCatalog.
 *
 * Usage:
 *   DerivedCatalogManager *catalog_manager = new DerivedCatalogManager();
 *   catalog_manager->Init();
 *   catalog_manager->Lookup(<inode>, &<result_entry>);
 */
template <class CatalogT>
class AbstractCatalogManager : public SingleCopy {
 public:
  typedef std::vector<CatalogT*> CatalogList;
  typedef CatalogT catalog_t;

  static const inode_t kInodeOffset = 255;
  explicit AbstractCatalogManager(perf::Statistics *statistics);
  virtual ~AbstractCatalogManager();

  void SetInodeAnnotation(InodeAnnotation *new_annotation);
  virtual bool Init();
  LoadError Remount(const bool dry_run);
  void DetachNested();

  bool LookupPath(const PathString &path, const LookupOptions options,
                  DirectoryEntry *entry);
  bool LookupPath(const std::string &path, const LookupOptions options,
                  DirectoryEntry *entry)
  {
    PathString p;
    p.Assign(&path[0], path.length());
    return LookupPath(p, options, entry);
  }
  bool LookupXattrs(const PathString &path, XattrList *xattrs);

  bool LookupNested(const PathString &path,
                    PathString *mountpoint,
                    shash::Any *hash,
                    uint64_t *size);
  bool ListCatalogSkein(const PathString &path,
                        std::vector<PathString> *result_list);

  bool Listing(const PathString &path, DirectoryEntryList *listing);
  bool Listing(const std::string &path, DirectoryEntryList *listing) {
    PathString p;
    p.Assign(&path[0], path.length());
    return Listing(p, listing);
  }
  bool ListingStat(const PathString &path, StatEntryList *listing);

  bool ListFileChunks(const PathString &path,
                      const shash::Algorithms interpret_hashes_as,
                      FileChunkList *chunks);
  void SetOwnerMaps(const OwnerMap &uid_map, const OwnerMap &gid_map);
  void SetCatalogWatermark(unsigned limit);

  shash::Any GetNestedCatalogHash(const PathString &mountpoint);

  Statistics statistics() const { return statistics_; }
  uint64_t inode_gauge() {
    ReadLock(); uint64_t r = inode_gauge_; Unlock(); return r;
  }
  bool volatile_flag() const { return volatile_flag_; }
  uint64_t GetRevision() const;
  uint64_t GetTTL() const;
  bool HasExplicitTTL() const;
  bool GetVOMSAuthz(std::string *authz) const;
  int GetNumCatalogs() const;
  std::string PrintHierarchy() const;
  std::string PrintAllMemStatistics() const;

  /**
   * Get the inode number of the root DirectoryEntry
   * ('root' means the root of the whole file system)
   * @return the root inode number
   */
  inline inode_t GetRootInode() const {
    return inode_annotation_ ?
      inode_annotation_->Annotate(kInodeOffset + 1) : kInodeOffset + 1;
  }
  inline CatalogT* GetRootCatalog() const { return catalogs_.front(); }
  /**
   * Inodes are ambiquitous under some circumstances, to prevent problems
   * they must be passed through this method first
   * @param inode the raw inode
   * @return the revised inode
   */
  inline inode_t MangleInode(const inode_t inode) const {
    return (inode <= kInodeOffset) ? GetRootInode() : inode;
  }

 protected:
  /**
   * Load the catalog and return a file name and the catalog hash. Derived
   * class can decide if it wants to use the hash or the path.
   * Both the input as well as the output hash can be 0.
   */
  virtual LoadError LoadCatalog(const PathString &mountpoint,
                                const shash::Any &hash,
                                std::string  *catalog_path,
                                shash::Any   *catalog_hash) = 0;
  virtual void UnloadCatalog(const CatalogT *catalog) { }
  virtual void ActivateCatalog(CatalogT *catalog) { }
  const std::vector<CatalogT*>& GetCatalogs() const { return catalogs_; }

  /**
   * Create a new Catalog object.
   * Every derived class has to implement this and return a newly
   * created (derived) Catalog structure of it's desired type.
   * @param mountpoint      the future mountpoint of the catalog to create
   * @param catalog_hash    the content hash of the catalog database
   * @param parent_catalog  the parent of the catalog to create
   * @return a newly created (derived) Catalog
   */
  virtual CatalogT* CreateCatalog(const PathString  &mountpoint,
                                  const shash::Any  &catalog_hash,
                                  CatalogT *parent_catalog) = 0;

  CatalogT *MountCatalog(const PathString &mountpoint, const shash::Any &hash,
                         CatalogT *parent_catalog);
  bool MountSubtree(const PathString &path, const CatalogT *entry_point,
                    CatalogT **leaf_catalog);

  bool AttachCatalog(const std::string &db_path, CatalogT *new_catalog);
  void DetachCatalog(CatalogT *catalog);
  void DetachSubtree(CatalogT *catalog);
  void DetachSiblings(const PathString &current_tree);
  void DetachAll() { if (!catalogs_.empty()) DetachSubtree(GetRootCatalog()); }
  bool IsAttached(const PathString &root_path,
                  CatalogT **attached_catalog) const;

  CatalogT *FindCatalog(const PathString &path) const;

  inline void ReadLock() const {
    int retval = pthread_rwlock_rdlock(rwlock_);
    assert(retval == 0);
  }
  inline void WriteLock() const {
    int retval = pthread_rwlock_wrlock(rwlock_);
    assert(retval == 0);
  }
  inline void Unlock() const {
    int retval = pthread_rwlock_unlock(rwlock_);
    assert(retval == 0);
  }
  virtual void EnforceSqliteMemLimit();

 private:
  void CheckInodeWatermark();

  /**
   * The flat list of all attached catalogs.
   */
  CatalogList catalogs_;
  int inode_watermark_status_;  /**< 0: OK, 1: > 32bit */
  uint64_t inode_gauge_;  /**< highest issued inode */
  uint64_t revision_cache_;
  /**
   * Try to keep number of nested catalogs below the given limit. Zero means no
   * limit. Surpassing the watermark on mounting a catalog triggers
   * a DetachSiblings() call.
   */
  unsigned catalog_watermark_;
  /**
   * Not protected by a read lock because it can only change when the root
   * catalog is exchanged (during big global lock of the file system).
   */
  bool volatile_flag_;
  /**
   * Saves the result of GetVOMSAuthz when a root catalog is attached
   */
  bool has_authz_cache_;
  /**
   * Saves the VOMS requirements when a root catalog is attached
   */
  std::string authz_cache_;
  /**
   * Counts how often the inodes have been invalidated.
   */
  uint64_t incarnation_;
  // TODO(molina) we could just add an atomic global counter instead
  InodeAnnotation *inode_annotation_;  /**< applied to all catalogs */
  pthread_rwlock_t *rwlock_;
  Statistics statistics_;
  pthread_key_t pkey_sqlitemem_;
  OwnerMap uid_map_;
  OwnerMap gid_map_;

  // Not needed anymore since there are the glue buffers
  // Catalog *Inode2Catalog(const inode_t inode);
  std::string PrintHierarchyRecursively(const CatalogT *catalog,
                                        const int level) const;
  std::string PrintMemStatsRecursively(const CatalogT *catalog) const;

  InodeRange AcquireInodes(uint64_t size);
  void ReleaseInodes(const InodeRange chunk);
};  // class CatalogManager

}  // namespace catalog

#include "catalog_mgr_impl.h"

#endif  // CVMFS_CATALOG_MGR_H_


Generated by: GCOVR (Version 4.1)

Line	Branch	Exec	Source
1			/**
2			* This file is part of the CernVM File System.
3			*/
4
5			#ifndef CVMFS_CATALOG_MGR_H_
6			#define CVMFS_CATALOG_MGR_H_
7
8			#ifndef __STDC_FORMAT_MACROS
9			#define __STDC_FORMAT_MACROS
10			#endif
11
12			#include <inttypes.h>
13			#include <pthread.h>
14
15			#include <cassert>
16			#include <map>
17			#include <string>
18			#include <vector>
19
20			#include "atomic.h"
21			#include "catalog.h"
22			#include "directory_entry.h"
23			#include "file_chunk.h"
24			#include "hash.h"
25			#include "logging.h"
26			#include "statistics.h"
27
28			class XattrList;
29
30			namespace catalog {
31
32			const unsigned kSqliteMemPerThread = 110241024;
33
34			/**
35			* Lookup a directory entry including its parent entry or not.
36			*/
37			enum LookupOptions {
38			kLookupSole = 0x01,
39			// kLookupFull = 0x02 not used anymore
40			kLookupRawSymlink = 0x10,
41			};
42
43
44			/**
45			* Results upon loading a catalog file.
46			*/
47			enum LoadError {
48			kLoadNew = 0,
49			kLoadUp2Date,
50			kLoadNoSpace,
51			kLoadFail,
52
53			kLoadNumEntries
54			};
55
56		11	inline const char *Code2Ascii(const LoadError error) {
57			const char *texts[kLoadNumEntries + 1];
58		11	texts[0] = "loaded new catalog";
59		11	texts[1] = "catalog was up to date";
60		11	texts[2] = "not enough space to load catalog";
61		11	texts[3] = "failed to load catalog";
62		11	texts[4] = "no text";
63		11	return texts[error];
64			}
65
66
67			struct Statistics {
68			perf::Counter *n_lookup_inode;
69			perf::Counter *n_lookup_path;
70			perf::Counter *n_lookup_path_negative;
71			perf::Counter *n_lookup_xattrs;
72			perf::Counter *n_listing;
73			perf::Counter *n_nested_listing;
74
75		84	explicit Statistics(perf::Statistics *statistics) {
76			n_lookup_inode = statistics->Register("catalog_mgr.n_lookup_inode",
77		84	"Number of inode lookups");
78			n_lookup_path = statistics->Register("catalog_mgr.n_lookup_path",
79		84	"Number of path lookups");
80			n_lookup_path_negative = statistics->Register(
81			"catalog_mgr.n_lookup_path_negative",
82		84	"Number of negative path lookups");
83			n_lookup_xattrs = statistics->Register("catalog_mgr.n_lookup_xattrs",
84		84	"Number of xattrs lookups");
85			n_listing = statistics->Register("catalog_mgr.n_listing",
86		84	"Number of listings");
87			n_nested_listing = statistics->Register("catalog_mgr.n_nested_listing",
88		84	"Number of listings of nested catalogs");
89		84	}
90			};
91
92
93			class InodeGenerationAnnotation : public InodeAnnotation {
94			public:
95		58	InodeGenerationAnnotation() { inode_offset_ = 0; }
96	✗✓	115	~InodeGenerationAnnotation() { }
97			bool ValidInode(const uint64_t inode) {
98			return inode >= inode_offset_;
99			}
100		1	inode_t Annotate(const inode_t raw_inode) {
101		1	return raw_inode + inode_offset_;
102			}
103			inode_t Strip(const inode_t annotated_inode) {
104			return annotated_inode - inode_offset_;
105			}
106		1	void IncGeneration(const uint64_t by) {
107		1	inode_offset_ += by;
108			LogCvmfs(kLogCatalog, kLogDebug, "set inode generation to %lu",
109		1	inode_offset_);
110		1	}
111		36	inode_t GetGeneration() { return inode_offset_; }
112
113			private:
114			uint64_t inode_offset_;
115			};
116
117			template <class CatalogT>
118			class AbstractCatalogManager;
119
120
121			/**
122			* This class provides the read-only interface to a tree of catalogs
123			* representing a (subtree of a) repository.
124			* Mostly lookup functions filling DirectoryEntry objects.
125			* Reloading of expired catalogs, attaching of nested catalogs and delegating
126			* of lookups to the appropriate catalog is done transparently.
127			*
128			* The loading / creating of catalogs is up to derived classes.
129			*
130			* CatalogT is either Catalog or MockCatalog.
131			*
132			* Usage:
133			* DerivedCatalogManager *catalog_manager = new DerivedCatalogManager();
134			* catalog_manager->Init();
135			* catalog_manager->Lookup(<inode>, &<result_entry>);
136			*/
137			template <class CatalogT>
138			class AbstractCatalogManager : public SingleCopy {
139			public:
140			typedef std::vector<CatalogT*> CatalogList;
141			typedef CatalogT catalog_t;
142
143			static const inode_t kInodeOffset = 255;
144			explicit AbstractCatalogManager(perf::Statistics *statistics);
145			virtual ~AbstractCatalogManager();
146
147			void SetInodeAnnotation(InodeAnnotation *new_annotation);
148			virtual bool Init();
149			LoadError Remount(const bool dry_run);
150			void DetachNested();
151
152			bool LookupPath(const PathString &path, const LookupOptions options,
153			DirectoryEntry *entry);
154		15	bool LookupPath(const std::string &path, const LookupOptions options,
155			DirectoryEntry *entry)
156			{
157		15	PathString p;
158		15	p.Assign(&path[0], path.length());
159		15	return LookupPath(p, options, entry);
160			}
161			bool LookupXattrs(const PathString &path, XattrList *xattrs);
162
163			bool LookupNested(const PathString &path,
164			PathString *mountpoint,
165			shash::Any *hash,
166			uint64_t *size);
167			bool ListCatalogSkein(const PathString &path,
168			std::vector<PathString> *result_list);
169
170			bool Listing(const PathString &path, DirectoryEntryList *listing);
171		17	bool Listing(const std::string &path, DirectoryEntryList *listing) {
172		17	PathString p;
173		17	p.Assign(&path[0], path.length());
174		17	return Listing(p, listing);
175			}
176			bool ListingStat(const PathString &path, StatEntryList *listing);
177
178			bool ListFileChunks(const PathString &path,
179			const shash::Algorithms interpret_hashes_as,
180			FileChunkList *chunks);
181			void SetOwnerMaps(const OwnerMap &uid_map, const OwnerMap &gid_map);
182			void SetCatalogWatermark(unsigned limit);
183
184			shash::Any GetNestedCatalogHash(const PathString &mountpoint);
185
186		1	Statistics statistics() const { return statistics_; }
187			uint64_t inode_gauge() {
188			ReadLock(); uint64_t r = inode_gauge_; Unlock(); return r;
189			}
190		38	bool volatile_flag() const { return volatile_flag_; }
191			uint64_t GetRevision() const;
192			uint64_t GetTTL() const;
193			bool HasExplicitTTL() const;
194			bool GetVOMSAuthz(std::string *authz) const;
195			int GetNumCatalogs() const;
196			std::string PrintHierarchy() const;
197			std::string PrintAllMemStatistics() const;
198
199			/**
200			* Get the inode number of the root DirectoryEntry
201			* ('root' means the root of the whole file system)
202			* @return the root inode number
203			*/
204		2	inline inode_t GetRootInode() const {
205			return inode_annotation_ ?
206	✓✓	2	inode_annotation_->Annotate(kInodeOffset + 1) : kInodeOffset + 1;
207			}
208		290	inline CatalogT* GetRootCatalog() const { return catalogs_.front(); }
209			/**
210			* Inodes are ambiquitous under some circumstances, to prevent problems
211			* they must be passed through this method first
212			* @param inode the raw inode
213			* @return the revised inode
214			*/
215			inline inode_t MangleInode(const inode_t inode) const {
216			return (inode <= kInodeOffset) ? GetRootInode() : inode;
217			}
218
219			protected:
220			/**
221			* Load the catalog and return a file name and the catalog hash. Derived
222			* class can decide if it wants to use the hash or the path.
223			* Both the input as well as the output hash can be 0.
224			*/
225			virtual LoadError LoadCatalog(const PathString &mountpoint,
226			const shash::Any &hash,
227			std::string *catalog_path,
228			shash::Any *catalog_hash) = 0;
229		98	virtual void UnloadCatalog(const CatalogT *catalog) { }
230		31	virtual void ActivateCatalog(CatalogT *catalog) { }
231		1	const std::vector<CatalogT*>& GetCatalogs() const { return catalogs_; }
232
233			/**
234			* Create a new Catalog object.
235			* Every derived class has to implement this and return a newly
236			* created (derived) Catalog structure of it's desired type.
237			* @param mountpoint the future mountpoint of the catalog to create
238			* @param catalog_hash the content hash of the catalog database
239			* @param parent_catalog the parent of the catalog to create
240			* @return a newly created (derived) Catalog
241			*/
242			virtual CatalogT* CreateCatalog(const PathString &mountpoint,
243			const shash::Any &catalog_hash,
244			CatalogT *parent_catalog) = 0;
245
246			CatalogT *MountCatalog(const PathString &mountpoint, const shash::Any &hash,
247			CatalogT *parent_catalog);
248			bool MountSubtree(const PathString &path, const CatalogT *entry_point,
249			CatalogT **leaf_catalog);
250
251			bool AttachCatalog(const std::string &db_path, CatalogT *new_catalog);
252			void DetachCatalog(CatalogT *catalog);
253			void DetachSubtree(CatalogT *catalog);
254			void DetachSiblings(const PathString &current_tree);
255	✓✓	85	void DetachAll() { if (!catalogs_.empty()) DetachSubtree(GetRootCatalog()); }
256			bool IsAttached(const PathString &root_path,
257			CatalogT **attached_catalog) const;
258
259			CatalogT *FindCatalog(const PathString &path) const;
260
261		178	inline void ReadLock() const {
262		178	int retval = pthread_rwlock_rdlock(rwlock_);
263	✗✓	178	assert(retval == 0);
264		178	}
265		97	inline void WriteLock() const {
266		97	int retval = pthread_rwlock_wrlock(rwlock_);
267	✗✓	97	assert(retval == 0);
268		97	}
269		275	inline void Unlock() const {
270		275	int retval = pthread_rwlock_unlock(rwlock_);
271	✗✓	275	assert(retval == 0);
272		275	}
273			virtual void EnforceSqliteMemLimit();
274
275			private:
276			void CheckInodeWatermark();
277
278			/**
279			* The flat list of all attached catalogs.
280			*/
281			CatalogList catalogs_;
282			int inode_watermark_status_; /*< 0: OK, 1: > 32bit /
283			uint64_t inode_gauge_; /*< highest issued inode /
284			uint64_t revision_cache_;
285			/**
286			* Try to keep number of nested catalogs below the given limit. Zero means no
287			* limit. Surpassing the watermark on mounting a catalog triggers
288			* a DetachSiblings() call.
289			*/
290			unsigned catalog_watermark_;
291			/**
292			* Not protected by a read lock because it can only change when the root
293			* catalog is exchanged (during big global lock of the file system).
294			*/
295			bool volatile_flag_;
296			/**
297			* Saves the result of GetVOMSAuthz when a root catalog is attached
298			*/
299			bool has_authz_cache_;
300			/**
301			* Saves the VOMS requirements when a root catalog is attached
302			*/
303			std::string authz_cache_;
304			/**
305			* Counts how often the inodes have been invalidated.
306			*/
307			uint64_t incarnation_;
308			// TODO(molina) we could just add an atomic global counter instead
309			InodeAnnotation inode_annotation_; /< applied to all catalogs /
310			pthread_rwlock_t *rwlock_;
311			Statistics statistics_;
312			pthread_key_t pkey_sqlitemem_;
313			OwnerMap uid_map_;
314			OwnerMap gid_map_;
315
316			// Not needed anymore since there are the glue buffers
317			// Catalog *Inode2Catalog(const inode_t inode);
318			std::string PrintHierarchyRecursively(const CatalogT *catalog,
319			const int level) const;
320			std::string PrintMemStatsRecursively(const CatalogT *catalog) const;
321
322			InodeRange AcquireInodes(uint64_t size);
323			void ReleaseInodes(const InodeRange chunk);
324			}; // class CatalogManager
325
326			} // namespace catalog
327
328			#include "catalog_mgr_impl.h"
329
330			#endif // CVMFS_CATALOG_MGR_H_