CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
catalog.h
Go to the documentation of this file.
1 
5 #ifndef CVMFS_CATALOG_H_
6 #define CVMFS_CATALOG_H_
7 
8 #include <pthread.h>
9 #include <stdint.h>
10 
11 #include <cassert>
12 #include <map>
13 #include <string>
14 #include <vector>
15 
16 #include "catalog_counters.h"
17 #include "catalog_sql.h"
18 #include "directory_entry.h"
19 #include "file_chunk.h"
20 #include "gtest/gtest_prod.h"
21 #include "hash.h"
22 #include "shortstring.h"
23 #include "sql.h"
24 #include "uid_map.h"
25 #include "xattr.h"
26 
27 namespace swissknife {
28 class CommandMigrate;
29 }
30 
31 namespace catalog {
32 
33 template <class CatalogT>
35 
36 class Catalog;
37 
38 class Counters;
39 
40 typedef std::vector<Catalog *> CatalogList;
41 typedef IntegerMap<uint64_t> OwnerMap; // used to map uid/gid
42 
43 
49 struct InodeRange {
50  uint64_t offset;
51  uint64_t size;
52 
53  InodeRange() : offset(0), size(0) { }
54 
55  inline bool ContainsInode(const inode_t inode) const {
56  return ((inode > offset) && (inode <= size + offset));
57  }
58 
59  inline void MakeDummy() { offset = 1; }
60 
61  inline bool IsInitialized() const { return offset > 0; }
62  inline bool IsDummy() const { return IsInitialized() && size == 0; }
63 };
64 
65 
73  public:
74  virtual ~InodeAnnotation() { }
75  virtual inode_t Annotate(const inode_t raw_inode) = 0;
76  virtual void IncGeneration(const uint64_t by) = 0;
77  virtual inode_t GetGeneration() = 0;
78  virtual bool ValidInode(const uint64_t inode) = 0;
79  virtual inode_t Strip(const inode_t annotated_inode) = 0;
80 };
81 
82 
92  FRIEND_TEST(T_Catalog, NormalizePath);
93  FRIEND_TEST(T_Catalog, PlantPath);
94  friend class swissknife::CommandMigrate; // for catalog version migration
95 
96  public:
97  typedef std::vector<shash::Any> HashVector;
98 
104  static const uint64_t kDefaultTTL = 240;
111  const shash::Any &catalog_hash,
112  Catalog *parent,
113  const bool is_nested = false);
114  virtual ~Catalog();
115 
116  static Catalog *AttachFreely(const std::string &imaginary_mountpoint,
117  const std::string &file,
118  const shash::Any &catalog_hash,
119  Catalog *parent = NULL,
120  const bool is_nested = false);
121 
122  bool OpenDatabase(const std::string &db_path);
123 
124  inline bool LookupPath(const PathString &path, DirectoryEntry *dirent) const {
125  return LookupMd5Path(NormalizePath(path), dirent);
126  }
127  bool LookupRawSymlink(const PathString &path, LinkString *raw_symlink) const;
128  bool LookupXattrsPath(const PathString &path, XattrList *xattrs) const {
129  return LookupXattrsMd5Path(NormalizePath(path), xattrs);
130  }
131 
132  inline bool ListingPath(const PathString &path,
133  DirectoryEntryList *listing,
134  const bool expand_symlink = true) const
135  {
136  return ListingMd5Path(NormalizePath(path), listing, expand_symlink);
137  }
138  bool ListingPathStat(const PathString &path,
139  StatEntryList *listing) const
140  {
141  return ListingMd5PathStat(NormalizePath(path), listing);
142  }
143  bool AllChunksBegin();
144  bool AllChunksNext(shash::Any *hash, zlib::Algorithms *compression_alg);
145  bool AllChunksEnd();
146 
147  inline bool ListPathChunks(const PathString &path,
148  const shash::Algorithms interpret_hashes_as,
149  FileChunkList *chunks) const
150  {
151  return ListMd5PathChunks(NormalizePath(path), interpret_hashes_as, chunks);
152  }
153 
154  CatalogList GetChildren() const;
155  Catalog* FindSubtree(const PathString &path) const;
156  Catalog* FindChild(const PathString &mountpoint) const;
157  void AddChild(Catalog *child);
158  void RemoveChild(Catalog *child);
159 
160  const HashVector& GetReferencedObjects() const;
163  bool OwnsDatabaseFile() const {
164  return ((database_ != NULL) && database_->OwnsFile()) || managed_database_;
165  }
166 
167  uint64_t GetTTL() const;
168  bool HasExplicitTTL() const;
169  uint64_t GetRevision() const;
170  bool GetVOMSAuthz(std::string *authz) const;
171  uint64_t GetLastModified() const;
172  uint64_t GetNumEntries() const;
173  uint64_t GetNumChunks() const;
175  const Counters& GetCounters() const { return counters_; }
176  std::string PrintMemStatistics() const;
177 
178  inline float schema() const { return database().schema_version(); }
179  inline PathString mountpoint() const { return mountpoint_; }
180  inline Catalog* parent() const { return parent_; }
181  inline uint64_t max_row_id() const { return max_row_id_; }
182  inline InodeRange inode_range() const { return inode_range_; }
183  inline void set_inode_range(const InodeRange value) { inode_range_ = value; }
184  inline std::string database_path() const { return database_->filename(); }
185  inline PathString root_prefix() const { return root_prefix_; }
186  inline shash::Any hash() const { return catalog_hash_; }
187  inline bool volatile_flag() const { return volatile_flag_; }
188  inline uint64_t revision() const { return GetRevision(); }
189 
190  inline bool IsInitialized() const {
192  }
193  inline bool IsRoot() const { return is_root_; }
194  bool IsAutogenerated() const {
195  DirectoryEntry dirent;
197  return LookupPath(PathString(
198  mountpoint_.ToString() + "/.cvmfsautocatalog"), &dirent);
199  }
200  inline bool HasParent() const { return parent_ != NULL; }
201  inline virtual bool IsWritable() const { return false; }
202 
203  typedef struct {
206  uint64_t size;
207  } NestedCatalog;
208  typedef std::vector<NestedCatalog> NestedCatalogList;
209  const NestedCatalogList& ListNestedCatalogs() const;
211  bool FindNested(const PathString &mountpoint,
212  shash::Any *hash, uint64_t *size) const;
213 
214  void SetInodeAnnotation(InodeAnnotation *new_annotation);
215  inode_t GetMangledInode(const uint64_t row_id,
216  const uint64_t hardlink_group) const;
217 
218  void SetOwnerMaps(const OwnerMap *uid_map, const OwnerMap *gid_map);
219  uint64_t MapUid(const uint64_t uid) const {
220  if (uid_map_) { return uid_map_->Map(uid); }
221  return uid;
222  }
223  uint64_t MapGid(const uint64_t gid) const {
224  if (gid_map_) { return gid_map_->Map(gid); }
225  return gid;
226  }
227 
228  protected:
229  typedef std::map<uint64_t, inode_t> HardlinkGroupMap;
231 
232  pthread_mutex_t *lock_;
233 
234  bool InitStandalone(const std::string &database_file);
235  bool ReadCatalogCounters();
236 
242  }
243 
244  virtual void InitPreparedStatements();
246 
248 
249  inline const CatalogDatabase &database() const { return *database_; }
250  inline CatalogDatabase &database() { return *database_; }
251  inline void set_parent(Catalog *catalog) { parent_ = catalog; }
252 
254 
255  bool LookupMd5Path(const shash::Md5 &md5path, DirectoryEntry *dirent) const;
256 
257  private:
258  typedef std::map<PathString, Catalog*> NestedCatalogMap;
259 
264  static const shash::Md5 kMd5PathEmpty;
265 
267  kVomsUnknown, // Not yet looked up
268  kVomsNone, // No voms_authz key in properties table
269  kVomsPresent, // voms_authz property available
270  };
271 
272  shash::Md5 NormalizePath(const PathString &path) const;
273  PathString NormalizePath2(const PathString &path) const;
274  PathString PlantPath(const PathString &path) const;
275 
276  void FixTransitionPoint(const shash::Md5 &md5path,
277  DirectoryEntry *dirent) const;
278 
279  bool LookupXattrsMd5Path(const shash::Md5 &md5path, XattrList *xattrs) const;
280  bool ListMd5PathChunks(const shash::Md5 &md5path,
281  const shash::Algorithms interpret_hashes_as,
282  FileChunkList *chunks) const;
283  bool ListingMd5Path(const shash::Md5 &md5path,
284  DirectoryEntryList *listing,
285  const bool expand_symlink = true) const;
286  bool ListingMd5PathStat(const shash::Md5 &md5path,
287  StatEntryList *listing) const;
288  bool LookupEntry(const shash::Md5 &md5path, const bool expand_symlink,
289  DirectoryEntry *dirent) const;
290 
292 
308  const bool is_root_;
310 
315 
317  mutable std::string voms_authz_;
318 
321  uint64_t max_row_id_;
324  // Point to the maps in the catalog manager
327 
336 
338 }; // class Catalog
339 
340 } // namespace catalog
341 
342 #endif // CVMFS_CATALOG_H_
InodeRange inode_range() const
Definition: catalog.h:182
PathString PlantPath(const PathString &path) const
Definition: catalog.cc:284
const Counters & GetCounters() const
Definition: catalog.h:175
SqlAllChunks * sql_all_chunks_
Definition: catalog.h:333
NestedCatalogMap children_
Definition: catalog.h:312
bool AllChunksNext(shash::Any *hash, zlib::Algorithms *compression_alg)
Definition: catalog.cc:427
HardlinkGroupMap hardlink_groups_
Definition: catalog.h:230
bool IsRoot() const
Definition: catalog.h:193
std::string database_path() const
Definition: catalog.h:184
uint64_t MapUid(const uint64_t uid) const
Definition: catalog.h:219
uint64_t revision() const
Definition: catalog.h:188
virtual ~InodeAnnotation()
Definition: catalog.h:74
const std::string & filename() const
Definition: sql.h:148
CatalogDatabase * database_
Definition: catalog.h:291
NestedCatalogList nested_catalog_cache_
Definition: catalog.h:313
bool ListPathChunks(const PathString &path, const shash::Algorithms interpret_hashes_as, FileChunkList *chunks) const
Definition: catalog.h:147
bool HasParent() const
Definition: catalog.h:200
Catalog(const PathString &mountpoint, const shash::Any &catalog_hash, Catalog *parent, const bool is_nested=false)
Definition: catalog.cc:49
bool LookupXattrsMd5Path(const shash::Md5 &md5path, XattrList *xattrs) const
Definition: catalog.cc:345
const OwnerMap * gid_map_
Definition: catalog.h:326
bool OpenDatabase(const std::string &db_path)
Definition: catalog.cc:164
const OwnerMap * uid_map_
Definition: catalog.h:325
SqlOwnNestedCatalogListing * sql_own_list_nested_
Definition: catalog.h:332
void MakeDummy()
Definition: catalog.h:59
PathString NormalizePath2(const PathString &path) const
Definition: catalog.cc:267
Catalog * parent_
Definition: catalog.h:311
bool LookupPath(const PathString &path, DirectoryEntry *dirent) const
Definition: catalog.h:124
static const shash::Md5 kMd5PathEmpty
Definition: catalog.h:264
std::map< PathString, Catalog * > NestedCatalogMap
Definition: catalog.h:258
bool ListingPath(const PathString &path, DirectoryEntryList *listing, const bool expand_symlink=true) const
Definition: catalog.h:132
bool IsInitialized() const
Definition: catalog.h:61
virtual inode_t Strip(const inode_t annotated_inode)=0
bool AllChunksBegin()
Definition: catalog.cc:422
bool managed_database_
Definition: catalog.h:309
float schema() const
Definition: catalog.h:178
Catalog * FindSubtree(const PathString &path) const
Definition: catalog.cc:773
virtual void InitPreparedStatements()
Definition: catalog.cc:98
pthread_mutex_t * lock_
Definition: catalog.h:232
shash::Any GetPreviousRevision() const
Definition: catalog.cc:555
uint64_t inode_t
assert((mem||(size==0))&&"Out Of Memory")
uint64_t GetTTL() const
Definition: catalog.cc:495
float schema_version() const
Definition: sql.h:149
CatalogList GetChildren() const
Definition: catalog.cc:752
bool InitStandalone(const std::string &database_file)
Definition: catalog.cc:122
Catalog * parent() const
Definition: catalog.h:180
std::string PrintMemStatistics() const
Definition: catalog.cc:565
bool ListMd5PathChunks(const shash::Md5 &md5path, const shash::Algorithms interpret_hashes_as, FileChunkList *chunks) const
Definition: catalog.cc:443
bool OwnsDatabaseFile() const
Definition: catalog.h:163
Catalog * FindChild(const PathString &mountpoint) const
Definition: catalog.cc:808
void SetInodeAnnotation(InodeAnnotation *new_annotation)
Definition: catalog.cc:711
bool IsDummy() const
Definition: catalog.h:62
const NestedCatalogList & ListNestedCatalogs() const
Definition: catalog.cc:630
SqlLookupXattrs * sql_lookup_xattrs_
Definition: catalog.h:335
bool ListingMd5Path(const shash::Md5 &md5path, DirectoryEntryList *listing, const bool expand_symlink=true) const
Definition: catalog.cc:402
bool LookupEntry(const shash::Md5 &md5path, const bool expand_symlink, DirectoryEntry *dirent) const
Definition: catalog.cc:303
IntegerMap< uint64_t > OwnerMap
Definition: catalog.h:41
void FinalizePreparedStatements()
Definition: catalog.cc:110
SqlListing * sql_listing_
Definition: catalog.h:328
InodeRange inode_range_
Definition: catalog.h:320
PathString root_prefix_
Definition: catalog.h:294
const shash::Any catalog_hash_
Definition: catalog.h:293
uint64_t size
Definition: catalog.h:51
InodeAnnotation * inode_annotation_
Definition: catalog.h:322
bool GetVOMSAuthz(std::string *authz) const
Definition: catalog.cc:507
Algorithms
Definition: hash.h:39
std::vector< shash::Any > HashVector
Definition: catalog.h:97
uint64_t GetNumEntries() const
Definition: catalog.cc:546
uint64_t GetRevision() const
Definition: catalog.cc:528
bool IsAutogenerated() const
Definition: catalog.h:194
std::vector< DirectoryEntry > DirectoryEntryList
CatalogDatabase & database()
Definition: catalog.h:250
uint64_t GetLastModified() const
Definition: catalog.cc:533
Algorithms
Definition: compression.h:44
bool HasExplicitTTL() const
Definition: catalog.cc:501
bool AllChunksEnd()
Definition: catalog.cc:433
bool volatile_flag() const
Definition: catalog.h:187
virtual CatalogDatabase::OpenMode DatabaseOpenMode() const
Definition: catalog.h:240
void set_parent(Catalog *catalog)
Definition: catalog.h:251
bool LookupXattrsPath(const PathString &path, XattrList *xattrs) const
Definition: catalog.h:128
FRIEND_TEST(T_Catalog, NormalizePath)
SqlNestedCatalogListing * sql_list_nested_
Definition: catalog.h:331
const HashVector & GetReferencedObjects() const
Definition: catalog.cc:464
bool LookupMd5Path(const shash::Md5 &md5path, DirectoryEntry *dirent) const
Definition: catalog.cc:327
void AddChild(Catalog *child)
Definition: catalog.cc:730
uint64_t max_row_id() const
Definition: catalog.h:181
virtual inode_t GetGeneration()=0
shash::Md5 NormalizePath(const PathString &path) const
Definition: catalog.cc:240
void FixTransitionPoint(const shash::Md5 &md5path, DirectoryEntry *dirent) const
Definition: catalog.cc:827
void ResetNestedCatalogCacheUnprotected()
Definition: catalog.cc:681
bool ListingMd5PathStat(const shash::Md5 &md5path, StatEntryList *listing) const
Definition: catalog.cc:369
PathString mountpoint() const
Definition: catalog.h:179
PathString mountpoint_
Definition: catalog.h:299
void TakeDatabaseFileOwnership()
Definition: catalog.cc:479
SqlLookupPathHash * sql_lookup_md5path_
Definition: catalog.h:329
bool ListingPathStat(const PathString &path, StatEntryList *listing) const
Definition: catalog.h:138
std::map< uint64_t, inode_t > HardlinkGroupMap
Definition: catalog.h:229
void RemoveChild(Catalog *child)
Definition: catalog.cc:743
PathString root_prefix() const
Definition: catalog.h:185
std::vector< Catalog * > CatalogList
Definition: catalog.h:38
bool is_regular_mountpoint_
Definition: catalog.h:303
bool OwnsFile() const
Definition: sql.h:221
bool IsInitialized() const
Definition: catalog.h:190
void set_inode_range(const InodeRange value)
Definition: catalog.h:183
std::string ToString() const
Definition: shortstring.h:114
inode_t GetMangledInode(const uint64_t row_id, const uint64_t hardlink_group) const
Definition: catalog.cc:593
std::vector< NestedCatalog > NestedCatalogList
Definition: catalog.h:208
const NestedCatalogList ListOwnNestedCatalogs() const
Definition: catalog.cc:656
SqlChunksListing * sql_chunks_listing_
Definition: catalog.h:334
virtual inode_t Annotate(const inode_t raw_inode)=0
ShortString< kDefaultMaxPath, 0 > PathString
Definition: shortstring.h:190
virtual ~Catalog()
Definition: catalog.cc:84
uint64_t GetNumChunks() const
Definition: catalog.cc:541
void SetOwnerMaps(const OwnerMap *uid_map, const OwnerMap *gid_map)
Definition: catalog.cc:720
HashVector referenced_hashes_
Definition: catalog.h:337
static Catalog * AttachFreely(const std::string &imaginary_mountpoint, const std::string &file, const shash::Any &catalog_hash, Catalog *parent=NULL, const bool is_nested=false)
Definition: catalog.cc:29
const CatalogDatabase & database() const
Definition: catalog.h:249
bool FindNested(const PathString &mountpoint, shash::Any *hash, uint64_t *size) const
Definition: catalog.cc:690
std::string voms_authz_
Definition: catalog.h:317
bool LookupRawSymlink(const PathString &path, LinkString *raw_symlink) const
Definition: catalog.cc:334
virtual bool ValidInode(const uint64_t inode)=0
bool ReadCatalogCounters()
Definition: catalog.cc:135
void DropDatabaseFileOwnership()
Definition: catalog.cc:487
const bool is_root_
Definition: catalog.h:308
Counters & GetWritableCounters()
Definition: catalog.h:247
shash::Any hash() const
Definition: catalog.h:186
virtual void IncGeneration(const uint64_t by)=0
bool initialized_
Definition: catalog.h:319
uint64_t max_row_id_
Definition: catalog.h:321
uint64_t offset
Definition: catalog.h:50
bool volatile_flag_
Definition: catalog.h:304
virtual bool IsWritable() const
Definition: catalog.h:201
bool ContainsInode(const inode_t inode) const
Definition: catalog.h:55
static void size_t size
Definition: smalloc.h:47
bool nested_catalog_cache_dirty_
Definition: catalog.h:314
SqlNestedCatalogLookup * sql_lookup_nested_
Definition: catalog.h:330
VomsAuthzStatus voms_authz_status_
Definition: catalog.h:316
static const uint64_t kDefaultTTL
Definition: catalog.h:104
Counters counters_
Definition: catalog.h:323
uint64_t MapGid(const uint64_t gid) const
Definition: catalog.h:223