CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
catalog_sql.h
Go to the documentation of this file.
1 
16 #ifndef CVMFS_CATALOG_SQL_H_
17 #define CVMFS_CATALOG_SQL_H_
18 
19 #ifndef __STDC_FORMAT_MACROS
20 #define __STDC_FORMAT_MACROS
21 #endif
22 
23 #include <inttypes.h>
24 
25 #include <string>
26 
27 #include "compression.h"
28 #include "directory_entry.h"
29 #include "file_chunk.h"
30 #include "hash.h"
31 #include "shortstring.h"
32 #include "sql.h"
33 
34 class XattrList;
35 
36 namespace catalog {
37 
38 class Catalog;
39 
40 
41 class CatalogDatabase : public sqlite::Database<CatalogDatabase> {
42  public:
43  static const float kLatestSchema;
44  static const float kLatestSupportedSchema; // + 1.X catalogs (r/o)
45  // Backwards-compatible schema changes
46  static const unsigned kLatestSchemaRevision;
47 
48  bool CreateEmptyDatabase();
49  bool InsertInitialValues(const std::string &root_path,
50  const bool volatile_content,
51  const std::string &voms_authz,
52  const DirectoryEntry &root_entry
54 
57  bool CompactDatabase() const;
58 
59  double GetRowIdWasteRatio() const;
60  bool SetVOMSAuthz(const std::string&);
61 
62  protected:
63  // TODO(rmeusel): C++11 - constructor inheritance
65  CatalogDatabase(const std::string &filename,
66  const OpenMode open_mode)
67  : sqlite::Database<CatalogDatabase>(filename, open_mode) { }
68 };
69 
70 
71 //------------------------------------------------------------------------------
72 
73 
78 class SqlCatalog : public sqlite::Sql {
79  public:
85  SqlCatalog(const CatalogDatabase &database, const std::string &statement) {
86  Init(database.sqlite_db(), statement);
87  }
88 
95  inline shash::Md5 RetrieveMd5(const int idx_high, const int idx_low) const {
96  return shash::Md5(RetrieveInt64(idx_high), RetrieveInt64(idx_low));
97  }
98 
103  const int idx_column,
104  const shash::Algorithms hash_algo,
105  const char hash_suffix = shash::kSuffixNone) const
106  {
107  // Note: SQLite documentation advises to first define the data type of BLOB
108  // by calling sqlite3_column_XXX() on the column and _afterwards_ get
109  // the number of bytes using sqlite3_column_bytes().
110  //
111  // See: https://www.sqlite.org/c3ref/column_blob.html
112  const unsigned char *buffer = static_cast<const unsigned char *>(
113  RetrieveBlob(idx_column));
114  const int byte_count = RetrieveBytes(idx_column);
115  return (byte_count > 0) ? shash::Any(hash_algo, buffer, hash_suffix)
116  : shash::Any(hash_algo);
117  }
118 
123  const int idx_column,
124  const char hash_suffix = shash::kSuffixNone) const
125  {
126  const std::string hash_string = std::string(reinterpret_cast<const char *>(
127  RetrieveText(idx_column)));
128  return shash::MkFromHexPtr(shash::HexPtr(hash_string), hash_suffix);
129  }
130 
138  inline bool BindMd5(const int idx_high, const int idx_low,
139  const shash::Md5 &hash)
140  {
141  uint64_t high, low;
142  hash.ToIntPair(&high, &low);
143  const bool retval = BindInt64(idx_high, high) && BindInt64(idx_low, low);
144  return retval;
145  }
146 
154  inline bool BindHashBlob(const int idx_column, const shash::Any &hash) {
155  if (hash.IsNull()) {
156  return BindNull(idx_column);
157  } else {
158  return BindBlob(idx_column, hash.digest, hash.GetDigestSize());
159  }
160  }
161 
162  protected:
163  SqlCatalog() : sqlite::Sql() {}
164 };
165 
166 
167 //------------------------------------------------------------------------------
168 
169 
173 class SqlDirent : public SqlCatalog {
174  public:
175  // Definition of bit positions for the flags field of a DirectoryEntry
176  // All other bit positions are unused
177  static const int kFlagDir = 1;
178  // Link in the parent catalog
179  static const int kFlagDirNestedMountpoint = 2;
180  // Link in the child catalog
181  static const int kFlagDirNestedRoot = 32;
182  static const int kFlagFile = 4;
183  static const int kFlagLink = 8;
184  static const int kFlagFileSpecial = 16;
185  static const int kFlagFileChunk = 64;
191  static const int kFlagFileExternal = 128;
192  // as of 2^8: 3 bit for hashes
193  // - 0: SHA-1
194  // - 1: RIPEMD-160
195  // - ...
196  // Corresponds to shash::algorithms with offset in order to support future
197  // hashes
198  static const int kFlagPosHash = 8;
199  // Compression methods, 3 bits starting at 2^11
200  // Corresponds to zlib::Algorithms
201  static const int kFlagPosCompression = 11;
206  static const int kFlagDirBindMountpoint = 0x4000; // 2^14
211  static const int kFlagHidden = 0x8000; // 2^15
215  static const int kFlagDirectIo = 0x10000; // 2^16
216 
217 
218  protected:
225  unsigned CreateDatabaseFlags(const DirectoryEntry &entry) const;
226  void StoreHashAlgorithm(const shash::Algorithms algo, unsigned *flags) const;
227  shash::Algorithms RetrieveHashAlgorithm(const unsigned flags) const;
228  zlib::Algorithms RetrieveCompressionAlgorithm(const unsigned flags) const;
229 
235  uint32_t Hardlinks2Linkcount(const uint64_t hardlinks) const;
236  uint32_t Hardlinks2HardlinkGroup(const uint64_t hardlinks) const;
237  uint64_t MakeHardlinks(const uint32_t hardlink_group,
238  const uint32_t linkcount) const;
239 
245  void ExpandSymlink(LinkString *raw_symlink) const;
246 };
247 
248 
249 //------------------------------------------------------------------------------
250 
251 
252 class SqlDirentWrite : public SqlDirent {
253  public:
259  virtual bool BindDirent(const DirectoryEntry &entry) = 0;
260 
261  protected:
262  bool BindDirentFields(const int hash_idx,
263  const int hardlinks_idx,
264  const int size_idx,
265  const int mode_idx,
266  const int mtime_idx,
267  const int flags_idx,
268  const int name_idx,
269  const int symlink_idx,
270  const int uid_idx,
271  const int gid_idx,
272  const DirectoryEntry &entry);
273 };
274 
275 
276 //------------------------------------------------------------------------------
277 
278 
280  public:
281  explicit SqlListContentHashes(const CatalogDatabase &database);
282  shash::Any GetHash() const;
283 };
284 
285 
286 //------------------------------------------------------------------------------
287 
288 
289 class SqlLookup : public SqlDirent {
290  public:
296  DirectoryEntry GetDirent(const Catalog *catalog,
297  const bool expand_symlink = true) const;
298 
304  shash::Md5 GetPathHash() const;
305 
312 };
313 
314 
315 //------------------------------------------------------------------------------
316 
317 
318 class SqlListing : public SqlLookup {
319  public:
320  explicit SqlListing(const CatalogDatabase &database);
321  bool BindPathHash(const struct shash::Md5 &hash);
322 };
323 
324 
325 //------------------------------------------------------------------------------
326 
327 
328 class SqlLookupPathHash : public SqlLookup {
329  public:
330  explicit SqlLookupPathHash(const CatalogDatabase &database);
331  bool BindPathHash(const struct shash::Md5 &hash);
332 };
333 
334 
335 //------------------------------------------------------------------------------
336 
337 
338 class SqlLookupInode : public SqlLookup {
339  public:
340  explicit SqlLookupInode(const CatalogDatabase &database);
341  bool BindRowId(const uint64_t inode);
342 };
343 
344 
345 //------------------------------------------------------------------------------
346 
347 
364  public:
365  explicit SqlLookupDanglingMountpoints(const CatalogDatabase &database);
366 };
367 
368 
369 //------------------------------------------------------------------------------
370 
371 
380 class SqlDirentTouch : public SqlCatalog {
381  public:
382  explicit SqlDirentTouch(const CatalogDatabase &database);
383 
384  bool BindDirentBase(const DirectoryEntryBase &entry);
385  bool BindPathHash(const shash::Md5 &hash);
386  bool BindXattr(const XattrList &xattrs);
387  bool BindXattrEmpty();
388 };
389 
390 
391 //------------------------------------------------------------------------------
392 
393 
398  public:
399  explicit SqlNestedCatalogLookup(const CatalogDatabase &database);
400  bool BindSearchPath(const PathString &path);
401  shash::Any GetContentHash() const;
402  uint64_t GetSize() const;
403 };
404 
405 
406 //------------------------------------------------------------------------------
407 
408 
413  public:
414  explicit SqlNestedCatalogListing(const CatalogDatabase &database);
415  PathString GetPath() const;
416  shash::Any GetContentHash() const;
417  uint64_t GetSize() const;
418 };
419 
420 
421 //------------------------------------------------------------------------------
422 
423 
428  public:
429  explicit SqlOwnNestedCatalogListing(const CatalogDatabase &database);
430  PathString GetPath() const;
431  shash::Any GetContentHash() const;
432  uint64_t GetSize() const;
433 };
434 
435 
436 //------------------------------------------------------------------------------
437 
438 
440  public:
441  explicit SqlDirentInsert(const CatalogDatabase &database);
442  bool BindPathHash(const shash::Md5 &hash);
443  bool BindParentPathHash(const shash::Md5 &hash);
444  bool BindDirent(const DirectoryEntry &entry);
445  bool BindXattr(const XattrList &xattrs);
446  bool BindXattrEmpty();
447 };
448 
449 
450 //------------------------------------------------------------------------------
451 
452 
454  public:
455  explicit SqlDirentUpdate(const CatalogDatabase &database);
456  bool BindPathHash(const shash::Md5 &hash);
457  bool BindDirent(const DirectoryEntry &entry);
458 };
459 
460 
461 //------------------------------------------------------------------------------
462 
463 
464 class SqlDirentUnlink : public SqlCatalog {
465  public:
466  explicit SqlDirentUnlink(const CatalogDatabase &database);
467  bool BindPathHash(const shash::Md5 &hash);
468 };
469 
470 
471 //------------------------------------------------------------------------------
472 
473 
477 class SqlIncLinkcount : public SqlCatalog {
478  public:
479  explicit SqlIncLinkcount(const CatalogDatabase &database);
480  bool BindPathHash(const shash::Md5 &hash);
481  bool BindDelta(const int delta);
482 };
483 
484 
485 //------------------------------------------------------------------------------
486 
487 
488 class SqlChunkInsert : public SqlCatalog {
489  public:
490  explicit SqlChunkInsert(const CatalogDatabase &database);
491  bool BindPathHash(const shash::Md5 &hash);
492  bool BindFileChunk(const FileChunk &chunk);
493 };
494 
495 
496 //------------------------------------------------------------------------------
497 
498 
499 class SqlChunksRemove : public SqlCatalog {
500  public:
501  explicit SqlChunksRemove(const CatalogDatabase &database);
502  bool BindPathHash(const shash::Md5 &hash);
503 };
504 
505 
506 //------------------------------------------------------------------------------
507 
508 
509 class SqlChunksListing : public SqlCatalog {
510  public:
511  explicit SqlChunksListing(const CatalogDatabase &database);
512  bool BindPathHash(const shash::Md5 &hash);
513  FileChunk GetFileChunk(const shash::Algorithms interpret_hash_as) const;
514 };
515 
516 
517 //------------------------------------------------------------------------------
518 
519 
520 class SqlChunksCount : public SqlCatalog {
521  public:
522  explicit SqlChunksCount(const CatalogDatabase &database);
523  bool BindPathHash(const shash::Md5 &hash);
524  int GetChunkCount() const;
525 };
526 
527 
528 //------------------------------------------------------------------------------
529 
530 
532  public:
533  explicit SqlMaxHardlinkGroup(const CatalogDatabase &database);
534  uint32_t GetMaxGroupId() const;
535 };
536 
537 
538 //------------------------------------------------------------------------------
539 
540 
541 class SqlGetCounter : public SqlCatalog {
542  public:
543  explicit SqlGetCounter(const CatalogDatabase &database);
544  bool BindCounter(const std::string &counter);
545  uint64_t GetCounter() const;
546  private:
547  bool compat_;
548 };
549 
550 
551 //------------------------------------------------------------------------------
552 
553 
554 class SqlUpdateCounter : public SqlCatalog {
555  public:
556  explicit SqlUpdateCounter(const CatalogDatabase &database);
557  bool BindCounter(const std::string &counter);
558  bool BindDelta(const int64_t delta);
559 };
560 
561 
562 //------------------------------------------------------------------------------
563 
564 
565 class SqlCreateCounter : public SqlCatalog {
566  public:
567  explicit SqlCreateCounter(const CatalogDatabase &database);
568  bool BindCounter(const std::string &counter);
569  bool BindInitialValue(const int64_t value);
570 };
571 
572 
573 //------------------------------------------------------------------------------
574 
575 
576 class SqlAllChunks : public SqlCatalog {
577  public:
578  explicit SqlAllChunks(const CatalogDatabase &database);
579  bool Open();
580  bool Next(shash::Any *hash, zlib::Algorithms *compression_alg);
581  bool Close();
582 };
583 
584 
585 //------------------------------------------------------------------------------
586 
587 
588 class SqlLookupXattrs : public SqlCatalog {
589  public:
590  explicit SqlLookupXattrs(const CatalogDatabase &database);
591  bool BindPathHash(const shash::Md5 &hash);
593 };
594 
595 } // namespace catalog
596 
597 #endif // CVMFS_CATALOG_SQL_H_
bool BindSearchPath(const PathString &path)
Definition: catalog_sql.cc:901
static const int kFlagDirNestedMountpoint
Definition: catalog_sql.h:179
bool BindCounter(const std::string &counter)
shash::Any GetHash() const
Definition: catalog_sql.cc:626
SqlAllChunks(const CatalogDatabase &database)
bool IsNull() const
Definition: hash.h:379
shash::Md5 GetPathHash() const
Definition: catalog_sql.cc:683
SqlListing(const CatalogDatabase &database)
Definition: catalog_sql.cc:759
static const int kFlagPosCompression
Definition: catalog_sql.h:201
SqlIncLinkcount(const CatalogDatabase &database)
bool BindDirentBase(const DirectoryEntryBase &entry)
Definition: catalog_sql.cc:833
const std::string & filename() const
Definition: sql.h:148
shash::Algorithms RetrieveHashAlgorithm(const unsigned flags) const
Definition: catalog_sql.cc:463
bool Init(const sqlite3 *database, const std::string &statement)
Definition: sql.cc:132
static const int kFlagDirNestedRoot
Definition: catalog_sql.h:181
bool BindPathHash(const shash::Md5 &hash)
SqlLookupPathHash(const CatalogDatabase &database)
Definition: catalog_sql.cc:774
bool BindDirent(const DirectoryEntry &entry)
void ToIntPair(uint64_t *lo, uint64_t *hi) const
Definition: hash.cc:385
bool BindRowId(const uint64_t inode)
Definition: catalog_sql.cc:794
SqlCreateCounter(const CatalogDatabase &database)
const void * RetrieveBlob(const int idx_column) const
Definition: sql.h:431
bool BindFileChunk(const FileChunk &chunk)
SqlUpdateCounter(const CatalogDatabase &database)
bool BindPathHash(const shash::Md5 &hash)
Definition: catalog_sql.cc:846
static const int kFlagPosHash
Definition: catalog_sql.h:198
SqlGetCounter(const CatalogDatabase &database)
SqlCatalog(const CatalogDatabase &database, const std::string &statement)
Definition: catalog_sql.h:85
DirectoryEntry GetDirent(const Catalog *catalog, const bool expand_symlink=true) const
Definition: catalog_sql.cc:696
shash::Md5 RetrieveMd5(const int idx_high, const int idx_low) const
Definition: catalog_sql.h:95
SqlDirentInsert(const CatalogDatabase &database)
SqlLookupInode(const CatalogDatabase &database)
Definition: catalog_sql.cc:788
bool BindPathHash(const shash::Md5 &hash)
uint32_t Hardlinks2HardlinkGroup(const uint64_t hardlinks) const
Definition: catalog_sql.cc:487
SqlListContentHashes(const CatalogDatabase &database)
Definition: catalog_sql.cc:599
Database(const std::string &filename, const OpenMode open_mode)
Definition: sql_impl.h:21
static const int kFlagFileChunk
Definition: catalog_sql.h:185
SqlNestedCatalogListing(const CatalogDatabase &database)
Definition: catalog_sql.cc:922
shash::Any GetContentHash() const
Definition: catalog_sql.cc:906
bool BindPathHash(const shash::Md5 &hash)
SqlChunksListing(const CatalogDatabase &database)
int GetChunkCount() const
static const int kFlagFile
Definition: catalog_sql.h:182
uint64_t GetCounter() const
unsigned char digest[digest_size_]
Definition: hash.h:122
Sql()
Definition: sql.h:453
static const int kFlagLink
Definition: catalog_sql.h:183
bool BindCounter(const std::string &counter)
bool BindPathHash(const shash::Md5 &hash)
bool BindXattr(const XattrList &xattrs)
Definition: catalog_sql.cc:851
Algorithms
Definition: hash.h:39
zlib::Algorithms RetrieveCompressionAlgorithm(const unsigned flags) const
Definition: catalog_sql.cc:472
bool BindCounter(const std::string &counter)
bool BindDelta(const int delta)
Algorithms
Definition: compression.h:44
unsigned GetDigestSize() const
Definition: hash.h:164
static const int kFlagDirBindMountpoint
Definition: catalog_sql.h:206
bool BindPathHash(const shash::Md5 &hash)
SqlChunksRemove(const CatalogDatabase &database)
bool Next(shash::Any *hash, zlib::Algorithms *compression_alg)
SqlOwnNestedCatalogListing(const CatalogDatabase &database)
Definition: catalog_sql.cc:977
static const int kFlagFileSpecial
Definition: catalog_sql.h:184
static const int kFlagFileExternal
Definition: catalog_sql.h:191
SqlDirentTouch(const CatalogDatabase &database)
Definition: catalog_sql.cc:821
bool BindHashBlob(const int idx_column, const shash::Any &hash)
Definition: catalog_sql.h:154
bool BindXattr(const XattrList &xattrs)
Any MkFromHexPtr(const HexPtr hex, const char suffix)
Definition: hash.cc:83
bool InsertInitialValues(const std::string &root_path, const bool volatile_content, const std::string &voms_authz, const DirectoryEntry &root_entry=DirectoryEntry(kDirentNegative))
Definition: catalog_sql.cc:263
SqlNestedCatalogLookup(const CatalogDatabase &database)
Definition: catalog_sql.cc:869
sqlite3_int64 RetrieveInt64(const int idx_column) const
Definition: sql.h:440
bool SetVOMSAuthz(const std::string &)
Definition: catalog_sql.cc:361
uint64_t MakeHardlinks(const uint32_t hardlink_group, const uint32_t linkcount) const
Definition: catalog_sql.cc:492
bool BindPathHash(const shash::Md5 &hash)
bool BindPathHash(const struct shash::Md5 &hash)
Definition: catalog_sql.cc:780
static const int kFlagDirectIo
Definition: catalog_sql.h:215
SqlDirentUpdate(const CatalogDatabase &database)
sqlite3 * sqlite_db() const
Definition: sql.h:147
static const float kLatestSchema
Definition: catalog_sql.h:43
bool BindInt64(const int index, const sqlite3_int64 value)
Definition: sql.h:377
bool BindPathHash(const struct shash::Md5 &hash)
Definition: catalog_sql.cc:766
shash::Any RetrieveHashBlob(const int idx_column, const shash::Algorithms hash_algo, const char hash_suffix=shash::kSuffixNone) const
Definition: catalog_sql.h:102
shash::Any RetrieveHashHex(const int idx_column, const char hash_suffix=shash::kSuffixNone) const
Definition: catalog_sql.h:122
bool CompactDatabase() const
Definition: catalog_sql.cc:396
bool BindDirentFields(const int hash_idx, const int hardlinks_idx, const int size_idx, const int mode_idx, const int mtime_idx, const int flags_idx, const int name_idx, const int symlink_idx, const int uid_idx, const int gid_idx, const DirectoryEntry &entry)
Definition: catalog_sql.cc:565
const unsigned char * RetrieveText(const int idx_column) const
Definition: sql.h:443
SqlLookupDanglingMountpoints(const CatalogDatabase &database)
Definition: catalog_sql.cc:802
shash::Any GetContentHash() const
Definition: catalog_sql.cc:961
FileChunk GetFileChunk(const shash::Algorithms interpret_hash_as) const
virtual bool BindDirent(const DirectoryEntry &entry)=0
SqlLookupXattrs(const CatalogDatabase &database)
SqlChunkInsert(const CatalogDatabase &database)
static const int kFlagHidden
Definition: catalog_sql.h:211
bool BindBlob(const int index, const void *value, const int size)
Definition: sql.h:355
uint32_t Hardlinks2Linkcount(const uint64_t hardlinks) const
Definition: catalog_sql.cc:482
shash::Md5 GetParentPathHash() const
Definition: catalog_sql.cc:688
static const float kLatestSupportedSchema
Definition: catalog_sql.h:44
double GetRowIdWasteRatio() const
Definition: catalog_sql.cc:366
unsigned CreateDatabaseFlags(const DirectoryEntry &entry) const
Definition: catalog_sql.cc:417
bool BindDelta(const int64_t delta)
bool BindMd5(const int idx_high, const int idx_low, const shash::Md5 &hash)
Definition: catalog_sql.h:138
bool BindNull(const int index)
Definition: sql.h:382
bool BindParentPathHash(const shash::Md5 &hash)
static const unsigned kLatestSchemaRevision
Definition: catalog_sql.h:46
static const int kFlagDir
Definition: catalog_sql.h:177
CatalogDatabase(const std::string &filename, const OpenMode open_mode)
Definition: catalog_sql.h:65
const char kSuffixNone
Definition: hash.h:51
bool BindInitialValue(const int64_t value)
bool BindPathHash(const shash::Md5 &hash)
SqlChunksCount(const CatalogDatabase &database)
int RetrieveBytes(const int idx_column) const
Definition: sql.h:428
bool BindPathHash(const shash::Md5 &hash)
bool BindDirent(const DirectoryEntry &entry)
void StoreHashAlgorithm(const shash::Algorithms algo, unsigned *flags) const
Definition: catalog_sql.cc:454
void ExpandSymlink(LinkString *raw_symlink) const
Definition: catalog_sql.cc:504