CernVM-FS  2.13.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
catalog_sql.h
Go to the documentation of this file.
1 
16 #ifndef CVMFS_CATALOG_SQL_H_
17 #define CVMFS_CATALOG_SQL_H_
18 
19 #ifndef __STDC_FORMAT_MACROS
20 #define __STDC_FORMAT_MACROS
21 #endif
22 
23 #include <inttypes.h>
24 
25 #include <string>
26 
28 #include "crypto/hash.h"
29 #include "directory_entry.h"
30 #include "file_chunk.h"
31 #include "shortstring.h"
32 #include "sql.h"
33 
34 class XattrList;
35 
36 namespace catalog {
37 
38 class Catalog;
39 
40 
41 class CatalogDatabase : public sqlite::Database<CatalogDatabase> {
42  public:
43  static const float kLatestSchema;
44  static const float kLatestSupportedSchema; // + 1.X catalogs (r/o)
45  // Backwards-compatible schema changes
46  static const unsigned kLatestSchemaRevision;
47 
48  bool CreateEmptyDatabase();
50  const std::string &root_path,
51  const bool volatile_content,
52  const std::string &voms_authz,
53  const DirectoryEntry &root_entry = DirectoryEntry(kDirentNegative));
54 
57  bool CompactDatabase() const;
58 
59  double GetRowIdWasteRatio() const;
60  bool SetVOMSAuthz(const std::string &);
61 
62  protected:
63  // TODO(rmeusel): C++11 - constructor inheritance
65  CatalogDatabase(const std::string &filename, const OpenMode open_mode)
66  : sqlite::Database<CatalogDatabase>(filename, open_mode) { }
67 };
68 
69 
70 //------------------------------------------------------------------------------
71 
72 
77 class SqlCatalog : public sqlite::Sql {
78  public:
84  SqlCatalog(const CatalogDatabase &database, const std::string &statement) {
85  Init(database.sqlite_db(), statement);
86  }
87 
94  inline shash::Md5 RetrieveMd5(const int idx_high, const int idx_low) const {
95  return shash::Md5(RetrieveInt64(idx_high), RetrieveInt64(idx_low));
96  }
97 
102  const int idx_column,
103  const shash::Algorithms hash_algo,
104  const char hash_suffix = shash::kSuffixNone) const {
105  // Note: SQLite documentation advises to first define the data type of BLOB
106  // by calling sqlite3_column_XXX() on the column and _afterwards_ get
107  // the number of bytes using sqlite3_column_bytes().
108  //
109  // See: https://www.sqlite.org/c3ref/column_blob.html
110  const unsigned char *buffer = static_cast<const unsigned char *>(
111  RetrieveBlob(idx_column));
112  const int byte_count = RetrieveBytes(idx_column);
113  return (byte_count > 0) ? shash::Any(hash_algo, buffer, hash_suffix)
114  : shash::Any(hash_algo);
115  }
116 
121  const int idx_column, const char hash_suffix = shash::kSuffixNone) const {
122  const std::string hash_string = std::string(
123  reinterpret_cast<const char *>(RetrieveText(idx_column)));
124  return shash::MkFromHexPtr(shash::HexPtr(hash_string), hash_suffix);
125  }
126 
134  inline bool BindMd5(const int idx_high, const int idx_low,
135  const shash::Md5 &hash) {
136  uint64_t high, low;
137  hash.ToIntPair(&high, &low);
138  const bool retval = BindInt64(idx_high, static_cast<int64_t>(high))
139  && BindInt64(idx_low, static_cast<int64_t>(low));
140  return retval;
141  }
142 
150  inline bool BindHashBlob(const int idx_column, const shash::Any &hash) {
151  if (hash.IsNull()) {
152  return BindNull(idx_column);
153  } else {
154  return BindBlob(idx_column, hash.digest, hash.GetDigestSize());
155  }
156  }
157 
158  protected:
159  SqlCatalog() : sqlite::Sql() { }
160 };
161 
162 
163 //------------------------------------------------------------------------------
164 
165 
169 class SqlDirent : public SqlCatalog {
170  public:
171  // Definition of bit positions for the flags field of a DirectoryEntry
172  // All other bit positions are unused
173  static const int kFlagDir = 1;
174  // Link in the parent catalog
175  static const int kFlagDirNestedMountpoint = 2;
176  // Link in the child catalog
177  static const int kFlagDirNestedRoot = 32;
178  static const int kFlagFile = 4;
179  static const int kFlagLink = 8;
180  static const int kFlagFileSpecial = 16;
181  static const int kFlagFileChunk = 64;
187  static const int kFlagFileExternal = 128;
188  // as of 2^8: 3 bit for hashes
189  // - 0: SHA-1
190  // - 1: RIPEMD-160
191  // - ...
192  // Corresponds to shash::algorithms with offset in order to support future
193  // hashes
194  static const int kFlagPosHash = 8;
195  // Compression methods, 3 bits starting at 2^11
196  // Corresponds to zlib::Algorithms
197  static const int kFlagPosCompression = 11;
202  static const int kFlagDirBindMountpoint = 0x4000; // 2^14
207  static const int kFlagHidden = 0x8000; // 2^15
211  static const int kFlagDirectIo = 0x10000; // 2^16
212 
213 
214  protected:
221  unsigned CreateDatabaseFlags(const DirectoryEntry &entry) const;
222  void StoreHashAlgorithm(const shash::Algorithms algo, unsigned *flags) const;
223  shash::Algorithms RetrieveHashAlgorithm(const unsigned flags) const;
224  zlib::Algorithms RetrieveCompressionAlgorithm(const unsigned flags) const;
225 
231  uint32_t Hardlinks2Linkcount(const uint64_t hardlinks) const;
232  uint32_t Hardlinks2HardlinkGroup(const uint64_t hardlinks) const;
233  uint64_t MakeHardlinks(const uint32_t hardlink_group,
234  const uint32_t linkcount) const;
235 
241  void ExpandSymlink(LinkString *raw_symlink) const;
242 };
243 
244 
245 //------------------------------------------------------------------------------
246 
247 
248 class SqlDirentWrite : public SqlDirent {
249  public:
255  virtual bool BindDirent(const DirectoryEntry &entry) = 0;
256 
257  protected:
258  bool BindDirentFields(const int hash_idx,
259  const int hardlinks_idx,
260  const int size_idx,
261  const int mode_idx,
262  const int mtime_idx,
263  const int mtimens_idx,
264  const int flags_idx,
265  const int name_idx,
266  const int symlink_idx,
267  const int uid_idx,
268  const int gid_idx,
269  const DirectoryEntry &entry);
270 };
271 
272 
273 //------------------------------------------------------------------------------
274 
275 
277  public:
278  explicit SqlListContentHashes(const CatalogDatabase &database);
279  shash::Any GetHash() const;
280 };
281 
282 
283 //------------------------------------------------------------------------------
284 
285 
286 class SqlLookup : public SqlDirent {
287  public:
293  DirectoryEntry GetDirent(const Catalog *catalog,
294  const bool expand_symlink = true) const;
295 
301  shash::Md5 GetPathHash() const;
302 
309 };
310 
311 
312 //------------------------------------------------------------------------------
313 
314 
315 class SqlListing : public SqlLookup {
316  public:
317  explicit SqlListing(const CatalogDatabase &database);
318  bool BindPathHash(const struct shash::Md5 &hash);
319 };
320 
321 
322 //------------------------------------------------------------------------------
323 
324 
325 class SqlLookupPathHash : public SqlLookup {
326  public:
327  explicit SqlLookupPathHash(const CatalogDatabase &database);
328  bool BindPathHash(const struct shash::Md5 &hash);
329 };
330 
331 
332 //------------------------------------------------------------------------------
333 
334 
335 class SqlLookupInode : public SqlLookup {
336  public:
337  explicit SqlLookupInode(const CatalogDatabase &database);
338  bool BindRowId(const uint64_t inode);
339 };
340 
341 
342 //------------------------------------------------------------------------------
343 
344 
362  public:
363  explicit SqlLookupDanglingMountpoints(const CatalogDatabase &database);
364 };
365 
366 
367 //------------------------------------------------------------------------------
368 
369 
378 class SqlDirentTouch : public SqlCatalog {
379  public:
380  explicit SqlDirentTouch(const CatalogDatabase &database);
381 
382  bool BindDirentBase(const DirectoryEntryBase &entry);
383  bool BindPathHash(const shash::Md5 &hash);
384  bool BindXattr(const XattrList &xattrs);
385  bool BindXattrEmpty();
386 };
387 
388 
389 //------------------------------------------------------------------------------
390 
391 
396  public:
397  explicit SqlNestedCatalogLookup(const CatalogDatabase &database);
398  bool BindSearchPath(const PathString &path);
399  shash::Any GetContentHash() const;
400  uint64_t GetSize() const;
401 };
402 
403 
404 //------------------------------------------------------------------------------
405 
406 
411  public:
412  explicit SqlNestedCatalogListing(const CatalogDatabase &database);
413  PathString GetPath() const;
414  shash::Any GetContentHash() const;
415  uint64_t GetSize() const;
416 };
417 
418 
419 //------------------------------------------------------------------------------
420 
421 
426  public:
427  explicit SqlOwnNestedCatalogListing(const CatalogDatabase &database);
428  PathString GetPath() const;
429  shash::Any GetContentHash() const;
430  uint64_t GetSize() const;
431 };
432 
433 
434 //------------------------------------------------------------------------------
435 
436 
438  public:
439  explicit SqlDirentInsert(const CatalogDatabase &database);
440  bool BindPathHash(const shash::Md5 &hash);
441  bool BindParentPathHash(const shash::Md5 &hash);
442  bool BindDirent(const DirectoryEntry &entry);
443  bool BindXattr(const XattrList &xattrs);
444  bool BindXattrEmpty();
445 };
446 
447 
448 //------------------------------------------------------------------------------
449 
450 
452  public:
453  explicit SqlDirentUpdate(const CatalogDatabase &database);
454  bool BindPathHash(const shash::Md5 &hash);
455  bool BindDirent(const DirectoryEntry &entry);
456 };
457 
458 
459 //------------------------------------------------------------------------------
460 
461 
462 class SqlDirentUnlink : public SqlCatalog {
463  public:
464  explicit SqlDirentUnlink(const CatalogDatabase &database);
465  bool BindPathHash(const shash::Md5 &hash);
466 };
467 
468 
469 //------------------------------------------------------------------------------
470 
471 
475 class SqlIncLinkcount : public SqlCatalog {
476  public:
477  explicit SqlIncLinkcount(const CatalogDatabase &database);
478  bool BindPathHash(const shash::Md5 &hash);
479  bool BindDelta(const int delta);
480 };
481 
482 
483 //------------------------------------------------------------------------------
484 
485 
486 class SqlChunkInsert : public SqlCatalog {
487  public:
488  explicit SqlChunkInsert(const CatalogDatabase &database);
489  bool BindPathHash(const shash::Md5 &hash);
490  bool BindFileChunk(const FileChunk &chunk);
491 };
492 
493 
494 //------------------------------------------------------------------------------
495 
496 
497 class SqlChunksRemove : public SqlCatalog {
498  public:
499  explicit SqlChunksRemove(const CatalogDatabase &database);
500  bool BindPathHash(const shash::Md5 &hash);
501 };
502 
503 
504 //------------------------------------------------------------------------------
505 
506 
507 class SqlChunksListing : public SqlCatalog {
508  public:
509  explicit SqlChunksListing(const CatalogDatabase &database);
510  bool BindPathHash(const shash::Md5 &hash);
511  FileChunk GetFileChunk(const shash::Algorithms interpret_hash_as) const;
512 };
513 
514 
515 //------------------------------------------------------------------------------
516 
517 
518 class SqlChunksCount : public SqlCatalog {
519  public:
520  explicit SqlChunksCount(const CatalogDatabase &database);
521  bool BindPathHash(const shash::Md5 &hash);
522  int GetChunkCount() const;
523 };
524 
525 
526 //------------------------------------------------------------------------------
527 
528 
530  public:
531  explicit SqlMaxHardlinkGroup(const CatalogDatabase &database);
532  uint32_t GetMaxGroupId() const;
533 };
534 
535 
536 //------------------------------------------------------------------------------
537 
538 
539 class SqlGetCounter : public SqlCatalog {
540  public:
541  explicit SqlGetCounter(const CatalogDatabase &database);
542  bool BindCounter(const std::string &counter);
543  uint64_t GetCounter() const;
544 
545  private:
546  bool compat_;
547 };
548 
549 
550 //------------------------------------------------------------------------------
551 
552 
553 class SqlUpdateCounter : public SqlCatalog {
554  public:
555  explicit SqlUpdateCounter(const CatalogDatabase &database);
556  bool BindCounter(const std::string &counter);
557  bool BindDelta(const int64_t delta);
558 };
559 
560 
561 //------------------------------------------------------------------------------
562 
563 
564 class SqlCreateCounter : public SqlCatalog {
565  public:
566  explicit SqlCreateCounter(const CatalogDatabase &database);
567  bool BindCounter(const std::string &counter);
568  bool BindInitialValue(const int64_t value);
569 };
570 
571 
572 //------------------------------------------------------------------------------
573 
574 
575 class SqlAllChunks : public SqlCatalog {
576  public:
577  explicit SqlAllChunks(const CatalogDatabase &database);
578  bool Open();
579  bool Next(shash::Any *hash, zlib::Algorithms *compression_alg);
580  bool Close();
581 };
582 
583 
584 //------------------------------------------------------------------------------
585 
586 
587 class SqlLookupXattrs : public SqlCatalog {
588  public:
589  explicit SqlLookupXattrs(const CatalogDatabase &database);
590  bool BindPathHash(const shash::Md5 &hash);
592 };
593 
594 } // namespace catalog
595 
596 #endif // CVMFS_CATALOG_SQL_H_
bool BindSearchPath(const PathString &path)
Definition: catalog_sql.cc:953
static const int kFlagDirNestedMountpoint
Definition: catalog_sql.h:175
bool BindCounter(const std::string &counter)
shash::Any GetHash() const
Definition: catalog_sql.cc:664
SqlAllChunks(const CatalogDatabase &database)
bool IsNull() const
Definition: hash.h:371
shash::Md5 GetPathHash() const
Definition: catalog_sql.cc:730
SqlListing(const CatalogDatabase &database)
Definition: catalog_sql.cc:810
static const int kFlagPosCompression
Definition: catalog_sql.h:197
SqlIncLinkcount(const CatalogDatabase &database)
bool BindDirentBase(const DirectoryEntryBase &entry)
Definition: catalog_sql.cc:887
const std::string & filename() const
Definition: sql.h:146
shash::Algorithms RetrieveHashAlgorithm(const unsigned flags) const
Definition: catalog_sql.cc:498
bool Init(const sqlite3 *database, const std::string &statement)
Definition: sql.cc:133
static const int kFlagDirNestedRoot
Definition: catalog_sql.h:177
bool BindPathHash(const shash::Md5 &hash)
SqlLookupPathHash(const CatalogDatabase &database)
Definition: catalog_sql.cc:825
bool BindDirent(const DirectoryEntry &entry)
void ToIntPair(uint64_t *lo, uint64_t *hi) const
Definition: hash.cc:384
bool BindRowId(const uint64_t inode)
Definition: catalog_sql.cc:845
SqlCreateCounter(const CatalogDatabase &database)
const void * RetrieveBlob(const int idx_column) const
Definition: sql.h:431
bool BindDirentFields(const int hash_idx, const int hardlinks_idx, const int size_idx, const int mode_idx, const int mtime_idx, const int mtimens_idx, const int flags_idx, const int name_idx, const int symlink_idx, const int uid_idx, const int gid_idx, const DirectoryEntry &entry)
Definition: catalog_sql.cc:598
bool BindFileChunk(const FileChunk &chunk)
SqlUpdateCounter(const CatalogDatabase &database)
bool BindPathHash(const shash::Md5 &hash)
Definition: catalog_sql.cc:903
static const int kFlagPosHash
Definition: catalog_sql.h:194
SqlGetCounter(const CatalogDatabase &database)
SqlCatalog(const CatalogDatabase &database, const std::string &statement)
Definition: catalog_sql.h:84
DirectoryEntry GetDirent(const Catalog *catalog, const bool expand_symlink=true) const
Definition: catalog_sql.cc:739
shash::Md5 RetrieveMd5(const int idx_high, const int idx_low) const
Definition: catalog_sql.h:94
SqlDirentInsert(const CatalogDatabase &database)
SqlLookupInode(const CatalogDatabase &database)
Definition: catalog_sql.cc:839
bool BindPathHash(const shash::Md5 &hash)
uint32_t Hardlinks2HardlinkGroup(const uint64_t hardlinks) const
Definition: catalog_sql.cc:521
SqlListContentHashes(const CatalogDatabase &database)
Definition: catalog_sql.cc:638
Database(const std::string &filename, const OpenMode open_mode)
Definition: sql_impl.h:21
static const int kFlagFileChunk
Definition: catalog_sql.h:181
SqlNestedCatalogListing(const CatalogDatabase &database)
Definition: catalog_sql.cc:972
shash::Any GetContentHash() const
Definition: catalog_sql.cc:958
bool BindPathHash(const shash::Md5 &hash)
SqlChunksListing(const CatalogDatabase &database)
int GetChunkCount() const
static const int kFlagFile
Definition: catalog_sql.h:178
uint64_t GetCounter() const
unsigned char digest[digest_size_]
Definition: hash.h:121
Sql()
Definition: sql.h:458
static const int kFlagLink
Definition: catalog_sql.h:179
bool BindCounter(const std::string &counter)
bool BindPathHash(const shash::Md5 &hash)
bool BindXattr(const XattrList &xattrs)
Definition: catalog_sql.cc:908
Algorithms
Definition: hash.h:41
zlib::Algorithms RetrieveCompressionAlgorithm(const unsigned flags) const
Definition: catalog_sql.cc:507
bool BindCounter(const std::string &counter)
bool BindDelta(const int delta)
Algorithms
Definition: compression.h:44
unsigned GetDigestSize() const
Definition: hash.h:164
static const int kFlagDirBindMountpoint
Definition: catalog_sql.h:202
bool BindPathHash(const shash::Md5 &hash)
SqlChunksRemove(const CatalogDatabase &database)
bool Next(shash::Any *hash, zlib::Algorithms *compression_alg)
SqlOwnNestedCatalogListing(const CatalogDatabase &database)
static const int kFlagFileSpecial
Definition: catalog_sql.h:180
static const int kFlagFileExternal
Definition: catalog_sql.h:187
SqlDirentTouch(const CatalogDatabase &database)
Definition: catalog_sql.cc:872
bool BindHashBlob(const int idx_column, const shash::Any &hash)
Definition: catalog_sql.h:150
bool BindXattr(const XattrList &xattrs)
bool InsertInitialValues(const std::string &root_path, const bool volatile_content, const std::string &voms_authz, const DirectoryEntry &root_entry=DirectoryEntry(kDirentNegative))
Definition: catalog_sql.cc:299
SqlNestedCatalogLookup(const CatalogDatabase &database)
Definition: catalog_sql.cc:924
sqlite3_int64 RetrieveInt64(const int idx_column) const
Definition: sql.h:445
bool BindBlob(const int index, const void *value, const unsigned size)
Definition: sql.h:352
bool SetVOMSAuthz(const std::string &)
Definition: catalog_sql.cc:394
uint64_t MakeHardlinks(const uint32_t hardlink_group, const uint32_t linkcount) const
Definition: catalog_sql.cc:526
bool BindPathHash(const shash::Md5 &hash)
bool BindPathHash(const struct shash::Md5 &hash)
Definition: catalog_sql.cc:831
static const int kFlagDirectIo
Definition: catalog_sql.h:211
SqlDirentUpdate(const CatalogDatabase &database)
sqlite3 * sqlite_db() const
Definition: sql.h:145
static const float kLatestSchema
Definition: catalog_sql.h:43
bool BindInt64(const int index, const sqlite3_int64 value)
Definition: sql.h:376
bool BindPathHash(const struct shash::Md5 &hash)
Definition: catalog_sql.cc:817
shash::Any RetrieveHashBlob(const int idx_column, const shash::Algorithms hash_algo, const char hash_suffix=shash::kSuffixNone) const
Definition: catalog_sql.h:101
shash::Any RetrieveHashHex(const int idx_column, const char hash_suffix=shash::kSuffixNone) const
Definition: catalog_sql.h:120
bool CompactDatabase() const
Definition: catalog_sql.cc:430
const unsigned char * RetrieveText(const int idx_column) const
Definition: sql.h:448
SqlLookupDanglingMountpoints(const CatalogDatabase &database)
Definition: catalog_sql.cc:853
shash::Any GetContentHash() const
FileChunk GetFileChunk(const shash::Algorithms interpret_hash_as) const
virtual bool BindDirent(const DirectoryEntry &entry)=0
SqlLookupXattrs(const CatalogDatabase &database)
SqlChunkInsert(const CatalogDatabase &database)
static const int kFlagHidden
Definition: catalog_sql.h:207
Any MkFromHexPtr(const HexPtr hex, const char suffix)
Definition: hash.cc:82
uint32_t Hardlinks2Linkcount(const uint64_t hardlinks) const
Definition: catalog_sql.cc:516
shash::Md5 GetParentPathHash() const
Definition: catalog_sql.cc:733
static const float kLatestSupportedSchema
Definition: catalog_sql.h:44
double GetRowIdWasteRatio() const
Definition: catalog_sql.cc:399
unsigned CreateDatabaseFlags(const DirectoryEntry &entry) const
Definition: catalog_sql.cc:453
bool BindDelta(const int64_t delta)
bool BindMd5(const int idx_high, const int idx_low, const shash::Md5 &hash)
Definition: catalog_sql.h:134
bool BindNull(const int index)
Definition: sql.h:381
bool BindParentPathHash(const shash::Md5 &hash)
static const unsigned kLatestSchemaRevision
Definition: catalog_sql.h:46
static const int kFlagDir
Definition: catalog_sql.h:173
CatalogDatabase(const std::string &filename, const OpenMode open_mode)
Definition: catalog_sql.h:65
const char kSuffixNone
Definition: hash.h:53
bool BindInitialValue(const int64_t value)
bool BindPathHash(const shash::Md5 &hash)
SqlChunksCount(const CatalogDatabase &database)
int RetrieveBytes(const int idx_column) const
Definition: sql.h:428
bool BindPathHash(const shash::Md5 &hash)
bool BindDirent(const DirectoryEntry &entry)
void StoreHashAlgorithm(const shash::Algorithms algo, unsigned *flags) const
Definition: catalog_sql.cc:490
void ExpandSymlink(LinkString *raw_symlink) const
Definition: catalog_sql.cc:537