GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/catalog_sql.h
Date: 2024-04-28 02:33:07
Exec Total Coverage
Lines: 19 21 90.5%
Branches: 11 18 61.1%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM file system.
3 *
4 * This file provides classes to wrap often used catalog SQL statements.
5 * In particular, it wraps around sqlite3 prepared statement syntax.
6 *
7 * Usage example:
8 * SqlLookup statement(<database>);
9 * statement.BindPathHash(<hash>);
10 * if (statement.FetchRow()) {
11 * statement.GetDirectoryEntry(<catalog>);
12 * }
13 * statement.Reset();
14 */
15
16 #ifndef CVMFS_CATALOG_SQL_H_
17 #define CVMFS_CATALOG_SQL_H_
18
19 #ifndef __STDC_FORMAT_MACROS
20 #define __STDC_FORMAT_MACROS
21 #endif
22
23 #include <inttypes.h>
24
25 #include <string>
26
27 #include "compression.h"
28 #include "crypto/hash.h"
29 #include "directory_entry.h"
30 #include "file_chunk.h"
31 #include "shortstring.h"
32 #include "sql.h"
33
34 class XattrList;
35
36 namespace catalog {
37
38 class Catalog;
39
40
41 class CatalogDatabase : public sqlite::Database<CatalogDatabase> {
42 public:
43 static const float kLatestSchema;
44 static const float kLatestSupportedSchema; // + 1.X catalogs (r/o)
45 // Backwards-compatible schema changes
46 static const unsigned kLatestSchemaRevision;
47
48 bool CreateEmptyDatabase();
49 bool InsertInitialValues(const std::string &root_path,
50 const bool volatile_content,
51 const std::string &voms_authz,
52 const DirectoryEntry &root_entry
53 = DirectoryEntry(kDirentNegative));
54
55 bool CheckSchemaCompatibility();
56 bool LiveSchemaUpgradeIfNecessary();
57 bool CompactDatabase() const;
58
59 double GetRowIdWasteRatio() const;
60 bool SetVOMSAuthz(const std::string&);
61
62 protected:
63 // TODO(rmeusel): C++11 - constructor inheritance
64 friend class sqlite::Database<CatalogDatabase>;
65 240 CatalogDatabase(const std::string &filename,
66 const OpenMode open_mode)
67 240 : sqlite::Database<CatalogDatabase>(filename, open_mode) { }
68 };
69
70
71 //------------------------------------------------------------------------------
72
73
74 /**
75 * Base class for all SQL statement classes. It wraps a single SQL statement
76 * and all necessary calls of the sqlite3 API to deal with this statement.
77 */
78 class SqlCatalog : public sqlite::Sql {
79 public:
80 /**
81 * Basic constructor to use this class for a specific statement.
82 * @param database the database to use the query on
83 * @param statement the statement to prepare
84 */
85 1003 SqlCatalog(const CatalogDatabase &database, const std::string &statement) {
86
2/4
✓ Branch 1 taken 1003 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1003 times.
✗ Branch 5 not taken.
 1003 Init(database.sqlite_db(), statement);
87 1003 }
88
89 /**
90 * Wrapper for retrieving MD5-ified path names.
91 * @param idx_high offset of most significant bits in database query
92 * @param idx_low offset of least significant bits in database query
93 * @result the retrieved MD5 hash
94 */
95 ✗ inline shash::Md5 RetrieveMd5(const int idx_high, const int idx_low) const {
96 ✗ return shash::Md5(RetrieveInt64(idx_high), RetrieveInt64(idx_low));
97 }
98
99 /**
100 * Wrapper for retrieving a cryptographic hash from a blob field.
101 */
102 730 inline shash::Any RetrieveHashBlob(
103 const int idx_column,
104 const shash::Algorithms hash_algo,
105 const char hash_suffix = shash::kSuffixNone) const
106 {
107 // Note: SQLite documentation advises to first define the data type of BLOB
108 // by calling sqlite3_column_XXX() on the column and _afterwards_ get
109 // the number of bytes using sqlite3_column_bytes().
110 //
111 // See: https://www.sqlite.org/c3ref/column_blob.html
112 const unsigned char *buffer = static_cast<const unsigned char *>(
113 730 RetrieveBlob(idx_column));
114 730 const int byte_count = RetrieveBytes(idx_column);
115 return (byte_count > 0) ? shash::Any(hash_algo, buffer, hash_suffix)
116
2/2
✓ Branch 0 taken 228 times.
✓ Branch 1 taken 502 times.
 730 : shash::Any(hash_algo);
117 }
118
119 /**
120 * Wrapper for retrieving a cryptographic hash from a text field.
121 */
122 inline shash::Any RetrieveHashHex(
123 const int idx_column,
124 const char hash_suffix = shash::kSuffixNone) const
125 {
126 const std::string hash_string = std::string(reinterpret_cast<const char *>(
127 RetrieveText(idx_column)));
128 return shash::MkFromHexPtr(shash::HexPtr(hash_string), hash_suffix);
129 }
130
131 /**
132 * Wrapper for binding a MD5-ified path name.
133 * @param idx_high offset of most significant bits in database query
134 * @param idx_low offset of least significant bits in database query
135 * @param hash the hash to bind in the query
136 * @result true on success, false otherwise
137 */
138 2019 inline bool BindMd5(const int idx_high, const int idx_low,
139 const shash::Md5 &hash)
140 {
141 uint64_t high, low;
142
1/2
✓ Branch 1 taken 2019 times.
✗ Branch 2 not taken.
 2019 hash.ToIntPair(&high, &low);
143
2/4
✓ Branch 1 taken 2019 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2019 times.
✗ Branch 4 not taken.
 4038 const bool retval = BindInt64(idx_high, static_cast<int64_t>(high)) &&
144
2/4
✓ Branch 1 taken 2019 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2019 times.
✗ Branch 4 not taken.
 2019 BindInt64(idx_low, static_cast<int64_t>(low));
145 2019 return retval;
146 }
147
148 /**
149 * Wrapper for binding a cryptographic hash. Algorithm of hash has to be
150 * elsewhere (in flags).
151 * @param idx_column offset of the blob field in database query
152 * @param hash the hash to bind in the query
153 * @result true on success, false otherwise
154 */
155 723 inline bool BindHashBlob(const int idx_column, const shash::Any &hash) {
156
2/2
✓ Branch 1 taken 446 times.
✓ Branch 2 taken 277 times.
 723 if (hash.IsNull()) {
157 446 return BindNull(idx_column);
158 } else {
159 277 return BindBlob(idx_column, hash.digest, hash.GetDigestSize());
160 }
161 }
162
163 protected:
164 2394 SqlCatalog() : sqlite::Sql() {}
165 };
166
167
168 //------------------------------------------------------------------------------
169
170
171 /**
172 * Common ancestor of SQL statements that deal with directory entries.
173 */
174 class SqlDirent : public SqlCatalog {
175 public:
176 // Definition of bit positions for the flags field of a DirectoryEntry
177 // All other bit positions are unused
178 static const int kFlagDir = 1;
179 // Link in the parent catalog
180 static const int kFlagDirNestedMountpoint = 2;
181 // Link in the child catalog
182 static const int kFlagDirNestedRoot = 32;
183 static const int kFlagFile = 4;
184 static const int kFlagLink = 8;
185 static const int kFlagFileSpecial = 16;
186 static const int kFlagFileChunk = 64;
187 /**
188 * The file is not natively stored in cvmfs but on a different storage system,
189 * for instance on HTTPS data federation services.
190 * NOTE: used as magic number in SqlListContentHashes::SqlListContentHashes
191 */
192 static const int kFlagFileExternal = 128;
193 // as of 2^8: 3 bit for hashes
194 // - 0: SHA-1
195 // - 1: RIPEMD-160
196 // - ...
197 // Corresponds to shash::algorithms with offset in order to support future
198 // hashes
199 static const int kFlagPosHash = 8;
200 // Compression methods, 3 bits starting at 2^11
201 // Corresponds to zlib::Algorithms
202 static const int kFlagPosCompression = 11;
203 /**
204 * A transition point to a root catalog (instead of a nested catalog). Used
205 * to link previous snapshots into the catalog structure.
206 */
207 static const int kFlagDirBindMountpoint = 0x4000; // 2^14
208 /**
209 * An entry that should not appear in listings. Used for the /.cvmfs
210 * directory.
211 */
212 static const int kFlagHidden = 0x8000; // 2^15
213 /**
214 * For regular files, indicates that the file should be opened with direct I/O
215 */
216 static const int kFlagDirectIo = 0x10000; // 2^16
217
218
219 protected:
220 /**
221 * Take the meta data from the DirectoryEntry and transform it
222 * into a valid flags field ready to be saved in the database.
223 * @param entry the DirectoryEntry to encode
224 * @return an integer containing the bitmap of the flags field
225 */
226 unsigned CreateDatabaseFlags(const DirectoryEntry &entry) const;
227 void StoreHashAlgorithm(const shash::Algorithms algo, unsigned *flags) const;
228 shash::Algorithms RetrieveHashAlgorithm(const unsigned flags) const;
229 zlib::Algorithms RetrieveCompressionAlgorithm(const unsigned flags) const;
230
231 /**
232 * The hardlink information (hardlink group ID and linkcount) is saved in one
233 * uint_64t field in the CVMFS Catalogs. Therefore we need to do bitshifting
234 * in these helper methods.
235 */
236 uint32_t Hardlinks2Linkcount(const uint64_t hardlinks) const;
237 uint32_t Hardlinks2HardlinkGroup(const uint64_t hardlinks) const;
238 uint64_t MakeHardlinks(const uint32_t hardlink_group,
239 const uint32_t linkcount) const;
240
241 /**
242 * Replaces place holder variables in a symbolic link by actual path elements.
243 * @param raw_symlink the raw symlink path (may) containing place holders
244 * @return the expanded symlink
245 */
246 void ExpandSymlink(LinkString *raw_symlink) const;
247 };
248
249
250 //------------------------------------------------------------------------------
251
252
253 class SqlDirentWrite : public SqlDirent {
254 public:
255 /**
256 * To bind an entire DirectoryEntry
257 * @param entry the DirectoryEntry to bind in the SQL statement
258 * @return true on success, false otherwise
259 */
260 virtual bool BindDirent(const DirectoryEntry &entry) = 0;
261
262 protected:
263 bool BindDirentFields(const int hash_idx,
264 const int hardlinks_idx,
265 const int size_idx,
266 const int mode_idx,
267 const int mtime_idx,
268 const int flags_idx,
269 const int name_idx,
270 const int symlink_idx,
271 const int uid_idx,
272 const int gid_idx,
273 const DirectoryEntry &entry);
274 };
275
276
277 //------------------------------------------------------------------------------
278
279
280 class SqlListContentHashes : public SqlDirent {
281 public:
282 explicit SqlListContentHashes(const CatalogDatabase &database);
283 shash::Any GetHash() const;
284 };
285
286
287 //------------------------------------------------------------------------------
288
289
290 class SqlLookup : public SqlDirent {
291 public:
292 /**
293 * Retrieves a DirectoryEntry from a freshly performed SqlLookup statement.
294 * @param catalog the catalog in which the DirectoryEntry resides
295 * @return the retrieved DirectoryEntry
296 */
297 DirectoryEntry GetDirent(const Catalog *catalog,
298 const bool expand_symlink = true) const;
299
300 /**
301 * DirectoryEntrys do not contain their path hash.
302 * This method retrieves the saved path hash from the database
303 * @return the MD5 path hash of a freshly performed lookup
304 */
305 shash::Md5 GetPathHash() const;
306
307 /**
308 * DirectoryEntries do not contain their parent path hash.
309 * This method retrieves the saved parent path hash from the database
310 * @return the MD5 parent path hash of a freshly performed lookup
311 */
312 shash::Md5 GetParentPathHash() const;
313 };
314
315
316 //------------------------------------------------------------------------------
317
318
319 class SqlListing : public SqlLookup {
320 public:
321 explicit SqlListing(const CatalogDatabase &database);
322 bool BindPathHash(const struct shash::Md5 &hash);
323 };
324
325
326 //------------------------------------------------------------------------------
327
328
329 class SqlLookupPathHash : public SqlLookup {
330 public:
331 explicit SqlLookupPathHash(const CatalogDatabase &database);
332 bool BindPathHash(const struct shash::Md5 &hash);
333 };
334
335
336 //------------------------------------------------------------------------------
337
338
339 class SqlLookupInode : public SqlLookup {
340 public:
341 explicit SqlLookupInode(const CatalogDatabase &database);
342 bool BindRowId(const uint64_t inode);
343 };
344
345
346 //------------------------------------------------------------------------------
347
348
349 /**
350 * This SQL statement is only used for legacy catalog migrations and has been
351 * moved here as it needs to use a locally defined macro inside catalog_sql.cc
352 *
353 * Queries a single catalog and looks for DirectoryEntrys that have direct
354 * children in the same catalog but are marked as 'nested catalog mountpoints'.
355 * This is an inconsistent situation, since a mountpoint is supposed to be empty
356 * and it's children are stored in the corresponding referenced nested catalog.
357 *
358 * Note: the user code needs to check if there is a corresponding nested catalog
359 * reference for the found dangling mountpoints. If so, we also have a
360 * bogus state, but it is not reliably fixable automatically. The child-
361 * DirectoryEntrys would be masked by the mounting nested catalog but it
362 * is not clear if we can simply delete them or if this would destroy data.
363 */
364 class SqlLookupDanglingMountpoints : public catalog::SqlLookup {
365 public:
366 explicit SqlLookupDanglingMountpoints(const CatalogDatabase &database);
367 };
368
369
370 //------------------------------------------------------------------------------
371
372
373 /**
374 * Filesystem like _touch_ of a DirectoryEntry. Only file system specific meta
375 * data will be modified. All CVMFS-specific administrative data stays
376 * unchanged.
377 * NOTE: This is not a subclass of SqlDirent since it works on
378 * DirectoryEntryBase objects, which are restricted to file system meta
379 * data.
380 */
381 class SqlDirentTouch : public SqlCatalog {
382 public:
383 explicit SqlDirentTouch(const CatalogDatabase &database);
384
385 bool BindDirentBase(const DirectoryEntryBase &entry);
386 bool BindPathHash(const shash::Md5 &hash);
387 bool BindXattr(const XattrList &xattrs);
388 bool BindXattrEmpty();
389 };
390
391
392 //------------------------------------------------------------------------------
393
394
395 /**
396 * Nested catalogs and bind mountpoints.
397 */
398 class SqlNestedCatalogLookup : public SqlCatalog {
399 public:
400 explicit SqlNestedCatalogLookup(const CatalogDatabase &database);
401 bool BindSearchPath(const PathString &path);
402 shash::Any GetContentHash() const;
403 uint64_t GetSize() const;
404 };
405
406
407 //------------------------------------------------------------------------------
408
409
410 /**
411 * Nested catalogs and bind mountpoints.
412 */
413 class SqlNestedCatalogListing : public SqlCatalog {
414 public:
415 explicit SqlNestedCatalogListing(const CatalogDatabase &database);
416 PathString GetPath() const;
417 shash::Any GetContentHash() const;
418 uint64_t GetSize() const;
419 };
420
421
422 //------------------------------------------------------------------------------
423
424
425 /**
426 * Only nested catalogs, no bind mountpoints. Used for replication and GC.
427 */
428 class SqlOwnNestedCatalogListing : public SqlCatalog {
429 public:
430 explicit SqlOwnNestedCatalogListing(const CatalogDatabase &database);
431 PathString GetPath() const;
432 shash::Any GetContentHash() const;
433 uint64_t GetSize() const;
434 };
435
436
437 //------------------------------------------------------------------------------
438
439
440 class SqlDirentInsert : public SqlDirentWrite {
441 public:
442 explicit SqlDirentInsert(const CatalogDatabase &database);
443 bool BindPathHash(const shash::Md5 &hash);
444 bool BindParentPathHash(const shash::Md5 &hash);
445 bool BindDirent(const DirectoryEntry &entry);
446 bool BindXattr(const XattrList &xattrs);
447 bool BindXattrEmpty();
448 };
449
450
451 //------------------------------------------------------------------------------
452
453
454 class SqlDirentUpdate : public SqlDirentWrite {
455 public:
456 explicit SqlDirentUpdate(const CatalogDatabase &database);
457 bool BindPathHash(const shash::Md5 &hash);
458 bool BindDirent(const DirectoryEntry &entry);
459 };
460
461
462 //------------------------------------------------------------------------------
463
464
465 class SqlDirentUnlink : public SqlCatalog {
466 public:
467 explicit SqlDirentUnlink(const CatalogDatabase &database);
468 bool BindPathHash(const shash::Md5 &hash);
469 };
470
471
472 //------------------------------------------------------------------------------
473
474
475 /**
476 * Changes the linkcount for all files in a hardlink group.
477 */
478 class SqlIncLinkcount : public SqlCatalog {
479 public:
480 explicit SqlIncLinkcount(const CatalogDatabase &database);
481 bool BindPathHash(const shash::Md5 &hash);
482 bool BindDelta(const int delta);
483 };
484
485
486 //------------------------------------------------------------------------------
487
488
489 class SqlChunkInsert : public SqlCatalog {
490 public:
491 explicit SqlChunkInsert(const CatalogDatabase &database);
492 bool BindPathHash(const shash::Md5 &hash);
493 bool BindFileChunk(const FileChunk &chunk);
494 };
495
496
497 //------------------------------------------------------------------------------
498
499
500 class SqlChunksRemove : public SqlCatalog {
501 public:
502 explicit SqlChunksRemove(const CatalogDatabase &database);
503 bool BindPathHash(const shash::Md5 &hash);
504 };
505
506
507 //------------------------------------------------------------------------------
508
509
510 class SqlChunksListing : public SqlCatalog {
511 public:
512 explicit SqlChunksListing(const CatalogDatabase &database);
513 bool BindPathHash(const shash::Md5 &hash);
514 FileChunk GetFileChunk(const shash::Algorithms interpret_hash_as) const;
515 };
516
517
518 //------------------------------------------------------------------------------
519
520
521 class SqlChunksCount : public SqlCatalog {
522 public:
523 explicit SqlChunksCount(const CatalogDatabase &database);
524 bool BindPathHash(const shash::Md5 &hash);
525 int GetChunkCount() const;
526 };
527
528
529 //------------------------------------------------------------------------------
530
531
532 class SqlMaxHardlinkGroup : public SqlCatalog {
533 public:
534 explicit SqlMaxHardlinkGroup(const CatalogDatabase &database);
535 uint32_t GetMaxGroupId() const;
536 };
537
538
539 //------------------------------------------------------------------------------
540
541
542 class SqlGetCounter : public SqlCatalog {
543 public:
544 explicit SqlGetCounter(const CatalogDatabase &database);
545 bool BindCounter(const std::string &counter);
546 uint64_t GetCounter() const;
547 private:
548 bool compat_;
549 };
550
551
552 //------------------------------------------------------------------------------
553
554
555 class SqlUpdateCounter : public SqlCatalog {
556 public:
557 explicit SqlUpdateCounter(const CatalogDatabase &database);
558 bool BindCounter(const std::string &counter);
559 bool BindDelta(const int64_t delta);
560 };
561
562
563 //------------------------------------------------------------------------------
564
565
566 class SqlCreateCounter : public SqlCatalog {
567 public:
568 explicit SqlCreateCounter(const CatalogDatabase &database);
569 bool BindCounter(const std::string &counter);
570 bool BindInitialValue(const int64_t value);
571 };
572
573
574 //------------------------------------------------------------------------------
575
576
577 class SqlAllChunks : public SqlCatalog {
578 public:
579 explicit SqlAllChunks(const CatalogDatabase &database);
580 bool Open();
581 bool Next(shash::Any *hash, zlib::Algorithms *compression_alg);
582 bool Close();
583 };
584
585
586 //------------------------------------------------------------------------------
587
588
589 class SqlLookupXattrs : public SqlCatalog {
590 public:
591 explicit SqlLookupXattrs(const CatalogDatabase &database);
592 bool BindPathHash(const shash::Md5 &hash);
593 XattrList GetXattrs();
594 };
595
596 } // namespace catalog
597
598 #endif // CVMFS_CATALOG_SQL_H_
599