GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/catalog_sql.h
Date: 2026-08-30 02:40:36
Exec Total Coverage
Lines: 19 21 90.5%
Branches: 11 18 61.1%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM file system.
3 *
4 * This file provides classes to wrap often used catalog SQL statements.
5 * In particular, it wraps around sqlite3 prepared statement syntax.
6 *
7 * Usage example:
8 * SqlLookup statement(<database>);
9 * statement.BindPathHash(<hash>);
10 * if (statement.FetchRow()) {
11 * statement.GetDirectoryEntry(<catalog>);
12 * }
13 * statement.Reset();
14 */
15
16 #ifndef CVMFS_CATALOG_SQL_H_
17 #define CVMFS_CATALOG_SQL_H_
18
19 #include <inttypes.h>
20
21 #include <string>
22
23 #include "compression/compression.h"
24 #include "crypto/hash.h"
25 #include "directory_entry.h"
26 #include "file_chunk.h"
27 #include "shortstring.h"
28 #include "sql.h"
29
30 class XattrList;
31
32 namespace catalog {
33
34 // Set to false via CVMFS_NO_IGNORE_LEGACY_BULKHASHES env var to restore the
35 // pre-2.14 behavior of including bulk hashes for chunked files.
36 extern bool g_ignore_legacy_bulk_hashes;
37
38 class Catalog;
39
40
41 class CatalogDatabase : public sqlite::Database<CatalogDatabase> {
42 public:
43 static const float kLatestSchema;
44 static const float kLatestSupportedSchema; // + 1.X catalogs (r/o)
45 // Backwards-compatible schema changes
46 static const unsigned kLatestSchemaRevision;
47
48 bool CreateEmptyDatabase();
49 bool InsertInitialValues(
50 const std::string &root_path,
51 const bool volatile_content,
52 const std::string &voms_authz,
53 const DirectoryEntry &root_entry = DirectoryEntry(kDirentNegative));
54
55 bool CheckSchemaCompatibility();
56 bool LiveSchemaUpgradeIfNecessary();
57 bool CompactDatabase() const;
58
59 double GetRowIdWasteRatio() const;
60 bool SetVOMSAuthz(const std::string &);
61
62 protected:
63 // TODO(rmeusel): C++11 - constructor inheritance
64 friend class sqlite::Database<CatalogDatabase>;
65 10986 CatalogDatabase(const std::string &filename, const OpenMode open_mode)
66 10986 : sqlite::Database<CatalogDatabase>(filename, open_mode) { }
67 };
68
69
70 //------------------------------------------------------------------------------
71
72
73 /**
74 * Base class for all SQL statement classes. It wraps a single SQL statement
75 * and all necessary calls of the sqlite3 API to deal with this statement.
76 */
77 class SqlCatalog : public sqlite::Sql {
78 public:
79 /**
80 * Basic constructor to use this class for a specific statement.
81 * @param database the database to use the query on
82 * @param statement the statement to prepare
83 */
84 44812 SqlCatalog(const CatalogDatabase &database, const std::string &statement) {
85
2/4
✓ Branch 1 taken 44812 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 44812 times.
✗ Branch 5 not taken.
 44812 Init(database.sqlite_db(), statement);
86 44812 }
87
88 /**
89 * Wrapper for retrieving MD5-ified path names.
90 * @param idx_high offset of most significant bits in database query
91 * @param idx_low offset of least significant bits in database query
92 * @result the retrieved MD5 hash
93 */
94 ✗ inline shash::Md5 RetrieveMd5(const int idx_high, const int idx_low) const {
95 ✗ return shash::Md5(RetrieveInt64(idx_high), RetrieveInt64(idx_low));
96 }
97
98 /**
99 * Wrapper for retrieving a cryptographic hash from a blob field.
100 */
101 30246 inline shash::Any RetrieveHashBlob(
102 const int idx_column,
103 const shash::Algorithms hash_algo,
104 const char hash_suffix = shash::kSuffixNone) const {
105 // Note: SQLite documentation advises to first define the data type of BLOB
106 // by calling sqlite3_column_XXX() on the column and _afterwards_ get
107 // the number of bytes using sqlite3_column_bytes().
108 //
109 // See: https://www.sqlite.org/c3ref/column_blob.html
110 const unsigned char *buffer = static_cast<const unsigned char *>(
111 30246 RetrieveBlob(idx_column));
112 30246 const int byte_count = RetrieveBytes(idx_column);
113 return (byte_count > 0) ? shash::Any(hash_algo, buffer, hash_suffix)
114
2/2
✓ Branch 0 taken 9134 times.
✓ Branch 1 taken 21112 times.
 30246 : shash::Any(hash_algo);
115 }
116
117 /**
118 * Wrapper for retrieving a cryptographic hash from a text field.
119 */
120 inline shash::Any RetrieveHashHex(
121 const int idx_column, const char hash_suffix = shash::kSuffixNone) const {
122 const std::string hash_string = std::string(
123 reinterpret_cast<const char *>(RetrieveText(idx_column)));
124 return shash::MkFromHexPtr(shash::HexPtr(hash_string), hash_suffix);
125 }
126
127 /**
128 * Wrapper for binding a MD5-ified path name.
129 * @param idx_high offset of most significant bits in database query
130 * @param idx_low offset of least significant bits in database query
131 * @param hash the hash to bind in the query
132 * @result true on success, false otherwise
133 */
134 84503 inline bool BindMd5(const int idx_high, const int idx_low,
135 const shash::Md5 &hash) {
136 uint64_t high, low;
137
1/2
✓ Branch 1 taken 84503 times.
✗ Branch 2 not taken.
 84503 hash.ToIntPair(&high, &low);
138
1/2
✓ Branch 1 taken 84503 times.
✗ Branch 2 not taken.
 84503 const bool retval = BindInt64(idx_high, static_cast<int64_t>(high))
139
3/6
✓ Branch 0 taken 84503 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 84503 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 84503 times.
✗ Branch 6 not taken.
 84503 && BindInt64(idx_low, static_cast<int64_t>(low));
140 84503 return retval;
141 }
142
143 /**
144 * Wrapper for binding a cryptographic hash. Algorithm of hash has to be
145 * elsewhere (in flags).
146 * @param idx_column offset of the blob field in database query
147 * @param hash the hash to bind in the query
148 * @result true on success, false otherwise
149 */
150 30378 inline bool BindHashBlob(const int idx_column, const shash::Any &hash) {
151
2/2
✓ Branch 1 taken 19261 times.
✓ Branch 2 taken 11117 times.
 30378 if (hash.IsNull()) {
152 19261 return BindNull(idx_column);
153 } else {
154 11117 return BindBlob(idx_column, hash.digest, hash.GetDigestSize());
155 }
156 }
157
158 protected:
159 112532 SqlCatalog() : sqlite::Sql() { }
160 };
161
162
163 //------------------------------------------------------------------------------
164
165
166 /**
167 * Common ancestor of SQL statements that deal with directory entries.
168 */
169 class SqlDirent : public SqlCatalog {
170 public:
171 // Definition of bit positions for the flags field of a DirectoryEntry
172 // All other bit positions are unused
173 static const int kFlagDir = 1;
174 // Link in the parent catalog
175 static const int kFlagDirNestedMountpoint = 2;
176 // Link in the child catalog
177 static const int kFlagDirNestedRoot = 32;
178 static const int kFlagFile = 4;
179 static const int kFlagLink = 8;
180 static const int kFlagFileSpecial = 16;
181 static const int kFlagFileChunk = 64;
182 /**
183 * The file is not natively stored in cvmfs but on a different storage system,
184 * for instance on HTTPS data federation services.
185 * NOTE: used as magic number in SqlListContentHashes::SqlListContentHashes
186 */
187 static const int kFlagFileExternal = 128;
188 // as of 2^8: 3 bit for hashes
189 // - 0: SHA-1
190 // - 1: RIPEMD-160
191 // - ...
192 // Corresponds to shash::algorithms with offset in order to support future
193 // hashes
194 static const int kFlagPosHash = 8;
195 // Compression methods, 3 bits starting at 2^11
196 // Corresponds to zlib::Algorithms
197 static const int kFlagPosCompression = 11;
198 /**
199 * A transition point to a root catalog (instead of a nested catalog). Used
200 * to link previous snapshots into the catalog structure.
201 */
202 static const int kFlagDirBindMountpoint = 0x4000; // 2^14
203 /**
204 * An entry that should not appear in listings. Used for the /.cvmfs
205 * directory.
206 */
207 static const int kFlagHidden = 0x8000; // 2^15
208 /**
209 * For regular files, indicates that the file should be opened with direct I/O
210 */
211 static const int kFlagDirectIo = 0x10000; // 2^16
212 /**
213 * For regular files, indicates that this file has an attached file bundle
214 * list, that should be prefetched upon opening the file.
215 */
216 static const int kFlagBundleTrigger = 0x20000; // 2^17
217 /**
218 * For regular files, indicates that the cached object should be evicted
219 * before non-volatile objects (per-file equivalent of the repo-level
220 * 'volatile' property).
221 */
222 static const int kFlagFileVolatile = 0x40000; // 2^18
223
224
225 protected:
226 /**
227 * Take the meta data from the DirectoryEntry and transform it
228 * into a valid flags field ready to be saved in the database.
229 * @param entry the DirectoryEntry to encode
230 * @return an integer containing the bitmap of the flags field
231 */
232 unsigned CreateDatabaseFlags(const DirectoryEntry &entry) const;
233 void StoreHashAlgorithm(const shash::Algorithms algo, unsigned *flags) const;
234 shash::Algorithms RetrieveHashAlgorithm(const unsigned flags) const;
235 zlib::Algorithms RetrieveCompressionAlgorithm(const unsigned flags) const;
236
237 /**
238 * The hardlink information (hardlink group ID and linkcount) is saved in one
239 * uint_64t field in the CVMFS Catalogs. Therefore we need to do bitshifting
240 * in these helper methods.
241 */
242 uint32_t Hardlinks2Linkcount(const uint64_t hardlinks) const;
243 uint32_t Hardlinks2HardlinkGroup(const uint64_t hardlinks) const;
244 uint64_t MakeHardlinks(const uint32_t hardlink_group,
245 const uint32_t linkcount) const;
246
247 /**
248 * Replaces place holder variables in a symbolic link by actual path elements.
249 * @param raw_symlink the raw symlink path (may) containing place holders
250 * @return the expanded symlink
251 */
252 void ExpandSymlink(LinkString *raw_symlink) const;
253 };
254
255
256 //------------------------------------------------------------------------------
257
258
259 class SqlDirentWrite : public SqlDirent {
260 public:
261 /**
262 * To bind an entire DirectoryEntry
263 * @param entry the DirectoryEntry to bind in the SQL statement
264 * @return true on success, false otherwise
265 */
266 virtual bool BindDirent(const DirectoryEntry &entry) = 0;
267
268 protected:
269 bool BindDirentFields(const int hash_idx,
270 const int hardlinks_idx,
271 const int size_idx,
272 const int mode_idx,
273 const int mtime_idx,
274 const int mtimens_idx,
275 const int flags_idx,
276 const int name_idx,
277 const int symlink_idx,
278 const int uid_idx,
279 const int gid_idx,
280 const DirectoryEntry &entry);
281 };
282
283
284 //------------------------------------------------------------------------------
285
286
287 class SqlListContentHashes : public SqlDirent {
288 public:
289 explicit SqlListContentHashes(const CatalogDatabase &database);
290 shash::Any GetHash() const;
291 };
292
293
294 //------------------------------------------------------------------------------
295
296
297 class SqlLookup : public SqlDirent {
298 public:
299 /**
300 * Retrieves a DirectoryEntry from a freshly performed SqlLookup statement.
301 * @param catalog the catalog in which the DirectoryEntry resides
302 * @return the retrieved DirectoryEntry
303 */
304 DirectoryEntry GetDirent(const Catalog *catalog,
305 const bool expand_symlink = true) const;
306
307 /**
308 * DirectoryEntrys do not contain their path hash.
309 * This method retrieves the saved path hash from the database
310 * @return the MD5 path hash of a freshly performed lookup
311 */
312 shash::Md5 GetPathHash() const;
313
314 /**
315 * DirectoryEntries do not contain their parent path hash.
316 * This method retrieves the saved parent path hash from the database
317 * @return the MD5 parent path hash of a freshly performed lookup
318 */
319 shash::Md5 GetParentPathHash() const;
320 };
321
322
323 //------------------------------------------------------------------------------
324
325
326 class SqlListing : public SqlLookup {
327 public:
328 explicit SqlListing(const CatalogDatabase &database);
329 bool BindPathHash(const struct shash::Md5 &hash);
330 };
331
332
333 //------------------------------------------------------------------------------
334
335
336 class SqlLookupPathHash : public SqlLookup {
337 public:
338 explicit SqlLookupPathHash(const CatalogDatabase &database);
339 bool BindPathHash(const struct shash::Md5 &hash);
340 };
341
342
343 //------------------------------------------------------------------------------
344
345
346 class SqlLookupInode : public SqlLookup {
347 public:
348 explicit SqlLookupInode(const CatalogDatabase &database);
349 bool BindRowId(const uint64_t inode);
350 };
351
352
353 //------------------------------------------------------------------------------
354
355
356 /**
357 * This SQL statement is only used for legacy catalog migrations and has been
358 * moved here as it needs to use a locally defined macro inside catalog_sql.cc
359 *
360 * Queries a single catalog and looks for DirectoryEntrys that have direct
361 * children in the same catalog but are marked as 'nested catalog mountpoints'.
362 * This is an inconsistent situation, since a mountpoint is supposed to be empty
363 * and it's children are stored in the corresponding referenced nested catalog.
364 *
365 * Note: the user code needs to check if there is a corresponding nested catalog
366 * reference for the found dangling mountpoints. If so, we also have a
367 * bogus state, but it is not reliably fixable automatically. The child-
368 * DirectoryEntrys would be masked by the mounting nested catalog but it
369 * is not clear if we can simply delete them or if this would destroy
370 * data.
371 */
372 class SqlLookupDanglingMountpoints : public catalog::SqlLookup {
373 public:
374 explicit SqlLookupDanglingMountpoints(const CatalogDatabase &database);
375 };
376
377
378 //------------------------------------------------------------------------------
379
380
381 /**
382 * Filesystem like _touch_ of a DirectoryEntry. Only file system specific meta
383 * data will be modified. All CVMFS-specific administrative data stays
384 * unchanged.
385 * NOTE: This is not a subclass of SqlDirent since it works on
386 * DirectoryEntryBase objects, which are restricted to file system meta
387 * data.
388 */
389 class SqlDirentTouch : public SqlCatalog {
390 public:
391 explicit SqlDirentTouch(const CatalogDatabase &database);
392
393 bool BindDirentBase(const DirectoryEntryBase &entry);
394 bool BindPathHash(const shash::Md5 &hash);
395 bool BindXattr(const XattrList &xattrs);
396 bool BindXattrEmpty();
397 };
398
399
400 //------------------------------------------------------------------------------
401
402
403 /**
404 * Nested catalogs and bind mountpoints.
405 */
406 class SqlNestedCatalogLookup : public SqlCatalog {
407 public:
408 explicit SqlNestedCatalogLookup(const CatalogDatabase &database);
409 bool BindSearchPath(const PathString &path);
410 shash::Any GetContentHash() const;
411 uint64_t GetSize() const;
412 };
413
414
415 //------------------------------------------------------------------------------
416
417
418 /**
419 * Nested catalogs and bind mountpoints.
420 */
421 class SqlNestedCatalogListing : public SqlCatalog {
422 public:
423 explicit SqlNestedCatalogListing(const CatalogDatabase &database);
424 PathString GetPath() const;
425 shash::Any GetContentHash() const;
426 uint64_t GetSize() const;
427 };
428
429
430 //------------------------------------------------------------------------------
431
432
433 /**
434 * Only nested catalogs, no bind mountpoints. Used for replication and GC.
435 */
436 class SqlOwnNestedCatalogListing : public SqlCatalog {
437 public:
438 explicit SqlOwnNestedCatalogListing(const CatalogDatabase &database);
439 PathString GetPath() const;
440 shash::Any GetContentHash() const;
441 uint64_t GetSize() const;
442 };
443
444
445 //------------------------------------------------------------------------------
446
447
448 class SqlDirentInsert : public SqlDirentWrite {
449 public:
450 explicit SqlDirentInsert(const CatalogDatabase &database);
451 bool BindPathHash(const shash::Md5 &hash);
452 bool BindParentPathHash(const shash::Md5 &hash);
453 bool BindDirent(const DirectoryEntry &entry);
454 bool BindXattr(const XattrList &xattrs);
455 bool BindXattrEmpty();
456 };
457
458
459 //------------------------------------------------------------------------------
460
461
462 class SqlDirentUpdate : public SqlDirentWrite {
463 public:
464 explicit SqlDirentUpdate(const CatalogDatabase &database);
465 bool BindPathHash(const shash::Md5 &hash);
466 bool BindDirent(const DirectoryEntry &entry);
467 };
468
469
470 //------------------------------------------------------------------------------
471
472
473 class SqlDirentUnlink : public SqlCatalog {
474 public:
475 explicit SqlDirentUnlink(const CatalogDatabase &database);
476 bool BindPathHash(const shash::Md5 &hash);
477 };
478
479
480 //------------------------------------------------------------------------------
481
482
483 /**
484 * Changes the linkcount for all files in a hardlink group.
485 */
486 class SqlIncLinkcount : public SqlCatalog {
487 public:
488 explicit SqlIncLinkcount(const CatalogDatabase &database);
489 bool BindPathHash(const shash::Md5 &hash);
490 bool BindDelta(const int delta);
491 };
492
493
494 //------------------------------------------------------------------------------
495
496
497 class SqlChunkInsert : public SqlCatalog {
498 public:
499 explicit SqlChunkInsert(const CatalogDatabase &database);
500 bool BindPathHash(const shash::Md5 &hash);
501 bool BindFileChunk(const FileChunk &chunk);
502 };
503
504
505 //------------------------------------------------------------------------------
506
507
508 class SqlChunksRemove : public SqlCatalog {
509 public:
510 explicit SqlChunksRemove(const CatalogDatabase &database);
511 bool BindPathHash(const shash::Md5 &hash);
512 };
513
514
515 //------------------------------------------------------------------------------
516
517
518 class SqlChunksListing : public SqlCatalog {
519 public:
520 explicit SqlChunksListing(const CatalogDatabase &database);
521 bool BindPathHash(const shash::Md5 &hash);
522 FileChunk GetFileChunk(const shash::Algorithms interpret_hash_as) const;
523 };
524
525
526 //------------------------------------------------------------------------------
527
528
529 class SqlChunksCount : public SqlCatalog {
530 public:
531 explicit SqlChunksCount(const CatalogDatabase &database);
532 bool BindPathHash(const shash::Md5 &hash);
533 int GetChunkCount() const;
534 };
535
536
537 //------------------------------------------------------------------------------
538
539
540 class SqlMaxHardlinkGroup : public SqlCatalog {
541 public:
542 explicit SqlMaxHardlinkGroup(const CatalogDatabase &database);
543 uint32_t GetMaxGroupId() const;
544 };
545
546
547 //------------------------------------------------------------------------------
548
549
550 class SqlGetCounter : public SqlCatalog {
551 public:
552 explicit SqlGetCounter(const CatalogDatabase &database);
553 bool BindCounter(const std::string &counter);
554 uint64_t GetCounter() const;
555
556 private:
557 bool compat_;
558 };
559
560
561 //------------------------------------------------------------------------------
562
563
564 class SqlUpdateCounter : public SqlCatalog {
565 public:
566 explicit SqlUpdateCounter(const CatalogDatabase &database);
567 bool BindCounter(const std::string &counter);
568 bool BindDelta(const int64_t delta);
569 };
570
571
572 //------------------------------------------------------------------------------
573
574
575 class SqlCreateCounter : public SqlCatalog {
576 public:
577 explicit SqlCreateCounter(const CatalogDatabase &database);
578 bool BindCounter(const std::string &counter);
579 bool BindInitialValue(const int64_t value);
580 };
581
582
583 //------------------------------------------------------------------------------
584
585
586 class SqlAllChunks : public SqlCatalog {
587 public:
588 explicit SqlAllChunks(const CatalogDatabase &database);
589 bool Open();
590 bool Next(shash::Any *hash, zlib::Algorithms *compression_alg);
591 bool Close();
592 };
593
594
595 //------------------------------------------------------------------------------
596
597
598 class SqlLookupXattrs : public SqlCatalog {
599 public:
600 explicit SqlLookupXattrs(const CatalogDatabase &database);
601 bool BindPathHash(const shash::Md5 &hash);
602 XattrList GetXattrs();
603 };
604
605 } // namespace catalog
606
607 #endif // CVMFS_CATALOG_SQL_H_
608