GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/catalog_sql.h
Date: 2026-04-05 02:35:23
Exec Total Coverage
Lines: 19 21 90.5%
Branches: 11 18 61.1%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM file system.
3 *
4 * This file provides classes to wrap often used catalog SQL statements.
5 * In particular, it wraps around sqlite3 prepared statement syntax.
6 *
7 * Usage example:
8 * SqlLookup statement(<database>);
9 * statement.BindPathHash(<hash>);
10 * if (statement.FetchRow()) {
11 * statement.GetDirectoryEntry(<catalog>);
12 * }
13 * statement.Reset();
14 */
15
16 #ifndef CVMFS_CATALOG_SQL_H_
17 #define CVMFS_CATALOG_SQL_H_
18
19 #ifndef __STDC_FORMAT_MACROS
20 #define __STDC_FORMAT_MACROS
21 #endif
22
23 #include <inttypes.h>
24
25 #include <string>
26
27 #include "compression/compression.h"
28 #include "crypto/hash.h"
29 #include "directory_entry.h"
30 #include "file_chunk.h"
31 #include "shortstring.h"
32 #include "sql.h"
33
34 class XattrList;
35
36 namespace catalog {
37
38 // Set to false via CVMFS_NO_IGNORE_LEGACY_BULKHASHES env var to restore the
39 // pre-2.14 behavior of including bulk hashes for chunked files.
40 extern bool g_ignore_legacy_bulk_hashes;
41
42 class Catalog;
43
44
45 class CatalogDatabase : public sqlite::Database<CatalogDatabase> {
46 public:
47 static const float kLatestSchema;
48 static const float kLatestSupportedSchema; // + 1.X catalogs (r/o)
49 // Backwards-compatible schema changes
50 static const unsigned kLatestSchemaRevision;
51
52 bool CreateEmptyDatabase();
53 bool InsertInitialValues(
54 const std::string &root_path,
55 const bool volatile_content,
56 const std::string &voms_authz,
57 const DirectoryEntry &root_entry = DirectoryEntry(kDirentNegative));
58
59 bool CheckSchemaCompatibility();
60 bool LiveSchemaUpgradeIfNecessary();
61 bool CompactDatabase() const;
62
63 double GetRowIdWasteRatio() const;
64 bool SetVOMSAuthz(const std::string &);
65
66 protected:
67 // TODO(rmeusel): C++11 - constructor inheritance
68 friend class sqlite::Database<CatalogDatabase>;
69 3141 CatalogDatabase(const std::string &filename, const OpenMode open_mode)
70 3141 : sqlite::Database<CatalogDatabase>(filename, open_mode) { }
71 };
72
73
74 //------------------------------------------------------------------------------
75
76
77 /**
78 * Base class for all SQL statement classes. It wraps a single SQL statement
79 * and all necessary calls of the sqlite3 API to deal with this statement.
80 */
81 class SqlCatalog : public sqlite::Sql {
82 public:
83 /**
84 * Basic constructor to use this class for a specific statement.
85 * @param database the database to use the query on
86 * @param statement the statement to prepare
87 */
88 12410 SqlCatalog(const CatalogDatabase &database, const std::string &statement) {
89
2/4
✓ Branch 1 taken 12410 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 12410 times.
✗ Branch 5 not taken.
 12410 Init(database.sqlite_db(), statement);
90 12410 }
91
92 /**
93 * Wrapper for retrieving MD5-ified path names.
94 * @param idx_high offset of most significant bits in database query
95 * @param idx_low offset of least significant bits in database query
96 * @result the retrieved MD5 hash
97 */
98 ✗ inline shash::Md5 RetrieveMd5(const int idx_high, const int idx_low) const {
99 ✗ return shash::Md5(RetrieveInt64(idx_high), RetrieveInt64(idx_low));
100 }
101
102 /**
103 * Wrapper for retrieving a cryptographic hash from a blob field.
104 */
105 8609 inline shash::Any RetrieveHashBlob(
106 const int idx_column,
107 const shash::Algorithms hash_algo,
108 const char hash_suffix = shash::kSuffixNone) const {
109 // Note: SQLite documentation advises to first define the data type of BLOB
110 // by calling sqlite3_column_XXX() on the column and _afterwards_ get
111 // the number of bytes using sqlite3_column_bytes().
112 //
113 // See: https://www.sqlite.org/c3ref/column_blob.html
114 const unsigned char *buffer = static_cast<const unsigned char *>(
115 8609 RetrieveBlob(idx_column));
116 8609 const int byte_count = RetrieveBytes(idx_column);
117 return (byte_count > 0) ? shash::Any(hash_algo, buffer, hash_suffix)
118
2/2
✓ Branch 0 taken 2531 times.
✓ Branch 1 taken 6078 times.
 8609 : shash::Any(hash_algo);
119 }
120
121 /**
122 * Wrapper for retrieving a cryptographic hash from a text field.
123 */
124 inline shash::Any RetrieveHashHex(
125 const int idx_column, const char hash_suffix = shash::kSuffixNone) const {
126 const std::string hash_string = std::string(
127 reinterpret_cast<const char *>(RetrieveText(idx_column)));
128 return shash::MkFromHexPtr(shash::HexPtr(hash_string), hash_suffix);
129 }
130
131 /**
132 * Wrapper for binding a MD5-ified path name.
133 * @param idx_high offset of most significant bits in database query
134 * @param idx_low offset of least significant bits in database query
135 * @param hash the hash to bind in the query
136 * @result true on success, false otherwise
137 */
138 23047 inline bool BindMd5(const int idx_high, const int idx_low,
139 const shash::Md5 &hash) {
140 uint64_t high, low;
141
1/2
✓ Branch 1 taken 23047 times.
✗ Branch 2 not taken.
 23047 hash.ToIntPair(&high, &low);
142
1/2
✓ Branch 1 taken 23047 times.
✗ Branch 2 not taken.
 23047 const bool retval = BindInt64(idx_high, static_cast<int64_t>(high))
143
3/6
✓ Branch 0 taken 23047 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 23047 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 23047 times.
✗ Branch 6 not taken.
 23047 && BindInt64(idx_low, static_cast<int64_t>(low));
144 23047 return retval;
145 }
146
147 /**
148 * Wrapper for binding a cryptographic hash. Algorithm of hash has to be
149 * elsewhere (in flags).
150 * @param idx_column offset of the blob field in database query
151 * @param hash the hash to bind in the query
152 * @result true on success, false otherwise
153 */
154 7961 inline bool BindHashBlob(const int idx_column, const shash::Any &hash) {
155
2/2
✓ Branch 1 taken 5177 times.
✓ Branch 2 taken 2784 times.
 7961 if (hash.IsNull()) {
156 5177 return BindNull(idx_column);
157 } else {
158 2784 return BindBlob(idx_column, hash.digest, hash.GetDigestSize());
159 }
160 }
161
162 protected:
163 28928 SqlCatalog() : sqlite::Sql() { }
164 };
165
166
167 //------------------------------------------------------------------------------
168
169
170 /**
171 * Common ancestor of SQL statements that deal with directory entries.
172 */
173 class SqlDirent : public SqlCatalog {
174 public:
175 // Definition of bit positions for the flags field of a DirectoryEntry
176 // All other bit positions are unused
177 static const int kFlagDir = 1;
178 // Link in the parent catalog
179 static const int kFlagDirNestedMountpoint = 2;
180 // Link in the child catalog
181 static const int kFlagDirNestedRoot = 32;
182 static const int kFlagFile = 4;
183 static const int kFlagLink = 8;
184 static const int kFlagFileSpecial = 16;
185 static const int kFlagFileChunk = 64;
186 /**
187 * The file is not natively stored in cvmfs but on a different storage system,
188 * for instance on HTTPS data federation services.
189 * NOTE: used as magic number in SqlListContentHashes::SqlListContentHashes
190 */
191 static const int kFlagFileExternal = 128;
192 // as of 2^8: 3 bit for hashes
193 // - 0: SHA-1
194 // - 1: RIPEMD-160
195 // - ...
196 // Corresponds to shash::algorithms with offset in order to support future
197 // hashes
198 static const int kFlagPosHash = 8;
199 // Compression methods, 3 bits starting at 2^11
200 // Corresponds to zlib::Algorithms
201 static const int kFlagPosCompression = 11;
202 /**
203 * A transition point to a root catalog (instead of a nested catalog). Used
204 * to link previous snapshots into the catalog structure.
205 */
206 static const int kFlagDirBindMountpoint = 0x4000; // 2^14
207 /**
208 * An entry that should not appear in listings. Used for the /.cvmfs
209 * directory.
210 */
211 static const int kFlagHidden = 0x8000; // 2^15
212 /**
213 * For regular files, indicates that the file should be opened with direct I/O
214 */
215 static const int kFlagDirectIo = 0x10000; // 2^16
216
217
218 protected:
219 /**
220 * Take the meta data from the DirectoryEntry and transform it
221 * into a valid flags field ready to be saved in the database.
222 * @param entry the DirectoryEntry to encode
223 * @return an integer containing the bitmap of the flags field
224 */
225 unsigned CreateDatabaseFlags(const DirectoryEntry &entry) const;
226 void StoreHashAlgorithm(const shash::Algorithms algo, unsigned *flags) const;
227 shash::Algorithms RetrieveHashAlgorithm(const unsigned flags) const;
228 zlib::Algorithms RetrieveCompressionAlgorithm(const unsigned flags) const;
229
230 /**
231 * The hardlink information (hardlink group ID and linkcount) is saved in one
232 * uint_64t field in the CVMFS Catalogs. Therefore we need to do bitshifting
233 * in these helper methods.
234 */
235 uint32_t Hardlinks2Linkcount(const uint64_t hardlinks) const;
236 uint32_t Hardlinks2HardlinkGroup(const uint64_t hardlinks) const;
237 uint64_t MakeHardlinks(const uint32_t hardlink_group,
238 const uint32_t linkcount) const;
239
240 /**
241 * Replaces place holder variables in a symbolic link by actual path elements.
242 * @param raw_symlink the raw symlink path (may) containing place holders
243 * @return the expanded symlink
244 */
245 void ExpandSymlink(LinkString *raw_symlink) const;
246 };
247
248
249 //------------------------------------------------------------------------------
250
251
252 class SqlDirentWrite : public SqlDirent {
253 public:
254 /**
255 * To bind an entire DirectoryEntry
256 * @param entry the DirectoryEntry to bind in the SQL statement
257 * @return true on success, false otherwise
258 */
259 virtual bool BindDirent(const DirectoryEntry &entry) = 0;
260
261 protected:
262 bool BindDirentFields(const int hash_idx,
263 const int hardlinks_idx,
264 const int size_idx,
265 const int mode_idx,
266 const int mtime_idx,
267 const int mtimens_idx,
268 const int flags_idx,
269 const int name_idx,
270 const int symlink_idx,
271 const int uid_idx,
272 const int gid_idx,
273 const DirectoryEntry &entry);
274 };
275
276
277 //------------------------------------------------------------------------------
278
279
280 class SqlListContentHashes : public SqlDirent {
281 public:
282 explicit SqlListContentHashes(const CatalogDatabase &database);
283 shash::Any GetHash() const;
284 };
285
286
287 //------------------------------------------------------------------------------
288
289
290 class SqlLookup : public SqlDirent {
291 public:
292 /**
293 * Retrieves a DirectoryEntry from a freshly performed SqlLookup statement.
294 * @param catalog the catalog in which the DirectoryEntry resides
295 * @return the retrieved DirectoryEntry
296 */
297 DirectoryEntry GetDirent(const Catalog *catalog,
298 const bool expand_symlink = true) const;
299
300 /**
301 * DirectoryEntrys do not contain their path hash.
302 * This method retrieves the saved path hash from the database
303 * @return the MD5 path hash of a freshly performed lookup
304 */
305 shash::Md5 GetPathHash() const;
306
307 /**
308 * DirectoryEntries do not contain their parent path hash.
309 * This method retrieves the saved parent path hash from the database
310 * @return the MD5 parent path hash of a freshly performed lookup
311 */
312 shash::Md5 GetParentPathHash() const;
313 };
314
315
316 //------------------------------------------------------------------------------
317
318
319 class SqlListing : public SqlLookup {
320 public:
321 explicit SqlListing(const CatalogDatabase &database);
322 bool BindPathHash(const struct shash::Md5 &hash);
323 };
324
325
326 //------------------------------------------------------------------------------
327
328
329 class SqlLookupPathHash : public SqlLookup {
330 public:
331 explicit SqlLookupPathHash(const CatalogDatabase &database);
332 bool BindPathHash(const struct shash::Md5 &hash);
333 };
334
335
336 //------------------------------------------------------------------------------
337
338
339 class SqlLookupInode : public SqlLookup {
340 public:
341 explicit SqlLookupInode(const CatalogDatabase &database);
342 bool BindRowId(const uint64_t inode);
343 };
344
345
346 //------------------------------------------------------------------------------
347
348
349 /**
350 * This SQL statement is only used for legacy catalog migrations and has been
351 * moved here as it needs to use a locally defined macro inside catalog_sql.cc
352 *
353 * Queries a single catalog and looks for DirectoryEntrys that have direct
354 * children in the same catalog but are marked as 'nested catalog mountpoints'.
355 * This is an inconsistent situation, since a mountpoint is supposed to be empty
356 * and it's children are stored in the corresponding referenced nested catalog.
357 *
358 * Note: the user code needs to check if there is a corresponding nested catalog
359 * reference for the found dangling mountpoints. If so, we also have a
360 * bogus state, but it is not reliably fixable automatically. The child-
361 * DirectoryEntrys would be masked by the mounting nested catalog but it
362 * is not clear if we can simply delete them or if this would destroy
363 * data.
364 */
365 class SqlLookupDanglingMountpoints : public catalog::SqlLookup {
366 public:
367 explicit SqlLookupDanglingMountpoints(const CatalogDatabase &database);
368 };
369
370
371 //------------------------------------------------------------------------------
372
373
374 /**
375 * Filesystem like _touch_ of a DirectoryEntry. Only file system specific meta
376 * data will be modified. All CVMFS-specific administrative data stays
377 * unchanged.
378 * NOTE: This is not a subclass of SqlDirent since it works on
379 * DirectoryEntryBase objects, which are restricted to file system meta
380 * data.
381 */
382 class SqlDirentTouch : public SqlCatalog {
383 public:
384 explicit SqlDirentTouch(const CatalogDatabase &database);
385
386 bool BindDirentBase(const DirectoryEntryBase &entry);
387 bool BindPathHash(const shash::Md5 &hash);
388 bool BindXattr(const XattrList &xattrs);
389 bool BindXattrEmpty();
390 };
391
392
393 //------------------------------------------------------------------------------
394
395
396 /**
397 * Nested catalogs and bind mountpoints.
398 */
399 class SqlNestedCatalogLookup : public SqlCatalog {
400 public:
401 explicit SqlNestedCatalogLookup(const CatalogDatabase &database);
402 bool BindSearchPath(const PathString &path);
403 shash::Any GetContentHash() const;
404 uint64_t GetSize() const;
405 };
406
407
408 //------------------------------------------------------------------------------
409
410
411 /**
412 * Nested catalogs and bind mountpoints.
413 */
414 class SqlNestedCatalogListing : public SqlCatalog {
415 public:
416 explicit SqlNestedCatalogListing(const CatalogDatabase &database);
417 PathString GetPath() const;
418 shash::Any GetContentHash() const;
419 uint64_t GetSize() const;
420 };
421
422
423 //------------------------------------------------------------------------------
424
425
426 /**
427 * Only nested catalogs, no bind mountpoints. Used for replication and GC.
428 */
429 class SqlOwnNestedCatalogListing : public SqlCatalog {
430 public:
431 explicit SqlOwnNestedCatalogListing(const CatalogDatabase &database);
432 PathString GetPath() const;
433 shash::Any GetContentHash() const;
434 uint64_t GetSize() const;
435 };
436
437
438 //------------------------------------------------------------------------------
439
440
441 class SqlDirentInsert : public SqlDirentWrite {
442 public:
443 explicit SqlDirentInsert(const CatalogDatabase &database);
444 bool BindPathHash(const shash::Md5 &hash);
445 bool BindParentPathHash(const shash::Md5 &hash);
446 bool BindDirent(const DirectoryEntry &entry);
447 bool BindXattr(const XattrList &xattrs);
448 bool BindXattrEmpty();
449 };
450
451
452 //------------------------------------------------------------------------------
453
454
455 class SqlDirentUpdate : public SqlDirentWrite {
456 public:
457 explicit SqlDirentUpdate(const CatalogDatabase &database);
458 bool BindPathHash(const shash::Md5 &hash);
459 bool BindDirent(const DirectoryEntry &entry);
460 };
461
462
463 //------------------------------------------------------------------------------
464
465
466 class SqlDirentUnlink : public SqlCatalog {
467 public:
468 explicit SqlDirentUnlink(const CatalogDatabase &database);
469 bool BindPathHash(const shash::Md5 &hash);
470 };
471
472
473 //------------------------------------------------------------------------------
474
475
476 /**
477 * Changes the linkcount for all files in a hardlink group.
478 */
479 class SqlIncLinkcount : public SqlCatalog {
480 public:
481 explicit SqlIncLinkcount(const CatalogDatabase &database);
482 bool BindPathHash(const shash::Md5 &hash);
483 bool BindDelta(const int delta);
484 };
485
486
487 //------------------------------------------------------------------------------
488
489
490 class SqlChunkInsert : public SqlCatalog {
491 public:
492 explicit SqlChunkInsert(const CatalogDatabase &database);
493 bool BindPathHash(const shash::Md5 &hash);
494 bool BindFileChunk(const FileChunk &chunk);
495 };
496
497
498 //------------------------------------------------------------------------------
499
500
501 class SqlChunksRemove : public SqlCatalog {
502 public:
503 explicit SqlChunksRemove(const CatalogDatabase &database);
504 bool BindPathHash(const shash::Md5 &hash);
505 };
506
507
508 //------------------------------------------------------------------------------
509
510
511 class SqlChunksListing : public SqlCatalog {
512 public:
513 explicit SqlChunksListing(const CatalogDatabase &database);
514 bool BindPathHash(const shash::Md5 &hash);
515 FileChunk GetFileChunk(const shash::Algorithms interpret_hash_as) const;
516 };
517
518
519 //------------------------------------------------------------------------------
520
521
522 class SqlChunksCount : public SqlCatalog {
523 public:
524 explicit SqlChunksCount(const CatalogDatabase &database);
525 bool BindPathHash(const shash::Md5 &hash);
526 int GetChunkCount() const;
527 };
528
529
530 //------------------------------------------------------------------------------
531
532
533 class SqlMaxHardlinkGroup : public SqlCatalog {
534 public:
535 explicit SqlMaxHardlinkGroup(const CatalogDatabase &database);
536 uint32_t GetMaxGroupId() const;
537 };
538
539
540 //------------------------------------------------------------------------------
541
542
543 class SqlGetCounter : public SqlCatalog {
544 public:
545 explicit SqlGetCounter(const CatalogDatabase &database);
546 bool BindCounter(const std::string &counter);
547 uint64_t GetCounter() const;
548
549 private:
550 bool compat_;
551 };
552
553
554 //------------------------------------------------------------------------------
555
556
557 class SqlUpdateCounter : public SqlCatalog {
558 public:
559 explicit SqlUpdateCounter(const CatalogDatabase &database);
560 bool BindCounter(const std::string &counter);
561 bool BindDelta(const int64_t delta);
562 };
563
564
565 //------------------------------------------------------------------------------
566
567
568 class SqlCreateCounter : public SqlCatalog {
569 public:
570 explicit SqlCreateCounter(const CatalogDatabase &database);
571 bool BindCounter(const std::string &counter);
572 bool BindInitialValue(const int64_t value);
573 };
574
575
576 //------------------------------------------------------------------------------
577
578
579 class SqlAllChunks : public SqlCatalog {
580 public:
581 explicit SqlAllChunks(const CatalogDatabase &database);
582 bool Open();
583 bool Next(shash::Any *hash, zlib::Algorithms *compression_alg);
584 bool Close();
585 };
586
587
588 //------------------------------------------------------------------------------
589
590
591 class SqlLookupXattrs : public SqlCatalog {
592 public:
593 explicit SqlLookupXattrs(const CatalogDatabase &database);
594 bool BindPathHash(const shash::Md5 &hash);
595 XattrList GetXattrs();
596 };
597
598 } // namespace catalog
599
600 #endif // CVMFS_CATALOG_SQL_H_
601