GCC Code Coverage Report
Directory: cvmfs/ Exec Total Coverage
File: cvmfs/catalog_sql.h Lines: 43 47 91.5 %
Date: 2019-02-03 02:48:13 Branches: 31 65 47.7 %

Line Branch Exec Source
1
/**
2
 * This file is part of the CernVM file system.
3
 *
4
 * This file provides classes to wrap often used catalog SQL statements.
5
 * In particular, it wraps around sqlite3 prepared statement syntax.
6
 *
7
 * Usage example:
8
 *   SqlLookup statement(<database>);
9
 *   statement.BindPathHash(<hash>);
10
 *   if (statement.FetchRow()) {
11
 *     statement.GetDirectoryEntry(<catalog>);
12
 *   }
13
 *   statement.Reset();
14
 */
15
16
#ifndef CVMFS_CATALOG_SQL_H_
17
#define CVMFS_CATALOG_SQL_H_
18
19
#ifndef __STDC_FORMAT_MACROS
20
#define __STDC_FORMAT_MACROS
21
#endif
22
23
#include <inttypes.h>
24
25
#include <string>
26
27
#include "compression.h"
28
#include "directory_entry.h"
29
#include "file_chunk.h"
30
#include "hash.h"
31
#include "shortstring.h"
32
#include "sql.h"
33
34
class XattrList;
35
36
namespace catalog {
37
38
class Catalog;
39
40
41
167
class CatalogDatabase : public sqlite::Database<CatalogDatabase> {
42
 public:
43
  static const float kLatestSchema;
44
  static const float kLatestSupportedSchema;  // + 1.X catalogs (r/o)
45
  // Backwards-compatible schema changes
46
  static const unsigned kLatestSchemaRevision;
47
48
  bool CreateEmptyDatabase();
49
  bool InsertInitialValues(const std::string     &root_path,
50
                           const bool             volatile_content,
51
                           const std::string     &voms_authz,
52
                           const DirectoryEntry  &root_entry
53
                                             = DirectoryEntry(kDirentNegative));
54
55
  bool CheckSchemaCompatibility();
56
  bool LiveSchemaUpgradeIfNecessary();
57
  bool CompactDatabase() const;
58
59
  double GetRowIdWasteRatio() const;
60
  bool SetVOMSAuthz(const std::string&);
61
62
 protected:
63
  // TODO(rmeusel): C++11 - constructor inheritance
64
  friend class sqlite::Database<CatalogDatabase>;
65
167
  CatalogDatabase(const std::string  &filename,
66
                  const OpenMode      open_mode)
67
167
    : sqlite::Database<CatalogDatabase>(filename, open_mode) { }
68
};
69
70
71
//------------------------------------------------------------------------------
72
73
74
/**
75
 * Base class for all SQL statement classes.  It wraps a single SQL statement
76
 * and all neccessary calls of the sqlite3 API to deal with this statement.
77
 */
78
1982
class SqlCatalog : public sqlite::Sql {
79
 public:
80
  /**
81
   * Basic constructor to use this class for a specific statement.
82
   * @param database the database to use the query on
83
   * @param statement the statement to prepare
84
   */
85
593
  SqlCatalog(const CatalogDatabase &database, const std::string &statement) {
86
593
    Init(database.sqlite_db(), statement);
87
  }
88
89
  /**
90
   * Wrapper for retrieving MD5-ified path names.
91
   * @param idx_high offset of most significant bits in database query
92
   * @param idx_low offset of least significant bits in database query
93
   * @result the retrieved MD5 hash
94
   */
95
  inline shash::Md5 RetrieveMd5(const int idx_high, const int idx_low) const {
96
    return shash::Md5(RetrieveInt64(idx_high), RetrieveInt64(idx_low));
97
  }
98
99
  /**
100
   * Wrapper for retrieving a cryptographic hash from a blob field.
101
   */
102
212
  inline shash::Any RetrieveHashBlob(
103
    const int                idx_column,
104
    const shash::Algorithms  hash_algo,
105
    const char               hash_suffix = shash::kSuffixNone) const
106
  {
107
    // Note: SQLite documentation advises to first define the data type of BLOB
108
    //       by calling sqlite3_column_XXX() on the column and _afterwards_ get
109
    //       the number of bytes using sqlite3_column_bytes().
110
    //
111
    //  See: https://www.sqlite.org/c3ref/column_blob.html
112
    const unsigned char *buffer = static_cast<const unsigned char *>(
113
212
      RetrieveBlob(idx_column));
114
212
    const int byte_count = RetrieveBytes(idx_column);
115
    return (byte_count > 0) ? shash::Any(hash_algo, buffer, hash_suffix)
116
212
                            : shash::Any(hash_algo);
117
  }
118
119
  /**
120
   * Wrapper for retrieving a cryptographic hash from a text field.
121
   */
122
  inline shash::Any RetrieveHashHex(
123
    const int  idx_column,
124
    const char hash_suffix = shash::kSuffixNone) const
125
  {
126
    const std::string hash_string = std::string(reinterpret_cast<const char *>(
127
      RetrieveText(idx_column)));
128
    return shash::MkFromHexPtr(shash::HexPtr(hash_string), hash_suffix);
129
  }
130
131
  /**
132
   * Wrapper for binding a MD5-ified path name.
133
   * @param idx_high offset of most significant bits in database query
134
   * @param idx_low offset of least significant bits in database query
135
   * @param hash the hash to bind in the query
136
   * @result true on success, false otherwise
137
   */
138
722
  inline bool BindMd5(const int idx_high, const int idx_low,
139
                      const shash::Md5 &hash)
140
  {
141
    uint64_t high, low;
142
722
    hash.ToIntPair(&high, &low);
143

722
    const bool retval = BindInt64(idx_high, high) && BindInt64(idx_low, low);
144
722
    return retval;
145
  }
146
147
  /**
148
   * Wrapper for binding a cryptographic hash.  Algorithm of hash has to be
149
   * elsewhere (in flags).
150
   * @param idx_column offset of the blob field in database query
151
   * @param hash the hash to bind in the query
152
   * @result true on success, false otherwise
153
   */
154
281
  inline bool BindHashBlob(const int idx_column, const shash::Any &hash) {
155
281
    if (hash.IsNull()) {
156
190
      return BindNull(idx_column);
157
    } else {
158
91
      return BindBlob(idx_column, hash.digest, hash.GetDigestSize());
159
    }
160
  }
161
162
 protected:
163
1389
  SqlCatalog() : sqlite::Sql() {}
164
};
165
166
167
//------------------------------------------------------------------------------
168
169
170
/**
171
 * Common ancestor of SQL statemnts that deal with directory entries.
172
 */
173
644
class SqlDirent : public SqlCatalog {
174
 public:
175
  // Definition of bit positions for the flags field of a DirectoryEntry
176
  // All other bit positions are unused
177
  static const int kFlagDir                 = 1;
178
  // Link in the parent catalog
179
  static const int kFlagDirNestedMountpoint = 2;
180
  // Link in the child catalog
181
  static const int kFlagDirNestedRoot       = 32;
182
  static const int kFlagFile                = 4;
183
  static const int kFlagLink                = 8;
184
  static const int kFlagFileSpecial         = 16;
185
  static const int kFlagFileChunk           = 64;
186
  /**
187
   * The file is not natively stored in cvmfs but on a different storage system,
188
   * for instance on HTTPS data federation services.
189
   * NOTE: used as magic number in SqlListContentHashes::SqlListContentHashes
190
   */
191
  static const int kFlagFileExternal        = 128;
192
  // as of 2^8: 3 bit for hashes
193
  //   - 0: SHA-1
194
  //   - 1: RIPEMD-160
195
  //   - ...
196
  // Corresponds to shash::algorithms with offset in order to support future
197
  // hashes
198
  static const int kFlagPosHash             = 8;
199
  // Compression methods, 3 bits starting at 2^11
200
  // Corresponds to zlib::Algorithms
201
  static const int kFlagPosCompression      = 11;
202
  /**
203
   * A transition point to a root catalog (instead of a nested catalog).  Used
204
   * to link previous snapshots into the catalog structure.
205
   */
206
  static const int kFlagDirBindMountpoint   = 0x4000;  // 2^14
207
  /**
208
   * An entry that should not appear in listings.  Used for the /.cvmfs
209
   * directory.
210
   */
211
  static const int kFlagHidden              = 0x8000;  // 2^15
212
213
214
 protected:
215
  /**
216
   * Take the meta data from the DirectoryEntry and transform it
217
   * into a valid flags field ready to be saved in the database.
218
   * @param entry the DirectoryEntry to encode
219
   * @return an integer containing the bitmap of the flags field
220
   */
221
  unsigned CreateDatabaseFlags(const DirectoryEntry &entry) const;
222
  void StoreHashAlgorithm(const shash::Algorithms algo, unsigned *flags) const;
223
  shash::Algorithms RetrieveHashAlgorithm(const unsigned flags) const;
224
  zlib::Algorithms RetrieveCompressionAlgorithm(const unsigned flags) const;
225
226
  /**
227
   * The hardlink information (hardlink group ID and linkcount) is saved in one
228
   * uint_64t field in the CVMFS Catalogs. Therefore we need to do bitshifting
229
   * in these helper methods.
230
   */
231
  uint32_t Hardlinks2Linkcount(const uint64_t hardlinks) const;
232
  uint32_t Hardlinks2HardlinkGroup(const uint64_t hardlinks) const;
233
  uint64_t MakeHardlinks(const uint32_t hardlink_group,
234
                         const uint32_t linkcount) const;
235
236
  /**
237
   * Replaces place holder variables in a symbolic link by actual path elements.
238
   * @param raw_symlink the raw symlink path (may) containing place holders
239
   * @return the expanded symlink
240
   */
241
  void ExpandSymlink(LinkString *raw_symlink) const;
242
};
243
244
245
//------------------------------------------------------------------------------
246
247
248
250
class SqlDirentWrite : public SqlDirent {
249
 public:
250
  /**
251
   * To bind an entire DirectoryEntry
252
   * @param entry the DirectoryEntry to bind in the SQL statement
253
   * @return true on success, false otherwise
254
   */
255
  virtual bool BindDirent(const DirectoryEntry &entry) = 0;
256
257
 protected:
258
  bool BindDirentFields(const int hash_idx,
259
                        const int hardlinks_idx,
260
                        const int size_idx,
261
                        const int mode_idx,
262
                        const int mtime_idx,
263
                        const int flags_idx,
264
                        const int name_idx,
265
                        const int symlink_idx,
266
                        const int uid_idx,
267
                        const int gid_idx,
268
                        const DirectoryEntry &entry);
269
};
270
271
272
//------------------------------------------------------------------------------
273
274
275
1
class SqlListContentHashes : public SqlDirent {
276
 public:
277
  explicit SqlListContentHashes(const CatalogDatabase &database);
278
  shash::Any GetHash() const;
279
};
280
281
282
//------------------------------------------------------------------------------
283
284
285
392
class SqlLookup : public SqlDirent {
286
 public:
287
  /**
288
   * Retrieves a DirectoryEntry from a freshly performed SqlLookup statement.
289
   * @param catalog the catalog in which the DirectoryEntry resides
290
   * @return the retrieved DirectoryEntry
291
   */
292
  DirectoryEntry GetDirent(const Catalog *catalog,
293
                           const bool expand_symlink = true) const;
294
295
  /**
296
   * DirectoryEntrys do not contain their path hash.
297
   * This method retrieves the saved path hash from the database
298
   * @return the MD5 path hash of a freshly performed lookup
299
   */
300
  shash::Md5 GetPathHash() const;
301
302
  /**
303
   * DirectoryEntries do not contain their parent path hash.
304
   * This method retrieves the saved parent path hash from the database
305
   * @return the MD5 parent path hash of a freshly performed lookup
306
   */
307
  shash::Md5 GetParentPathHash() const;
308
};
309
310
311
//------------------------------------------------------------------------------
312
313
314
196
class SqlListing : public SqlLookup {
315
 public:
316
  explicit SqlListing(const CatalogDatabase &database);
317
  bool BindPathHash(const struct shash::Md5 &hash);
318
};
319
320
321
//------------------------------------------------------------------------------
322
323
324
196
class SqlLookupPathHash : public SqlLookup {
325
 public:
326
  explicit SqlLookupPathHash(const CatalogDatabase &database);
327
  bool BindPathHash(const struct shash::Md5 &hash);
328
};
329
330
331
//------------------------------------------------------------------------------
332
333
334
class SqlLookupInode : public SqlLookup {
335
 public:
336
  explicit SqlLookupInode(const CatalogDatabase &database);
337
  bool BindRowId(const uint64_t inode);
338
};
339
340
341
//------------------------------------------------------------------------------
342
343
344
/**
345
 * This SQL statement is only used for legacy catalog migrations and has been
346
 * moved here as it needs to use a locally defined macro inside catalog_sql.cc
347
 *
348
 * Queries a single catalog and looks for DirectoryEntrys that have direct
349
 * children in the same catalog but are marked as 'nested catalog mountpoints'.
350
 * This is an inconsistent situation, since a mountpoint is supposed to be empty
351
 * and it's children are stored in the corresponding referenced nested catalog.
352
 *
353
 * Note: the user code needs to check if there is a corresponding nested catalog
354
 *       reference for the found dangling mountpoints. If so, we also have a
355
 *       bogus state, but it is not reliably fixable automatically. The child-
356
 *       DirectoryEntrys would be masked by the mounting nested catalog but it
357
 *       is not clear if we can simply delete them or if this would destroy data.
358
 */
359
class SqlLookupDanglingMountpoints : public catalog::SqlLookup {
360
 public:
361
  explicit SqlLookupDanglingMountpoints(const CatalogDatabase &database);
362
};
363
364
365
//------------------------------------------------------------------------------
366
367
368
/**
369
 * Filesystem like _touch_ of a DirectoryEntry. Only file system specific meta
370
 * data will be modified.  All CVMFS-specific administrative data stays
371
 * unchanged.
372
 * NOTE: This is not a subclass of SqlDirent since it works on
373
 *       DirectoryEntryBase objects, which are restricted to file system meta
374
 *       data.
375
 */
376
78
class SqlDirentTouch : public SqlCatalog {
377
 public:
378
  explicit SqlDirentTouch(const CatalogDatabase &database);
379
380
  bool BindDirentBase(const DirectoryEntryBase &entry);
381
  bool BindPathHash(const shash::Md5 &hash);
382
};
383
384
385
//------------------------------------------------------------------------------
386
387
388
/**
389
 * Nested catalogs and bind mountpoints.
390
 */
391
196
class SqlNestedCatalogLookup : public SqlCatalog {
392
 public:
393
  explicit SqlNestedCatalogLookup(const CatalogDatabase &database);
394
  bool BindSearchPath(const PathString &path);
395
  shash::Any GetContentHash() const;
396
  uint64_t GetSize() const;
397
};
398
399
400
//------------------------------------------------------------------------------
401
402
403
/**
404
 * Nested catalogs and bind mountpoints.
405
 */
406
196
class SqlNestedCatalogListing : public SqlCatalog {
407
 public:
408
  explicit SqlNestedCatalogListing(const CatalogDatabase &database);
409
  PathString GetPath() const;
410
  shash::Any GetContentHash() const;
411
  uint64_t GetSize() const;
412
};
413
414
415
//------------------------------------------------------------------------------
416
417
418
/**
419
 * Only nested catalogs, no bind mountpoints. Used for replication and GC.
420
 */
421
196
class SqlOwnNestedCatalogListing : public SqlCatalog {
422
 public:
423
  explicit SqlOwnNestedCatalogListing(const CatalogDatabase &database);
424
  PathString GetPath() const;
425
  shash::Any GetContentHash() const;
426
  uint64_t GetSize() const;
427
};
428
429
430
//------------------------------------------------------------------------------
431
432
433
125
class SqlDirentInsert : public SqlDirentWrite {
434
 public:
435
  explicit SqlDirentInsert(const CatalogDatabase &database);
436
  bool BindPathHash(const shash::Md5 &hash);
437
  bool BindParentPathHash(const shash::Md5 &hash);
438
  bool BindDirent(const DirectoryEntry &entry);
439
  bool BindXattr(const XattrList &xattrs);
440
  bool BindXattrEmpty();
441
};
442
443
444
//------------------------------------------------------------------------------
445
446
447
78
class SqlDirentUpdate : public SqlDirentWrite {
448
 public:
449
  explicit SqlDirentUpdate(const CatalogDatabase &database);
450
  bool BindPathHash(const shash::Md5 &hash);
451
  bool BindDirent(const DirectoryEntry &entry);
452
};
453
454
455
//------------------------------------------------------------------------------
456
457
458
78
class SqlDirentUnlink : public SqlCatalog {
459
 public:
460
  explicit SqlDirentUnlink(const CatalogDatabase &database);
461
  bool BindPathHash(const shash::Md5 &hash);
462
};
463
464
465
//------------------------------------------------------------------------------
466
467
468
/**
469
 * Changes the linkcount for all files in a hardlink group.
470
 */
471
78
class SqlIncLinkcount : public SqlCatalog {
472
 public:
473
  explicit SqlIncLinkcount(const CatalogDatabase &database);
474
  bool BindPathHash(const shash::Md5 &hash);
475
  bool BindDelta(const int delta);
476
};
477
478
479
//------------------------------------------------------------------------------
480
481
482
78
class SqlChunkInsert : public SqlCatalog {
483
 public:
484
  explicit SqlChunkInsert(const CatalogDatabase &database);
485
  bool BindPathHash(const shash::Md5 &hash);
486
  bool BindFileChunk(const FileChunk &chunk);
487
};
488
489
490
//------------------------------------------------------------------------------
491
492
493
78
class SqlChunksRemove : public SqlCatalog {
494
 public:
495
  explicit SqlChunksRemove(const CatalogDatabase &database);
496
  bool BindPathHash(const shash::Md5 &hash);
497
};
498
499
500
//------------------------------------------------------------------------------
501
502
503
196
class SqlChunksListing : public SqlCatalog {
504
 public:
505
  explicit SqlChunksListing(const CatalogDatabase &database);
506
  bool BindPathHash(const shash::Md5 &hash);
507
  FileChunk GetFileChunk(const shash::Algorithms interpret_hash_as) const;
508
};
509
510
511
//------------------------------------------------------------------------------
512
513
514
78
class SqlChunksCount : public SqlCatalog {
515
 public:
516
  explicit SqlChunksCount(const CatalogDatabase &database);
517
  bool BindPathHash(const shash::Md5 &hash);
518
  int GetChunkCount() const;
519
};
520
521
522
//------------------------------------------------------------------------------
523
524
525
78
class SqlMaxHardlinkGroup : public SqlCatalog {
526
 public:
527
  explicit SqlMaxHardlinkGroup(const CatalogDatabase &database);
528
  uint32_t GetMaxGroupId() const;
529
};
530
531
532
//------------------------------------------------------------------------------
533
534
535
120
class SqlGetCounter : public SqlCatalog {
536
 public:
537
  explicit SqlGetCounter(const CatalogDatabase &database);
538
  bool BindCounter(const std::string &counter);
539
  uint64_t GetCounter() const;
540
 private:
541
  bool compat_;
542
};
543
544
545
//------------------------------------------------------------------------------
546
547
548
22
class SqlUpdateCounter : public SqlCatalog {
549
 public:
550
  explicit SqlUpdateCounter(const CatalogDatabase &database);
551
  bool BindCounter(const std::string &counter);
552
  bool BindDelta(const int64_t delta);
553
};
554
555
556
//------------------------------------------------------------------------------
557
558
559
64
class SqlCreateCounter : public SqlCatalog {
560
 public:
561
  explicit SqlCreateCounter(const CatalogDatabase &database);
562
  bool BindCounter(const std::string &counter);
563
  bool BindInitialValue(const int64_t value);
564
};
565
566
567
//------------------------------------------------------------------------------
568
569
570
196
class SqlAllChunks : public SqlCatalog {
571
 public:
572
  explicit SqlAllChunks(const CatalogDatabase &database);
573
  bool Open();
574
  bool Next(shash::Any *hash, zlib::Algorithms *compression_alg);
575
  bool Close();
576
};
577
578
579
//------------------------------------------------------------------------------
580
581
582
196
class SqlLookupXattrs : public SqlCatalog {
583
 public:
584
  explicit SqlLookupXattrs(const CatalogDatabase &database);
585
  bool BindPathHash(const shash::Md5 &hash);
586
  XattrList GetXattrs();
587
};
588
589
}  // namespace catalog
590
591
#endif  // CVMFS_CATALOG_SQL_H_