CernVM-FS  2.12.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
catalog_sql.cc
Go to the documentation of this file.
1 
5 #include "cvmfs_config.h"
6 #include "catalog_sql.h"
7 
8 #include <cstdlib>
9 #include <cstring>
10 
11 #include "catalog.h"
12 #include "globals.h"
13 #include "util/logging.h"
14 #include "util/posix.h"
15 #include "xattr.h"
16 
17 using namespace std; // NOLINT
18 
19 namespace catalog {
20 
34 // ChangeLog
35 // 2.5 (Jun 26 2013 - Git: e79baec22c6abd6ddcdf8f8d7d33921027a052ab)
36 // * add (backward compatible) schema revision - see below
37 // * add statistics counters for chunked files
38 // Note: this was retrofitted and needed a catalog migration step
39 //
40 // 2.4 (Aug 15 2012 - Git: 17de8fc782b5b8dc4404dda925627b5ec2b552e1)
41 // 2.3 (Aug 15 2012 - Git: ab77688cdb2f851af3fe983bf3694dc2465e65be)
42 // 2.2 (never existed)
43 // 2.1 (Aug 7 2012 - Git: beba36c12d2b1123ffbb169f865a861e570adc68)
44 // * add 'chunks' table for file chunks
45 // * add 'statistics' table for accumulative counters
46 // * rename 'inode' field to 'hardlinks'
47 // * containing both hardlink group ID and linkcount
48 // * .cvmfscatalog files become first-class entries in the catalogs
49 //
50 // 2.0 (Aug 6 2012 - Git: c8a81ede603e57fbe4324b6ab6bc8c41e3a2fa5f)
51 // * beginning of CernVM-FS 2.1.x branch ('modern' era)
52 //
53 // 1.x (earlier - code base still in SVN)
54 // * pre-historic times
55 // 0.9 (some time 2011, artificial version)
56 // * 1.0 catalogs that lack the SHA-1 value for nested catalogs
57 const float CatalogDatabase::kLatestSchema = 2.5;
58 const float CatalogDatabase::kLatestSupportedSchema = 2.5; // + 1.X (r/o)
59 
60 // ChangeLog
61 // 0 --> 1: (Jan 6 2014 - Git: 3667fe7a669d0d65e07275b753a7c6f23fc267df)
62 // * add size column to nested catalog table,
63 // * add schema_revision property
64 // 1 --> 2: (Jan 22 2014 - Git: 85e6680e52cfe56dc1213a5ad74a5cc62fd50ead):
65 // * add xattr column to catalog table
66 // * add self_xattr and subtree_xattr statistics counters
67 // 2 --> 3: (Sep 28 2015 - Git: f4171234b13ea448589820c1524ee52eae141bb4):
68 // * add kFlagFileExternal to entries in catalog table
69 // * add self_external and subtree_external statistics counters
70 // * store compression algorithm in flags
71 // 3 --> 4: (Nov 11 2016 - Git):
72 // * add kFlagDirBindMountpoint
73 // * add kFlagHidden
74 // * add table bind_mountpoints
75 // 4 --> 5: (Dec 07 2017):
76 // * add kFlagFileSpecial (rebranded unused kFlagFileStat)
77 // * add self_special and subtree_special statistics counters
78 // 5 --> 6: (Jul 01 2021):
79 // * Add kFlagDirectIo
80 const unsigned CatalogDatabase::kLatestSchemaRevision = 6;
81 
82 bool CatalogDatabase::CheckSchemaCompatibility() {
83  return !( (schema_version() >= 2.0-kSchemaEpsilon) &&
84  (!IsEqualSchema(schema_version(), kLatestSupportedSchema)) &&
85  (!IsEqualSchema(schema_version(), 2.4) ||
86  !IsEqualSchema(kLatestSupportedSchema, 2.5)) );
87 }
88 
89 
90 bool CatalogDatabase::LiveSchemaUpgradeIfNecessary() {
91  assert(read_write());
92 
93  if (IsEqualSchema(schema_version(), 2.5) && (schema_revision() == 0)) {
94  LogCvmfs(kLogCatalog, kLogDebug, "upgrading schema revision (0 --> 1)");
95 
96  SqlCatalog sql_upgrade(*this, "ALTER TABLE nested_catalogs "
97  "ADD size INTEGER;");
98  if (!sql_upgrade.Execute()) {
99  LogCvmfs(kLogCatalog, kLogDebug, "failed to upgrade nested_catalogs");
100  return false;
101  }
102 
103  set_schema_revision(1);
104  if (!StoreSchemaRevision()) {
105  LogCvmfs(kLogCatalog, kLogDebug, "failed to upgrade schema revision");
106  return false;
107  }
108  }
109 
110  if (IsEqualSchema(schema_version(), 2.5) && (schema_revision() == 1)) {
111  LogCvmfs(kLogCatalog, kLogDebug, "upgrading schema revision (1 --> 2)");
112 
113  SqlCatalog sql_upgrade1(*this, "ALTER TABLE catalog ADD xattr BLOB;");
114  SqlCatalog sql_upgrade2(*this,
115  "INSERT INTO statistics (counter, value) VALUES ('self_xattr', 0);");
116  SqlCatalog sql_upgrade3(*this,
117  "INSERT INTO statistics (counter, value) VALUES ('subtree_xattr', 0);");
118  if (!sql_upgrade1.Execute() || !sql_upgrade2.Execute() ||
119  !sql_upgrade3.Execute())
120  {
121  LogCvmfs(kLogCatalog, kLogDebug, "failed to upgrade catalogs (1 --> 2)");
122  return false;
123  }
124 
125  set_schema_revision(2);
126  if (!StoreSchemaRevision()) {
127  LogCvmfs(kLogCatalog, kLogDebug, "failed to upgrade schema revision");
128  return false;
129  }
130  }
131 
132  if (IsEqualSchema(schema_version(), 2.5) && (schema_revision() == 2)) {
133  LogCvmfs(kLogCatalog, kLogDebug, "upgrading schema revision (2 --> 3)");
134 
135  SqlCatalog sql_upgrade4(*this,
136  "INSERT INTO statistics (counter, value) VALUES "
137  "('self_external', 0);");
138  SqlCatalog sql_upgrade5(*this,
139  "INSERT INTO statistics (counter, value) VALUES "
140  "('self_external_file_size', 0);");
141  SqlCatalog sql_upgrade6(*this,
142  "INSERT INTO statistics (counter, value) VALUES "
143  "('subtree_external', 0);");
144  SqlCatalog sql_upgrade7(*this,
145  "INSERT INTO statistics (counter, value) VALUES "
146  "('subtree_external_file_size', 0);");
147  if (!sql_upgrade4.Execute() || !sql_upgrade5.Execute() ||
148  !sql_upgrade6.Execute() || !sql_upgrade7.Execute())
149  {
150  LogCvmfs(kLogCatalog, kLogDebug, "failed to upgrade catalogs (2 --> 3)");
151  return false;
152  }
153 
154  set_schema_revision(3);
155  if (!StoreSchemaRevision()) {
156  LogCvmfs(kLogCatalog, kLogDebug, "failed to upgrade schema revision");
157  return false;
158  }
159  }
160 
161  if (IsEqualSchema(schema_version(), 2.5) && (schema_revision() == 3)) {
162  LogCvmfs(kLogCatalog, kLogDebug, "upgrading schema revision (3 --> 4)");
163 
164  SqlCatalog sql_upgrade8(*this,
165  "CREATE TABLE bind_mountpoints (path TEXT, sha1 TEXT, size INTEGER, "
166  "CONSTRAINT pk_bind_mountpoints PRIMARY KEY (path));");
167  if (!sql_upgrade8.Execute()) {
168  LogCvmfs(kLogCatalog, kLogDebug, "failed to upgrade catalogs (3 --> 4)");
169  return false;
170  }
171 
172  set_schema_revision(4);
173  if (!StoreSchemaRevision()) {
174  LogCvmfs(kLogCatalog, kLogDebug, "failed to upgrade schema revision");
175  return false;
176  }
177  }
178 
179 
180  if (IsEqualSchema(schema_version(), 2.5) && (schema_revision() == 4)) {
181  LogCvmfs(kLogCatalog, kLogDebug, "upgrading schema revision (4 --> 5)");
182 
183  SqlCatalog sql_upgrade9(*this,
184  "INSERT INTO statistics (counter, value) VALUES "
185  "('self_special', 0);");
186  SqlCatalog sql_upgrade10(*this,
187  "INSERT INTO statistics (counter, value) VALUES "
188  "('subtree_special', 0);");
189  if (!sql_upgrade9.Execute() || !sql_upgrade10.Execute()) {
190  LogCvmfs(kLogCatalog, kLogDebug, "failed to upgrade catalogs (4 --> 5)");
191  return false;
192  }
193 
194  set_schema_revision(5);
195  if (!StoreSchemaRevision()) {
196  LogCvmfs(kLogCatalog, kLogDebug, "failed to upgrade schema revision");
197  return false;
198  }
199  }
200 
201 
202  if (IsEqualSchema(schema_version(), 2.5) && (schema_revision() == 5)) {
203  LogCvmfs(kLogCatalog, kLogDebug, "upgrading schema revision (5 --> 6)");
204 
205  set_schema_revision(6);
206  if (!StoreSchemaRevision()) {
207  LogCvmfs(kLogCatalog, kLogDebug, "failed to upgrade schema revision");
208  return false;
209  }
210  }
211 
212  return true;
213 }
214 
215 
216 bool CatalogDatabase::CreateEmptyDatabase() {
217  assert(read_write());
218 
219  // generate the catalog table and index structure
220  const bool retval =
221  SqlCatalog(*this,
222  "CREATE TABLE catalog "
223  "(md5path_1 INTEGER, md5path_2 INTEGER, parent_1 INTEGER, parent_2 INTEGER,"
224  " hardlinks INTEGER, hash BLOB, size INTEGER, mode INTEGER, mtime INTEGER,"
225  " flags INTEGER, name TEXT, symlink TEXT, uid INTEGER, gid INTEGER, "
226  " xattr BLOB, "
227  " CONSTRAINT pk_catalog PRIMARY KEY (md5path_1, md5path_2));").Execute() &&
228  SqlCatalog(*this,
229  "CREATE INDEX idx_catalog_parent "
230  "ON catalog (parent_1, parent_2);") .Execute() &&
231  SqlCatalog(*this,
232  "CREATE TABLE chunks "
233  "(md5path_1 INTEGER, md5path_2 INTEGER, offset INTEGER, size INTEGER, "
234  " hash BLOB, "
235  " CONSTRAINT pk_chunks PRIMARY KEY (md5path_1, md5path_2, offset, size), "
236  " FOREIGN KEY (md5path_1, md5path_2) REFERENCES "
237  " catalog(md5path_1, md5path_2));") .Execute() &&
238  SqlCatalog(*this,
239  "CREATE TABLE nested_catalogs (path TEXT, sha1 TEXT, size INTEGER, "
240  "CONSTRAINT pk_nested_catalogs PRIMARY KEY (path));") .Execute() &&
241  // Bind mountpoints and nested catalogs are almost the same. We put them in
242  // separate tables to
243  // - not confuse previous client versions, which would crash on bind
244  // mountpoints
245  // - prevent catalogs referenced as bind mountpoints from being replicated,
246  // which would cause exhaustive recursive catalog tree walking
247  // - don't walk into bind mountpoints in catalog traversal (e.g. GC)
248  SqlCatalog(*this,
249  "CREATE TABLE bind_mountpoints (path TEXT, sha1 TEXT, size INTEGER, "
250  "CONSTRAINT pk_bind_mountpoints PRIMARY KEY (path));") .Execute() &&
251  SqlCatalog(*this,
252  "CREATE TABLE statistics (counter TEXT, value INTEGER, "
253  "CONSTRAINT pk_statistics PRIMARY KEY (counter));") .Execute();
254 
255  if (!retval) {
256  PrintSqlError("failed to create catalog database tables.");
257  }
258 
259  return retval;
260 }
261 
262 
263 bool CatalogDatabase::InsertInitialValues(
264  const std::string &root_path,
265  const bool volatile_content,
266  const std::string &voms_authz,
267  const DirectoryEntry &root_entry)
268 {
269  assert(read_write());
270  bool retval = false;
271 
272  // Path hashes
273  shash::Md5 root_path_hash = shash::Md5(shash::AsciiPtr(root_path));
274  shash::Md5 root_parent_hash = (root_path == "")
275  ? shash::Md5()
277 
278  // Start initial filling transaction
279  retval = BeginTransaction();
280  if (!retval) {
281  PrintSqlError("failed to enter initial filling transaction");
282  return false;
283  }
284 
285  // Insert initial values to properties
286  if (!this->SetProperty("revision", 0)) {
287  PrintSqlError(
288  "failed to insert default initial values into the newly created "
289  "catalog tables.");
290  return false;
291  }
292 
293  if (volatile_content) {
294  if (!this->SetProperty("volatile", 1)) {
295  PrintSqlError("failed to insert volatile flag into the newly created "
296  "catalog tables.");
297  return false;
298  }
299  }
300 
301  if (!voms_authz.empty()) {
302  if (!SetVOMSAuthz(voms_authz)) {
303  PrintSqlError("failed to insert VOMS authz flag into the newly created "
304  "catalog tables.");
305  return false;
306  }
307  }
308 
309  // Create initial statistics counters
310  catalog::Counters counters;
311 
312  // Insert root entry (when given)
313  if (!root_entry.IsNegative()) {
314  SqlDirentInsert sql_insert(*this);
315  retval = sql_insert.BindPathHash(root_path_hash) &&
316  sql_insert.BindParentPathHash(root_parent_hash) &&
317  sql_insert.BindDirent(root_entry) &&
318  sql_insert.Execute();
319  if (!retval) {
320  PrintSqlError("failed to insert root entry into newly created catalog.");
321  return false;
322  }
323 
324  // account for the created root entry
325  counters.self.directories = 1;
326  }
327 
328  // Save initial statistics counters
329  if (!counters.InsertIntoDatabase(*this)) {
330  PrintSqlError("failed to insert initial catalog statistics counters.");
331  return false;
332  }
333 
334  // Insert root path (when given)
335  if (!root_path.empty()) {
336  if (!this->SetProperty("root_prefix", root_path)) {
337  PrintSqlError(
338  "failed to store root prefix in the newly created catalog.");
339  return false;
340  }
341  }
342 
343  // Set creation timestamp
344  if (!this->SetProperty("last_modified", static_cast<uint64_t>(time(NULL)))) {
345  PrintSqlError("failed to store creation timestamp in the new catalog.");
346  return false;
347  }
348 
349  // Commit initial filling transaction
350  retval = CommitTransaction();
351  if (!retval) {
352  PrintSqlError("failed to commit initial filling transaction");
353  return false;
354  }
355 
356  return true;
357 }
358 
359 
360 bool
361 CatalogDatabase::SetVOMSAuthz(const std::string &voms_authz) {
362  return this->SetProperty("voms_authz", voms_authz);
363 }
364 
365 
366 double CatalogDatabase::GetRowIdWasteRatio() const {
367  SqlCatalog rowid_waste_ratio_query(*this,
368  "SELECT 1.0 - CAST(COUNT(*) AS DOUBLE) / MAX(rowid) "
369  "AS ratio FROM catalog;");
370  const bool retval = rowid_waste_ratio_query.FetchRow();
371  assert(retval);
372 
373  return rowid_waste_ratio_query.RetrieveDouble(0);
374 }
375 
396 bool CatalogDatabase::CompactDatabase() const {
397  assert(read_write());
398 
399  return SqlCatalog(*this, "PRAGMA foreign_keys = OFF;").Execute() &&
400  BeginTransaction() &&
401  SqlCatalog(*this, "CREATE TEMPORARY TABLE duplicate AS "
402  " SELECT * FROM catalog "
403  " ORDER BY rowid ASC;").Execute() &&
404  SqlCatalog(*this, "DELETE FROM catalog;").Execute() &&
405  SqlCatalog(*this, "INSERT INTO catalog "
406  " SELECT * FROM duplicate "
407  " ORDER BY rowid").Execute() &&
408  SqlCatalog(*this, "DROP TABLE duplicate;").Execute() &&
409  CommitTransaction() &&
410  SqlCatalog(*this, "PRAGMA foreign_keys = ON;").Execute();
411 }
412 
413 
414 //------------------------------------------------------------------------------
415 
416 
417 unsigned SqlDirent::CreateDatabaseFlags(const DirectoryEntry &entry) const {
418  unsigned int database_flags = 0;
419 
420  if (entry.IsNestedCatalogRoot())
421  database_flags |= kFlagDirNestedRoot;
422  else if (entry.IsNestedCatalogMountpoint())
423  database_flags |= kFlagDirNestedMountpoint;
424  else if (entry.IsBindMountpoint())
425  database_flags |= kFlagDirBindMountpoint;
426 
427  if (entry.IsDirectory()) {
428  database_flags |= kFlagDir;
429  } else if (entry.IsLink()) {
430  database_flags |= kFlagFile | kFlagLink;
431  } else if (entry.IsSpecial()) {
432  database_flags |= kFlagFile | kFlagFileSpecial;
433  } else {
434  database_flags |= kFlagFile;
435  database_flags |= entry.compression_algorithm() << kFlagPosCompression;
436  if (entry.IsChunkedFile())
437  database_flags |= kFlagFileChunk;
438  if (entry.IsExternalFile())
439  database_flags |= kFlagFileExternal;
440  if (entry.IsDirectIo())
441  database_flags |= kFlagDirectIo;
442  }
443 
444  if (!entry.checksum_ptr()->IsNull() || entry.IsChunkedFile())
445  StoreHashAlgorithm(entry.checksum_ptr()->algorithm, &database_flags);
446 
447  if (entry.IsHidden())
448  database_flags |= kFlagHidden;
449 
450  return database_flags;
451 }
452 
453 
454 void SqlDirent::StoreHashAlgorithm(const shash::Algorithms algo,
455  unsigned *flags) const
456 {
457  assert(algo != shash::kAny);
458  // Md5 unusable for content hashes
459  *flags |= (algo - 1) << kFlagPosHash;
460 }
461 
462 
463 shash::Algorithms SqlDirent::RetrieveHashAlgorithm(const unsigned flags) const {
464  unsigned in_flags = ((7 << kFlagPosHash) & flags) >> kFlagPosHash;
465  // Skip Md5
466  in_flags++;
467  assert(in_flags < shash::kAny);
468  return static_cast<shash::Algorithms>(in_flags);
469 }
470 
471 
472 zlib::Algorithms SqlDirent::RetrieveCompressionAlgorithm(const unsigned flags)
473  const
474 {
475  // 3 bits, so use 7 (111) to only pull out the flags we want
476  unsigned in_flags =
477  ((7 << kFlagPosCompression) & flags) >> kFlagPosCompression;
478  return static_cast<zlib::Algorithms>(in_flags);
479 }
480 
481 
482 uint32_t SqlDirent::Hardlinks2Linkcount(const uint64_t hardlinks) const {
483  return (hardlinks << 32) >> 32;
484 }
485 
486 
487 uint32_t SqlDirent::Hardlinks2HardlinkGroup(const uint64_t hardlinks) const {
488  return hardlinks >> 32;
489 }
490 
491 
492 uint64_t SqlDirent::MakeHardlinks(const uint32_t hardlink_group,
493  const uint32_t linkcount) const
494 {
495  assert(linkcount > 0);
496  return (static_cast<uint64_t>(hardlink_group) << 32) | linkcount;
497 }
498 
499 
504 void SqlDirent::ExpandSymlink(LinkString *raw_symlink) const {
505  const char *c = raw_symlink->GetChars();
506  const char *cEnd = c+raw_symlink->GetLength();
507  for (; c < cEnd; ++c) {
508  if (*c == '$')
509  goto expand_symlink;
510  }
511  return;
512 
513  expand_symlink:
514  LinkString result;
515  for (c = raw_symlink->GetChars(); c < cEnd; ++c) {
516  if ((*c == '$') && (c < cEnd-2) && (*(c+1) == '(')) {
517  c += 2;
518  const char *rpar = c;
519  while (rpar < cEnd) {
520  if (*rpar == ')')
521  goto expand_symlink_getenv;
522  rpar++;
523  }
524  // right parenthesis missing
525  result.Append("$(", 2);
526  result.Append(c, 1);
527  continue;
528 
529  expand_symlink_getenv:
530  // Check for default value
531  const char *default_separator = c;
532  const char *default_value = rpar;
533  while (default_separator != rpar) {
534  if ((*default_separator == ':') && (*(default_separator + 1) == '-')) {
535  default_value = default_separator+2;
536  break;
537  }
538  default_separator++;
539  }
540 
541  const unsigned environ_var_length = default_separator-c;
542  char environ_var[environ_var_length+1];
543  environ_var[environ_var_length] = '\0';
544  memcpy(environ_var, c, environ_var_length);
545  const char *environ_value = getenv(environ_var); // Don't free!
546  if (environ_value) {
547  result.Append(environ_value, strlen(environ_value));
548  } else {
549  const unsigned default_length = rpar-default_value;
550  result.Append(default_value, default_length);
551  }
552  c = rpar;
553  continue;
554  }
555  result.Append(c, 1);
556  }
557  raw_symlink->Assign(result);
558  return;
559 }
560 
561 
562 //------------------------------------------------------------------------------
563 
564 
565 bool SqlDirentWrite::BindDirentFields(const int hash_idx,
566  const int hardlinks_idx,
567  const int size_idx,
568  const int mode_idx,
569  const int mtime_idx,
570  const int flags_idx,
571  const int name_idx,
572  const int symlink_idx,
573  const int uid_idx,
574  const int gid_idx,
575  const DirectoryEntry &entry)
576 {
577  const uint64_t hardlinks =
578  MakeHardlinks(entry.hardlink_group_,
579  entry.linkcount_);
580 
581  return (
582  BindHashBlob(hash_idx, entry.checksum_) &&
583  BindInt64(hardlinks_idx, hardlinks) &&
584  BindInt64(size_idx, entry.size_) &&
585  BindInt(mode_idx, entry.mode_) &&
586  BindInt64(uid_idx, entry.uid_) &&
587  BindInt64(gid_idx, entry.gid_) &&
588  BindInt64(mtime_idx, entry.mtime_) &&
589  BindInt(flags_idx, CreateDatabaseFlags(entry)) &&
590  BindText(name_idx, entry.name_.GetChars(), entry.name_.GetLength()) &&
591  BindText(symlink_idx, entry.symlink_.GetChars(), entry.symlink_.GetLength())
592  ); // NOLINT
593 }
594 
595 
596 //------------------------------------------------------------------------------
597 
598 
599 SqlListContentHashes::SqlListContentHashes(const CatalogDatabase &database) {
600  static const char *stmt_lt_2_4 =
601  "SELECT hash, flags, 0 "
602  " FROM catalog "
603  " WHERE length(hash) > 0;";
604 
605  static const char *stmt_ge_2_4 =
606  "SELECT hash, flags, 0 "
607  " FROM catalog "
608  " WHERE (length(catalog.hash) > 0) AND "
609  " ((flags & 128) = 0) " // kFlagFileExternal
610  "UNION "
611  "SELECT chunks.hash, catalog.flags, 1 "
612  " FROM catalog "
613  " LEFT JOIN chunks "
614  " ON catalog.md5path_1 = chunks.md5path_1 AND "
615  " catalog.md5path_2 = chunks.md5path_2 "
616  " WHERE (catalog.flags & 128) = 0;"; // kFlagFileExternal
617 
618  if (database.schema_version() < 2.4-CatalogDatabase::kSchemaEpsilon) {
619  DeferredInit(database.sqlite_db(), stmt_lt_2_4);
620  } else {
621  DeferredInit(database.sqlite_db(), stmt_ge_2_4);
622  }
623 }
624 
625 
626 shash::Any SqlListContentHashes::GetHash() const {
627  const unsigned int db_flags = RetrieveInt(1);
628  const shash::Algorithms hash_algorithm = RetrieveHashAlgorithm(db_flags);
629  shash::Any hash = RetrieveHashBlob(0, hash_algorithm);
630  if (RetrieveInt(2) == 1) {
632  }
633 
634  return hash;
635 }
636 
637 
638 //------------------------------------------------------------------------------
639 
640 #define DB_FIELDS_LT_V2_1 \
641  "catalog.hash, catalog.inode, catalog.size, " \
642  "catalog.mode, catalog.mtime, catalog.flags, " \
643  "catalog.name, catalog.symlink, catalog.md5path_1, " \
644  "catalog.md5path_2, catalog.parent_1, catalog.parent_2, " \
645  "catalog.rowid"
646 #define DB_FIELDS_GE_V2_1_LT_R2 \
647  "catalog.hash, catalog.hardlinks, catalog.size, " \
648  "catalog.mode, catalog.mtime, catalog.flags, " \
649  "catalog.name, catalog.symlink, catalog.md5path_1, " \
650  "catalog.md5path_2, catalog.parent_1, catalog.parent_2, " \
651  "catalog.rowid, catalog.uid, catalog.gid, " \
652  "0"
653 #define DB_FIELDS_GE_V2_1_GE_R2 \
654  "catalog.hash, catalog.hardlinks, catalog.size, " \
655  "catalog.mode, catalog.mtime, catalog.flags, " \
656  "catalog.name, catalog.symlink, catalog.md5path_1, " \
657  "catalog.md5path_2, catalog.parent_1, catalog.parent_2, " \
658  "catalog.rowid, catalog.uid, catalog.gid, " \
659  "catalog.xattr IS NOT NULL"
660 
661 #define MAKE_STATEMENT(STMT_TMPL, REV) \
662  static const std::string REV = \
663  ReplaceAll(STMT_TMPL, "@DB_FIELDS@", DB_FIELDS_ ## REV)
664 
665 #define MAKE_STATEMENTS(STMT_TMPL) \
666  MAKE_STATEMENT(STMT_TMPL, LT_V2_1); \
667  MAKE_STATEMENT(STMT_TMPL, GE_V2_1_LT_R2); \
668  MAKE_STATEMENT(STMT_TMPL, GE_V2_1_GE_R2)
669 
670 #define DEFERRED_INIT(DB, REV) \
671  DeferredInit((DB).sqlite_db(), (REV).c_str())
672 
673 #define DEFERRED_INITS(DB) \
674  if ((DB).schema_version() < 2.1 - CatalogDatabase::kSchemaEpsilon) { \
675  DEFERRED_INIT((DB), LT_V2_1); \
676  } else if ((DB).schema_revision() < 2) { \
677  DEFERRED_INIT((DB), GE_V2_1_LT_R2); \
678  } else { \
679  DEFERRED_INIT((DB), GE_V2_1_GE_R2); \
680  }
681 
682 
683 shash::Md5 SqlLookup::GetPathHash() const {
684  return RetrieveMd5(8, 9);
685 }
686 
687 
688 shash::Md5 SqlLookup::GetParentPathHash() const {
689  return RetrieveMd5(10, 11);
690 }
691 
692 
696 DirectoryEntry SqlLookup::GetDirent(const Catalog *catalog,
697  const bool expand_symlink) const
698 {
699  DirectoryEntry result;
700 
701  const unsigned database_flags = RetrieveInt(5);
702  result.is_nested_catalog_root_ = (database_flags & kFlagDirNestedRoot);
704  (database_flags & kFlagDirNestedMountpoint);
705  const char *name = reinterpret_cast<const char *>(RetrieveText(6));
706  const char *symlink = reinterpret_cast<const char *>(RetrieveText(7));
707 
708  // Retrieve the hardlink information from the hardlinks database field
709  if (catalog->schema() < 2.1 - CatalogDatabase::kSchemaEpsilon) {
710  result.linkcount_ = 1;
711  result.hardlink_group_ = 0;
712  result.inode_ = catalog->GetMangledInode(RetrieveInt64(12), 0);
713  result.is_chunked_file_ = false;
714  result.has_xattrs_ = false;
715  result.checksum_ = RetrieveHashBlob(0, shash::kSha1);
716  result.uid_ = g_uid;
717  result.gid_ = g_gid;
718  } else {
719  const uint64_t hardlinks = RetrieveInt64(1);
720  result.linkcount_ = Hardlinks2Linkcount(hardlinks);
721  result.hardlink_group_ = Hardlinks2HardlinkGroup(hardlinks);
722  result.inode_ =
723  catalog->GetMangledInode(RetrieveInt64(12), result.hardlink_group_);
724  result.is_bind_mountpoint_ = (database_flags & kFlagDirBindMountpoint);
725  result.is_chunked_file_ = (database_flags & kFlagFileChunk);
726  result.is_hidden_ = (database_flags & kFlagHidden);
727  result.is_direct_io_ = (database_flags & kFlagDirectIo);
728  result.is_external_file_ = (database_flags & kFlagFileExternal);
729  result.has_xattrs_ = RetrieveInt(15) != 0;
730  result.checksum_ =
731  RetrieveHashBlob(0, RetrieveHashAlgorithm(database_flags));
732  result.compression_algorithm_ =
733  RetrieveCompressionAlgorithm(database_flags);
734 
735  if (g_claim_ownership) {
736  result.uid_ = g_uid;
737  result.gid_ = g_gid;
738  } else {
739  result.uid_ = catalog->MapUid(RetrieveInt64(13));
740  result.gid_ = catalog->MapGid(RetrieveInt64(14));
741  }
742  }
743 
744  result.mode_ = RetrieveInt(3);
745  result.size_ = RetrieveInt64(2);
746  result.mtime_ = RetrieveInt64(4);
747  result.name_.Assign(name, strlen(name));
748  result.symlink_.Assign(symlink, strlen(symlink));
749  if (expand_symlink && !g_raw_symlinks)
750  ExpandSymlink(&result.symlink_);
751 
752  if (g_world_readable) {
753  if (S_ISDIR(result.mode_)) {
754  result.mode_ |= 0555;
755  } else {
756  result.mode_ |= 0444;
757  }
758  }
759 
760  return result;
761 }
762 
763 
764 //------------------------------------------------------------------------------
765 
766 
767 SqlListing::SqlListing(const CatalogDatabase &database) {
768  MAKE_STATEMENTS("SELECT @DB_FIELDS@ FROM catalog "
769  "WHERE (parent_1 = :p_1) AND (parent_2 = :p_2);");
770  DEFERRED_INITS(database);
771 }
772 
773 
774 bool SqlListing::BindPathHash(const struct shash::Md5 &hash) {
775  return BindMd5(1, 2, hash);
776 }
777 
778 
779 //------------------------------------------------------------------------------
780 
781 
782 SqlLookupPathHash::SqlLookupPathHash(const CatalogDatabase &database) {
783  MAKE_STATEMENTS("SELECT @DB_FIELDS@ FROM catalog "
784  "WHERE (md5path_1 = :md5_1) AND (md5path_2 = :md5_2);");
785  DEFERRED_INITS(database);
786 }
787 
788 bool SqlLookupPathHash::BindPathHash(const struct shash::Md5 &hash) {
789  return BindMd5(1, 2, hash);
790 }
791 
792 
793 //------------------------------------------------------------------------------
794 
795 
796 SqlLookupInode::SqlLookupInode(const CatalogDatabase &database) {
797  MAKE_STATEMENTS("SELECT @DB_FIELDS@ FROM catalog WHERE rowid = :rowid;");
798  DEFERRED_INITS(database);
799 }
800 
801 
802 bool SqlLookupInode::BindRowId(const uint64_t inode) {
803  return BindInt64(1, inode);
804 }
805 
806 
807 //------------------------------------------------------------------------------
808 
809 
810 SqlLookupDanglingMountpoints::SqlLookupDanglingMountpoints(
811  const catalog::CatalogDatabase &database) {
812  MAKE_STATEMENTS("SELECT DISTINCT @DB_FIELDS@ FROM catalog "
813  "JOIN catalog AS c2 "
814  "ON catalog.md5path_1 = c2.parent_1 AND "
815  " catalog.md5path_2 = c2.parent_2 "
816  "WHERE catalog.flags & :nested_mountpoint_flag");
817  DEFERRED_INITS(database);
818 
819  // this pretty much removes the advantage of a deferred init but the statement
820  // is anyway only used directly.
821  const bool success = BindInt64(1, SqlDirent::kFlagDirNestedMountpoint);
822  assert(success);
823 }
824 
825 
826 //------------------------------------------------------------------------------
827 
828 
829 SqlDirentTouch::SqlDirentTouch(const CatalogDatabase &database) {
830  DeferredInit(database.sqlite_db(),
831  "UPDATE catalog "
832  "SET hash = :hash, size = :size, mode = :mode, mtime = :mtime, "
833 // 1 2 3 4
834  "name = :name, symlink = :symlink, uid = :uid, gid = :gid, xattr = :xattr "
835 // 5 6 7 8 9
836  "WHERE (md5path_1 = :md5_1) AND (md5path_2 = :md5_2);");
837 // 10 11
838 }
839 
840 
841 bool SqlDirentTouch::BindDirentBase(const DirectoryEntryBase &entry) {
842  return (
843  BindHashBlob(1, entry.checksum_) &&
844  BindInt64(2, entry.size_) &&
845  BindInt(3, entry.mode_) &&
846  BindInt64(4, entry.mtime_) &&
847  BindText(5, entry.name_.GetChars(), entry.name_.GetLength()) &&
848  BindText(6, entry.symlink_.GetChars(), entry.symlink_.GetLength()) &&
849  BindInt64(7, entry.uid_) &&
850  BindInt64(8, entry.gid_));
851 }
852 
853 
854 bool SqlDirentTouch::BindPathHash(const shash::Md5 &hash) {
855  return BindMd5(10, 11, hash);
856 }
857 
858 
859 bool SqlDirentTouch::BindXattr(const XattrList &xattrs) {
860  unsigned char *packed_xattrs;
861  unsigned size;
862  xattrs.Serialize(&packed_xattrs, &size);
863  if (packed_xattrs == NULL)
864  return BindNull(9);
865  return BindBlobTransient(9, packed_xattrs, size);
866 }
867 
868 
869 bool SqlDirentTouch::BindXattrEmpty() {
870  return BindNull(9);
871 }
872 
873 
874 //------------------------------------------------------------------------------
875 
876 
877 SqlNestedCatalogLookup::SqlNestedCatalogLookup(const CatalogDatabase &database)
878 {
879  // We cannot access nested catalogs where the content hash is missing
880  static const char *stmt_0_9 =
881  "SELECT '', 0 FROM nested_catalogs;";
882  static const char *stmt_2_5_ge_4 =
883  "SELECT sha1, size FROM nested_catalogs WHERE path=:path "
884  "UNION ALL SELECT sha1, size FROM bind_mountpoints WHERE path=:path;";
885  static const char *stmt_2_5_ge_1_lt_4 =
886  "SELECT sha1, size FROM nested_catalogs WHERE path=:path;";
887  // Internally converts NULL to 0 for size
888  static const char *stmt_2_5_lt_1 =
889  "SELECT sha1, 0 FROM nested_catalogs WHERE path=:path;";
890 
891  if (database.IsEqualSchema(database.schema_version(), 2.5) &&
892  (database.schema_revision() >= 4))
893  {
894  DeferredInit(database.sqlite_db(), stmt_2_5_ge_4);
895  } else if (database.IsEqualSchema(database.schema_version(), 2.5) &&
896  (database.schema_revision() >= 1))
897  {
898  DeferredInit(database.sqlite_db(), stmt_2_5_ge_1_lt_4);
899  } else {
900  if (database.IsEqualSchema(database.schema_version(), 0.9)) {
901  DeferredInit(database.sqlite_db(), stmt_0_9);
902  } else {
903  DeferredInit(database.sqlite_db(), stmt_2_5_lt_1);
904  }
905  }
906 }
907 
908 
909 bool SqlNestedCatalogLookup::BindSearchPath(const PathString &path) {
910  return BindText(1, path.GetChars(), path.GetLength());
911 }
912 
913 
914 shash::Any SqlNestedCatalogLookup::GetContentHash() const {
915  const string hash = string(reinterpret_cast<const char *>(RetrieveText(0)));
916  return (hash.empty()) ? shash::Any(shash::kAny) :
919 }
920 
921 
922 uint64_t SqlNestedCatalogLookup::GetSize() const {
923  return RetrieveInt64(1);
924 }
925 
926 
927 //------------------------------------------------------------------------------
928 
929 
930 SqlNestedCatalogListing::SqlNestedCatalogListing(
931  const CatalogDatabase &database)
932 {
933  // We cannot access nested catalogs where the content hash is missing
934  static const char *stmt_0_9 =
935  "SELECT '', '', 0 FROM nested_catalogs;";
936  static const char *stmt_2_5_ge_4 =
937  "SELECT path, sha1, size FROM nested_catalogs "
938  "UNION ALL SELECT path, sha1, size FROM bind_mountpoints;";
939  static const char *stmt_2_5_ge_1_lt_4 =
940  "SELECT path, sha1, size FROM nested_catalogs;";
941  // Internally converts NULL to 0 for size
942  static const char *stmt_2_5_lt_1 =
943  "SELECT path, sha1, 0 FROM nested_catalogs;";
944 
945  if (database.IsEqualSchema(database.schema_version(), 2.5) &&
946  (database.schema_revision() >= 4))
947  {
948  DeferredInit(database.sqlite_db(), stmt_2_5_ge_4);
949  } else if (database.IsEqualSchema(database.schema_version(), 2.5) &&
950  (database.schema_revision() >= 1))
951  {
952  DeferredInit(database.sqlite_db(), stmt_2_5_ge_1_lt_4);
953  } else {
954  if (database.IsEqualSchema(database.schema_version(), 0.9)) {
955  DeferredInit(database.sqlite_db(), stmt_0_9);
956  } else {
957  DeferredInit(database.sqlite_db(), stmt_2_5_lt_1);
958  }
959  }
960 }
961 
962 
963 PathString SqlNestedCatalogListing::GetPath() const {
964  const char *path = reinterpret_cast<const char *>(RetrieveText(0));
965  return PathString(path, strlen(path));
966 }
967 
968 
969 shash::Any SqlNestedCatalogListing::GetContentHash() const {
970  const string hash = string(reinterpret_cast<const char *>(RetrieveText(1)));
971  return (hash.empty()) ? shash::Any(shash::kAny) :
974 }
975 
976 
977 uint64_t SqlNestedCatalogListing::GetSize() const {
978  return RetrieveInt64(2);
979 }
980 
981 
982 //------------------------------------------------------------------------------
983 
984 
985 SqlOwnNestedCatalogListing::SqlOwnNestedCatalogListing(
986  const CatalogDatabase &database)
987 {
988  // We cannot access nested catalogs where the content hash is missing
989  static const char *stmt_0_9 =
990  "SELECT '', '', 0 FROM nested_catalogs;";
991  static const char *stmt_2_5_ge_1 =
992  "SELECT path, sha1, size FROM nested_catalogs;";
993  // Internally converts NULL to 0 for size
994  static const char *stmt_2_5_lt_1 =
995  "SELECT path, sha1, 0 FROM nested_catalogs;";
996 
997  if (database.IsEqualSchema(database.schema_version(), 2.5) &&
998  (database.schema_revision() >= 1))
999  {
1000  DeferredInit(database.sqlite_db(), stmt_2_5_ge_1);
1001  } else {
1002  if (database.IsEqualSchema(database.schema_version(), 0.9)) {
1003  DeferredInit(database.sqlite_db(), stmt_0_9);
1004  } else {
1005  DeferredInit(database.sqlite_db(), stmt_2_5_lt_1);
1006  }
1007  }
1008 }
1009 
1010 
1011 PathString SqlOwnNestedCatalogListing::GetPath() const {
1012  const char *path = reinterpret_cast<const char *>(RetrieveText(0));
1013  return PathString(path, strlen(path));
1014 }
1015 
1016 
1017 shash::Any SqlOwnNestedCatalogListing::GetContentHash() const {
1018  const string hash = string(reinterpret_cast<const char *>(RetrieveText(1)));
1019  return (hash.empty()) ? shash::Any(shash::kAny) :
1022 }
1023 
1024 
1025 uint64_t SqlOwnNestedCatalogListing::GetSize() const {
1026  return RetrieveInt64(2);
1027 }
1028 
1029 
1030 //------------------------------------------------------------------------------
1031 
1032 
1033 SqlDirentInsert::SqlDirentInsert(const CatalogDatabase &database) {
1034  DeferredInit(database.sqlite_db(),
1035  "INSERT INTO catalog "
1036  "(md5path_1, md5path_2, parent_1, parent_2, hash, hardlinks, size, mode,"
1037  // 1 2 3 4 5 6 7 8
1038  "mtime, flags, name, symlink, uid, gid, xattr) "
1039  // 9, 10 11 12 13 14 15
1040  "VALUES (:md5_1, :md5_2, :p_1, :p_2, :hash, :links, :size, :mode, :mtime,"
1041  " :flags, :name, :symlink, :uid, :gid, :xattr);");
1042 }
1043 
1044 
1045 bool SqlDirentInsert::BindPathHash(const shash::Md5 &hash) {
1046  return BindMd5(1, 2, hash);
1047 }
1048 
1049 
1050 bool SqlDirentInsert::BindParentPathHash(const shash::Md5 &hash) {
1051  return BindMd5(3, 4, hash);
1052 }
1053 
1054 
1055 bool SqlDirentInsert::BindDirent(const DirectoryEntry &entry) {
1056  return BindDirentFields(5, 6, 7, 8, 9, 10, 11, 12, 13, 14, entry);
1057 }
1058 
1059 
1060 bool SqlDirentInsert::BindXattr(const XattrList &xattrs) {
1061  unsigned char *packed_xattrs;
1062  unsigned size;
1063  xattrs.Serialize(&packed_xattrs, &size);
1064  if (packed_xattrs == NULL)
1065  return BindNull(15);
1066  return BindBlobTransient(15, packed_xattrs, size);
1067 }
1068 
1069 
1070 bool SqlDirentInsert::BindXattrEmpty() {
1071  return BindNull(15);
1072 }
1073 
1074 
1075 //------------------------------------------------------------------------------
1076 
1077 
1078 SqlDirentUpdate::SqlDirentUpdate(const CatalogDatabase &database) {
1079  DeferredInit(database.sqlite_db(),
1080  "UPDATE catalog "
1081  "SET hash = :hash, size = :size, mode = :mode, mtime = :mtime, "
1082 // 1 2 3 4
1083  "flags = :flags, name = :name, symlink = :symlink, hardlinks = :hardlinks, "
1084 // 5 6 7 8
1085  "uid = :uid, gid = :gid "
1086 // 9 10
1087  "WHERE (md5path_1 = :md5_1) AND (md5path_2 = :md5_2);");
1088 // 11 12
1089 }
1090 
1091 
1092 bool SqlDirentUpdate::BindPathHash(const shash::Md5 &hash) {
1093  return BindMd5(11, 12, hash);
1094 }
1095 
1096 
1097 bool SqlDirentUpdate::BindDirent(const DirectoryEntry &entry) {
1098  return BindDirentFields(1, 8, 2, 3, 4, 5, 6, 7, 9, 10, entry);
1099 }
1100 
1101 
1102 //------------------------------------------------------------------------------
1103 
1104 
1105 SqlDirentUnlink::SqlDirentUnlink(const CatalogDatabase &database) {
1106  DeferredInit(database.sqlite_db(),
1107  "DELETE FROM catalog "
1108  "WHERE (md5path_1 = :md5_1) AND (md5path_2 = :md5_2);");
1109 }
1110 
1111 bool SqlDirentUnlink::BindPathHash(const shash::Md5 &hash) {
1112  return BindMd5(1, 2, hash);
1113 }
1114 
1115 
1116 //------------------------------------------------------------------------------
1117 
1118 
1119 SqlIncLinkcount::SqlIncLinkcount(const CatalogDatabase &database) {
1120  // This command changes the linkcount of a whole hardlink group at once!
1121  // We can do this, since the 'hardlinks'-field contains the hardlink group ID
1122  // in the higher 32bit as well as the 'linkcount' in the lower 32bit.
1123  // This field will be equal for all entries belonging to the same hardlink
1124  // group while adding/subtracting small values from it will only effect the
1125  // linkcount in the lower 32bit.
1126  // Take a deep breath!
1127  DeferredInit(database.sqlite_db(),
1128  "UPDATE catalog SET hardlinks = hardlinks + :delta "
1129  "WHERE hardlinks = (SELECT hardlinks from catalog "
1130  "WHERE md5path_1 = :md5_1 AND md5path_2 = :md5_2);");
1131 }
1132 
1133 
1134 bool SqlIncLinkcount::BindPathHash(const shash::Md5 &hash) {
1135  return BindMd5(2, 3, hash);
1136 }
1137 
1138 
1139 bool SqlIncLinkcount::BindDelta(const int delta) {
1140  return BindInt(1, delta);
1141 }
1142 
1143 
1144 //------------------------------------------------------------------------------
1145 
1146 
1147 SqlChunkInsert::SqlChunkInsert(const CatalogDatabase &database) {
1148  DeferredInit(database.sqlite_db(),
1149  "INSERT INTO chunks (md5path_1, md5path_2, offset, size, hash) "
1150  // 1 2 3 4 5
1151  "VALUES (:md5_1, :md5_2, :offset, :size, :hash);");
1152 }
1153 
1154 
1155 bool SqlChunkInsert::BindPathHash(const shash::Md5 &hash) {
1156  return BindMd5(1, 2, hash);
1157 }
1158 
1159 
1160 bool SqlChunkInsert::BindFileChunk(const FileChunk &chunk) {
1161  return
1162  BindInt64(3, chunk.offset()) &&
1163  BindInt64(4, chunk.size()) &&
1164  BindHashBlob(5, chunk.content_hash());
1165 }
1166 
1167 
1168 //------------------------------------------------------------------------------
1169 
1170 
1171 SqlChunksRemove::SqlChunksRemove(const CatalogDatabase &database) {
1172  DeferredInit(database.sqlite_db(),
1173  "DELETE FROM chunks "
1174  "WHERE (md5path_1 = :md5_1) AND (md5path_2 = :md5_2);");
1175 }
1176 
1177 
1178 bool SqlChunksRemove::BindPathHash(const shash::Md5 &hash) {
1179  return BindMd5(1, 2, hash);
1180 }
1181 
1182 
1183 //------------------------------------------------------------------------------
1184 
1185 
1186 SqlChunksListing::SqlChunksListing(const CatalogDatabase &database) {
1187  DeferredInit(database.sqlite_db(),
1188  "SELECT offset, size, hash FROM chunks "
1189  // 0 1 2
1190  "WHERE (md5path_1 = :md5_1) AND (md5path_2 = :md5_2) "
1191  // 1 2
1192  "ORDER BY offset ASC;");
1193 }
1194 
1195 
1196 bool SqlChunksListing::BindPathHash(const shash::Md5 &hash) {
1197  return BindMd5(1, 2, hash);
1198 }
1199 
1200 
1201 FileChunk SqlChunksListing::GetFileChunk(
1202  const shash::Algorithms interpret_hash_as) const
1203 {
1204  return FileChunk(
1205  RetrieveHashBlob(2, interpret_hash_as, shash::kSuffixPartial),
1206  RetrieveInt64(0),
1207  RetrieveInt64(1));
1208 }
1209 
1210 
1211 //------------------------------------------------------------------------------
1212 
1213 
1214 SqlChunksCount::SqlChunksCount(const CatalogDatabase &database) {
1215  DeferredInit(database.sqlite_db(),
1216  "SELECT count(*) FROM chunks "
1217  // 0
1218  "WHERE (md5path_1 = :md5_1) AND (md5path_2 = :md5_2)");
1219  // 1 2
1220 }
1221 
1222 
1223 bool SqlChunksCount::BindPathHash(const shash::Md5 &hash) {
1224  return BindMd5(1, 2, hash);
1225 }
1226 
1227 
1228 int SqlChunksCount::GetChunkCount() const {
1229  return RetrieveInt64(0);
1230 }
1231 
1232 
1233 //------------------------------------------------------------------------------
1234 
1235 
1236 SqlMaxHardlinkGroup::SqlMaxHardlinkGroup(const CatalogDatabase &database) {
1237  DeferredInit(database.sqlite_db(), "SELECT max(hardlinks) FROM catalog;");
1238 }
1239 
1240 
1241 uint32_t SqlMaxHardlinkGroup::GetMaxGroupId() const {
1242  return RetrieveInt64(0) >> 32;
1243 }
1244 
1245 
1246 //------------------------------------------------------------------------------
1247 
1248 
1249 SqlGetCounter::SqlGetCounter(const CatalogDatabase &database) {
1250  static const char *stmt_ge_2_4 =
1251  "SELECT value from statistics WHERE counter = :counter;";
1252  static const char *stmt_lt_2_4 =
1253  "SELECT 0;";
1254 
1255  if (database.schema_version() >= 2.4 - CatalogDatabase::kSchemaEpsilon) {
1256  compat_ = false;
1257  DeferredInit(database.sqlite_db(), stmt_ge_2_4);
1258  } else {
1259  compat_ = true;
1260  DeferredInit(database.sqlite_db(), stmt_lt_2_4);
1261  }
1262 }
1263 
1264 
1265 bool SqlGetCounter::BindCounter(const std::string &counter) {
1266  if (compat_) return true;
1267  return BindText(1, counter);
1268 }
1269 
1270 
1271 uint64_t SqlGetCounter::GetCounter() const {
1272  if (compat_) return 0;
1273  return RetrieveInt64(0);
1274 }
1275 
1276 
1277 //------------------------------------------------------------------------------
1278 
1279 
1280 SqlUpdateCounter::SqlUpdateCounter(const CatalogDatabase &database) {
1281  DeferredInit(database.sqlite_db(),
1282  "UPDATE statistics SET value=value+:val WHERE counter=:counter;");
1283 }
1284 
1285 
1286 bool SqlUpdateCounter::BindCounter(const std::string &counter) {
1287  return BindText(2, counter);
1288 }
1289 
1290 
1291 bool SqlUpdateCounter::BindDelta(const int64_t delta) {
1292  return BindInt64(1, delta);
1293 }
1294 
1295 
1296 //------------------------------------------------------------------------------
1297 
1298 
1299 SqlCreateCounter::SqlCreateCounter(const CatalogDatabase &database) {
1300  DeferredInit(database.sqlite_db(),
1301  "INSERT OR REPLACE INTO statistics (counter, value) "
1302  "VALUES (:counter, :value);");
1303 }
1304 
1305 
1306 bool SqlCreateCounter::BindCounter(const std::string &counter) {
1307  return BindText(1, counter);
1308 }
1309 
1310 
1311 bool SqlCreateCounter::BindInitialValue(const int64_t value) {
1312  return BindInt64(2, value);
1313 }
1314 
1315 
1316 //------------------------------------------------------------------------------
1317 
1318 
1319 SqlAllChunks::SqlAllChunks(const CatalogDatabase &database) {
1320  int hash_mask = 7 << SqlDirent::kFlagPosHash;
1321  string flags2hash =
1322  " ((flags&" + StringifyInt(hash_mask) + ") >> " +
1323  StringifyInt(SqlDirent::kFlagPosHash) + ")+1 AS hash_algorithm ";
1324 
1325  int compression_mask = 7 << SqlDirent::kFlagPosCompression;
1326  string flags2compression =
1327  " ((flags&" + StringifyInt(compression_mask) + ") >> " +
1328  StringifyInt(SqlDirent::kFlagPosCompression) + ") " +
1329  "AS compression_algorithm ";
1330 
1331  // TODO(reneme): this depends on shash::kSuffix* being a char!
1332  // it should be more generic or replaced entirely
1333  // TODO(reneme): this is practically the same as SqlListContentHashes and
1334  // should be consolidated
1335  string sql = "SELECT DISTINCT hash, "
1336  "CASE WHEN flags & " + StringifyInt(SqlDirent::kFlagFile) + " THEN " +
1338  "WHEN flags & " + StringifyInt(SqlDirent::kFlagDir) + " THEN " +
1340  "AS chunk_type, " + flags2hash + "," + flags2compression +
1341  "FROM catalog WHERE (hash IS NOT NULL) AND "
1342  "(flags & " + StringifyInt(SqlDirent::kFlagFileExternal) + " = 0)";
1343  if (database.schema_version() >= 2.4 - CatalogDatabase::kSchemaEpsilon) {
1344  sql +=
1345  " UNION "
1346  "SELECT DISTINCT chunks.hash, " + StringifyInt(shash::kSuffixPartial) +
1347  ", " + flags2hash + "," + flags2compression +
1348  "FROM chunks, catalog WHERE "
1349  "chunks.md5path_1=catalog.md5path_1 AND "
1350  "chunks.md5path_2=catalog.md5path_2 AND "
1351  "(catalog.flags & " + StringifyInt(SqlDirent::kFlagFileExternal) +
1352  " = 0)";
1353  }
1354  sql += ";";
1355  Init(database.sqlite_db(), sql);
1356 }
1357 
1358 
1360  return true;
1361 }
1362 
1363 
1364 bool SqlAllChunks::Next(shash::Any *hash, zlib::Algorithms *compression_alg) {
1365  if (!FetchRow()) {
1366  return false;
1367  }
1368 
1369  *hash = RetrieveHashBlob(0, static_cast<shash::Algorithms>(RetrieveInt(2)),
1370  static_cast<shash::Suffix>(RetrieveInt(1)));
1371  *compression_alg = static_cast<zlib::Algorithms>(RetrieveInt(3));
1372  return true;
1373 }
1374 
1375 
1376 bool SqlAllChunks::Close() {
1377  return Reset();
1378 }
1379 
1380 
1381 //------------------------------------------------------------------------------
1382 
1383 
1384 SqlLookupXattrs::SqlLookupXattrs(const CatalogDatabase &database) {
1385  DeferredInit(database.sqlite_db(),
1386  "SELECT xattr FROM catalog "
1387  "WHERE (md5path_1 = :md5_1) AND (md5path_2 = :md5_2);");
1388 }
1389 
1390 
1391 bool SqlLookupXattrs::BindPathHash(const shash::Md5 &hash) {
1392  return BindMd5(1, 2, hash);
1393 }
1394 
1395 
1396 XattrList SqlLookupXattrs::GetXattrs() {
1397  const unsigned char *packed_xattrs =
1398  reinterpret_cast<const unsigned char *>(RetrieveBlob(0));
1399  if (packed_xattrs == NULL)
1400  return XattrList();
1401 
1402  int size = RetrieveBytes(0);
1403  assert(size >= 0);
1404  UniquePtr<XattrList> xattrs(XattrList::Deserialize(packed_xattrs, size));
1405  if (!xattrs.IsValid()) {
1406  LogCvmfs(kLogCatalog, kLogDebug, "corrupted xattr data");
1407  return XattrList();
1408  }
1409  return *xattrs;
1410 }
1411 
1412 } // namespace catalog
bool InsertIntoDatabase(const CatalogDatabase &database) const
bool IsExternalFile() const
bool IsNull() const
Definition: hash.h:383
bool IsSpecial() const
bool Execute()
Definition: sql.cc:42
bool IsDirectory() const
bool FetchRow()
Definition: sql.cc:62
uint64_t MapUid(const uint64_t uid) const
Definition: catalog.h:219
bool IsHidden() const
bool IsChunkedFile() const
inode_t inode_
bool is_hidden_
bool BindDirent(const DirectoryEntry &entry)
void Assign(const char *chars, const unsigned length)
Definition: shortstring.h:61
void Open()
double RetrieveDouble(const int idx_column) const
Definition: sql.h:439
gid_t gid_
const char kSuffixMicroCatalog
Definition: hash.h:56
const shash::Any & content_hash() const
Definition: file_chunk.h:41
float schema() const
Definition: catalog.h:178
assert((mem||(size==0))&&"Out Of Memory")
float schema_version() const
Definition: sql.h:149
bool IsDirectIo() const
#define MAKE_STATEMENTS(STMT_TMPL)
Definition: catalog_sql.cc:665
Algorithms algorithm
Definition: hash.h:125
bool has_xattrs_
uint64_t size_
unsigned schema_revision() const
Definition: sql.h:150
#define DEFERRED_INITS(DB)
Definition: catalog_sql.cc:673
bool IsNestedCatalogMountpoint() const
bool BindPathHash(const shash::Md5 &hash)
Algorithms
Definition: hash.h:41
bool IsNestedCatalogRoot() const
NameString name_
zlib::Algorithms compression_algorithm_
bool IsLink() const
Algorithms
Definition: compression.h:44
gid_t g_gid
Definition: globals.cc:14
const char kSuffixPartial
Definition: hash.h:57
void Serialize(unsigned char **outbuf, unsigned *size, const std::vector< std::string > *blacklist=NULL) const
Definition: xattr.cc:196
const char kSuffixCatalog
Definition: hash.h:54
uint32_t linkcount_
off_t offset() const
Definition: file_chunk.h:42
bool is_nested_catalog_mountpoint_
bool is_external_file_
zlib::Algorithms compression_algorithm() const
uid_t g_uid
Definition: globals.cc:13
uint32_t hardlink_group_
time_t mtime_
static int Init(const loader::LoaderExports *loader_exports)
Definition: cvmfs.cc:2296
void Append(const char *chars, const unsigned length)
Definition: shortstring.h:80
string StringifyInt(const int64_t value)
Definition: string.cc:78
sqlite3 * sqlite_db() const
Definition: sql.h:147
bool IsBindMountpoint() const
bool is_chunked_file_
bool g_raw_symlinks
Definition: globals.cc:12
bool IsEqualSchema(const float value, const float compare) const
Definition: sql.h:131
bool g_world_readable
Definition: globals.cc:15
inode_t GetMangledInode(const uint64_t row_id, const uint64_t hardlink_group) const
Definition: catalog.cc:593
static XattrList * Deserialize(const unsigned char *inbuf, const unsigned size)
Definition: xattr.cc:67
unsigned int mode_
ShortString< kDefaultMaxPath, 0 > PathString
Definition: shortstring.h:217
size_t size() const
Definition: file_chunk.h:43
PathString GetParentPath(const PathString &path)
Definition: shortstring.cc:15
Any MkFromHexPtr(const HexPtr hex, const char suffix)
Definition: hash.cc:83
unsigned GetLength() const
Definition: shortstring.h:131
shash::Any checksum_
Suffix suffix
Definition: hash.h:126
bool is_direct_io_
bool IsNegative() const
bool is_nested_catalog_root_
const char * GetChars() const
Definition: shortstring.h:123
bool is_bind_mountpoint_
LinkString symlink_
bool BindParentPathHash(const shash::Md5 &hash)
static void size_t size
Definition: smalloc.h:54
const char kSuffixNone
Definition: hash.h:53
bool g_claim_ownership
Definition: globals.cc:11
const shash::Any * checksum_ptr() const
uint64_t MapGid(const uint64_t gid) const
Definition: catalog.h:223
uid_t uid_
CVMFS_EXPORT void LogCvmfs(const LogSource source, const int mask, const char *format,...)
Definition: logging.cc:528