CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
catalog_sql.cc
Go to the documentation of this file.
1 
5 #include "cvmfs_config.h"
6 #include "catalog_sql.h"
7 
8 #include <cstdlib>
9 #include <cstring>
10 
11 #include "catalog.h"
12 #include "globals.h"
13 #include "logging.h"
14 #include "util/posix.h"
15 #include "xattr.h"
16 
17 using namespace std; // NOLINT
18 
19 namespace catalog {
20 
34 // ChangeLog
35 // 2.5 (Jun 26 2013 - Git: e79baec22c6abd6ddcdf8f8d7d33921027a052ab)
36 // * add (backward compatible) schema revision - see below
37 // * add statistics counters for chunked files
38 // Note: this was retrofitted and needed a catalog migration step
39 //
40 // 2.4 (Aug 15 2012 - Git: 17de8fc782b5b8dc4404dda925627b5ec2b552e1)
41 // 2.3 (Aug 15 2012 - Git: ab77688cdb2f851af3fe983bf3694dc2465e65be)
42 // 2.2 (never existed)
43 // 2.1 (Aug 7 2012 - Git: beba36c12d2b1123ffbb169f865a861e570adc68)
44 // * add 'chunks' table for file chunks
45 // * add 'statistics' table for accumulative counters
46 // * rename 'inode' field to 'hardlinks'
47 // * containing both hardlink group ID and linkcount
48 // * .cvmfscatalog files become first-class entries in the catalogs
49 //
50 // 2.0 (Aug 6 2012 - Git: c8a81ede603e57fbe4324b6ab6bc8c41e3a2fa5f)
51 // * beginning of CernVM-FS 2.1.x branch ('modern' era)
52 //
53 // 1.x (earlier - code base still in SVN)
54 // * pre-historic times
55 // 0.9 (some time 2011, artifical version)
56 // * 1.0 catalogs that lack the SHA-1 value for nested catalogs
57 const float CatalogDatabase::kLatestSchema = 2.5;
58 const float CatalogDatabase::kLatestSupportedSchema = 2.5; // + 1.X (r/o)
59 
60 // ChangeLog
61 // 0 --> 1: (Jan 6 2014 - Git: 3667fe7a669d0d65e07275b753a7c6f23fc267df)
62 // * add size column to nested catalog table,
63 // * add schema_revision property
64 // 1 --> 2: (Jan 22 2014 - Git: 85e6680e52cfe56dc1213a5ad74a5cc62fd50ead):
65 // * add xattr column to catalog table
66 // * add self_xattr and subtree_xattr statistics counters
67 // 2 --> 3: (Sep 28 2015 - Git: f4171234b13ea448589820c1524ee52eae141bb4):
68 // * add kFlagFileExternal to entries in catalog table
69 // * add self_external and subtree_external statistics counters
70 // * store compression algorithm in flags
71 // 3 --> 4: (Nov 11 2016 - Git):
72 // * add kFlagDirBindMountpoint
73 // * add kFlagHidden
74 // * add table bind_mountpoints
75 // 4 --> 5: (Dec 07 2017):
76 // * add kFlagFileSpecial (rebranded unused kFlagFileStat)
77 // * add self_special and subtree_special statistics counters
78 // 5 --> 6: (Jul 01 2021):
79 // * Add kFlagDirectIo
80 const unsigned CatalogDatabase::kLatestSchemaRevision = 6;
81 
82 bool CatalogDatabase::CheckSchemaCompatibility() {
83  return !( (schema_version() >= 2.0-kSchemaEpsilon) &&
84  (!IsEqualSchema(schema_version(), kLatestSupportedSchema)) &&
85  (!IsEqualSchema(schema_version(), 2.4) ||
86  !IsEqualSchema(kLatestSupportedSchema, 2.5)) );
87 }
88 
89 
90 bool CatalogDatabase::LiveSchemaUpgradeIfNecessary() {
91  assert(read_write());
92 
93  if (IsEqualSchema(schema_version(), 2.5) && (schema_revision() == 0)) {
94  LogCvmfs(kLogCatalog, kLogDebug, "upgrading schema revision (0 --> 1)");
95 
96  SqlCatalog sql_upgrade(*this, "ALTER TABLE nested_catalogs "
97  "ADD size INTEGER;");
98  if (!sql_upgrade.Execute()) {
99  LogCvmfs(kLogCatalog, kLogDebug, "failed to upgrade nested_catalogs");
100  return false;
101  }
102 
103  set_schema_revision(1);
104  if (!StoreSchemaRevision()) {
105  LogCvmfs(kLogCatalog, kLogDebug, "failed to upgrade schema revision");
106  return false;
107  }
108  }
109 
110  if (IsEqualSchema(schema_version(), 2.5) && (schema_revision() == 1)) {
111  LogCvmfs(kLogCatalog, kLogDebug, "upgrading schema revision (1 --> 2)");
112 
113  SqlCatalog sql_upgrade1(*this, "ALTER TABLE catalog ADD xattr BLOB;");
114  SqlCatalog sql_upgrade2(*this,
115  "INSERT INTO statistics (counter, value) VALUES ('self_xattr', 0);");
116  SqlCatalog sql_upgrade3(*this,
117  "INSERT INTO statistics (counter, value) VALUES ('subtree_xattr', 0);");
118  if (!sql_upgrade1.Execute() || !sql_upgrade2.Execute() ||
119  !sql_upgrade3.Execute())
120  {
121  LogCvmfs(kLogCatalog, kLogDebug, "failed to upgrade catalogs (1 --> 2)");
122  return false;
123  }
124 
125  set_schema_revision(2);
126  if (!StoreSchemaRevision()) {
127  LogCvmfs(kLogCatalog, kLogDebug, "failed to upgrade schema revision");
128  return false;
129  }
130  }
131 
132  if (IsEqualSchema(schema_version(), 2.5) && (schema_revision() == 2)) {
133  LogCvmfs(kLogCatalog, kLogDebug, "upgrading schema revision (2 --> 3)");
134 
135  SqlCatalog sql_upgrade4(*this,
136  "INSERT INTO statistics (counter, value) VALUES "
137  "('self_external', 0);");
138  SqlCatalog sql_upgrade5(*this,
139  "INSERT INTO statistics (counter, value) VALUES "
140  "('self_external_file_size', 0);");
141  SqlCatalog sql_upgrade6(*this,
142  "INSERT INTO statistics (counter, value) VALUES "
143  "('subtree_external', 0);");
144  SqlCatalog sql_upgrade7(*this,
145  "INSERT INTO statistics (counter, value) VALUES "
146  "('subtree_external_file_size', 0);");
147  if (!sql_upgrade4.Execute() || !sql_upgrade5.Execute() ||
148  !sql_upgrade6.Execute() || !sql_upgrade7.Execute())
149  {
150  LogCvmfs(kLogCatalog, kLogDebug, "failed to upgrade catalogs (2 --> 3)");
151  return false;
152  }
153 
154  set_schema_revision(3);
155  if (!StoreSchemaRevision()) {
156  LogCvmfs(kLogCatalog, kLogDebug, "failed to upgrade schema revision");
157  return false;
158  }
159  }
160 
161  if (IsEqualSchema(schema_version(), 2.5) && (schema_revision() == 3)) {
162  LogCvmfs(kLogCatalog, kLogDebug, "upgrading schema revision (3 --> 4)");
163 
164  SqlCatalog sql_upgrade8(*this,
165  "CREATE TABLE bind_mountpoints (path TEXT, sha1 TEXT, size INTEGER, "
166  "CONSTRAINT pk_bind_mountpoints PRIMARY KEY (path));");
167  if (!sql_upgrade8.Execute()) {
168  LogCvmfs(kLogCatalog, kLogDebug, "failed to upgrade catalogs (3 --> 4)");
169  return false;
170  }
171 
172  set_schema_revision(4);
173  if (!StoreSchemaRevision()) {
174  LogCvmfs(kLogCatalog, kLogDebug, "failed to upgrade schema revision");
175  return false;
176  }
177  }
178 
179 
180  if (IsEqualSchema(schema_version(), 2.5) && (schema_revision() == 4)) {
181  LogCvmfs(kLogCatalog, kLogDebug, "upgrading schema revision (4 --> 5)");
182 
183  SqlCatalog sql_upgrade9(*this,
184  "INSERT INTO statistics (counter, value) VALUES "
185  "('self_special', 0);");
186  SqlCatalog sql_upgrade10(*this,
187  "INSERT INTO statistics (counter, value) VALUES "
188  "('subtree_special', 0);");
189  if (!sql_upgrade9.Execute() || !sql_upgrade10.Execute()) {
190  LogCvmfs(kLogCatalog, kLogDebug, "failed to upgrade catalogs (4 --> 5)");
191  return false;
192  }
193 
194  set_schema_revision(5);
195  if (!StoreSchemaRevision()) {
196  LogCvmfs(kLogCatalog, kLogDebug, "failed to upgrade schema revision");
197  return false;
198  }
199  }
200 
201 
202  if (IsEqualSchema(schema_version(), 2.5) && (schema_revision() == 5)) {
203  LogCvmfs(kLogCatalog, kLogDebug, "upgrading schema revision (5 --> 6)");
204 
205  set_schema_revision(6);
206  if (!StoreSchemaRevision()) {
207  LogCvmfs(kLogCatalog, kLogDebug, "failed to upgrade schema revision");
208  return false;
209  }
210  }
211 
212  return true;
213 }
214 
215 
216 bool CatalogDatabase::CreateEmptyDatabase() {
217  assert(read_write());
218 
219  // generate the catalog table and index structure
220  const bool retval =
221  SqlCatalog(*this,
222  "CREATE TABLE catalog "
223  "(md5path_1 INTEGER, md5path_2 INTEGER, parent_1 INTEGER, parent_2 INTEGER,"
224  " hardlinks INTEGER, hash BLOB, size INTEGER, mode INTEGER, mtime INTEGER,"
225  " flags INTEGER, name TEXT, symlink TEXT, uid INTEGER, gid INTEGER, "
226  " xattr BLOB, "
227  " CONSTRAINT pk_catalog PRIMARY KEY (md5path_1, md5path_2));").Execute() &&
228  SqlCatalog(*this,
229  "CREATE INDEX idx_catalog_parent "
230  "ON catalog (parent_1, parent_2);") .Execute() &&
231  SqlCatalog(*this,
232  "CREATE TABLE chunks "
233  "(md5path_1 INTEGER, md5path_2 INTEGER, offset INTEGER, size INTEGER, "
234  " hash BLOB, "
235  " CONSTRAINT pk_chunks PRIMARY KEY (md5path_1, md5path_2, offset, size), "
236  " FOREIGN KEY (md5path_1, md5path_2) REFERENCES "
237  " catalog(md5path_1, md5path_2));") .Execute() &&
238  SqlCatalog(*this,
239  "CREATE TABLE nested_catalogs (path TEXT, sha1 TEXT, size INTEGER, "
240  "CONSTRAINT pk_nested_catalogs PRIMARY KEY (path));") .Execute() &&
241  // Bind mountpoints and nested catalogs are almost the same. We put them in
242  // separate tables to
243  // - not confuse previous client versions, which would crash on bind
244  // mountpoints
245  // - prevent catalogs referenced as bind mountpoints from being replicated,
246  // which would cause exhaustive recursive catalog tree walking
247  // - don't walk into bind mountpoints in catalog traversal (e.g. GC)
248  SqlCatalog(*this,
249  "CREATE TABLE bind_mountpoints (path TEXT, sha1 TEXT, size INTEGER, "
250  "CONSTRAINT pk_bind_mountpoints PRIMARY KEY (path));") .Execute() &&
251  SqlCatalog(*this,
252  "CREATE TABLE statistics (counter TEXT, value INTEGER, "
253  "CONSTRAINT pk_statistics PRIMARY KEY (counter));") .Execute();
254 
255  if (!retval) {
256  PrintSqlError("failed to create catalog database tables.");
257  }
258 
259  return retval;
260 }
261 
262 
263 bool CatalogDatabase::InsertInitialValues(
264  const std::string &root_path,
265  const bool volatile_content,
266  const std::string &voms_authz,
267  const DirectoryEntry &root_entry)
268 {
269  assert(read_write());
270  bool retval = false;
271 
272  // Path hashes
273  shash::Md5 root_path_hash = shash::Md5(shash::AsciiPtr(root_path));
274  shash::Md5 root_parent_hash = (root_path == "")
275  ? shash::Md5()
277 
278  // Start initial filling transaction
279  retval = BeginTransaction();
280  if (!retval) {
281  PrintSqlError("failed to enter initial filling transaction");
282  return false;
283  }
284 
285  // Insert initial values to properties
286  if (!this->SetProperty("revision", 0)) {
287  PrintSqlError(
288  "failed to insert default initial values into the newly created "
289  "catalog tables.");
290  return false;
291  }
292 
293  if (volatile_content) {
294  if (!this->SetProperty("volatile", 1)) {
295  PrintSqlError("failed to insert volatile flag into the newly created "
296  "catalog tables.");
297  return false;
298  }
299  }
300 
301  if (!voms_authz.empty()) {
302  if (!SetVOMSAuthz(voms_authz)) {
303  PrintSqlError("failed to insert VOMS authz flag into the newly created "
304  "catalog tables.");
305  return false;
306  }
307  }
308 
309  // Create initial statistics counters
310  catalog::Counters counters;
311 
312  // Insert root entry (when given)
313  if (!root_entry.IsNegative()) {
314  SqlDirentInsert sql_insert(*this);
315  retval = sql_insert.BindPathHash(root_path_hash) &&
316  sql_insert.BindParentPathHash(root_parent_hash) &&
317  sql_insert.BindDirent(root_entry) &&
318  sql_insert.Execute();
319  if (!retval) {
320  PrintSqlError("failed to insert root entry into newly created catalog.");
321  return false;
322  }
323 
324  // account for the created root entry
325  counters.self.directories = 1;
326  }
327 
328  // Save initial statistics counters
329  if (!counters.InsertIntoDatabase(*this)) {
330  PrintSqlError("failed to insert initial catalog statistics counters.");
331  return false;
332  }
333 
334  // Insert root path (when given)
335  if (!root_path.empty()) {
336  if (!this->SetProperty("root_prefix", root_path)) {
337  PrintSqlError(
338  "failed to store root prefix in the newly created catalog.");
339  return false;
340  }
341  }
342 
343  // Set creation timestamp
344  if (!this->SetProperty("last_modified", static_cast<uint64_t>(time(NULL)))) {
345  PrintSqlError("failed to store creation timestamp in the new catalog.");
346  return false;
347  }
348 
349  // Commit initial filling transaction
350  retval = CommitTransaction();
351  if (!retval) {
352  PrintSqlError("failed to commit initial filling transaction");
353  return false;
354  }
355 
356  return true;
357 }
358 
359 
360 bool
361 CatalogDatabase::SetVOMSAuthz(const std::string &voms_authz) {
362  return this->SetProperty("voms_authz", voms_authz);
363 }
364 
365 
366 double CatalogDatabase::GetRowIdWasteRatio() const {
367  SqlCatalog rowid_waste_ratio_query(*this,
368  "SELECT 1.0 - CAST(COUNT(*) AS DOUBLE) / MAX(rowid) "
369  "AS ratio FROM catalog;");
370  const bool retval = rowid_waste_ratio_query.FetchRow();
371  assert(retval);
372 
373  return rowid_waste_ratio_query.RetrieveDouble(0);
374 }
375 
396 bool CatalogDatabase::CompactDatabase() const {
397  assert(read_write());
398 
399  return SqlCatalog(*this, "PRAGMA foreign_keys = OFF;").Execute() &&
400  BeginTransaction() &&
401  SqlCatalog(*this, "CREATE TEMPORARY TABLE duplicate AS "
402  " SELECT * FROM catalog "
403  " ORDER BY rowid ASC;").Execute() &&
404  SqlCatalog(*this, "DELETE FROM catalog;").Execute() &&
405  SqlCatalog(*this, "INSERT INTO catalog "
406  " SELECT * FROM duplicate "
407  " ORDER BY rowid").Execute() &&
408  SqlCatalog(*this, "DROP TABLE duplicate;").Execute() &&
409  CommitTransaction() &&
410  SqlCatalog(*this, "PRAGMA foreign_keys = ON;").Execute();
411 }
412 
413 
414 //------------------------------------------------------------------------------
415 
416 
417 unsigned SqlDirent::CreateDatabaseFlags(const DirectoryEntry &entry) const {
418  unsigned int database_flags = 0;
419 
420  if (entry.IsNestedCatalogRoot())
421  database_flags |= kFlagDirNestedRoot;
422  else if (entry.IsNestedCatalogMountpoint())
423  database_flags |= kFlagDirNestedMountpoint;
424  else if (entry.IsBindMountpoint())
425  database_flags |= kFlagDirBindMountpoint;
426 
427  if (entry.IsDirectory()) {
428  database_flags |= kFlagDir;
429  } else if (entry.IsLink()) {
430  database_flags |= kFlagFile | kFlagLink;
431  } else if (entry.IsSpecial()) {
432  database_flags |= kFlagFile | kFlagFileSpecial;
433  } else {
434  database_flags |= kFlagFile;
435  database_flags |= entry.compression_algorithm() << kFlagPosCompression;
436  if (entry.IsChunkedFile())
437  database_flags |= kFlagFileChunk;
438  if (entry.IsExternalFile())
439  database_flags |= kFlagFileExternal;
440  if (entry.IsDirectIo())
441  database_flags |= kFlagDirectIo;
442  }
443 
444  if (!entry.checksum_ptr()->IsNull() || entry.IsChunkedFile())
445  StoreHashAlgorithm(entry.checksum_ptr()->algorithm, &database_flags);
446 
447  if (entry.IsHidden())
448  database_flags |= kFlagHidden;
449 
450  return database_flags;
451 }
452 
453 
454 void SqlDirent::StoreHashAlgorithm(const shash::Algorithms algo,
455  unsigned *flags) const
456 {
457  assert(algo != shash::kAny);
458  // Md5 unusable for content hashes
459  *flags |= (algo - 1) << kFlagPosHash;
460 }
461 
462 
463 shash::Algorithms SqlDirent::RetrieveHashAlgorithm(const unsigned flags) const {
464  unsigned in_flags = ((7 << kFlagPosHash) & flags) >> kFlagPosHash;
465  // Skip Md5
466  in_flags++;
467  assert(in_flags < shash::kAny);
468  return static_cast<shash::Algorithms>(in_flags);
469 }
470 
471 
472 zlib::Algorithms SqlDirent::RetrieveCompressionAlgorithm(const unsigned flags)
473  const
474 {
475  // 3 bits, so use 7 (111) to only pull out the flags we want
476  unsigned in_flags =
477  ((7 << kFlagPosCompression) & flags) >> kFlagPosCompression;
478  return static_cast<zlib::Algorithms>(in_flags);
479 }
480 
481 
482 uint32_t SqlDirent::Hardlinks2Linkcount(const uint64_t hardlinks) const {
483  return (hardlinks << 32) >> 32;
484 }
485 
486 
487 uint32_t SqlDirent::Hardlinks2HardlinkGroup(const uint64_t hardlinks) const {
488  return hardlinks >> 32;
489 }
490 
491 
492 uint64_t SqlDirent::MakeHardlinks(const uint32_t hardlink_group,
493  const uint32_t linkcount) const
494 {
495  assert(linkcount > 0);
496  return (static_cast<uint64_t>(hardlink_group) << 32) | linkcount;
497 }
498 
499 
504 void SqlDirent::ExpandSymlink(LinkString *raw_symlink) const {
505  const char *c = raw_symlink->GetChars();
506  const char *cEnd = c+raw_symlink->GetLength();
507  for (; c < cEnd; ++c) {
508  if (*c == '$')
509  goto expand_symlink;
510  }
511  return;
512 
513  expand_symlink:
514  LinkString result;
515  for (c = raw_symlink->GetChars(); c < cEnd; ++c) {
516  if ((*c == '$') && (c < cEnd-2) && (*(c+1) == '(')) {
517  c += 2;
518  const char *rpar = c;
519  while (rpar < cEnd) {
520  if (*rpar == ')')
521  goto expand_symlink_getenv;
522  rpar++;
523  }
524  // right parenthesis missing
525  result.Append("$(", 2);
526  result.Append(c, 1);
527  continue;
528 
529  expand_symlink_getenv:
530  // Check for default value
531  const char *default_separator = c;
532  const char *default_value = rpar;
533  while (default_separator != rpar) {
534  if ((*default_separator == ':') && (*(default_separator + 1) == '-')) {
535  default_value = default_separator+2;
536  break;
537  }
538  default_separator++;
539  }
540 
541  const unsigned environ_var_length = default_separator-c;
542  char environ_var[environ_var_length+1];
543  environ_var[environ_var_length] = '\0';
544  memcpy(environ_var, c, environ_var_length);
545  const char *environ_value = getenv(environ_var); // Don't free!
546  if (environ_value) {
547  result.Append(environ_value, strlen(environ_value));
548  } else {
549  const unsigned default_length = rpar-default_value;
550  result.Append(default_value, default_length);
551  }
552  c = rpar;
553  continue;
554  }
555  result.Append(c, 1);
556  }
557  raw_symlink->Assign(result);
558  return;
559 }
560 
561 
562 //------------------------------------------------------------------------------
563 
564 
565 bool SqlDirentWrite::BindDirentFields(const int hash_idx,
566  const int hardlinks_idx,
567  const int size_idx,
568  const int mode_idx,
569  const int mtime_idx,
570  const int flags_idx,
571  const int name_idx,
572  const int symlink_idx,
573  const int uid_idx,
574  const int gid_idx,
575  const DirectoryEntry &entry)
576 {
577  const uint64_t hardlinks =
578  MakeHardlinks(entry.hardlink_group_,
579  entry.linkcount_);
580 
581  return (
582  BindHashBlob(hash_idx, entry.checksum_) &&
583  BindInt64(hardlinks_idx, hardlinks) &&
584  BindInt64(size_idx, entry.size_) &&
585  BindInt(mode_idx, entry.mode_) &&
586  BindInt64(uid_idx, entry.uid_) &&
587  BindInt64(gid_idx, entry.gid_) &&
588  BindInt64(mtime_idx, entry.mtime_) &&
589  BindInt(flags_idx, CreateDatabaseFlags(entry)) &&
590  BindText(name_idx, entry.name_.GetChars(), entry.name_.GetLength()) &&
591  BindText(symlink_idx, entry.symlink_.GetChars(), entry.symlink_.GetLength())
592  ); // NOLINT
593 }
594 
595 
596 //------------------------------------------------------------------------------
597 
598 
599 SqlListContentHashes::SqlListContentHashes(const CatalogDatabase &database) {
600  static const char *stmt_lt_2_4 =
601  "SELECT hash, flags, 0 "
602  " FROM catalog "
603  " WHERE length(hash) > 0;";
604 
605  static const char *stmt_ge_2_4 =
606  "SELECT hash, flags, 0 "
607  " FROM catalog "
608  " WHERE (length(catalog.hash) > 0) AND "
609  " ((flags & 128) = 0) " // kFlagFileExternal
610  "UNION "
611  "SELECT chunks.hash, catalog.flags, 1 "
612  " FROM catalog "
613  " LEFT JOIN chunks "
614  " ON catalog.md5path_1 = chunks.md5path_1 AND "
615  " catalog.md5path_2 = chunks.md5path_2 "
616  " WHERE (catalog.flags & 128) = 0;"; // kFlagFileExternal
617 
618  if (database.schema_version() < 2.4-CatalogDatabase::kSchemaEpsilon) {
619  DeferredInit(database.sqlite_db(), stmt_lt_2_4);
620  } else {
621  DeferredInit(database.sqlite_db(), stmt_ge_2_4);
622  }
623 }
624 
625 
626 shash::Any SqlListContentHashes::GetHash() const {
627  const unsigned int db_flags = RetrieveInt(1);
628  const shash::Algorithms hash_algorithm = RetrieveHashAlgorithm(db_flags);
629  shash::Any hash = RetrieveHashBlob(0, hash_algorithm);
630  if (RetrieveInt(2) == 1) {
632  }
633 
634  return hash;
635 }
636 
637 
638 //------------------------------------------------------------------------------
639 
640 #define DB_FIELDS_LT_V2_1 \
641  "catalog.hash, catalog.inode, catalog.size, " \
642  "catalog.mode, catalog.mtime, catalog.flags, " \
643  "catalog.name, catalog.symlink, catalog.md5path_1, " \
644  "catalog.md5path_2, catalog.parent_1, catalog.parent_2, " \
645  "catalog.rowid"
646 #define DB_FIELDS_GE_V2_1_LT_R2 \
647  "catalog.hash, catalog.hardlinks, catalog.size, " \
648  "catalog.mode, catalog.mtime, catalog.flags, " \
649  "catalog.name, catalog.symlink, catalog.md5path_1, " \
650  "catalog.md5path_2, catalog.parent_1, catalog.parent_2, " \
651  "catalog.rowid, catalog.uid, catalog.gid, " \
652  "0"
653 #define DB_FIELDS_GE_V2_1_GE_R2 \
654  "catalog.hash, catalog.hardlinks, catalog.size, " \
655  "catalog.mode, catalog.mtime, catalog.flags, " \
656  "catalog.name, catalog.symlink, catalog.md5path_1, " \
657  "catalog.md5path_2, catalog.parent_1, catalog.parent_2, " \
658  "catalog.rowid, catalog.uid, catalog.gid, " \
659  "catalog.xattr IS NOT NULL"
660 
661 #define MAKE_STATEMENT(STMT_TMPL, REV) \
662  static const std::string REV = \
663  ReplaceAll(STMT_TMPL, "@DB_FIELDS@", DB_FIELDS_ ## REV)
664 
665 #define MAKE_STATEMENTS(STMT_TMPL) \
666  MAKE_STATEMENT(STMT_TMPL, LT_V2_1); \
667  MAKE_STATEMENT(STMT_TMPL, GE_V2_1_LT_R2); \
668  MAKE_STATEMENT(STMT_TMPL, GE_V2_1_GE_R2)
669 
670 #define DEFERRED_INIT(DB, REV) \
671  DeferredInit((DB).sqlite_db(), (REV).c_str())
672 
673 #define DEFERRED_INITS(DB) \
674  if ((DB).schema_version() < 2.1 - CatalogDatabase::kSchemaEpsilon) { \
675  DEFERRED_INIT((DB), LT_V2_1); \
676  } else if ((DB).schema_revision() < 2) { \
677  DEFERRED_INIT((DB), GE_V2_1_LT_R2); \
678  } else { \
679  DEFERRED_INIT((DB), GE_V2_1_GE_R2); \
680  }
681 
682 
683 shash::Md5 SqlLookup::GetPathHash() const {
684  return RetrieveMd5(8, 9);
685 }
686 
687 
688 shash::Md5 SqlLookup::GetParentPathHash() const {
689  return RetrieveMd5(10, 11);
690 }
691 
692 
696 DirectoryEntry SqlLookup::GetDirent(const Catalog *catalog,
697  const bool expand_symlink) const
698 {
699  DirectoryEntry result;
700 
701  const unsigned database_flags = RetrieveInt(5);
702  result.is_nested_catalog_root_ = (database_flags & kFlagDirNestedRoot);
704  (database_flags & kFlagDirNestedMountpoint);
705  const char *name = reinterpret_cast<const char *>(RetrieveText(6));
706  const char *symlink = reinterpret_cast<const char *>(RetrieveText(7));
707 
708  // Retrieve the hardlink information from the hardlinks database field
709  if (catalog->schema() < 2.1 - CatalogDatabase::kSchemaEpsilon) {
710  result.linkcount_ = 1;
711  result.hardlink_group_ = 0;
712  result.inode_ = catalog->GetMangledInode(RetrieveInt64(12), 0);
713  result.is_chunked_file_ = false;
714  result.has_xattrs_ = false;
715  result.checksum_ = RetrieveHashBlob(0, shash::kSha1);
716  result.uid_ = g_uid;
717  result.gid_ = g_gid;
718  } else {
719  const uint64_t hardlinks = RetrieveInt64(1);
720  result.linkcount_ = Hardlinks2Linkcount(hardlinks);
721  result.hardlink_group_ = Hardlinks2HardlinkGroup(hardlinks);
722  result.inode_ =
723  catalog->GetMangledInode(RetrieveInt64(12), result.hardlink_group_);
724  result.is_bind_mountpoint_ = (database_flags & kFlagDirBindMountpoint);
725  result.is_chunked_file_ = (database_flags & kFlagFileChunk);
726  result.is_hidden_ = (database_flags & kFlagHidden);
727  result.is_direct_io_ = (database_flags & kFlagDirectIo);
728  result.is_external_file_ = (database_flags & kFlagFileExternal);
729  result.has_xattrs_ = RetrieveInt(15) != 0;
730  result.checksum_ =
731  RetrieveHashBlob(0, RetrieveHashAlgorithm(database_flags));
732  result.compression_algorithm_ =
733  RetrieveCompressionAlgorithm(database_flags);
734 
735  if (g_claim_ownership) {
736  result.uid_ = g_uid;
737  result.gid_ = g_gid;
738  } else {
739  result.uid_ = catalog->MapUid(RetrieveInt64(13));
740  result.gid_ = catalog->MapGid(RetrieveInt64(14));
741  }
742  }
743 
744  result.mode_ = RetrieveInt(3);
745  result.size_ = RetrieveInt64(2);
746  result.mtime_ = RetrieveInt64(4);
747  result.name_.Assign(name, strlen(name));
748  result.symlink_.Assign(symlink, strlen(symlink));
749  if (expand_symlink && !g_raw_symlinks)
750  ExpandSymlink(&result.symlink_);
751 
752  return result;
753 }
754 
755 
756 //------------------------------------------------------------------------------
757 
758 
759 SqlListing::SqlListing(const CatalogDatabase &database) {
760  MAKE_STATEMENTS("SELECT @DB_FIELDS@ FROM catalog "
761  "WHERE (parent_1 = :p_1) AND (parent_2 = :p_2);");
762  DEFERRED_INITS(database);
763 }
764 
765 
766 bool SqlListing::BindPathHash(const struct shash::Md5 &hash) {
767  return BindMd5(1, 2, hash);
768 }
769 
770 
771 //------------------------------------------------------------------------------
772 
773 
774 SqlLookupPathHash::SqlLookupPathHash(const CatalogDatabase &database) {
775  MAKE_STATEMENTS("SELECT @DB_FIELDS@ FROM catalog "
776  "WHERE (md5path_1 = :md5_1) AND (md5path_2 = :md5_2);");
777  DEFERRED_INITS(database);
778 }
779 
780 bool SqlLookupPathHash::BindPathHash(const struct shash::Md5 &hash) {
781  return BindMd5(1, 2, hash);
782 }
783 
784 
785 //------------------------------------------------------------------------------
786 
787 
788 SqlLookupInode::SqlLookupInode(const CatalogDatabase &database) {
789  MAKE_STATEMENTS("SELECT @DB_FIELDS@ FROM catalog WHERE rowid = :rowid;");
790  DEFERRED_INITS(database);
791 }
792 
793 
794 bool SqlLookupInode::BindRowId(const uint64_t inode) {
795  return BindInt64(1, inode);
796 }
797 
798 
799 //------------------------------------------------------------------------------
800 
801 
802 SqlLookupDanglingMountpoints::SqlLookupDanglingMountpoints(
803  const catalog::CatalogDatabase &database) {
804  MAKE_STATEMENTS("SELECT DISTINCT @DB_FIELDS@ FROM catalog "
805  "JOIN catalog AS c2 "
806  "ON catalog.md5path_1 = c2.parent_1 AND "
807  " catalog.md5path_2 = c2.parent_2 "
808  "WHERE catalog.flags & :nested_mountpoint_flag");
809  DEFERRED_INITS(database);
810 
811  // this pretty much removes the advantage of a deferred init but the statement
812  // is anyway only used directly.
813  const bool success = BindInt64(1, SqlDirent::kFlagDirNestedMountpoint);
814  assert(success);
815 }
816 
817 
818 //------------------------------------------------------------------------------
819 
820 
821 SqlDirentTouch::SqlDirentTouch(const CatalogDatabase &database) {
822  DeferredInit(database.sqlite_db(),
823  "UPDATE catalog "
824  "SET hash = :hash, size = :size, mode = :mode, mtime = :mtime, "
825 // 1 2 3 4
826  "name = :name, symlink = :symlink, uid = :uid, gid = :gid, xattr = :xattr "
827 // 5 6 7 8 9
828  "WHERE (md5path_1 = :md5_1) AND (md5path_2 = :md5_2);");
829 // 10 11
830 }
831 
832 
833 bool SqlDirentTouch::BindDirentBase(const DirectoryEntryBase &entry) {
834  return (
835  BindHashBlob(1, entry.checksum_) &&
836  BindInt64(2, entry.size_) &&
837  BindInt(3, entry.mode_) &&
838  BindInt64(4, entry.mtime_) &&
839  BindText(5, entry.name_.GetChars(), entry.name_.GetLength()) &&
840  BindText(6, entry.symlink_.GetChars(), entry.symlink_.GetLength()) &&
841  BindInt64(7, entry.uid_) &&
842  BindInt64(8, entry.gid_));
843 }
844 
845 
846 bool SqlDirentTouch::BindPathHash(const shash::Md5 &hash) {
847  return BindMd5(10, 11, hash);
848 }
849 
850 
851 bool SqlDirentTouch::BindXattr(const XattrList &xattrs) {
852  unsigned char *packed_xattrs;
853  unsigned size;
854  xattrs.Serialize(&packed_xattrs, &size);
855  if (packed_xattrs == NULL)
856  return BindNull(9);
857  return BindBlobTransient(9, packed_xattrs, size);
858 }
859 
860 
861 bool SqlDirentTouch::BindXattrEmpty() {
862  return BindNull(9);
863 }
864 
865 
866 //------------------------------------------------------------------------------
867 
868 
869 SqlNestedCatalogLookup::SqlNestedCatalogLookup(const CatalogDatabase &database)
870 {
871  // We cannot access nested catalogs where the content hash is missing
872  static const char *stmt_0_9 =
873  "SELECT '', 0 FROM nested_catalogs;";
874  static const char *stmt_2_5_ge_4 =
875  "SELECT sha1, size FROM nested_catalogs WHERE path=:path "
876  "UNION ALL SELECT sha1, size FROM bind_mountpoints WHERE path=:path;";
877  static const char *stmt_2_5_ge_1_lt_4 =
878  "SELECT sha1, size FROM nested_catalogs WHERE path=:path;";
879  // Internally converts NULL to 0 for size
880  static const char *stmt_2_5_lt_1 =
881  "SELECT sha1, 0 FROM nested_catalogs WHERE path=:path;";
882 
883  if (database.IsEqualSchema(database.schema_version(), 2.5) &&
884  (database.schema_revision() >= 4))
885  {
886  DeferredInit(database.sqlite_db(), stmt_2_5_ge_4);
887  } else if (database.IsEqualSchema(database.schema_version(), 2.5) &&
888  (database.schema_revision() >= 1))
889  {
890  DeferredInit(database.sqlite_db(), stmt_2_5_ge_1_lt_4);
891  } else {
892  if (database.IsEqualSchema(database.schema_version(), 0.9)) {
893  DeferredInit(database.sqlite_db(), stmt_0_9);
894  } else {
895  DeferredInit(database.sqlite_db(), stmt_2_5_lt_1);
896  }
897  }
898 }
899 
900 
901 bool SqlNestedCatalogLookup::BindSearchPath(const PathString &path) {
902  return BindText(1, path.GetChars(), path.GetLength());
903 }
904 
905 
906 shash::Any SqlNestedCatalogLookup::GetContentHash() const {
907  const string hash = string(reinterpret_cast<const char *>(RetrieveText(0)));
908  return (hash.empty()) ? shash::Any(shash::kAny) :
911 }
912 
913 
914 uint64_t SqlNestedCatalogLookup::GetSize() const {
915  return RetrieveInt64(1);
916 }
917 
918 
919 //------------------------------------------------------------------------------
920 
921 
922 SqlNestedCatalogListing::SqlNestedCatalogListing(
923  const CatalogDatabase &database)
924 {
925  // We cannot access nested catalogs where the content hash is missing
926  static const char *stmt_0_9 =
927  "SELECT '', '', 0 FROM nested_catalogs;";
928  static const char *stmt_2_5_ge_4 =
929  "SELECT path, sha1, size FROM nested_catalogs "
930  "UNION ALL SELECT path, sha1, size FROM bind_mountpoints;";
931  static const char *stmt_2_5_ge_1_lt_4 =
932  "SELECT path, sha1, size FROM nested_catalogs;";
933  // Internally converts NULL to 0 for size
934  static const char *stmt_2_5_lt_1 =
935  "SELECT path, sha1, 0 FROM nested_catalogs;";
936 
937  if (database.IsEqualSchema(database.schema_version(), 2.5) &&
938  (database.schema_revision() >= 4))
939  {
940  DeferredInit(database.sqlite_db(), stmt_2_5_ge_4);
941  } else if (database.IsEqualSchema(database.schema_version(), 2.5) &&
942  (database.schema_revision() >= 1))
943  {
944  DeferredInit(database.sqlite_db(), stmt_2_5_ge_1_lt_4);
945  } else {
946  if (database.IsEqualSchema(database.schema_version(), 0.9)) {
947  DeferredInit(database.sqlite_db(), stmt_0_9);
948  } else {
949  DeferredInit(database.sqlite_db(), stmt_2_5_lt_1);
950  }
951  }
952 }
953 
954 
955 PathString SqlNestedCatalogListing::GetPath() const {
956  const char *path = reinterpret_cast<const char *>(RetrieveText(0));
957  return PathString(path, strlen(path));
958 }
959 
960 
961 shash::Any SqlNestedCatalogListing::GetContentHash() const {
962  const string hash = string(reinterpret_cast<const char *>(RetrieveText(1)));
963  return (hash.empty()) ? shash::Any(shash::kAny) :
966 }
967 
968 
969 uint64_t SqlNestedCatalogListing::GetSize() const {
970  return RetrieveInt64(2);
971 }
972 
973 
974 //------------------------------------------------------------------------------
975 
976 
977 SqlOwnNestedCatalogListing::SqlOwnNestedCatalogListing(
978  const CatalogDatabase &database)
979 {
980  // We cannot access nested catalogs where the content hash is missing
981  static const char *stmt_0_9 =
982  "SELECT '', '', 0 FROM nested_catalogs;";
983  static const char *stmt_2_5_ge_1 =
984  "SELECT path, sha1, size FROM nested_catalogs;";
985  // Internally converts NULL to 0 for size
986  static const char *stmt_2_5_lt_1 =
987  "SELECT path, sha1, 0 FROM nested_catalogs;";
988 
989  if (database.IsEqualSchema(database.schema_version(), 2.5) &&
990  (database.schema_revision() >= 1))
991  {
992  DeferredInit(database.sqlite_db(), stmt_2_5_ge_1);
993  } else {
994  if (database.IsEqualSchema(database.schema_version(), 0.9)) {
995  DeferredInit(database.sqlite_db(), stmt_0_9);
996  } else {
997  DeferredInit(database.sqlite_db(), stmt_2_5_lt_1);
998  }
999  }
1000 }
1001 
1002 
1003 PathString SqlOwnNestedCatalogListing::GetPath() const {
1004  const char *path = reinterpret_cast<const char *>(RetrieveText(0));
1005  return PathString(path, strlen(path));
1006 }
1007 
1008 
1009 shash::Any SqlOwnNestedCatalogListing::GetContentHash() const {
1010  const string hash = string(reinterpret_cast<const char *>(RetrieveText(1)));
1011  return (hash.empty()) ? shash::Any(shash::kAny) :
1014 }
1015 
1016 
1017 uint64_t SqlOwnNestedCatalogListing::GetSize() const {
1018  return RetrieveInt64(2);
1019 }
1020 
1021 
1022 //------------------------------------------------------------------------------
1023 
1024 
1025 SqlDirentInsert::SqlDirentInsert(const CatalogDatabase &database) {
1026  DeferredInit(database.sqlite_db(),
1027  "INSERT INTO catalog "
1028  "(md5path_1, md5path_2, parent_1, parent_2, hash, hardlinks, size, mode,"
1029  // 1 2 3 4 5 6 7 8
1030  "mtime, flags, name, symlink, uid, gid, xattr) "
1031  // 9, 10 11 12 13 14 15
1032  "VALUES (:md5_1, :md5_2, :p_1, :p_2, :hash, :links, :size, :mode, :mtime,"
1033  " :flags, :name, :symlink, :uid, :gid, :xattr);");
1034 }
1035 
1036 
1037 bool SqlDirentInsert::BindPathHash(const shash::Md5 &hash) {
1038  return BindMd5(1, 2, hash);
1039 }
1040 
1041 
1042 bool SqlDirentInsert::BindParentPathHash(const shash::Md5 &hash) {
1043  return BindMd5(3, 4, hash);
1044 }
1045 
1046 
1047 bool SqlDirentInsert::BindDirent(const DirectoryEntry &entry) {
1048  return BindDirentFields(5, 6, 7, 8, 9, 10, 11, 12, 13, 14, entry);
1049 }
1050 
1051 
1052 bool SqlDirentInsert::BindXattr(const XattrList &xattrs) {
1053  unsigned char *packed_xattrs;
1054  unsigned size;
1055  xattrs.Serialize(&packed_xattrs, &size);
1056  if (packed_xattrs == NULL)
1057  return BindNull(15);
1058  return BindBlobTransient(15, packed_xattrs, size);
1059 }
1060 
1061 
1062 bool SqlDirentInsert::BindXattrEmpty() {
1063  return BindNull(15);
1064 }
1065 
1066 
1067 //------------------------------------------------------------------------------
1068 
1069 
1070 SqlDirentUpdate::SqlDirentUpdate(const CatalogDatabase &database) {
1071  DeferredInit(database.sqlite_db(),
1072  "UPDATE catalog "
1073  "SET hash = :hash, size = :size, mode = :mode, mtime = :mtime, "
1074 // 1 2 3 4
1075  "flags = :flags, name = :name, symlink = :symlink, hardlinks = :hardlinks, "
1076 // 5 6 7 8
1077  "uid = :uid, gid = :gid "
1078 // 9 10
1079  "WHERE (md5path_1 = :md5_1) AND (md5path_2 = :md5_2);");
1080 // 11 12
1081 }
1082 
1083 
1084 bool SqlDirentUpdate::BindPathHash(const shash::Md5 &hash) {
1085  return BindMd5(11, 12, hash);
1086 }
1087 
1088 
1089 bool SqlDirentUpdate::BindDirent(const DirectoryEntry &entry) {
1090  return BindDirentFields(1, 8, 2, 3, 4, 5, 6, 7, 9, 10, entry);
1091 }
1092 
1093 
1094 //------------------------------------------------------------------------------
1095 
1096 
1097 SqlDirentUnlink::SqlDirentUnlink(const CatalogDatabase &database) {
1098  DeferredInit(database.sqlite_db(),
1099  "DELETE FROM catalog "
1100  "WHERE (md5path_1 = :md5_1) AND (md5path_2 = :md5_2);");
1101 }
1102 
1103 bool SqlDirentUnlink::BindPathHash(const shash::Md5 &hash) {
1104  return BindMd5(1, 2, hash);
1105 }
1106 
1107 
1108 //------------------------------------------------------------------------------
1109 
1110 
1111 SqlIncLinkcount::SqlIncLinkcount(const CatalogDatabase &database) {
1112  // This command changes the linkcount of a whole hardlink group at once!
1113  // We can do this, since the 'hardlinks'-field contains the hardlink group ID
1114  // in the higher 32bit as well as the 'linkcount' in the lower 32bit.
1115  // This field will be equal for all entries belonging to the same hardlink
1116  // group while adding/subtracting small values from it will only effect the
1117  // linkcount in the lower 32bit.
1118  // Take a deep breath!
1119  DeferredInit(database.sqlite_db(),
1120  "UPDATE catalog SET hardlinks = hardlinks + :delta "
1121  "WHERE hardlinks = (SELECT hardlinks from catalog "
1122  "WHERE md5path_1 = :md5_1 AND md5path_2 = :md5_2);");
1123 }
1124 
1125 
1126 bool SqlIncLinkcount::BindPathHash(const shash::Md5 &hash) {
1127  return BindMd5(2, 3, hash);
1128 }
1129 
1130 
1131 bool SqlIncLinkcount::BindDelta(const int delta) {
1132  return BindInt(1, delta);
1133 }
1134 
1135 
1136 //------------------------------------------------------------------------------
1137 
1138 
1139 SqlChunkInsert::SqlChunkInsert(const CatalogDatabase &database) {
1140  DeferredInit(database.sqlite_db(),
1141  "INSERT INTO chunks (md5path_1, md5path_2, offset, size, hash) "
1142  // 1 2 3 4 5
1143  "VALUES (:md5_1, :md5_2, :offset, :size, :hash);");
1144 }
1145 
1146 
1147 bool SqlChunkInsert::BindPathHash(const shash::Md5 &hash) {
1148  return BindMd5(1, 2, hash);
1149 }
1150 
1151 
1152 bool SqlChunkInsert::BindFileChunk(const FileChunk &chunk) {
1153  return
1154  BindInt64(3, chunk.offset()) &&
1155  BindInt64(4, chunk.size()) &&
1156  BindHashBlob(5, chunk.content_hash());
1157 }
1158 
1159 
1160 //------------------------------------------------------------------------------
1161 
1162 
1163 SqlChunksRemove::SqlChunksRemove(const CatalogDatabase &database) {
1164  DeferredInit(database.sqlite_db(),
1165  "DELETE FROM chunks "
1166  "WHERE (md5path_1 = :md5_1) AND (md5path_2 = :md5_2);");
1167 }
1168 
1169 
1170 bool SqlChunksRemove::BindPathHash(const shash::Md5 &hash) {
1171  return BindMd5(1, 2, hash);
1172 }
1173 
1174 
1175 //------------------------------------------------------------------------------
1176 
1177 
1178 SqlChunksListing::SqlChunksListing(const CatalogDatabase &database) {
1179  DeferredInit(database.sqlite_db(),
1180  "SELECT offset, size, hash FROM chunks "
1181  // 0 1 2
1182  "WHERE (md5path_1 = :md5_1) AND (md5path_2 = :md5_2) "
1183  // 1 2
1184  "ORDER BY offset ASC;");
1185 }
1186 
1187 
1188 bool SqlChunksListing::BindPathHash(const shash::Md5 &hash) {
1189  return BindMd5(1, 2, hash);
1190 }
1191 
1192 
1193 FileChunk SqlChunksListing::GetFileChunk(
1194  const shash::Algorithms interpret_hash_as) const
1195 {
1196  return FileChunk(
1197  RetrieveHashBlob(2, interpret_hash_as, shash::kSuffixPartial),
1198  RetrieveInt64(0),
1199  RetrieveInt64(1));
1200 }
1201 
1202 
1203 //------------------------------------------------------------------------------
1204 
1205 
1206 SqlChunksCount::SqlChunksCount(const CatalogDatabase &database) {
1207  DeferredInit(database.sqlite_db(),
1208  "SELECT count(*) FROM chunks "
1209  // 0
1210  "WHERE (md5path_1 = :md5_1) AND (md5path_2 = :md5_2)");
1211  // 1 2
1212 }
1213 
1214 
1215 bool SqlChunksCount::BindPathHash(const shash::Md5 &hash) {
1216  return BindMd5(1, 2, hash);
1217 }
1218 
1219 
1220 int SqlChunksCount::GetChunkCount() const {
1221  return RetrieveInt64(0);
1222 }
1223 
1224 
1225 //------------------------------------------------------------------------------
1226 
1227 
1228 SqlMaxHardlinkGroup::SqlMaxHardlinkGroup(const CatalogDatabase &database) {
1229  DeferredInit(database.sqlite_db(), "SELECT max(hardlinks) FROM catalog;");
1230 }
1231 
1232 
1233 uint32_t SqlMaxHardlinkGroup::GetMaxGroupId() const {
1234  return RetrieveInt64(0) >> 32;
1235 }
1236 
1237 
1238 //------------------------------------------------------------------------------
1239 
1240 
1241 SqlGetCounter::SqlGetCounter(const CatalogDatabase &database) {
1242  static const char *stmt_ge_2_4 =
1243  "SELECT value from statistics WHERE counter = :counter;";
1244  static const char *stmt_lt_2_4 =
1245  "SELECT 0;";
1246 
1247  if (database.schema_version() >= 2.4 - CatalogDatabase::kSchemaEpsilon) {
1248  compat_ = false;
1249  DeferredInit(database.sqlite_db(), stmt_ge_2_4);
1250  } else {
1251  compat_ = true;
1252  DeferredInit(database.sqlite_db(), stmt_lt_2_4);
1253  }
1254 }
1255 
1256 
1257 bool SqlGetCounter::BindCounter(const std::string &counter) {
1258  if (compat_) return true;
1259  return BindText(1, counter);
1260 }
1261 
1262 
1263 uint64_t SqlGetCounter::GetCounter() const {
1264  if (compat_) return 0;
1265  return RetrieveInt64(0);
1266 }
1267 
1268 
1269 //------------------------------------------------------------------------------
1270 
1271 
1272 SqlUpdateCounter::SqlUpdateCounter(const CatalogDatabase &database) {
1273  DeferredInit(database.sqlite_db(),
1274  "UPDATE statistics SET value=value+:val WHERE counter=:counter;");
1275 }
1276 
1277 
1278 bool SqlUpdateCounter::BindCounter(const std::string &counter) {
1279  return BindText(2, counter);
1280 }
1281 
1282 
1283 bool SqlUpdateCounter::BindDelta(const int64_t delta) {
1284  return BindInt64(1, delta);
1285 }
1286 
1287 
1288 //------------------------------------------------------------------------------
1289 
1290 
1291 SqlCreateCounter::SqlCreateCounter(const CatalogDatabase &database) {
1292  DeferredInit(database.sqlite_db(),
1293  "INSERT OR REPLACE INTO statistics (counter, value) "
1294  "VALUES (:counter, :value);");
1295 }
1296 
1297 
1298 bool SqlCreateCounter::BindCounter(const std::string &counter) {
1299  return BindText(1, counter);
1300 }
1301 
1302 
1303 bool SqlCreateCounter::BindInitialValue(const int64_t value) {
1304  return BindInt64(2, value);
1305 }
1306 
1307 
1308 //------------------------------------------------------------------------------
1309 
1310 
1311 SqlAllChunks::SqlAllChunks(const CatalogDatabase &database) {
1312  int hash_mask = 7 << SqlDirent::kFlagPosHash;
1313  string flags2hash =
1314  " ((flags&" + StringifyInt(hash_mask) + ") >> " +
1315  StringifyInt(SqlDirent::kFlagPosHash) + ")+1 AS hash_algorithm ";
1316 
1317  int compression_mask = 7 << SqlDirent::kFlagPosCompression;
1318  string flags2compression =
1319  " ((flags&" + StringifyInt(compression_mask) + ") >> " +
1320  StringifyInt(SqlDirent::kFlagPosCompression) + ") " +
1321  "AS compression_algorithm ";
1322 
1323  // TODO(reneme): this depends on shash::kSuffix* being a char!
1324  // it should be more generic or replaced entirely
1325  // TODO(reneme): this is practically the same as SqlListContentHashes and
1326  // should be consolidated
1327  string sql = "SELECT DISTINCT hash, "
1328  "CASE WHEN flags & " + StringifyInt(SqlDirent::kFlagFile) + " THEN " +
1330  "WHEN flags & " + StringifyInt(SqlDirent::kFlagDir) + " THEN " +
1332  "AS chunk_type, " + flags2hash + "," + flags2compression +
1333  "FROM catalog WHERE (hash IS NOT NULL) AND "
1334  "(flags & " + StringifyInt(SqlDirent::kFlagFileExternal) + " = 0)";
1335  if (database.schema_version() >= 2.4 - CatalogDatabase::kSchemaEpsilon) {
1336  sql +=
1337  " UNION "
1338  "SELECT DISTINCT chunks.hash, " + StringifyInt(shash::kSuffixPartial) +
1339  ", " + flags2hash + "," + flags2compression +
1340  "FROM chunks, catalog WHERE "
1341  "chunks.md5path_1=catalog.md5path_1 AND "
1342  "chunks.md5path_2=catalog.md5path_2 AND "
1343  "(catalog.flags & " + StringifyInt(SqlDirent::kFlagFileExternal) +
1344  " = 0)";
1345  }
1346  sql += ";";
1347  Init(database.sqlite_db(), sql);
1348 }
1349 
1350 
1352  return true;
1353 }
1354 
1355 
1356 bool SqlAllChunks::Next(shash::Any *hash, zlib::Algorithms *compression_alg) {
1357  if (!FetchRow()) {
1358  return false;
1359  }
1360 
1361  *hash = RetrieveHashBlob(0, static_cast<shash::Algorithms>(RetrieveInt(2)),
1362  static_cast<shash::Suffix>(RetrieveInt(1)));
1363  *compression_alg = static_cast<zlib::Algorithms>(RetrieveInt(3));
1364  return true;
1365 }
1366 
1367 
1368 bool SqlAllChunks::Close() {
1369  return Reset();
1370 }
1371 
1372 
1373 //------------------------------------------------------------------------------
1374 
1375 
1376 SqlLookupXattrs::SqlLookupXattrs(const CatalogDatabase &database) {
1377  DeferredInit(database.sqlite_db(),
1378  "SELECT xattr FROM catalog "
1379  "WHERE (md5path_1 = :md5_1) AND (md5path_2 = :md5_2);");
1380 }
1381 
1382 
1383 bool SqlLookupXattrs::BindPathHash(const shash::Md5 &hash) {
1384  return BindMd5(1, 2, hash);
1385 }
1386 
1387 
1388 XattrList SqlLookupXattrs::GetXattrs() {
1389  const unsigned char *packed_xattrs =
1390  reinterpret_cast<const unsigned char *>(RetrieveBlob(0));
1391  if (packed_xattrs == NULL)
1392  return XattrList();
1393 
1394  int size = RetrieveBytes(0);
1395  assert(size >= 0);
1396  UniquePtr<XattrList> xattrs(XattrList::Deserialize(packed_xattrs, size));
1397  if (!xattrs.IsValid()) {
1398  LogCvmfs(kLogCatalog, kLogDebug, "corrupted xattr data");
1399  return XattrList();
1400  }
1401  return *xattrs;
1402 }
1403 
1404 } // namespace catalog
bool InsertIntoDatabase(const CatalogDatabase &database) const
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
bool IsExternalFile() const
bool IsNull() const
Definition: hash.h:379
bool IsSpecial() const
bool Execute()
Definition: sql.cc:42
bool IsDirectory() const
bool FetchRow()
Definition: sql.cc:62
uint64_t MapUid(const uint64_t uid) const
Definition: catalog.h:219
bool IsHidden() const
bool IsChunkedFile() const
inode_t inode_
bool is_hidden_
bool BindDirent(const DirectoryEntry &entry)
void Assign(const char *chars, const unsigned length)
Definition: shortstring.h:53
void Open()
double RetrieveDouble(const int idx_column) const
Definition: sql.h:434
gid_t gid_
const char kSuffixMicroCatalog
Definition: hash.h:54
const shash::Any & content_hash() const
Definition: file_chunk.h:41
float schema() const
Definition: catalog.h:178
assert((mem||(size==0))&&"Out Of Memory")
float schema_version() const
Definition: sql.h:149
bool IsDirectIo() const
#define MAKE_STATEMENTS(STMT_TMPL)
Definition: catalog_sql.cc:665
std::string GetParentPath(const std::string &path)
Definition: posix.cc:129
Algorithms algorithm
Definition: hash.h:123
bool has_xattrs_
uint64_t size_
unsigned schema_revision() const
Definition: sql.h:150
#define DEFERRED_INITS(DB)
Definition: catalog_sql.cc:673
bool IsNestedCatalogMountpoint() const
bool BindPathHash(const shash::Md5 &hash)
Algorithms
Definition: hash.h:39
bool IsNestedCatalogRoot() const
NameString name_
zlib::Algorithms compression_algorithm_
bool IsLink() const
Algorithms
Definition: compression.h:44
gid_t g_gid
Definition: globals.cc:14
const char kSuffixPartial
Definition: hash.h:55
void Serialize(unsigned char **outbuf, unsigned *size, const std::vector< std::string > *blacklist=NULL) const
Definition: xattr.cc:196
const char kSuffixCatalog
Definition: hash.h:52
uint32_t linkcount_
off_t offset() const
Definition: file_chunk.h:42
bool is_nested_catalog_mountpoint_
bool is_external_file_
zlib::Algorithms compression_algorithm() const
uid_t g_uid
Definition: globals.cc:13
Any MkFromHexPtr(const HexPtr hex, const char suffix)
Definition: hash.cc:83
uint32_t hardlink_group_
time_t mtime_
static int Init(const loader::LoaderExports *loader_exports)
Definition: cvmfs.cc:1755
void Append(const char *chars, const unsigned length)
Definition: shortstring.h:70
string StringifyInt(const int64_t value)
Definition: string.cc:77
sqlite3 * sqlite_db() const
Definition: sql.h:147
bool IsBindMountpoint() const
bool is_chunked_file_
bool g_raw_symlinks
Definition: globals.cc:12
bool IsEqualSchema(const float value, const float compare) const
Definition: sql.h:131
inode_t GetMangledInode(const uint64_t row_id, const uint64_t hardlink_group) const
Definition: catalog.cc:593
static XattrList * Deserialize(const unsigned char *inbuf, const unsigned size)
Definition: xattr.cc:67
unsigned int mode_
ShortString< kDefaultMaxPath, 0 > PathString
Definition: shortstring.h:189
size_t size() const
Definition: file_chunk.h:43
unsigned GetLength() const
Definition: shortstring.h:103
shash::Any checksum_
Suffix suffix
Definition: hash.h:124
bool is_direct_io_
bool IsNegative() const
bool is_nested_catalog_root_
const char * GetChars() const
Definition: shortstring.h:96
bool is_bind_mountpoint_
LinkString symlink_
bool BindParentPathHash(const shash::Md5 &hash)
static void size_t size
Definition: smalloc.h:47
const char kSuffixNone
Definition: hash.h:51
bool g_claim_ownership
Definition: globals.cc:11
const shash::Any * checksum_ptr() const
uint64_t MapGid(const uint64_t gid) const
Definition: catalog.h:223
uid_t uid_