CernVM-FS  2.12.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
sql_impl.h
Go to the documentation of this file.
1 
5 #ifndef CVMFS_SQL_IMPL_H_
6 #define CVMFS_SQL_IMPL_H_
7 
8 #include <fcntl.h>
9 
10 #include <cassert>
11 #include <cerrno>
12 #include <string>
13 
14 #include "sqlitemem.h"
15 #include "util/logging.h"
16 #include "util/platform.h"
17 
18 namespace sqlite {
19 
20 template <class DerivedT>
21 Database<DerivedT>::Database(const std::string &filename,
22  const OpenMode open_mode)
23  : database_(filename, this)
24  , read_write_(kOpenReadWrite == open_mode)
25  , schema_version_(0.0f)
26  , schema_revision_(0) {}
27 
28 
29 template <class DerivedT>
30 DerivedT* Database<DerivedT>::Create(const std::string &filename) {
31  UniquePtr<DerivedT> database(new DerivedT(filename, kOpenReadWrite));
32 
33  if (!database.IsValid()) {
34  LogCvmfs(kLogSql, kLogDebug, "Failed to create new database object");
35  return NULL;
36  }
37 
38  database->set_schema_version(DerivedT::kLatestSchema);
39  database->set_schema_revision(DerivedT::kLatestSchemaRevision);
40 
41  const int open_flags = SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_READWRITE |
42  SQLITE_OPEN_CREATE;
43  if (!database->OpenDatabase(open_flags)) {
44  LogCvmfs(kLogSql, kLogDebug, "Failed to create new database file");
45  return NULL;
46  }
47 
48  if (!database->CreatePropertiesTable()) {
49  database->PrintSqlError("Failed to create common properties table");
50  return NULL;
51  }
52 
53  if (!database->CreateEmptyDatabase()) {
54  database->PrintSqlError("Failed to create empty database");
55  return NULL;
56  }
57 
58  if (!database->PrepareCommonQueries()) {
59  database->PrintSqlError("Failed to initialize properties queries");
60  return NULL;
61  }
62 
63  if (!database->StoreSchemaRevision()) {
64  database->PrintSqlError("Failed to store initial schema revision");
65  return NULL;
66  }
67 
68  return database.Release();
69 }
70 
71 
72 template <class DerivedT>
73 DerivedT* Database<DerivedT>::Open(const std::string &filename,
74  const OpenMode open_mode) {
75  UniquePtr<DerivedT> database(new DerivedT(filename, open_mode));
76 
77  if (!database.IsValid()) {
79  "Failed to open database file '%s' - errno: %d",
80  filename.c_str(), errno);
81  return NULL;
82  }
83 
84  if (!database->Initialize()) {
85  return NULL;
86  }
87 
88  return database.Release();
89 }
90 
91 
92 template <class DerivedT>
94  const int flags = (read_write_) ? SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_READWRITE
95  : SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_READONLY;
96 
97  bool successful = OpenDatabase(flags) &&
98  Configure() &&
99  FileReadAhead() &&
100  PrepareCommonQueries();
101  if (!successful) {
102  LogCvmfs(kLogSql, kLogDebug, "failed to open database file '%s'",
103  filename().c_str());
104  return false;
105  }
106 
107  ReadSchemaRevision();
108  LogCvmfs(kLogSql, kLogDebug, "opened database with schema version %f "
109  "and revision %u",
110  schema_version_, schema_revision_);
111 
112  if (!static_cast<DerivedT*>(this)->CheckSchemaCompatibility()) {
113  LogCvmfs(kLogSql, kLogDebug, "schema version %f not supported (%s)",
114  schema_version_, filename().c_str());
115  return false;
116  }
117 
118  if (read_write_ &&
119  !static_cast<DerivedT*>(this)->LiveSchemaUpgradeIfNecessary()) {
120  LogCvmfs(kLogSql, kLogDebug, "failed tp upgrade schema revision");
121  return false;
122  }
123 
124  return true;
125 }
126 
127 
128 template <class DerivedT>
129 bool Database<DerivedT>::OpenDatabase(const int flags) {
130  // Open database file (depending on the flags read-only or read-write)
131  LogCvmfs(kLogSql, kLogDebug, "opening database file %s",
132  filename().c_str());
133  int retval = sqlite3_open_v2(filename().c_str(),
134  &database_.sqlite_db,
135  flags | SQLITE_OPEN_EXRESCODE,
136  NULL);
137  if (retval != SQLITE_OK) {
138  LogCvmfs(kLogSql, kLogDebug, "cannot open database file %s (%d - %d)",
139  filename().c_str(), retval, errno);
140  return false;
141  }
142 
143  return true;
144 }
145 
146 
147 template <class DerivedT>
149  if (NULL != sqlite_db) {
150  const bool close_successful = Close();
151  assert(close_successful);
152  }
153 }
154 
155 
156 template <class DerivedT>
158  assert(NULL != sqlite_db);
159 
160  LogCvmfs(kLogSql, kLogDebug, "closing SQLite database '%s' (unlink: %s)",
161  filename().c_str(),
162  (db_file_guard.IsEnabled() ? "yes" : "no"));
163  const int result = sqlite3_close(sqlite_db);
164 
165  if (result != SQLITE_OK) {
167  "failed to close SQLite database '%s' (%d - %s)",
168  filename().c_str(), result,
169  delegate_->GetLastErrorMsg().c_str());
170  return false;
171  }
172 
173  sqlite_db = NULL;
174  if (lookaside_buffer != NULL) {
176  lookaside_buffer);
177  lookaside_buffer = NULL;
178  }
179  return true;
180 }
181 
182 
183 template <class DerivedT>
185  // Read-only databases should store temporary files in memory. This avoids
186  // unexpected open read-write file descriptors in the cache directory like
187  // etilqs_<number>. They also use the optimized memory manager, if it is
188  // available.
189  if (!read_write_) {
193  }
194 
195  return Sql(sqlite_db() , "PRAGMA temp_store=2;").Execute() &&
196  Sql(sqlite_db() , "PRAGMA locking_mode=EXCLUSIVE;").Execute();
197  }
198  return true;
199 }
200 
201 
202 template <class DerivedT>
204  // Read-ahead into file system buffers
205  // TODO(jblomer): mmap, re-readahead
206  assert(filename().length() > 1);
207  int fd_readahead;
208  if (filename()[0] != '@') {
209  fd_readahead = open(filename().c_str(), O_RDONLY);
210  if (fd_readahead < 0) {
211  LogCvmfs(kLogSql, kLogDebug, "failed to open %s for read-ahead (%d)",
212  filename().c_str(), errno);
213  return false;
214  }
215  const ssize_t retval = platform_readahead(fd_readahead);
216  close(fd_readahead);
217 
218  // Read-ahead is known to fail on tmpfs with EINVAL
219  // EINVAL = "readahead() cannot be applied to that a file type"
220  // Don't consider it a fatal error.
221  if (retval != 0 && errno != EINVAL) {
223  "failed to read-ahead %s: invalid file descrp. or not open for reading",
224  filename().c_str());
225  return false;
226  }
227  }
228 
229  return true;
230 }
231 
232 
233 template <class DerivedT>
235  sqlite3 *db = sqlite_db();
236  begin_transaction_ = new Sql(db, "BEGIN;");
237  commit_transaction_ = new Sql(db, "COMMIT;");
238  has_property_ = new Sql(db, "SELECT count(*) FROM properties "
239  "WHERE key = :key;");
240  get_property_ = new Sql(db, "SELECT value FROM properties "
241  "WHERE key = :key;");
242  set_property_ = new Sql(db, "INSERT OR REPLACE INTO properties "
243  "(key, value) VALUES (:key, :value);");
244  return (begin_transaction_.IsValid() &&
245  commit_transaction_.IsValid() &&
246  has_property_.IsValid() &&
247  get_property_.IsValid() &&
248  set_property_.IsValid());
249 }
250 
251 
252 template <class DerivedT>
255  ? this->GetProperty<double>(kSchemaVersionKey)
256  : 1.0;
258  ? this->GetProperty<int>(kSchemaRevisionKey)
259  : 0;
260 }
261 
262 
263 template <class DerivedT>
267 }
268 
269 
270 template <class DerivedT>
272  return begin_transaction_->Execute() &&
273  begin_transaction_->Reset();
274 }
275 
276 
277 template <class DerivedT>
279  return commit_transaction_->Execute() &&
280  commit_transaction_->Reset();
281 }
282 
283 
284 template <class DerivedT>
286  return Sql(sqlite_db(),
287  "CREATE TABLE properties (key TEXT, value TEXT, "
288  "CONSTRAINT pk_properties PRIMARY KEY (key));").Execute();
289 }
290 
291 
292 template <class DerivedT>
293 bool Database<DerivedT>::HasProperty(const std::string &key) const {
294  assert(has_property_.IsValid());
295  const bool retval = has_property_->BindText(1, key) &&
296  has_property_->FetchRow();
297  assert(retval);
298  const bool result = has_property_->RetrieveInt64(0) > 0;
299  has_property_->Reset();
300  return result;
301 }
302 
303 template <class DerivedT>
304 template <typename T>
305 T Database<DerivedT>::GetProperty(const std::string &key) const {
306  assert(get_property_.IsValid());
307  const bool retval = get_property_->BindText(1, key) &&
308  get_property_->FetchRow();
309  assert(retval);
310  const T result = get_property_->Retrieve<T>(0);
311  get_property_->Reset();
312  return result;
313 }
314 
315 template <class DerivedT>
316 template <typename T>
317 T Database<DerivedT>::GetPropertyDefault(const std::string &key,
318  const T default_value) const {
319  return (HasProperty(key)) ? GetProperty<T>(key)
320  : default_value;
321 }
322 
323 template <class DerivedT>
324 template <typename T>
325 bool Database<DerivedT>::SetProperty(const std::string &key,
326  const T value) {
327  assert(set_property_.IsValid());
328  return set_property_->BindText(1, key) &&
329  set_property_->Bind(2, value) &&
330  set_property_->Execute() &&
331  set_property_->Reset();
332 }
333 
334 template <class DerivedT>
336  std::string msg = sqlite3_errmsg(sqlite_db());
337  return msg;
338 }
339 
340 
341 template <class DerivedT>
344  LogCvmfs(kLogSql, kLogDebug, "Database object took ownership of '%s'",
345  database_.filename().c_str());
346 }
347 
348 
349 template <class DerivedT>
352  LogCvmfs(kLogSql, kLogDebug, "Database object dropped ownership of '%s'",
353  database_.filename().c_str());
354 }
355 
356 
357 template <class DerivedT>
359  const int modified_rows = sqlite3_total_changes(sqlite_db());
360  assert(modified_rows >= 0);
361  return static_cast<unsigned>(modified_rows);
362 }
363 
367 template <class DerivedT>
369  const int reset = 0;
370  int current;
371  int highwater;
372  int retval = SQLITE_OK;
373  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_USED,
374  &current, &highwater, reset);
375  stats->lookaside_slots_used = current;
376  stats->lookaside_slots_max = highwater;
377  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_HIT,
378  &current, &highwater, reset);
379  stats->lookaside_hit = highwater;
380  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE,
381  &current, &highwater, reset);
382  stats->lookaside_miss_size = highwater;
383  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL,
384  &current, &highwater, reset);
385  stats->lookaside_miss_full = highwater;
386  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_CACHE_USED,
387  &current, &highwater, reset);
388  stats->page_cache_used = current;
389  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_CACHE_HIT,
390  &current, &highwater, reset);
391  stats->page_cache_hit = current;
392  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_CACHE_MISS,
393  &current, &highwater, reset);
394  stats->page_cache_miss = current;
395  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_SCHEMA_USED,
396  &current, &highwater, reset);
397  stats->schema_used = current;
398  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_STMT_USED,
399  &current, &highwater, reset);
400  stats->stmt_used = current;
401  assert(retval == SQLITE_OK);
402 }
403 
404 
405 template <class DerivedT>
407  Sql free_page_count_query(this->sqlite_db(), "PRAGMA freelist_count;");
408  Sql page_count_query(this->sqlite_db(), "PRAGMA page_count;");
409 
410  const bool retval = page_count_query.FetchRow() &&
411  free_page_count_query.FetchRow();
412  assert(retval);
413 
414  int64_t pages = page_count_query.RetrieveInt64(0);
415  int64_t free_pages = free_page_count_query.RetrieveInt64(0);
416  assert(pages > 0);
417 
418  return (static_cast<double>(free_pages) / static_cast<double>(pages));
419 }
420 
421 
422 template <class DerivedT>
425  return static_cast<const DerivedT*>(this)->CompactDatabase() &&
426  Sql(this->sqlite_db(), "VACUUM;").Execute();
427 }
428 
429 
430 template <class DerivedT>
431 void Database<DerivedT>::PrintSqlError(const std::string &error_msg) {
432  LogCvmfs(kLogSql, kLogStderr, "%s\nSQLite said: '%s'",
433  error_msg.c_str(), this->GetLastErrorMsg().c_str());
434 }
435 
436 template <class DerivedT>
437 const float Database<DerivedT>::kSchemaEpsilon = 0.0005;
438 template <class DerivedT>
439 const char *Database<DerivedT>::kSchemaVersionKey = "schema";
440 template <class DerivedT>
441 const char *Database<DerivedT>::kSchemaRevisionKey = "schema_revision";
442 
443 
444 //
445 // # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
446 //
447 
448 
449 template <>
450 inline bool Sql::Bind(const int index, const int &value) {
451  return this->BindInt64(index, value);
452 }
453 
454 template <>
455 inline bool Sql::Bind(const int index, const unsigned int &value) {
456  return this->BindInt64(index, static_cast<int>(value));
457 }
458 
459 template <>
460 inline bool Sql::Bind(const int index, const uint64_t &value) {
461  return this->BindInt64(index, static_cast<int64_t>(value));
462 }
463 
464 template <>
465 inline bool Sql::Bind(const int index, const sqlite3_int64 &value) {
466  return this->BindInt64(index, value);
467 }
468 
469 template <>
470 inline bool Sql::Bind(const int index, const std::string &value) {
471  return this->BindTextTransient(index, value);
472 }
473 
474 template <>
475 inline bool Sql::Bind(const int index, const float &value) {
476  return this->BindDouble(index, value);
477 }
478 
479 template <>
480 inline bool Sql::Bind(const int index, const double &value) {
481  return this->BindDouble(index, value);
482 }
483 
484 
485 template <>
486 inline int Sql::Retrieve(const int index) {
487  return static_cast<int>(this->RetrieveInt64(index));
488 }
489 
490 template <>
491 inline bool Sql::Retrieve(const int index) {
492  return static_cast<bool>(this->RetrieveInt(index));
493 }
494 
495 template <>
496 inline sqlite3_int64 Sql::Retrieve(const int index) {
497  return this->RetrieveInt64(index);
498 }
499 
500 template <>
501 inline uint64_t Sql::Retrieve(const int index) {
502  return static_cast<uint64_t>(this->RetrieveInt64(index));
503 }
504 
505 template <>
506 inline std::string Sql::Retrieve(const int index) {
507  return RetrieveString(index);
508 }
509 
510 template <>
511 inline float Sql::Retrieve(const int index) {
512  return static_cast<float>(this->RetrieveDouble(index));
513 }
514 
515 template <>
516 inline double Sql::Retrieve(const int index) {
517  return this->RetrieveDouble(index);
518 }
519 
520 } // namespace sqlite
521 
522 #endif // CVMFS_SQL_IMPL_H_
bool Bind(const int index, const T &value)
void DropFileOwnership()
Definition: sql_impl.h:350
const std::string & filename() const
Definition: sql.h:271
std::string GetLastErrorMsg() const
Definition: sql_impl.h:335
bool Execute()
Definition: sql.cc:42
double GetFreePageRatio() const
Definition: sql_impl.h:406
int page_cache_used
Bytes used for caching pages.
Definition: sql.h:35
bool FetchRow()
Definition: sql.cc:62
const std::string & filename() const
Definition: sql.h:148
bool PrepareCommonQueries()
Definition: sql_impl.h:234
void TakeFileOwnership()
Definition: sql_impl.h:342
int lookaside_miss_size
Definition: sql.h:33
bool Initialize()
Definition: sql_impl.h:93
float schema_version_
Definition: sql.h:291
bool BeginTransaction() const
Definition: sql_impl.h:271
void ReadSchemaRevision()
Definition: sql_impl.h:253
int lookaside_slots_used
Definition: sql.h:30
Database(const std::string &filename, const OpenMode open_mode)
Definition: sql_impl.h:21
assert((mem||(size==0))&&"Out Of Memory")
int stmt_used
Bytes used for prepared statmements (lookaside + heap)
Definition: sql.h:39
bool Configure()
Definition: sql_impl.h:184
static DerivedT * Open(const std::string &filename, const OpenMode open_mode)
Definition: sql_impl.h:73
bool CreatePropertiesTable()
Definition: sql_impl.h:285
T GetProperty(const std::string &key) const
Definition: sql_impl.h:305
static bool HasInstance()
Definition: sqlitemem.h:143
T GetPropertyDefault(const std::string &key, const T default_value) const
Definition: sql_impl.h:317
UniquePtr< Sql > set_property_
Definition: sql.h:298
unsigned GetModifiedRowCount() const
Definition: sql_impl.h:358
const bool read_write_
Definition: sql.h:290
ssize_t platform_readahead(int filedes)
bool Vacuum() const
Definition: sql_impl.h:423
sqlite3_int64 RetrieveInt64(const int idx_column) const
Definition: sql.h:445
static SqliteMemoryManager * GetInstance()
Definition: sqlitemem.h:137
T Retrieve(const int index)
UniquePtr< Sql > commit_transaction_
Definition: sql.h:295
UniquePtr< Sql > has_property_
Definition: sql.h:297
sqlite3 * sqlite_db() const
Definition: sql.h:147
bool CommitTransaction() const
Definition: sql_impl.h:278
int page_cache_miss
Definition: sql.h:37
bool SetProperty(const std::string &key, const T value)
Definition: sql_impl.h:325
bool OpenDatabase(const int sqlite_open_flags)
Definition: sql_impl.h:129
bool HasProperty(const std::string &key) const
Definition: sql_impl.h:293
static const char * kSchemaRevisionKey
Definition: sql.h:286
unsigned schema_revision_
Definition: sql.h:292
bool FileReadAhead()
Definition: sql_impl.h:203
int schema_used
Bytes used to store db schema.
Definition: sql.h:38
int lookaside_miss_full
Definition: sql.h:34
int page_cache_hit
Definition: sql.h:36
int lookaside_slots_max
Definition: sql.h:31
void GetMemStatistics(MemStatistics *stats) const
Definition: sql_impl.h:368
void PrintSqlError(const std::string &error_msg)
Definition: sql_impl.h:431
DatabaseRaiiWrapper database_
Definition: sql.h:288
static const char * kSchemaVersionKey
Definition: sql.h:285
UniquePtr< Sql > get_property_
Definition: sql.h:299
bool StoreSchemaRevision()
Definition: sql_impl.h:264
UniquePtr< Sql > begin_transaction_
Definition: sql.h:294
void ReleaseLookasideBuffer(void *buffer)
Definition: sqlitemem.cc:317
static DerivedT * Create(const std::string &filename)
Definition: sql_impl.h:30
void * AssignLookasideBuffer(sqlite3 *db)
Definition: sqlitemem.cc:158
CVMFS_EXPORT void LogCvmfs(const LogSource source, const int mask, const char *format,...)
Definition: logging.cc:528