CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
sql_impl.h
Go to the documentation of this file.
1 
5 #ifndef CVMFS_SQL_IMPL_H_
6 #define CVMFS_SQL_IMPL_H_
7 
8 #include <fcntl.h>
9 
10 #include <cassert>
11 #include <cerrno>
12 #include <string>
13 
14 #include "logging.h"
15 #include "platform.h"
16 #include "sqlitemem.h"
17 
18 namespace sqlite {
19 
20 template <class DerivedT>
21 Database<DerivedT>::Database(const std::string &filename,
22  const OpenMode open_mode)
23  : database_(filename, this)
24  , read_write_(kOpenReadWrite == open_mode)
25  , schema_version_(0.0f)
26  , schema_revision_(0) {}
27 
28 
29 template <class DerivedT>
30 DerivedT* Database<DerivedT>::Create(const std::string &filename) {
31  UniquePtr<DerivedT> database(new DerivedT(filename, kOpenReadWrite));
32 
33  if (!database.IsValid()) {
34  LogCvmfs(kLogSql, kLogDebug, "Failed to create new database object");
35  return NULL;
36  }
37 
38  database->set_schema_version(DerivedT::kLatestSchema);
39  database->set_schema_revision(DerivedT::kLatestSchemaRevision);
40 
41  const int open_flags = SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_READWRITE |
42  SQLITE_OPEN_CREATE;
43  if (!database->OpenDatabase(open_flags)) {
44  LogCvmfs(kLogSql, kLogDebug, "Failed to create new database file");
45  return NULL;
46  }
47 
48  if (!database->CreatePropertiesTable()) {
49  database->PrintSqlError("Failed to create common properties table");
50  return NULL;
51  }
52 
53  if (!database->CreateEmptyDatabase()) {
54  database->PrintSqlError("Failed to create empty database");
55  return NULL;
56  }
57 
58  if (!database->PrepareCommonQueries()) {
59  database->PrintSqlError("Failed to initialize properties queries");
60  return NULL;
61  }
62 
63  if (!database->StoreSchemaRevision()) {
64  database->PrintSqlError("Failed to store initial schema revision");
65  return NULL;
66  }
67 
68  return database.Release();
69 }
70 
71 
72 template <class DerivedT>
73 DerivedT* Database<DerivedT>::Open(const std::string &filename,
74  const OpenMode open_mode) {
75  UniquePtr<DerivedT> database(new DerivedT(filename, open_mode));
76 
77  if (!database.IsValid()) {
79  "Failed to open database file '%s' - errno: %d",
80  filename.c_str(), errno);
81  return NULL;
82  }
83 
84  if (!database->Initialize()) {
85  return NULL;
86  }
87 
88  return database.Release();
89 }
90 
91 
92 template <class DerivedT>
94  const int flags = (read_write_) ? SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_READWRITE
95  : SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_READONLY;
96 
97  bool successful = OpenDatabase(flags) &&
98  Configure() &&
99  FileReadAhead() &&
100  PrepareCommonQueries();
101  if (!successful) {
102  LogCvmfs(kLogSql, kLogDebug, "failed to open database file '%s'",
103  filename().c_str());
104  return false;
105  }
106 
107  ReadSchemaRevision();
108  LogCvmfs(kLogSql, kLogDebug, "opened database with schema version %f "
109  "and revision %u",
110  schema_version_, schema_revision_);
111 
112  if (!static_cast<DerivedT*>(this)->CheckSchemaCompatibility()) {
113  LogCvmfs(kLogSql, kLogDebug, "schema version %f not supported (%s)",
114  schema_version_, filename().c_str());
115  return false;
116  }
117 
118  if (read_write_ &&
119  !static_cast<DerivedT*>(this)->LiveSchemaUpgradeIfNecessary()) {
120  LogCvmfs(kLogSql, kLogDebug, "failed tp upgrade schema revision");
121  return false;
122  }
123 
124  return true;
125 }
126 
127 
128 template <class DerivedT>
129 bool Database<DerivedT>::OpenDatabase(const int flags) {
130  // Open database file (depending on the flags read-only or read-write)
131  LogCvmfs(kLogSql, kLogDebug, "opening database file %s",
132  filename().c_str());
133  int retval = sqlite3_open_v2(filename().c_str(),
134  &database_.sqlite_db, flags, NULL);
135  if (retval != SQLITE_OK) {
136  LogCvmfs(kLogSql, kLogDebug, "cannot open database file %s (%d - %d)",
137  filename().c_str(), retval, errno);
138  return false;
139  }
140 
141  retval = sqlite3_extended_result_codes(sqlite_db(), 1);
142  assert(SQLITE_OK == retval);
143 
144  return true;
145 }
146 
147 
148 template <class DerivedT>
150  if (NULL != sqlite_db) {
151  const bool close_successful = Close();
152  assert(close_successful);
153  }
154 }
155 
156 
157 template <class DerivedT>
159  assert(NULL != sqlite_db);
160 
161  LogCvmfs(kLogSql, kLogDebug, "closing SQLite database '%s' (unlink: %s)",
162  filename().c_str(),
163  (db_file_guard.IsEnabled() ? "yes" : "no"));
164  const int result = sqlite3_close(sqlite_db);
165 
166  if (result != SQLITE_OK) {
168  "failed to close SQLite database '%s' (%d - %s)",
169  filename().c_str(), result,
170  delegate_->GetLastErrorMsg().c_str());
171  return false;
172  }
173 
174  sqlite_db = NULL;
175  if (lookaside_buffer != NULL) {
177  lookaside_buffer);
178  lookaside_buffer = NULL;
179  }
180  return true;
181 }
182 
183 
184 template <class DerivedT>
186  // Read-only databases should store temporary files in memory. This avoids
187  // unexpected open read-write file descriptors in the cache directory like
188  // etilqs_<number>. They also use the optimized memory manager, if it is
189  // available.
190  if (!read_write_) {
194  }
195 
196  return Sql(sqlite_db() , "PRAGMA temp_store=2;").Execute() &&
197  Sql(sqlite_db() , "PRAGMA locking_mode=EXCLUSIVE;").Execute();
198  }
199  return true;
200 }
201 
202 
203 template <class DerivedT>
205  // Read-ahead into file system buffers
206  // TODO(jblomer): mmap, re-readahead
207  assert(filename().length() > 1);
208  int fd_readahead;
209  if (filename()[0] != '@') {
210  fd_readahead = open(filename().c_str(), O_RDONLY);
211  if (fd_readahead < 0) {
212  LogCvmfs(kLogSql, kLogDebug, "failed to open %s for read-ahead (%d)",
213  filename().c_str(), errno);
214  return false;
215  }
216  const int retval = platform_readahead(fd_readahead);
217  close(fd_readahead);
218  if (retval != 0) {
220  "failed to read-ahead %s (%d)", filename().c_str(), errno);
221  // Read-ahead is known to fail on tmpfs. Don't consider it as a fatal
222  // error.
223  // return false;
224  }
225  }
226 
227  return true;
228 }
229 
230 
231 template <class DerivedT>
233  sqlite3 *db = sqlite_db();
234  begin_transaction_ = new Sql(db, "BEGIN;");
235  commit_transaction_ = new Sql(db, "COMMIT;");
236  has_property_ = new Sql(db, "SELECT count(*) FROM properties "
237  "WHERE key = :key;");
238  get_property_ = new Sql(db, "SELECT value FROM properties "
239  "WHERE key = :key;");
240  set_property_ = new Sql(db, "INSERT OR REPLACE INTO properties "
241  "(key, value) VALUES (:key, :value);");
244 }
245 
246 
247 template <class DerivedT>
250  ? this->GetProperty<double>(kSchemaVersionKey)
251  : 1.0;
253  ? this->GetProperty<int>(kSchemaRevisionKey)
254  : 0;
255 }
256 
257 
258 template <class DerivedT>
262 }
263 
264 
265 template <class DerivedT>
267  return begin_transaction_->Execute() &&
268  begin_transaction_->Reset();
269 }
270 
271 
272 template <class DerivedT>
274  return commit_transaction_->Execute() &&
275  commit_transaction_->Reset();
276 }
277 
278 
279 template <class DerivedT>
281  return Sql(sqlite_db(),
282  "CREATE TABLE properties (key TEXT, value TEXT, "
283  "CONSTRAINT pk_properties PRIMARY KEY (key));").Execute();
284 }
285 
286 
287 template <class DerivedT>
288 bool Database<DerivedT>::HasProperty(const std::string &key) const {
290  const bool retval = has_property_->BindText(1, key) &&
291  has_property_->FetchRow();
292  assert(retval);
293  const bool result = has_property_->RetrieveInt64(0) > 0;
294  has_property_->Reset();
295  return result;
296 }
297 
298 template <class DerivedT>
299 template <typename T>
300 T Database<DerivedT>::GetProperty(const std::string &key) const {
302  const bool retval = get_property_->BindText(1, key) &&
303  get_property_->FetchRow();
304  assert(retval);
305  const T result = get_property_->Retrieve<T>(0);
306  get_property_->Reset();
307  return result;
308 }
309 
310 template <class DerivedT>
311 template <typename T>
312 T Database<DerivedT>::GetPropertyDefault(const std::string &key,
313  const T default_value) const {
314  return (HasProperty(key)) ? GetProperty<T>(key)
315  : default_value;
316 }
317 
318 template <class DerivedT>
319 template <typename T>
320 bool Database<DerivedT>::SetProperty(const std::string &key,
321  const T value) {
323  return set_property_->BindText(1, key) &&
324  set_property_->Bind(2, value) &&
325  set_property_->Execute() &&
326  set_property_->Reset();
327 }
328 
329 template <class DerivedT>
331  const std::string msg = sqlite3_errmsg(sqlite_db());
332  return msg;
333 }
334 
335 
336 template <class DerivedT>
339  LogCvmfs(kLogSql, kLogDebug, "Database object took ownership of '%s'",
340  database_.filename().c_str());
341 }
342 
343 
344 template <class DerivedT>
347  LogCvmfs(kLogSql, kLogDebug, "Database object dropped ownership of '%s'",
348  database_.filename().c_str());
349 }
350 
351 
352 template <class DerivedT>
354  const int modified_rows = sqlite3_total_changes(sqlite_db());
355  assert(modified_rows >= 0);
356  return static_cast<unsigned>(modified_rows);
357 }
358 
362 template <class DerivedT>
364  const int reset = 0;
365  int current;
366  int highwater;
367  int retval = SQLITE_OK;
368  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_USED,
369  &current, &highwater, reset);
370  stats->lookaside_slots_used = current;
371  stats->lookaside_slots_max = highwater;
372  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_HIT,
373  &current, &highwater, reset);
374  stats->lookaside_hit = highwater;
375  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE,
376  &current, &highwater, reset);
377  stats->lookaside_miss_size = highwater;
378  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL,
379  &current, &highwater, reset);
380  stats->lookaside_miss_full = highwater;
381  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_CACHE_USED,
382  &current, &highwater, reset);
383  stats->page_cache_used = current;
384  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_CACHE_HIT,
385  &current, &highwater, reset);
386  stats->page_cache_hit = current;
387  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_CACHE_MISS,
388  &current, &highwater, reset);
389  stats->page_cache_miss = current;
390  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_SCHEMA_USED,
391  &current, &highwater, reset);
392  stats->schema_used = current;
393  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_STMT_USED,
394  &current, &highwater, reset);
395  stats->stmt_used = current;
396  assert(retval == SQLITE_OK);
397 }
398 
399 
400 template <class DerivedT>
402  Sql free_page_count_query(this->sqlite_db(), "PRAGMA freelist_count;");
403  Sql page_count_query(this->sqlite_db(), "PRAGMA page_count;");
404 
405  const bool retval = page_count_query.FetchRow() &&
406  free_page_count_query.FetchRow();
407  assert(retval);
408 
409  int64_t pages = page_count_query.RetrieveInt64(0);
410  int64_t free_pages = free_page_count_query.RetrieveInt64(0);
411  assert(pages > 0);
412 
413  return (static_cast<double>(free_pages) / static_cast<double>(pages));
414 }
415 
416 
417 template <class DerivedT>
420  return static_cast<const DerivedT*>(this)->CompactDatabase() &&
421  Sql(this->sqlite_db(), "VACUUM;").Execute();
422 }
423 
424 
425 template <class DerivedT>
426 void Database<DerivedT>::PrintSqlError(const std::string &error_msg) {
427  LogCvmfs(kLogSql, kLogStderr, "%s\nSQLite said: '%s'",
428  error_msg.c_str(), this->GetLastErrorMsg().c_str());
429 }
430 
431 template <class DerivedT>
432 const float Database<DerivedT>::kSchemaEpsilon = 0.0005;
433 template <class DerivedT>
434 const std::string Database<DerivedT>::kSchemaVersionKey = "schema";
435 template <class DerivedT>
436 const std::string Database<DerivedT>::kSchemaRevisionKey = "schema_revision";
437 
438 
439 //
440 // # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
441 //
442 
443 
444 template <>
445 inline bool Sql::Bind(const int index, const int value) {
446  return this->BindInt64(index, value);
447 }
448 
449 template <>
450 inline bool Sql::Bind(const int index, const unsigned int value) {
451  return this->BindInt64(index, static_cast<int>(value));
452 }
453 
454 template <>
455 inline bool Sql::Bind(const int index, const uint64_t value) {
456  return this->BindInt64(index, value);
457 }
458 
459 template <>
460 inline bool Sql::Bind(const int index, const sqlite3_int64 value) {
461  return this->BindInt64(index, value);
462 }
463 
464 template <>
465 inline bool Sql::Bind(const int index, const std::string value) {
466  return this->BindTextTransient(index, value);
467 }
468 
469 template <>
470 inline bool Sql::Bind(const int index, const char *value) {
471  return this->BindTextTransient(index, value, strlen(value));
472 }
473 
474 template <>
475 inline bool Sql::Bind(const int index, const float value) {
476  return this->BindDouble(index, value);
477 }
478 
479 template <>
480 inline bool Sql::Bind(const int index, const double value) {
481  return this->BindDouble(index, value);
482 }
483 
484 
485 template <>
486 inline int Sql::Retrieve(const int index) {
487  return this->RetrieveInt64(index);
488 }
489 
490 template <>
491 inline bool Sql::Retrieve(const int index) {
492  return static_cast<bool>(this->RetrieveInt(index));
493 }
494 
495 template <>
496 inline sqlite3_int64 Sql::Retrieve(const int index) {
497  return this->RetrieveInt64(index);
498 }
499 
500 template <>
501 inline uint64_t Sql::Retrieve(const int index) {
502  return static_cast<uint64_t>(this->RetrieveInt64(index));
503 }
504 
505 template <>
506 inline std::string Sql::Retrieve(const int index) {
507  return RetrieveString(index);
508 }
509 
510 template <>
511 inline float Sql::Retrieve(const int index) {
512  return this->RetrieveDouble(index);
513 }
514 
515 template <>
516 inline double Sql::Retrieve(const int index) {
517  return this->RetrieveDouble(index);
518 }
519 
520 } // namespace sqlite
521 
522 #endif // CVMFS_SQL_IMPL_H_
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
void DropFileOwnership()
Definition: sql_impl.h:345
const std::string & filename() const
Definition: sql.h:271
std::string GetLastErrorMsg() const
Definition: sql_impl.h:330
bool Execute()
Definition: sql.cc:42
double GetFreePageRatio() const
Definition: sql_impl.h:401
int page_cache_used
Bytes used for caching pages.
Definition: sql.h:35
bool FetchRow()
Definition: sql.cc:62
const std::string & filename() const
Definition: sql.h:148
bool PrepareCommonQueries()
Definition: sql_impl.h:232
void TakeFileOwnership()
Definition: sql_impl.h:337
int lookaside_miss_size
Definition: sql.h:33
bool Initialize()
Definition: sql_impl.h:93
float schema_version_
Definition: sql.h:291
static const std::string kSchemaVersionKey
Definition: sql.h:285
bool BeginTransaction() const
Definition: sql_impl.h:266
void ReadSchemaRevision()
Definition: sql_impl.h:248
int platform_readahead(int filedes)
int lookaside_slots_used
Definition: sql.h:30
Database(const std::string &filename, const OpenMode open_mode)
Definition: sql_impl.h:21
assert((mem||(size==0))&&"Out Of Memory")
int stmt_used
Bytes used for prepared statmements (lookaside + heap)
Definition: sql.h:39
bool Configure()
Definition: sql_impl.h:185
static DerivedT * Open(const std::string &filename, const OpenMode open_mode)
Definition: sql_impl.h:73
bool CreatePropertiesTable()
Definition: sql_impl.h:280
T GetProperty(const std::string &key) const
Definition: sql_impl.h:300
static bool HasInstance()
Definition: sqlitemem.h:143
T GetPropertyDefault(const std::string &key, const T default_value) const
Definition: sql_impl.h:312
UniquePtr< Sql > set_property_
Definition: sql.h:298
unsigned GetModifiedRowCount() const
Definition: sql_impl.h:353
const bool read_write_
Definition: sql.h:290
bool Vacuum() const
Definition: sql_impl.h:418
sqlite3_int64 RetrieveInt64(const int idx_column) const
Definition: sql.h:440
static SqliteMemoryManager * GetInstance()
Definition: sqlitemem.h:137
T Retrieve(const int index)
UniquePtr< Sql > commit_transaction_
Definition: sql.h:295
UniquePtr< Sql > has_property_
Definition: sql.h:297
sqlite3 * sqlite_db() const
Definition: sql.h:147
bool CommitTransaction() const
Definition: sql_impl.h:273
int page_cache_miss
Definition: sql.h:37
bool SetProperty(const std::string &key, const T value)
Definition: sql_impl.h:320
bool OpenDatabase(const int sqlite_open_flags)
Definition: sql_impl.h:129
bool HasProperty(const std::string &key) const
Definition: sql_impl.h:288
unsigned schema_revision_
Definition: sql.h:292
bool FileReadAhead()
Definition: sql_impl.h:204
bool Bind(const int index, const T value)
int schema_used
Bytes used to store db schema.
Definition: sql.h:38
int lookaside_miss_full
Definition: sql.h:34
int page_cache_hit
Definition: sql.h:36
int lookaside_slots_max
Definition: sql.h:31
void GetMemStatistics(MemStatistics *stats) const
Definition: sql_impl.h:363
void PrintSqlError(const std::string &error_msg)
Definition: sql_impl.h:426
DatabaseRaiiWrapper database_
Definition: sql.h:288
UniquePtr< Sql > get_property_
Definition: sql.h:299
static const std::string kSchemaRevisionKey
Definition: sql.h:286
bool StoreSchemaRevision()
Definition: sql_impl.h:259
UniquePtr< Sql > begin_transaction_
Definition: sql.h:294
void ReleaseLookasideBuffer(void *buffer)
Definition: sqlitemem.cc:317
static DerivedT * Create(const std::string &filename)
Definition: sql_impl.h:30
void * AssignLookasideBuffer(sqlite3 *db)
Definition: sqlitemem.cc:158