CernVM-FS  2.10.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
sql_impl.h
Go to the documentation of this file.
1 
5 #ifndef CVMFS_SQL_IMPL_H_
6 #define CVMFS_SQL_IMPL_H_
7 
8 #include <fcntl.h>
9 
10 #include <cassert>
11 #include <cerrno>
12 #include <string>
13 
14 #include "logging.h"
15 #include "platform.h"
16 #include "sqlitemem.h"
17 
18 namespace sqlite {
19 
20 template <class DerivedT>
21 Database<DerivedT>::Database(const std::string &filename,
22  const OpenMode open_mode)
23  : database_(filename, this)
24  , read_write_(kOpenReadWrite == open_mode)
25  , schema_version_(0.0f)
26  , schema_revision_(0) {}
27 
28 
29 template <class DerivedT>
30 DerivedT* Database<DerivedT>::Create(const std::string &filename) {
31  UniquePtr<DerivedT> database(new DerivedT(filename, kOpenReadWrite));
32 
33  if (!database.IsValid()) {
34  LogCvmfs(kLogSql, kLogDebug, "Failed to create new database object");
35  return NULL;
36  }
37 
38  database->set_schema_version(DerivedT::kLatestSchema);
39  database->set_schema_revision(DerivedT::kLatestSchemaRevision);
40 
41  const int open_flags = SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_READWRITE |
42  SQLITE_OPEN_CREATE;
43  if (!database->OpenDatabase(open_flags)) {
44  LogCvmfs(kLogSql, kLogDebug, "Failed to create new database file");
45  return NULL;
46  }
47 
48  if (!database->CreatePropertiesTable()) {
49  database->PrintSqlError("Failed to create common properties table");
50  return NULL;
51  }
52 
53  if (!database->CreateEmptyDatabase()) {
54  database->PrintSqlError("Failed to create empty database");
55  return NULL;
56  }
57 
58  if (!database->PrepareCommonQueries()) {
59  database->PrintSqlError("Failed to initialize properties queries");
60  return NULL;
61  }
62 
63  if (!database->StoreSchemaRevision()) {
64  database->PrintSqlError("Failed to store initial schema revision");
65  return NULL;
66  }
67 
68  return database.Release();
69 }
70 
71 
72 template <class DerivedT>
73 DerivedT* Database<DerivedT>::Open(const std::string &filename,
74  const OpenMode open_mode) {
75  UniquePtr<DerivedT> database(new DerivedT(filename, open_mode));
76 
77  if (!database.IsValid()) {
79  "Failed to open database file '%s' - errno: %d",
80  filename.c_str(), errno);
81  return NULL;
82  }
83 
84  if (!database->Initialize()) {
85  return NULL;
86  }
87 
88  return database.Release();
89 }
90 
91 
92 template <class DerivedT>
94  const int flags = (read_write_) ? SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_READWRITE
95  : SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_READONLY;
96 
97  bool successful = OpenDatabase(flags) &&
98  Configure() &&
99  FileReadAhead() &&
100  PrepareCommonQueries();
101  if (!successful) {
102  LogCvmfs(kLogSql, kLogDebug, "failed to open database file '%s'",
103  filename().c_str());
104  return false;
105  }
106 
107  ReadSchemaRevision();
108  LogCvmfs(kLogSql, kLogDebug, "opened database with schema version %f "
109  "and revision %u",
110  schema_version_, schema_revision_);
111 
112  if (!static_cast<DerivedT*>(this)->CheckSchemaCompatibility()) {
113  LogCvmfs(kLogSql, kLogDebug, "schema version %f not supported (%s)",
114  schema_version_, filename().c_str());
115  return false;
116  }
117 
118  if (read_write_ &&
119  !static_cast<DerivedT*>(this)->LiveSchemaUpgradeIfNecessary()) {
120  LogCvmfs(kLogSql, kLogDebug, "failed tp upgrade schema revision");
121  return false;
122  }
123 
124  return true;
125 }
126 
127 
128 template <class DerivedT>
129 bool Database<DerivedT>::OpenDatabase(const int flags) {
130  // Open database file (depending on the flags read-only or read-write)
131  LogCvmfs(kLogSql, kLogDebug, "opening database file %s",
132  filename().c_str());
133  int retval = sqlite3_open_v2(filename().c_str(),
134  &database_.sqlite_db, flags, NULL);
135  if (retval != SQLITE_OK) {
136  LogCvmfs(kLogSql, kLogDebug, "cannot open database file %s (%d - %d)",
137  filename().c_str(), retval, errno);
138  return false;
139  }
140 
141  retval = sqlite3_extended_result_codes(sqlite_db(), 1);
142  assert(SQLITE_OK == retval);
143 
144  return true;
145 }
146 
147 
148 template <class DerivedT>
150  if (NULL != sqlite_db) {
151  const bool close_successful = Close();
152  assert(close_successful);
153  }
154 }
155 
156 
157 template <class DerivedT>
159  assert(NULL != sqlite_db);
160 
161  LogCvmfs(kLogSql, kLogDebug, "closing SQLite database '%s' (unlink: %s)",
162  filename().c_str(),
163  (db_file_guard.IsEnabled() ? "yes" : "no"));
164  const int result = sqlite3_close(sqlite_db);
165 
166  if (result != SQLITE_OK) {
168  "failed to close SQLite database '%s' (%d - %s)",
169  filename().c_str(), result,
170  delegate_->GetLastErrorMsg().c_str());
171  return false;
172  }
173 
174  sqlite_db = NULL;
175  if (lookaside_buffer != NULL) {
177  lookaside_buffer);
178  lookaside_buffer = NULL;
179  }
180  return true;
181 }
182 
183 
184 template <class DerivedT>
186  // Read-only databases should store temporary files in memory. This avoids
187  // unexpected open read-write file descriptors in the cache directory like
188  // etilqs_<number>. They also use the optimized memory manager, if it is
189  // available.
190  if (!read_write_) {
194  }
195 
196  return Sql(sqlite_db() , "PRAGMA temp_store=2;").Execute() &&
197  Sql(sqlite_db() , "PRAGMA locking_mode=EXCLUSIVE;").Execute();
198  }
199  return true;
200 }
201 
202 
203 template <class DerivedT>
205  // Read-ahead into file system buffers
206  // TODO(jblomer): mmap, re-readahead
207  assert(filename().length() > 1);
208  int fd_readahead;
209  if (filename()[0] != '@') {
210  fd_readahead = open(filename().c_str(), O_RDONLY);
211  if (fd_readahead < 0) {
212  LogCvmfs(kLogSql, kLogDebug, "failed to open %s for read-ahead (%d)",
213  filename().c_str(), errno);
214  return false;
215  }
216  const ssize_t retval = platform_readahead(fd_readahead);
217  close(fd_readahead);
218  if (retval != 0) {
220  "failed to read-ahead %s (%d)", filename().c_str(), errno);
221  // Read-ahead is known to fail on tmpfs. Don't consider it as a fatal
222  // error.
223  // return false;
224  }
225  }
226 
227  return true;
228 }
229 
230 
231 template <class DerivedT>
233  sqlite3 *db = sqlite_db();
234  begin_transaction_ = new Sql(db, "BEGIN;");
235  commit_transaction_ = new Sql(db, "COMMIT;");
236  has_property_ = new Sql(db, "SELECT count(*) FROM properties "
237  "WHERE key = :key;");
238  get_property_ = new Sql(db, "SELECT value FROM properties "
239  "WHERE key = :key;");
240  set_property_ = new Sql(db, "INSERT OR REPLACE INTO properties "
241  "(key, value) VALUES (:key, :value);");
242  return (begin_transaction_.IsValid() &&
243  commit_transaction_.IsValid() &&
244  has_property_.IsValid() &&
245  get_property_.IsValid() &&
246  set_property_.IsValid());
247 }
248 
249 
250 template <class DerivedT>
253  ? this->GetProperty<double>(kSchemaVersionKey)
254  : 1.0;
256  ? this->GetProperty<int>(kSchemaRevisionKey)
257  : 0;
258 }
259 
260 
261 template <class DerivedT>
265 }
266 
267 
268 template <class DerivedT>
270  return begin_transaction_->Execute() &&
271  begin_transaction_->Reset();
272 }
273 
274 
275 template <class DerivedT>
277  return commit_transaction_->Execute() &&
278  commit_transaction_->Reset();
279 }
280 
281 
282 template <class DerivedT>
284  return Sql(sqlite_db(),
285  "CREATE TABLE properties (key TEXT, value TEXT, "
286  "CONSTRAINT pk_properties PRIMARY KEY (key));").Execute();
287 }
288 
289 
290 template <class DerivedT>
291 bool Database<DerivedT>::HasProperty(const std::string &key) const {
292  assert(has_property_.IsValid());
293  const bool retval = has_property_->BindText(1, key) &&
294  has_property_->FetchRow();
295  assert(retval);
296  const bool result = has_property_->RetrieveInt64(0) > 0;
297  has_property_->Reset();
298  return result;
299 }
300 
301 template <class DerivedT>
302 template <typename T>
303 T Database<DerivedT>::GetProperty(const std::string &key) const {
304  assert(get_property_.IsValid());
305  const bool retval = get_property_->BindText(1, key) &&
306  get_property_->FetchRow();
307  assert(retval);
308  const T result = get_property_->Retrieve<T>(0);
309  get_property_->Reset();
310  return result;
311 }
312 
313 template <class DerivedT>
314 template <typename T>
315 T Database<DerivedT>::GetPropertyDefault(const std::string &key,
316  const T default_value) const {
317  return (HasProperty(key)) ? GetProperty<T>(key)
318  : default_value;
319 }
320 
321 template <class DerivedT>
322 template <typename T>
323 bool Database<DerivedT>::SetProperty(const std::string &key,
324  const T value) {
325  assert(set_property_.IsValid());
326  return set_property_->BindText(1, key) &&
327  set_property_->Bind(2, value) &&
328  set_property_->Execute() &&
329  set_property_->Reset();
330 }
331 
332 template <class DerivedT>
334  std::string msg = sqlite3_errmsg(sqlite_db());
335  return msg;
336 }
337 
338 
339 template <class DerivedT>
342  LogCvmfs(kLogSql, kLogDebug, "Database object took ownership of '%s'",
343  database_.filename().c_str());
344 }
345 
346 
347 template <class DerivedT>
350  LogCvmfs(kLogSql, kLogDebug, "Database object dropped ownership of '%s'",
351  database_.filename().c_str());
352 }
353 
354 
355 template <class DerivedT>
357  const int modified_rows = sqlite3_total_changes(sqlite_db());
358  assert(modified_rows >= 0);
359  return static_cast<unsigned>(modified_rows);
360 }
361 
365 template <class DerivedT>
367  const int reset = 0;
368  int current;
369  int highwater;
370  int retval = SQLITE_OK;
371  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_USED,
372  &current, &highwater, reset);
373  stats->lookaside_slots_used = current;
374  stats->lookaside_slots_max = highwater;
375  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_HIT,
376  &current, &highwater, reset);
377  stats->lookaside_hit = highwater;
378  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE,
379  &current, &highwater, reset);
380  stats->lookaside_miss_size = highwater;
381  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL,
382  &current, &highwater, reset);
383  stats->lookaside_miss_full = highwater;
384  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_CACHE_USED,
385  &current, &highwater, reset);
386  stats->page_cache_used = current;
387  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_CACHE_HIT,
388  &current, &highwater, reset);
389  stats->page_cache_hit = current;
390  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_CACHE_MISS,
391  &current, &highwater, reset);
392  stats->page_cache_miss = current;
393  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_SCHEMA_USED,
394  &current, &highwater, reset);
395  stats->schema_used = current;
396  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_STMT_USED,
397  &current, &highwater, reset);
398  stats->stmt_used = current;
399  assert(retval == SQLITE_OK);
400 }
401 
402 
403 template <class DerivedT>
405  Sql free_page_count_query(this->sqlite_db(), "PRAGMA freelist_count;");
406  Sql page_count_query(this->sqlite_db(), "PRAGMA page_count;");
407 
408  const bool retval = page_count_query.FetchRow() &&
409  free_page_count_query.FetchRow();
410  assert(retval);
411 
412  int64_t pages = page_count_query.RetrieveInt64(0);
413  int64_t free_pages = free_page_count_query.RetrieveInt64(0);
414  assert(pages > 0);
415 
416  return (static_cast<double>(free_pages) / static_cast<double>(pages));
417 }
418 
419 
420 template <class DerivedT>
423  return static_cast<const DerivedT*>(this)->CompactDatabase() &&
424  Sql(this->sqlite_db(), "VACUUM;").Execute();
425 }
426 
427 
428 template <class DerivedT>
429 void Database<DerivedT>::PrintSqlError(const std::string &error_msg) {
430  LogCvmfs(kLogSql, kLogStderr, "%s\nSQLite said: '%s'",
431  error_msg.c_str(), this->GetLastErrorMsg().c_str());
432 }
433 
434 template <class DerivedT>
435 const float Database<DerivedT>::kSchemaEpsilon = 0.0005;
436 template <class DerivedT>
437 const char *Database<DerivedT>::kSchemaVersionKey = "schema";
438 template <class DerivedT>
439 const char *Database<DerivedT>::kSchemaRevisionKey = "schema_revision";
440 
441 
442 //
443 // # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
444 //
445 
446 
447 template <>
448 inline bool Sql::Bind(const int index, const int &value) {
449  return this->BindInt64(index, value);
450 }
451 
452 template <>
453 inline bool Sql::Bind(const int index, const unsigned int &value) {
454  return this->BindInt64(index, static_cast<int>(value));
455 }
456 
457 template <>
458 inline bool Sql::Bind(const int index, const uint64_t &value) {
459  return this->BindInt64(index, static_cast<int64_t>(value));
460 }
461 
462 template <>
463 inline bool Sql::Bind(const int index, const sqlite3_int64 &value) {
464  return this->BindInt64(index, value);
465 }
466 
467 template <>
468 inline bool Sql::Bind(const int index, const std::string &value) {
469  return this->BindTextTransient(index, value);
470 }
471 
472 template <>
473 inline bool Sql::Bind(const int index, const float &value) {
474  return this->BindDouble(index, value);
475 }
476 
477 template <>
478 inline bool Sql::Bind(const int index, const double &value) {
479  return this->BindDouble(index, value);
480 }
481 
482 
483 template <>
484 inline int Sql::Retrieve(const int index) {
485  return static_cast<int>(this->RetrieveInt64(index));
486 }
487 
488 template <>
489 inline bool Sql::Retrieve(const int index) {
490  return static_cast<bool>(this->RetrieveInt(index));
491 }
492 
493 template <>
494 inline sqlite3_int64 Sql::Retrieve(const int index) {
495  return this->RetrieveInt64(index);
496 }
497 
498 template <>
499 inline uint64_t Sql::Retrieve(const int index) {
500  return static_cast<uint64_t>(this->RetrieveInt64(index));
501 }
502 
503 template <>
504 inline std::string Sql::Retrieve(const int index) {
505  return RetrieveString(index);
506 }
507 
508 template <>
509 inline float Sql::Retrieve(const int index) {
510  return static_cast<float>(this->RetrieveDouble(index));
511 }
512 
513 template <>
514 inline double Sql::Retrieve(const int index) {
515  return this->RetrieveDouble(index);
516 }
517 
518 } // namespace sqlite
519 
520 #endif // CVMFS_SQL_IMPL_H_
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
bool Bind(const int index, const T &value)
void DropFileOwnership()
Definition: sql_impl.h:348
const std::string & filename() const
Definition: sql.h:271
std::string GetLastErrorMsg() const
Definition: sql_impl.h:333
bool Execute()
Definition: sql.cc:42
double GetFreePageRatio() const
Definition: sql_impl.h:404
int page_cache_used
Bytes used for caching pages.
Definition: sql.h:35
bool FetchRow()
Definition: sql.cc:62
const std::string & filename() const
Definition: sql.h:148
bool PrepareCommonQueries()
Definition: sql_impl.h:232
void TakeFileOwnership()
Definition: sql_impl.h:340
int lookaside_miss_size
Definition: sql.h:33
bool Initialize()
Definition: sql_impl.h:93
float schema_version_
Definition: sql.h:291
bool BeginTransaction() const
Definition: sql_impl.h:269
void ReadSchemaRevision()
Definition: sql_impl.h:251
int lookaside_slots_used
Definition: sql.h:30
Database(const std::string &filename, const OpenMode open_mode)
Definition: sql_impl.h:21
assert((mem||(size==0))&&"Out Of Memory")
int stmt_used
Bytes used for prepared statmements (lookaside + heap)
Definition: sql.h:39
bool Configure()
Definition: sql_impl.h:185
static DerivedT * Open(const std::string &filename, const OpenMode open_mode)
Definition: sql_impl.h:73
bool CreatePropertiesTable()
Definition: sql_impl.h:283
T GetProperty(const std::string &key) const
Definition: sql_impl.h:303
static bool HasInstance()
Definition: sqlitemem.h:143
T GetPropertyDefault(const std::string &key, const T default_value) const
Definition: sql_impl.h:315
UniquePtr< Sql > set_property_
Definition: sql.h:298
unsigned GetModifiedRowCount() const
Definition: sql_impl.h:356
const bool read_write_
Definition: sql.h:290
ssize_t platform_readahead(int filedes)
bool Vacuum() const
Definition: sql_impl.h:421
sqlite3_int64 RetrieveInt64(const int idx_column) const
Definition: sql.h:445
static SqliteMemoryManager * GetInstance()
Definition: sqlitemem.h:137
T Retrieve(const int index)
UniquePtr< Sql > commit_transaction_
Definition: sql.h:295
UniquePtr< Sql > has_property_
Definition: sql.h:297
sqlite3 * sqlite_db() const
Definition: sql.h:147
bool CommitTransaction() const
Definition: sql_impl.h:276
int page_cache_miss
Definition: sql.h:37
bool SetProperty(const std::string &key, const T value)
Definition: sql_impl.h:323
bool OpenDatabase(const int sqlite_open_flags)
Definition: sql_impl.h:129
bool HasProperty(const std::string &key) const
Definition: sql_impl.h:291
static const char * kSchemaRevisionKey
Definition: sql.h:286
unsigned schema_revision_
Definition: sql.h:292
bool FileReadAhead()
Definition: sql_impl.h:204
int schema_used
Bytes used to store db schema.
Definition: sql.h:38
int lookaside_miss_full
Definition: sql.h:34
int page_cache_hit
Definition: sql.h:36
int lookaside_slots_max
Definition: sql.h:31
void GetMemStatistics(MemStatistics *stats) const
Definition: sql_impl.h:366
void PrintSqlError(const std::string &error_msg)
Definition: sql_impl.h:429
DatabaseRaiiWrapper database_
Definition: sql.h:288
static const char * kSchemaVersionKey
Definition: sql.h:285
UniquePtr< Sql > get_property_
Definition: sql.h:299
bool StoreSchemaRevision()
Definition: sql_impl.h:262
UniquePtr< Sql > begin_transaction_
Definition: sql.h:294
void ReleaseLookasideBuffer(void *buffer)
Definition: sqlitemem.cc:317
static DerivedT * Create(const std::string &filename)
Definition: sql_impl.h:30
void * AssignLookasideBuffer(sqlite3 *db)
Definition: sqlitemem.cc:158