CernVM-FS  2.11.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
sql_impl.h
Go to the documentation of this file.
1 
5 #ifndef CVMFS_SQL_IMPL_H_
6 #define CVMFS_SQL_IMPL_H_
7 
8 #include <fcntl.h>
9 
10 #include <cassert>
11 #include <cerrno>
12 #include <string>
13 
14 #include "sqlitemem.h"
15 #include "util/logging.h"
16 #include "util/platform.h"
17 
18 namespace sqlite {
19 
20 template <class DerivedT>
21 Database<DerivedT>::Database(const std::string &filename,
22  const OpenMode open_mode)
23  : database_(filename, this)
24  , read_write_(kOpenReadWrite == open_mode)
25  , schema_version_(0.0f)
26  , schema_revision_(0) {}
27 
28 
29 template <class DerivedT>
30 DerivedT* Database<DerivedT>::Create(const std::string &filename) {
31  UniquePtr<DerivedT> database(new DerivedT(filename, kOpenReadWrite));
32 
33  if (!database.IsValid()) {
34  LogCvmfs(kLogSql, kLogDebug, "Failed to create new database object");
35  return NULL;
36  }
37 
38  database->set_schema_version(DerivedT::kLatestSchema);
39  database->set_schema_revision(DerivedT::kLatestSchemaRevision);
40 
41  const int open_flags = SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_READWRITE |
42  SQLITE_OPEN_CREATE;
43  if (!database->OpenDatabase(open_flags)) {
44  LogCvmfs(kLogSql, kLogDebug, "Failed to create new database file");
45  return NULL;
46  }
47 
48  if (!database->CreatePropertiesTable()) {
49  database->PrintSqlError("Failed to create common properties table");
50  return NULL;
51  }
52 
53  if (!database->CreateEmptyDatabase()) {
54  database->PrintSqlError("Failed to create empty database");
55  return NULL;
56  }
57 
58  if (!database->PrepareCommonQueries()) {
59  database->PrintSqlError("Failed to initialize properties queries");
60  return NULL;
61  }
62 
63  if (!database->StoreSchemaRevision()) {
64  database->PrintSqlError("Failed to store initial schema revision");
65  return NULL;
66  }
67 
68  return database.Release();
69 }
70 
71 
72 template <class DerivedT>
73 DerivedT* Database<DerivedT>::Open(const std::string &filename,
74  const OpenMode open_mode) {
75  UniquePtr<DerivedT> database(new DerivedT(filename, open_mode));
76 
77  if (!database.IsValid()) {
79  "Failed to open database file '%s' - errno: %d",
80  filename.c_str(), errno);
81  return NULL;
82  }
83 
84  if (!database->Initialize()) {
85  return NULL;
86  }
87 
88  return database.Release();
89 }
90 
91 
92 template <class DerivedT>
94  const int flags = (read_write_) ? SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_READWRITE
95  : SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_READONLY;
96 
97  bool successful = OpenDatabase(flags) &&
98  Configure() &&
99  FileReadAhead() &&
100  PrepareCommonQueries();
101  if (!successful) {
102  LogCvmfs(kLogSql, kLogDebug, "failed to open database file '%s'",
103  filename().c_str());
104  return false;
105  }
106 
107  ReadSchemaRevision();
108  LogCvmfs(kLogSql, kLogDebug, "opened database with schema version %f "
109  "and revision %u",
110  schema_version_, schema_revision_);
111 
112  if (!static_cast<DerivedT*>(this)->CheckSchemaCompatibility()) {
113  LogCvmfs(kLogSql, kLogDebug, "schema version %f not supported (%s)",
114  schema_version_, filename().c_str());
115  return false;
116  }
117 
118  if (read_write_ &&
119  !static_cast<DerivedT*>(this)->LiveSchemaUpgradeIfNecessary()) {
120  LogCvmfs(kLogSql, kLogDebug, "failed tp upgrade schema revision");
121  return false;
122  }
123 
124  return true;
125 }
126 
127 
128 template <class DerivedT>
129 bool Database<DerivedT>::OpenDatabase(const int flags) {
130  // Open database file (depending on the flags read-only or read-write)
131  LogCvmfs(kLogSql, kLogDebug, "opening database file %s",
132  filename().c_str());
133  int retval = sqlite3_open_v2(filename().c_str(),
134  &database_.sqlite_db,
135  flags | SQLITE_OPEN_EXRESCODE,
136  NULL);
137  if (retval != SQLITE_OK) {
138  LogCvmfs(kLogSql, kLogDebug, "cannot open database file %s (%d - %d)",
139  filename().c_str(), retval, errno);
140  return false;
141  }
142 
143  return true;
144 }
145 
146 
147 template <class DerivedT>
149  if (NULL != sqlite_db) {
150  const bool close_successful = Close();
151  assert(close_successful);
152  }
153 }
154 
155 
156 template <class DerivedT>
158  assert(NULL != sqlite_db);
159 
160  LogCvmfs(kLogSql, kLogDebug, "closing SQLite database '%s' (unlink: %s)",
161  filename().c_str(),
162  (db_file_guard.IsEnabled() ? "yes" : "no"));
163  const int result = sqlite3_close(sqlite_db);
164 
165  if (result != SQLITE_OK) {
167  "failed to close SQLite database '%s' (%d - %s)",
168  filename().c_str(), result,
169  delegate_->GetLastErrorMsg().c_str());
170  return false;
171  }
172 
173  sqlite_db = NULL;
174  if (lookaside_buffer != NULL) {
176  lookaside_buffer);
177  lookaside_buffer = NULL;
178  }
179  return true;
180 }
181 
182 
183 template <class DerivedT>
185  // Read-only databases should store temporary files in memory. This avoids
186  // unexpected open read-write file descriptors in the cache directory like
187  // etilqs_<number>. They also use the optimized memory manager, if it is
188  // available.
189  if (!read_write_) {
193  }
194 
195  return Sql(sqlite_db() , "PRAGMA temp_store=2;").Execute() &&
196  Sql(sqlite_db() , "PRAGMA locking_mode=EXCLUSIVE;").Execute();
197  }
198  return true;
199 }
200 
201 
202 template <class DerivedT>
204  // Read-ahead into file system buffers
205  // TODO(jblomer): mmap, re-readahead
206  assert(filename().length() > 1);
207  int fd_readahead;
208  if (filename()[0] != '@') {
209  fd_readahead = open(filename().c_str(), O_RDONLY);
210  if (fd_readahead < 0) {
211  LogCvmfs(kLogSql, kLogDebug, "failed to open %s for read-ahead (%d)",
212  filename().c_str(), errno);
213  return false;
214  }
215  const ssize_t retval = platform_readahead(fd_readahead);
216  close(fd_readahead);
217  if (retval != 0) {
219  "failed to read-ahead %s (%d)", filename().c_str(), errno);
220  // Read-ahead is known to fail on tmpfs. Don't consider it as a fatal
221  // error.
222  // return false;
223  }
224  }
225 
226  return true;
227 }
228 
229 
230 template <class DerivedT>
232  sqlite3 *db = sqlite_db();
233  begin_transaction_ = new Sql(db, "BEGIN;");
234  commit_transaction_ = new Sql(db, "COMMIT;");
235  has_property_ = new Sql(db, "SELECT count(*) FROM properties "
236  "WHERE key = :key;");
237  get_property_ = new Sql(db, "SELECT value FROM properties "
238  "WHERE key = :key;");
239  set_property_ = new Sql(db, "INSERT OR REPLACE INTO properties "
240  "(key, value) VALUES (:key, :value);");
241  return (begin_transaction_.IsValid() &&
242  commit_transaction_.IsValid() &&
243  has_property_.IsValid() &&
244  get_property_.IsValid() &&
245  set_property_.IsValid());
246 }
247 
248 
249 template <class DerivedT>
252  ? this->GetProperty<double>(kSchemaVersionKey)
253  : 1.0;
255  ? this->GetProperty<int>(kSchemaRevisionKey)
256  : 0;
257 }
258 
259 
260 template <class DerivedT>
264 }
265 
266 
267 template <class DerivedT>
269  return begin_transaction_->Execute() &&
270  begin_transaction_->Reset();
271 }
272 
273 
274 template <class DerivedT>
276  return commit_transaction_->Execute() &&
277  commit_transaction_->Reset();
278 }
279 
280 
281 template <class DerivedT>
283  return Sql(sqlite_db(),
284  "CREATE TABLE properties (key TEXT, value TEXT, "
285  "CONSTRAINT pk_properties PRIMARY KEY (key));").Execute();
286 }
287 
288 
289 template <class DerivedT>
290 bool Database<DerivedT>::HasProperty(const std::string &key) const {
291  assert(has_property_.IsValid());
292  const bool retval = has_property_->BindText(1, key) &&
293  has_property_->FetchRow();
294  assert(retval);
295  const bool result = has_property_->RetrieveInt64(0) > 0;
296  has_property_->Reset();
297  return result;
298 }
299 
300 template <class DerivedT>
301 template <typename T>
302 T Database<DerivedT>::GetProperty(const std::string &key) const {
303  assert(get_property_.IsValid());
304  const bool retval = get_property_->BindText(1, key) &&
305  get_property_->FetchRow();
306  assert(retval);
307  const T result = get_property_->Retrieve<T>(0);
308  get_property_->Reset();
309  return result;
310 }
311 
312 template <class DerivedT>
313 template <typename T>
314 T Database<DerivedT>::GetPropertyDefault(const std::string &key,
315  const T default_value) const {
316  return (HasProperty(key)) ? GetProperty<T>(key)
317  : default_value;
318 }
319 
320 template <class DerivedT>
321 template <typename T>
322 bool Database<DerivedT>::SetProperty(const std::string &key,
323  const T value) {
324  assert(set_property_.IsValid());
325  return set_property_->BindText(1, key) &&
326  set_property_->Bind(2, value) &&
327  set_property_->Execute() &&
328  set_property_->Reset();
329 }
330 
331 template <class DerivedT>
333  std::string msg = sqlite3_errmsg(sqlite_db());
334  return msg;
335 }
336 
337 
338 template <class DerivedT>
341  LogCvmfs(kLogSql, kLogDebug, "Database object took ownership of '%s'",
342  database_.filename().c_str());
343 }
344 
345 
346 template <class DerivedT>
349  LogCvmfs(kLogSql, kLogDebug, "Database object dropped ownership of '%s'",
350  database_.filename().c_str());
351 }
352 
353 
354 template <class DerivedT>
356  const int modified_rows = sqlite3_total_changes(sqlite_db());
357  assert(modified_rows >= 0);
358  return static_cast<unsigned>(modified_rows);
359 }
360 
364 template <class DerivedT>
366  const int reset = 0;
367  int current;
368  int highwater;
369  int retval = SQLITE_OK;
370  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_USED,
371  &current, &highwater, reset);
372  stats->lookaside_slots_used = current;
373  stats->lookaside_slots_max = highwater;
374  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_HIT,
375  &current, &highwater, reset);
376  stats->lookaside_hit = highwater;
377  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE,
378  &current, &highwater, reset);
379  stats->lookaside_miss_size = highwater;
380  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL,
381  &current, &highwater, reset);
382  stats->lookaside_miss_full = highwater;
383  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_CACHE_USED,
384  &current, &highwater, reset);
385  stats->page_cache_used = current;
386  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_CACHE_HIT,
387  &current, &highwater, reset);
388  stats->page_cache_hit = current;
389  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_CACHE_MISS,
390  &current, &highwater, reset);
391  stats->page_cache_miss = current;
392  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_SCHEMA_USED,
393  &current, &highwater, reset);
394  stats->schema_used = current;
395  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_STMT_USED,
396  &current, &highwater, reset);
397  stats->stmt_used = current;
398  assert(retval == SQLITE_OK);
399 }
400 
401 
402 template <class DerivedT>
404  Sql free_page_count_query(this->sqlite_db(), "PRAGMA freelist_count;");
405  Sql page_count_query(this->sqlite_db(), "PRAGMA page_count;");
406 
407  const bool retval = page_count_query.FetchRow() &&
408  free_page_count_query.FetchRow();
409  assert(retval);
410 
411  int64_t pages = page_count_query.RetrieveInt64(0);
412  int64_t free_pages = free_page_count_query.RetrieveInt64(0);
413  assert(pages > 0);
414 
415  return (static_cast<double>(free_pages) / static_cast<double>(pages));
416 }
417 
418 
419 template <class DerivedT>
422  return static_cast<const DerivedT*>(this)->CompactDatabase() &&
423  Sql(this->sqlite_db(), "VACUUM;").Execute();
424 }
425 
426 
427 template <class DerivedT>
428 void Database<DerivedT>::PrintSqlError(const std::string &error_msg) {
429  LogCvmfs(kLogSql, kLogStderr, "%s\nSQLite said: '%s'",
430  error_msg.c_str(), this->GetLastErrorMsg().c_str());
431 }
432 
433 template <class DerivedT>
434 const float Database<DerivedT>::kSchemaEpsilon = 0.0005;
435 template <class DerivedT>
436 const char *Database<DerivedT>::kSchemaVersionKey = "schema";
437 template <class DerivedT>
438 const char *Database<DerivedT>::kSchemaRevisionKey = "schema_revision";
439 
440 
441 //
442 // # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
443 //
444 
445 
446 template <>
447 inline bool Sql::Bind(const int index, const int &value) {
448  return this->BindInt64(index, value);
449 }
450 
451 template <>
452 inline bool Sql::Bind(const int index, const unsigned int &value) {
453  return this->BindInt64(index, static_cast<int>(value));
454 }
455 
456 template <>
457 inline bool Sql::Bind(const int index, const uint64_t &value) {
458  return this->BindInt64(index, static_cast<int64_t>(value));
459 }
460 
461 template <>
462 inline bool Sql::Bind(const int index, const sqlite3_int64 &value) {
463  return this->BindInt64(index, value);
464 }
465 
466 template <>
467 inline bool Sql::Bind(const int index, const std::string &value) {
468  return this->BindTextTransient(index, value);
469 }
470 
471 template <>
472 inline bool Sql::Bind(const int index, const float &value) {
473  return this->BindDouble(index, value);
474 }
475 
476 template <>
477 inline bool Sql::Bind(const int index, const double &value) {
478  return this->BindDouble(index, value);
479 }
480 
481 
482 template <>
483 inline int Sql::Retrieve(const int index) {
484  return static_cast<int>(this->RetrieveInt64(index));
485 }
486 
487 template <>
488 inline bool Sql::Retrieve(const int index) {
489  return static_cast<bool>(this->RetrieveInt(index));
490 }
491 
492 template <>
493 inline sqlite3_int64 Sql::Retrieve(const int index) {
494  return this->RetrieveInt64(index);
495 }
496 
497 template <>
498 inline uint64_t Sql::Retrieve(const int index) {
499  return static_cast<uint64_t>(this->RetrieveInt64(index));
500 }
501 
502 template <>
503 inline std::string Sql::Retrieve(const int index) {
504  return RetrieveString(index);
505 }
506 
507 template <>
508 inline float Sql::Retrieve(const int index) {
509  return static_cast<float>(this->RetrieveDouble(index));
510 }
511 
512 template <>
513 inline double Sql::Retrieve(const int index) {
514  return this->RetrieveDouble(index);
515 }
516 
517 } // namespace sqlite
518 
519 #endif // CVMFS_SQL_IMPL_H_
#define LogCvmfs(source, mask,...)
Definition: logging.h:22
bool Bind(const int index, const T &value)
void DropFileOwnership()
Definition: sql_impl.h:347
const std::string & filename() const
Definition: sql.h:271
std::string GetLastErrorMsg() const
Definition: sql_impl.h:332
bool Execute()
Definition: sql.cc:42
double GetFreePageRatio() const
Definition: sql_impl.h:403
int page_cache_used
Bytes used for caching pages.
Definition: sql.h:35
bool FetchRow()
Definition: sql.cc:62
const std::string & filename() const
Definition: sql.h:148
bool PrepareCommonQueries()
Definition: sql_impl.h:231
void TakeFileOwnership()
Definition: sql_impl.h:339
int lookaside_miss_size
Definition: sql.h:33
bool Initialize()
Definition: sql_impl.h:93
float schema_version_
Definition: sql.h:291
bool BeginTransaction() const
Definition: sql_impl.h:268
void ReadSchemaRevision()
Definition: sql_impl.h:250
int lookaside_slots_used
Definition: sql.h:30
Database(const std::string &filename, const OpenMode open_mode)
Definition: sql_impl.h:21
assert((mem||(size==0))&&"Out Of Memory")
int stmt_used
Bytes used for prepared statmements (lookaside + heap)
Definition: sql.h:39
bool Configure()
Definition: sql_impl.h:184
static DerivedT * Open(const std::string &filename, const OpenMode open_mode)
Definition: sql_impl.h:73
bool CreatePropertiesTable()
Definition: sql_impl.h:282
T GetProperty(const std::string &key) const
Definition: sql_impl.h:302
static bool HasInstance()
Definition: sqlitemem.h:143
T GetPropertyDefault(const std::string &key, const T default_value) const
Definition: sql_impl.h:314
UniquePtr< Sql > set_property_
Definition: sql.h:298
unsigned GetModifiedRowCount() const
Definition: sql_impl.h:355
const bool read_write_
Definition: sql.h:290
ssize_t platform_readahead(int filedes)
bool Vacuum() const
Definition: sql_impl.h:420
sqlite3_int64 RetrieveInt64(const int idx_column) const
Definition: sql.h:445
static SqliteMemoryManager * GetInstance()
Definition: sqlitemem.h:137
T Retrieve(const int index)
UniquePtr< Sql > commit_transaction_
Definition: sql.h:295
UniquePtr< Sql > has_property_
Definition: sql.h:297
sqlite3 * sqlite_db() const
Definition: sql.h:147
bool CommitTransaction() const
Definition: sql_impl.h:275
int page_cache_miss
Definition: sql.h:37
bool SetProperty(const std::string &key, const T value)
Definition: sql_impl.h:322
bool OpenDatabase(const int sqlite_open_flags)
Definition: sql_impl.h:129
bool HasProperty(const std::string &key) const
Definition: sql_impl.h:290
static const char * kSchemaRevisionKey
Definition: sql.h:286
unsigned schema_revision_
Definition: sql.h:292
bool FileReadAhead()
Definition: sql_impl.h:203
int schema_used
Bytes used to store db schema.
Definition: sql.h:38
int lookaside_miss_full
Definition: sql.h:34
int page_cache_hit
Definition: sql.h:36
int lookaside_slots_max
Definition: sql.h:31
void GetMemStatistics(MemStatistics *stats) const
Definition: sql_impl.h:365
void PrintSqlError(const std::string &error_msg)
Definition: sql_impl.h:428
DatabaseRaiiWrapper database_
Definition: sql.h:288
static const char * kSchemaVersionKey
Definition: sql.h:285
UniquePtr< Sql > get_property_
Definition: sql.h:299
bool StoreSchemaRevision()
Definition: sql_impl.h:261
UniquePtr< Sql > begin_transaction_
Definition: sql.h:294
void ReleaseLookasideBuffer(void *buffer)
Definition: sqlitemem.cc:317
static DerivedT * Create(const std::string &filename)
Definition: sql_impl.h:30
void * AssignLookasideBuffer(sqlite3 *db)
Definition: sqlitemem.cc:158