CernVM-FS  2.13.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
sql_impl.h
Go to the documentation of this file.
1 
5 #ifndef CVMFS_SQL_IMPL_H_
6 #define CVMFS_SQL_IMPL_H_
7 
8 #include <fcntl.h>
9 
10 #include <cassert>
11 #include <cerrno>
12 #include <string>
13 
14 #include "sqlitemem.h"
15 #include "util/logging.h"
16 #include "util/platform.h"
17 
18 namespace sqlite {
19 
20 template<class DerivedT>
21 Database<DerivedT>::Database(const std::string &filename,
22  const OpenMode open_mode)
23  : database_(filename, this)
24  , read_write_(kOpenReadWrite == open_mode)
25  , schema_version_(0.0f)
26  , schema_revision_(0) { }
27 
28 
29 template<class DerivedT>
30 DerivedT *Database<DerivedT>::Create(const std::string &filename) {
31  UniquePtr<DerivedT> database(new DerivedT(filename, kOpenReadWrite));
32 
33  if (!database.IsValid()) {
34  LogCvmfs(kLogSql, kLogDebug, "Failed to create new database object");
35  return NULL;
36  }
37 
38  database->set_schema_version(DerivedT::kLatestSchema);
39  database->set_schema_revision(DerivedT::kLatestSchemaRevision);
40 
41  const int open_flags = SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_READWRITE
42  | SQLITE_OPEN_CREATE;
43  if (!database->OpenDatabase(open_flags)) {
44  LogCvmfs(kLogSql, kLogDebug, "Failed to create new database file");
45  return NULL;
46  }
47 
48  if (!database->CreatePropertiesTable()) {
49  database->PrintSqlError("Failed to create common properties table");
50  return NULL;
51  }
52 
53  if (!database->CreateEmptyDatabase()) {
54  database->PrintSqlError("Failed to create empty database");
55  return NULL;
56  }
57 
58  if (!database->PrepareCommonQueries()) {
59  database->PrintSqlError("Failed to initialize properties queries");
60  return NULL;
61  }
62 
63  if (!database->StoreSchemaRevision()) {
64  database->PrintSqlError("Failed to store initial schema revision");
65  return NULL;
66  }
67 
68  return database.Release();
69 }
70 
71 
72 template<class DerivedT>
73 DerivedT *Database<DerivedT>::Open(const std::string &filename,
74  const OpenMode open_mode) {
75  UniquePtr<DerivedT> database(new DerivedT(filename, open_mode));
76 
77  if (!database.IsValid()) {
79  "Failed to open database file '%s' - errno: %d", filename.c_str(),
80  errno);
81  return NULL;
82  }
83 
84  if (!database->Initialize()) {
85  return NULL;
86  }
87 
88  return database.Release();
89 }
90 
91 
92 template<class DerivedT>
94  const int flags = (read_write_) ? SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_READWRITE
95  : SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_READONLY;
96 
97  bool successful = OpenDatabase(flags) && Configure() && FileReadAhead()
98  && PrepareCommonQueries();
99  if (!successful) {
100  LogCvmfs(kLogSql, kLogDebug, "failed to open database file '%s'",
101  filename().c_str());
102  return false;
103  }
104 
105  ReadSchemaRevision();
107  "opened database with schema version %f "
108  "and revision %u",
109  schema_version_, schema_revision_);
110 
111  if (!static_cast<DerivedT *>(this)->CheckSchemaCompatibility()) {
112  LogCvmfs(kLogSql, kLogDebug, "schema version %f not supported (%s)",
113  schema_version_, filename().c_str());
114  return false;
115  }
116 
117  if (read_write_
118  && !static_cast<DerivedT *>(this)->LiveSchemaUpgradeIfNecessary()) {
119  LogCvmfs(kLogSql, kLogDebug, "failed tp upgrade schema revision");
120  return false;
121  }
122 
123  return true;
124 }
125 
126 
127 template<class DerivedT>
128 bool Database<DerivedT>::OpenDatabase(const int flags) {
129  // Open database file (depending on the flags read-only or read-write)
130  LogCvmfs(kLogSql, kLogDebug, "opening database file %s", filename().c_str());
131  int retval = sqlite3_open_v2(filename().c_str(),
132  &database_.sqlite_db,
133  flags | SQLITE_OPEN_EXRESCODE,
134  NULL);
135  if (retval != SQLITE_OK) {
136  LogCvmfs(kLogSql, kLogDebug, "cannot open database file %s (%d - %d)",
137  filename().c_str(), retval, errno);
138  return false;
139  }
140 
141  return true;
142 }
143 
144 
145 template<class DerivedT>
147  if (NULL != sqlite_db) {
148  const bool close_successful = Close();
149  assert(close_successful);
150  }
151 }
152 
153 
154 template<class DerivedT>
156  assert(NULL != sqlite_db);
157 
158  LogCvmfs(kLogSql, kLogDebug, "closing SQLite database '%s' (unlink: %s)",
159  filename().c_str(), (db_file_guard.IsEnabled() ? "yes" : "no"));
160  const int result = sqlite3_close(sqlite_db);
161 
162  if (result != SQLITE_OK) {
164  "failed to close SQLite database '%s' (%d - %s)",
165  filename().c_str(), result, delegate_->GetLastErrorMsg().c_str());
166  return false;
167  }
168 
169  sqlite_db = NULL;
170  if (lookaside_buffer != NULL) {
172  lookaside_buffer);
173  lookaside_buffer = NULL;
174  }
175  return true;
176 }
177 
178 
179 template<class DerivedT>
181  // Read-only databases should store temporary files in memory. This avoids
182  // unexpected open read-write file descriptors in the cache directory like
183  // etilqs_<number>. They also use the optimized memory manager, if it is
184  // available.
185  if (!read_write_) {
189  }
190 
191  return Sql(sqlite_db(), "PRAGMA temp_store=2;").Execute()
192  && Sql(sqlite_db(), "PRAGMA locking_mode=EXCLUSIVE;").Execute();
193  }
194  return true;
195 }
196 
197 
198 template<class DerivedT>
200  // Read-ahead into file system buffers
201  // TODO(jblomer): mmap, re-readahead
202  assert(filename().length() > 1);
203  int fd_readahead;
204  if (filename()[0] != '@') {
205  fd_readahead = open(filename().c_str(), O_RDONLY);
206  if (fd_readahead < 0) {
207  LogCvmfs(kLogSql, kLogDebug, "failed to open %s for read-ahead (%d)",
208  filename().c_str(), errno);
209  return false;
210  }
211  const ssize_t retval = platform_readahead(fd_readahead);
212  close(fd_readahead);
213 
214  // Read-ahead is known to fail on tmpfs with EINVAL
215  // EINVAL = "readahead() cannot be applied to that a file type"
216  // It can also fail with ENOSYS on kernel interfaces that do not implement
217  // readahead(), such as LX-Brand zones on illumos.
218  // Don't consider either a fatal error.
219  if (retval != 0 && errno != EINVAL && errno != ENOSYS) {
221  "failed to read-ahead %s: invalid file descrp. or not open for "
222  "reading (%d)",
223  filename().c_str(), errno);
224  return false;
225  }
226  }
227 
228  return true;
229 }
230 
231 
232 template<class DerivedT>
234  sqlite3 *db = sqlite_db();
235  begin_transaction_ = new Sql(db, "BEGIN;");
236  commit_transaction_ = new Sql(db, "COMMIT;");
237  has_property_ = new Sql(db, "SELECT count(*) FROM properties "
238  "WHERE key = :key;");
239  get_property_ = new Sql(db, "SELECT value FROM properties "
240  "WHERE key = :key;");
241  set_property_ = new Sql(db, "INSERT OR REPLACE INTO properties "
242  "(key, value) VALUES (:key, :value);");
243  return (begin_transaction_.IsValid() && commit_transaction_.IsValid()
244  && has_property_.IsValid() && get_property_.IsValid()
245  && set_property_.IsValid());
246 }
247 
248 
249 template<class DerivedT>
252  ? this->GetProperty<double>(kSchemaVersionKey)
253  : 1.0;
255  ? this->GetProperty<int>(kSchemaRevisionKey)
256  : 0;
257 }
258 
259 
260 template<class DerivedT>
264 }
265 
266 
267 template<class DerivedT>
269  return begin_transaction_->Execute() && begin_transaction_->Reset();
270 }
271 
272 
273 template<class DerivedT>
275  return commit_transaction_->Execute() && commit_transaction_->Reset();
276 }
277 
278 
279 template<class DerivedT>
281  return Sql(sqlite_db(),
282  "CREATE TABLE properties (key TEXT, value TEXT, "
283  "CONSTRAINT pk_properties PRIMARY KEY (key));")
284  .Execute();
285 }
286 
287 
288 template<class DerivedT>
289 bool Database<DerivedT>::HasProperty(const std::string &key) const {
290  assert(has_property_.IsValid());
291  const bool retval = has_property_->BindText(1, key)
292  && has_property_->FetchRow();
293  assert(retval);
294  const bool result = has_property_->RetrieveInt64(0) > 0;
295  has_property_->Reset();
296  return result;
297 }
298 
299 template<class DerivedT>
300 template<typename T>
301 T Database<DerivedT>::GetProperty(const std::string &key) const {
302  assert(get_property_.IsValid());
303  const bool retval = get_property_->BindText(1, key)
304  && get_property_->FetchRow();
305  assert(retval);
306  const T result = get_property_->Retrieve<T>(0);
307  get_property_->Reset();
308  return result;
309 }
310 
311 template<class DerivedT>
312 template<typename T>
313 T Database<DerivedT>::GetPropertyDefault(const std::string &key,
314  const T default_value) const {
315  return (HasProperty(key)) ? GetProperty<T>(key) : default_value;
316 }
317 
318 template<class DerivedT>
319 template<typename T>
320 bool Database<DerivedT>::SetProperty(const std::string &key, const T value) {
321  assert(set_property_.IsValid());
322  return set_property_->BindText(1, key) && set_property_->Bind(2, value)
323  && set_property_->Execute() && set_property_->Reset();
324 }
325 
326 template<class DerivedT>
328  std::string msg = sqlite3_errmsg(sqlite_db());
329  return msg;
330 }
331 
332 
333 template<class DerivedT>
336  LogCvmfs(kLogSql, kLogDebug, "Database object took ownership of '%s'",
337  database_.filename().c_str());
338 }
339 
340 
341 template<class DerivedT>
344  LogCvmfs(kLogSql, kLogDebug, "Database object dropped ownership of '%s'",
345  database_.filename().c_str());
346 }
347 
348 
349 template<class DerivedT>
351  const int modified_rows = sqlite3_total_changes(sqlite_db());
352  assert(modified_rows >= 0);
353  return static_cast<unsigned>(modified_rows);
354 }
355 
359 template<class DerivedT>
361  const int reset = 0;
362  int current;
363  int highwater;
364  int retval = SQLITE_OK;
365  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_USED,
366  &current, &highwater, reset);
367  stats->lookaside_slots_used = current;
368  stats->lookaside_slots_max = highwater;
369  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_HIT,
370  &current, &highwater, reset);
371  stats->lookaside_hit = highwater;
372  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE,
373  &current, &highwater, reset);
374  stats->lookaside_miss_size = highwater;
375  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL,
376  &current, &highwater, reset);
377  stats->lookaside_miss_full = highwater;
378  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_CACHE_USED, &current,
379  &highwater, reset);
380  stats->page_cache_used = current;
381  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_CACHE_HIT, &current,
382  &highwater, reset);
383  stats->page_cache_hit = current;
384  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_CACHE_MISS, &current,
385  &highwater, reset);
386  stats->page_cache_miss = current;
387  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_SCHEMA_USED,
388  &current, &highwater, reset);
389  stats->schema_used = current;
390  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_STMT_USED, &current,
391  &highwater, reset);
392  stats->stmt_used = current;
393  assert(retval == SQLITE_OK);
394 }
395 
396 
397 template<class DerivedT>
399  Sql free_page_count_query(this->sqlite_db(), "PRAGMA freelist_count;");
400  Sql page_count_query(this->sqlite_db(), "PRAGMA page_count;");
401 
402  const bool retval = page_count_query.FetchRow()
403  && free_page_count_query.FetchRow();
404  assert(retval);
405 
406  int64_t pages = page_count_query.RetrieveInt64(0);
407  int64_t free_pages = free_page_count_query.RetrieveInt64(0);
408  assert(pages > 0);
409 
410  return (static_cast<double>(free_pages) / static_cast<double>(pages));
411 }
412 
413 
414 template<class DerivedT>
417  return static_cast<const DerivedT *>(this)->CompactDatabase()
418  && Sql(this->sqlite_db(), "VACUUM;").Execute();
419 }
420 
421 
422 template<class DerivedT>
423 void Database<DerivedT>::PrintSqlError(const std::string &error_msg) {
424  LogCvmfs(kLogSql, kLogStderr, "%s\nSQLite said: '%s'", error_msg.c_str(),
425  this->GetLastErrorMsg().c_str());
426 }
427 
428 template<class DerivedT>
429 const float Database<DerivedT>::kSchemaEpsilon = 0.0005;
430 template<class DerivedT>
431 const char *Database<DerivedT>::kSchemaVersionKey = "schema";
432 template<class DerivedT>
433 const char *Database<DerivedT>::kSchemaRevisionKey = "schema_revision";
434 
435 
436 //
437 // # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
438 //
439 
440 
441 template<>
442 inline bool Sql::Bind(const int index, const int &value) {
443  return this->BindInt64(index, value);
444 }
445 
446 template<>
447 inline bool Sql::Bind(const int index, const unsigned int &value) {
448  return this->BindInt64(index, static_cast<int>(value));
449 }
450 
451 template<>
452 inline bool Sql::Bind(const int index, const uint64_t &value) {
453  return this->BindInt64(index, static_cast<int64_t>(value));
454 }
455 
456 template<>
457 inline bool Sql::Bind(const int index, const sqlite3_int64 &value) {
458  return this->BindInt64(index, value);
459 }
460 
461 template<>
462 inline bool Sql::Bind(const int index, const std::string &value) {
463  return this->BindTextTransient(index, value);
464 }
465 
466 template<>
467 inline bool Sql::Bind(const int index, const float &value) {
468  return this->BindDouble(index, value);
469 }
470 
471 template<>
472 inline bool Sql::Bind(const int index, const double &value) {
473  return this->BindDouble(index, value);
474 }
475 
476 
477 template<>
478 inline int Sql::Retrieve(const int index) {
479  return static_cast<int>(this->RetrieveInt64(index));
480 }
481 
482 template<>
483 inline bool Sql::Retrieve(const int index) {
484  return static_cast<bool>(this->RetrieveInt(index));
485 }
486 
487 template<>
488 inline sqlite3_int64 Sql::Retrieve(const int index) {
489  return this->RetrieveInt64(index);
490 }
491 
492 template<>
493 inline uint64_t Sql::Retrieve(const int index) {
494  return static_cast<uint64_t>(this->RetrieveInt64(index));
495 }
496 
497 template<>
498 inline std::string Sql::Retrieve(const int index) {
499  return RetrieveString(index);
500 }
501 
502 template<>
503 inline float Sql::Retrieve(const int index) {
504  return static_cast<float>(this->RetrieveDouble(index));
505 }
506 
507 template<>
508 inline double Sql::Retrieve(const int index) {
509  return this->RetrieveDouble(index);
510 }
511 
512 } // namespace sqlite
513 
514 #endif // CVMFS_SQL_IMPL_H_
bool Bind(const int index, const T &value)
void DropFileOwnership()
Definition: sql_impl.h:342
const std::string & filename() const
Definition: sql.h:268
std::string GetLastErrorMsg() const
Definition: sql_impl.h:327
bool Execute()
Definition: sql.cc:41
double GetFreePageRatio() const
Definition: sql_impl.h:398
int page_cache_used
Bytes used for caching pages.
Definition: sql.h:34
bool FetchRow()
Definition: sql.cc:61
const std::string & filename() const
Definition: sql.h:146
bool PrepareCommonQueries()
Definition: sql_impl.h:233
void TakeFileOwnership()
Definition: sql_impl.h:334
int lookaside_miss_size
Definition: sql.h:32
bool Initialize()
Definition: sql_impl.h:93
float schema_version_
Definition: sql.h:288
bool BeginTransaction() const
Definition: sql_impl.h:268
void ReadSchemaRevision()
Definition: sql_impl.h:250
int lookaside_slots_used
Definition: sql.h:29
Database(const std::string &filename, const OpenMode open_mode)
Definition: sql_impl.h:21
assert((mem||(size==0))&&"Out Of Memory")
int stmt_used
Bytes used for prepared statmements (lookaside + heap)
Definition: sql.h:38
bool Configure()
Definition: sql_impl.h:180
static DerivedT * Open(const std::string &filename, const OpenMode open_mode)
Definition: sql_impl.h:73
bool CreatePropertiesTable()
Definition: sql_impl.h:280
T GetProperty(const std::string &key) const
Definition: sql_impl.h:301
static bool HasInstance()
Definition: sqlitemem.h:143
T GetPropertyDefault(const std::string &key, const T default_value) const
Definition: sql_impl.h:313
UniquePtr< Sql > set_property_
Definition: sql.h:295
unsigned GetModifiedRowCount() const
Definition: sql_impl.h:350
const bool read_write_
Definition: sql.h:287
ssize_t platform_readahead(int filedes)
bool Vacuum() const
Definition: sql_impl.h:415
sqlite3_int64 RetrieveInt64(const int idx_column) const
Definition: sql.h:445
static SqliteMemoryManager * GetInstance()
Definition: sqlitemem.h:137
T Retrieve(const int index)
UniquePtr< Sql > commit_transaction_
Definition: sql.h:292
UniquePtr< Sql > has_property_
Definition: sql.h:294
sqlite3 * sqlite_db() const
Definition: sql.h:145
bool CommitTransaction() const
Definition: sql_impl.h:274
int page_cache_miss
Definition: sql.h:36
bool SetProperty(const std::string &key, const T value)
Definition: sql_impl.h:320
bool OpenDatabase(const int sqlite_open_flags)
Definition: sql_impl.h:128
bool HasProperty(const std::string &key) const
Definition: sql_impl.h:289
static const char * kSchemaRevisionKey
Definition: sql.h:283
unsigned schema_revision_
Definition: sql.h:289
bool FileReadAhead()
Definition: sql_impl.h:199
int schema_used
Bytes used to store db schema.
Definition: sql.h:37
int lookaside_miss_full
Definition: sql.h:33
int page_cache_hit
Definition: sql.h:35
int lookaside_slots_max
Definition: sql.h:30
void GetMemStatistics(MemStatistics *stats) const
Definition: sql_impl.h:360
void PrintSqlError(const std::string &error_msg)
Definition: sql_impl.h:423
DatabaseRaiiWrapper database_
Definition: sql.h:285
static const char * kSchemaVersionKey
Definition: sql.h:282
UniquePtr< Sql > get_property_
Definition: sql.h:296
bool StoreSchemaRevision()
Definition: sql_impl.h:261
UniquePtr< Sql > begin_transaction_
Definition: sql.h:291
void ReleaseLookasideBuffer(void *buffer)
Definition: sqlitemem.cc:311
static DerivedT * Create(const std::string &filename)
Definition: sql_impl.h:30
void * AssignLookasideBuffer(sqlite3 *db)
Definition: sqlitemem.cc:153
CVMFS_EXPORT void LogCvmfs(const LogSource source, const int mask, const char *format,...)
Definition: logging.cc:545