GCC Code Coverage Report
Directory: cvmfs/ Exec Total Coverage
File: cvmfs/sql_impl.h Lines: 186 212 87.7 %
Date: 2019-02-03 02:48:13 Branches: 157 307 51.1 %

Line Branch Exec Source
1
/**
2
 * This file is part of the CernVM file system.
3
 */
4
5
#ifndef CVMFS_SQL_IMPL_H_
6
#define CVMFS_SQL_IMPL_H_
7
8
#include <fcntl.h>
9
10
#include <cassert>
11
#include <cerrno>
12
#include <string>
13
14
#include "logging.h"
15
#include "platform.h"
16
#include "sqlitemem.h"
17
18
namespace sqlite {
19
20
template <class DerivedT>
21
369
Database<DerivedT>::Database(const std::string  &filename,
22
                             const OpenMode      open_mode)
23
  : database_(filename, this)
24
  , read_write_(kOpenReadWrite == open_mode)
25
  , schema_version_(0.0f)
26
369
  , schema_revision_(0) {}
27
28
29
template <class DerivedT>
30
197
DerivedT* Database<DerivedT>::Create(const std::string &filename) {
31
197
  UniquePtr<DerivedT> database(new DerivedT(filename, kOpenReadWrite));
32
33
197
  if (!database.IsValid()) {
34
    LogCvmfs(kLogSql, kLogDebug, "Failed to create new database object");
35
    return NULL;
36
  }
37
38
197
  database->set_schema_version(DerivedT::kLatestSchema);
39
197
  database->set_schema_revision(DerivedT::kLatestSchemaRevision);
40
41
  const int open_flags = SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_READWRITE |
42
197
                         SQLITE_OPEN_CREATE;
43
197
  if (!database->OpenDatabase(open_flags)) {
44
    LogCvmfs(kLogSql, kLogDebug, "Failed to create new database file");
45
    return NULL;
46
  }
47
48
197
  if (!database->CreatePropertiesTable()) {
49
    database->PrintSqlError("Failed to create common properties table");
50
    return NULL;
51
  }
52
53
197
  if (!database->CreateEmptyDatabase()) {
54
    database->PrintSqlError("Failed to create empty database");
55
    return NULL;
56
  }
57
58
197
  if (!database->PrepareCommonQueries()) {
59
    database->PrintSqlError("Failed to initialize properties queries");
60
    return NULL;
61
  }
62
63
197
  if (!database->StoreSchemaRevision()) {
64
    database->PrintSqlError("Failed to store initial schema revision");
65
    return NULL;
66
  }
67
68
197
  return database.Release();
69
}
70
71
72
template <class DerivedT>
73
172
DerivedT* Database<DerivedT>::Open(const std::string  &filename,
74
                                   const OpenMode      open_mode) {
75
172
  UniquePtr<DerivedT> database(new DerivedT(filename, open_mode));
76
77
172
  if (!database.IsValid()) {
78
    LogCvmfs(kLogSql, kLogDebug,
79
             "Failed to open database file '%s' - errno: %d",
80
             filename.c_str(), errno);
81
    return NULL;
82
  }
83
84
172
  if (!database->Initialize()) {
85
4
    return NULL;
86
  }
87
88
168
  return database.Release();
89
}
90
91
92
template <class DerivedT>
93
172
bool Database<DerivedT>::Initialize() {
94
  const int flags = (read_write_) ? SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_READWRITE
95
172
                                  : SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_READONLY;
96
97
  bool successful = OpenDatabase(flags) &&
98
    Configure()         &&
99
    FileReadAhead()     &&
100


172
    PrepareCommonQueries();
101
172
  if (!successful) {
102
2
    LogCvmfs(kLogSql, kLogDebug, "failed to open database file '%s'",
103
                                 filename().c_str());
104
2
    return false;
105
  }
106
107
170
  ReadSchemaRevision();
108
170
  LogCvmfs(kLogSql, kLogDebug, "opened database with schema version %f "
109
                               "and revision %u",
110
                               schema_version_, schema_revision_);
111
112
170
  if (!static_cast<DerivedT*>(this)->CheckSchemaCompatibility()) {
113
1
    LogCvmfs(kLogSql, kLogDebug, "schema version %f not supported (%s)",
114
             schema_version_, filename().c_str());
115
1
    return false;
116
  }
117
118

169
  if (read_write_ &&
119
      !static_cast<DerivedT*>(this)->LiveSchemaUpgradeIfNecessary()) {
120
1
    LogCvmfs(kLogSql, kLogDebug, "failed tp upgrade schema revision");
121
1
    return false;
122
  }
123
124
168
  return true;
125
}
126
127
128
template <class DerivedT>
129
369
bool Database<DerivedT>::OpenDatabase(const int flags) {
130
  // Open database file (depending on the flags read-only or read-write)
131
369
  LogCvmfs(kLogSql, kLogDebug, "opening database file %s",
132
           filename().c_str());
133
369
  if (SQLITE_OK != sqlite3_open_v2(filename().c_str(),
134
                                   &database_.sqlite_db, flags, NULL)) {
135
2
    LogCvmfs(kLogSql, kLogDebug, "cannot open database file %s",
136
             filename().c_str());
137
2
    return false;
138
  }
139
140
367
  const int retval = sqlite3_extended_result_codes(sqlite_db(), 1);
141
367
  assert(SQLITE_OK == retval);
142
143
367
  return true;
144
}
145
146
147
template <class DerivedT>
148
368
Database<DerivedT>::DatabaseRaiiWrapper::~DatabaseRaiiWrapper() {
149

368
  if (NULL != sqlite_db) {
150
368
    const bool close_successful = Close();
151

368
    assert(close_successful);
152
  }
153
368
}
154
155
156
template <class DerivedT>
157
368
bool Database<DerivedT>::DatabaseRaiiWrapper::Close() {
158

368
  assert(NULL != sqlite_db);
159
160

368
  LogCvmfs(kLogSql, kLogDebug, "closing SQLite database '%s' (unlink: %s)",
161
           filename().c_str(),
162
           (db_file_guard.IsEnabled() ? "yes" : "no"));
163
368
  const int result = sqlite3_close(sqlite_db);
164
165

368
  if (result != SQLITE_OK) {
166
    LogCvmfs(kLogSql, kLogDebug,
167
             "failed to close SQLite database '%s' (%d - %s)",
168
             filename().c_str(), result,
169
             delegate_->GetLastErrorMsg().c_str());
170
    return false;
171
  }
172
173
368
  sqlite_db = NULL;
174

368
  if (lookaside_buffer != NULL) {
175
52
    SqliteMemoryManager::GetInstance()->ReleaseLookasideBuffer(
176
      lookaside_buffer);
177
52
    lookaside_buffer = NULL;
178
  }
179
368
  return true;
180
}
181
182
183
template <class DerivedT>
184
170
bool Database<DerivedT>::Configure() {
185
  // Read-only databases should store temporary files in memory.  This avoids
186
  // unexpected open read-write file descriptors in the cache directory like
187
  // etilqs_<number>.  They also use the optimized memory manager, if it is
188
  // available.
189
170
  if (!read_write_) {
190
95
    if (SqliteMemoryManager::HasInstance()) {
191
52
      database_.lookaside_buffer =
192
        SqliteMemoryManager::GetInstance()->AssignLookasideBuffer(sqlite_db());
193
    }
194
195
    return Sql(sqlite_db() , "PRAGMA temp_store=2;").Execute() &&
196








95
           Sql(sqlite_db() , "PRAGMA locking_mode=EXCLUSIVE;").Execute();
197
  }
198
75
  return true;
199
}
200
201
202
template <class DerivedT>
203
170
bool Database<DerivedT>::FileReadAhead() {
204
  // Read-ahead into file system buffers
205
  // TODO(jblomer): mmap, re-readahead
206
170
  assert(filename().length() > 1);
207
  int fd_readahead;
208
170
  if (filename()[0] != '@') {
209
118
    fd_readahead = open(filename().c_str(), O_RDONLY);
210
118
    if (fd_readahead < 0) {
211
      LogCvmfs(kLogSql, kLogDebug, "failed to open %s for read-ahead (%d)",
212
               filename().c_str(), errno);
213
      return false;
214
    }
215
118
    const int retval = platform_readahead(fd_readahead);
216
118
    close(fd_readahead);
217
118
    if (retval != 0) {
218
      LogCvmfs(kLogSql, kLogDebug | kLogSyslogWarn,
219
               "failed to read-ahead %s (%d)", filename().c_str(), errno);
220
      // Read-ahead is known to fail on tmpfs.  Don't consider it as a fatal
221
      // error.
222
      // return false;
223
    }
224
  }
225
226
170
  return true;
227
}
228
229
230
template <class DerivedT>
231
367
bool Database<DerivedT>::PrepareCommonQueries() {
232
367
  sqlite3 *db = sqlite_db();
233
367
  begin_transaction_  = new Sql(db, "BEGIN;");
234
367
  commit_transaction_ = new Sql(db, "COMMIT;");
235
367
  has_property_       = new Sql(db, "SELECT count(*) FROM properties "
236
                                    "WHERE key = :key;");
237
367
  get_property_       = new Sql(db, "SELECT value FROM properties "
238
                                    "WHERE key = :key;");
239
367
  set_property_       = new Sql(db, "INSERT OR REPLACE INTO properties "
240
                                    "(key, value) VALUES (:key, :value);");
241
  return (begin_transaction_ && commit_transaction_ &&
242


367
          has_property_ && get_property_ && set_property_);
243
}
244
245
246
template <class DerivedT>
247
170
void Database<DerivedT>::ReadSchemaRevision() {
248
170
  schema_version_  = (this->HasProperty(kSchemaVersionKey))
249
                        ? this->GetProperty<double>(kSchemaVersionKey)
250
                        : 1.0;
251
170
  schema_revision_ = (this->HasProperty(kSchemaRevisionKey))
252
                        ? this->GetProperty<int>(kSchemaRevisionKey)
253
                        : 0;
254
170
}
255
256
257
template <class DerivedT>
258
210
bool Database<DerivedT>::StoreSchemaRevision() {
259
  return this->SetProperty(kSchemaVersionKey,  schema_version_)   &&
260

210
         this->SetProperty(kSchemaRevisionKey, schema_revision_);
261
}
262
263
264
template <class DerivedT>
265
126
bool Database<DerivedT>::BeginTransaction() const {
266
  return begin_transaction_->Execute() &&
267

126
         begin_transaction_->Reset();
268
}
269
270
271
template <class DerivedT>
272
122
bool Database<DerivedT>::CommitTransaction() const {
273
  return commit_transaction_->Execute() &&
274

122
         commit_transaction_->Reset();
275
}
276
277
278
template <class DerivedT>
279
197
bool Database<DerivedT>::CreatePropertiesTable() {
280
  return Sql(sqlite_db(),
281
    "CREATE TABLE properties (key TEXT, value TEXT, "
282
197
    "CONSTRAINT pk_properties PRIMARY KEY (key));").Execute();
283
}
284
285
286
template <class DerivedT>
287
758
bool Database<DerivedT>::HasProperty(const std::string &key) const {
288
758
  assert(has_property_);
289
  const bool retval = has_property_->BindText(1, key) &&
290

758
                      has_property_->FetchRow();
291
758
  assert(retval);
292
758
  const bool result = has_property_->RetrieveInt64(0) > 0;
293
758
  has_property_->Reset();
294
758
  return result;
295
}
296
297
template <class DerivedT>
298
template <typename T>
299
526
T Database<DerivedT>::GetProperty(const std::string &key) const {
300


526
  assert(get_property_);
301
  const bool retval = get_property_->BindText(1, key) &&
302





526
                      get_property_->FetchRow();
303


526
  assert(retval);
304
526
  const T result = get_property_->Retrieve<T>(0);
305
526
  get_property_->Reset();
306
526
  return result;
307
}
308
309
template <class DerivedT>
310
template <typename T>
311
203
T Database<DerivedT>::GetPropertyDefault(const std::string &key,
312
                                         const T default_value) const {
313
  return (HasProperty(key)) ? GetProperty<T>(key)
314


203
                            : default_value;
315
}
316
317
template <class DerivedT>
318
template <typename T>
319
876
bool Database<DerivedT>::SetProperty(const std::string &key,
320
                                     const T            value) {
321



876
  assert(set_property_);
322
  return set_property_->BindText(1, key) &&
323
         set_property_->Bind(2, value)   &&
324
         set_property_->Execute()        &&
325
















876
         set_property_->Reset();
326
}
327
328
template <class DerivedT>
329
std::string Database<DerivedT>::GetLastErrorMsg() const {
330
  const std::string msg = sqlite3_errmsg(sqlite_db());
331
  return msg;
332
}
333
334
335
template <class DerivedT>
336
53
void Database<DerivedT>::TakeFileOwnership() {
337
53
  database_.TakeFileOwnership();
338
53
  LogCvmfs(kLogSql, kLogDebug, "Database object took ownership of '%s'",
339
           database_.filename().c_str());
340
53
}
341
342
343
template <class DerivedT>
344
4
void Database<DerivedT>::DropFileOwnership() {
345
4
  database_.DropFileOwnership();
346
4
  LogCvmfs(kLogSql, kLogDebug, "Database object dropped ownership of '%s'",
347
           database_.filename().c_str());
348
4
}
349
350
351
template <class DerivedT>
352
11
unsigned Database<DerivedT>::GetModifiedRowCount() const {
353
11
  const int modified_rows = sqlite3_total_changes(sqlite_db());
354
11
  assert(modified_rows >= 0);
355
11
  return static_cast<unsigned>(modified_rows);
356
}
357
358
/**
359
 * Ask SQlite for per-connection memory statistics
360
 */
361
template <class DerivedT>
362
1
void Database<DerivedT>::GetMemStatistics(MemStatistics *stats) const {
363
1
  const int reset = 0;
364
  int current;
365
  int highwater;
366
1
  int retval = SQLITE_OK;
367
1
  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_USED,
368
                              &current, &highwater, reset);
369
1
  stats->lookaside_slots_used = current;
370
1
  stats->lookaside_slots_max = highwater;
371
1
  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_HIT,
372
                              &current, &highwater, reset);
373
1
  stats->lookaside_hit = highwater;
374
1
  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE,
375
                              &current, &highwater, reset);
376
1
  stats->lookaside_miss_size = highwater;
377
1
  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL,
378
                              &current, &highwater, reset);
379
1
  stats->lookaside_miss_full = highwater;
380
1
  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_CACHE_USED,
381
                              &current, &highwater, reset);
382
1
  stats->page_cache_used = current;
383
1
  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_CACHE_HIT,
384
                              &current, &highwater, reset);
385
1
  stats->page_cache_hit = current;
386
1
  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_CACHE_MISS,
387
                              &current, &highwater, reset);
388
1
  stats->page_cache_miss = current;
389
1
  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_SCHEMA_USED,
390
                              &current, &highwater, reset);
391
1
  stats->schema_used = current;
392
1
  retval |= sqlite3_db_status(sqlite_db(), SQLITE_DBSTATUS_STMT_USED,
393
                              &current, &highwater, reset);
394
1
  stats->stmt_used = current;
395
1
  assert(retval == SQLITE_OK);
396
1
}
397
398
399
template <class DerivedT>
400
27
double Database<DerivedT>::GetFreePageRatio() const {
401
27
  Sql free_page_count_query(this->sqlite_db(), "PRAGMA freelist_count;");
402
27
  Sql page_count_query(this->sqlite_db(), "PRAGMA page_count;");
403
404
  const bool retval = page_count_query.FetchRow() &&
405

27
                      free_page_count_query.FetchRow();
406
27
  assert(retval);
407
408
27
  int64_t pages      = page_count_query.RetrieveInt64(0);
409
27
  int64_t free_pages = free_page_count_query.RetrieveInt64(0);
410
27
  assert(pages > 0);
411
412
27
  return (static_cast<double>(free_pages) / static_cast<double>(pages));
413
}
414
415
416
template <class DerivedT>
417
8
bool Database<DerivedT>::Vacuum() const {
418
8
  assert(read_write_);
419
  return static_cast<const DerivedT*>(this)->CompactDatabase() &&
420




8
         Sql(this->sqlite_db(), "VACUUM;").Execute();
421
}
422
423
424
template <class DerivedT>
425
void Database<DerivedT>::PrintSqlError(const std::string &error_msg) {
426
  LogCvmfs(kLogSql, kLogStderr, "%s\nSQLite said: '%s'",
427
           error_msg.c_str(), this->GetLastErrorMsg().c_str());
428
}
429
430
template <class DerivedT>
431
const float Database<DerivedT>::kSchemaEpsilon = 0.0005;
432
template <class DerivedT>
433
150
const std::string Database<DerivedT>::kSchemaVersionKey = "schema";
434
template <class DerivedT>
435
150
const std::string Database<DerivedT>::kSchemaRevisionKey = "schema_revision";
436
437
438
//
439
// # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
440
//
441
442
443
template <>
444
70
inline bool Sql::Bind(const int index, const int value) {
445
70
  return this->BindInt64(index, value);
446
}
447
448
template <>
449
230
inline bool Sql::Bind(const int index, const unsigned int value) {
450
230
  return this->BindInt64(index, static_cast<int>(value));
451
}
452
453
template <>
454
173
inline bool Sql::Bind(const int index, const uint64_t value) {
455
173
  return this->BindInt64(index, value);
456
}
457
458
template <>
459
inline bool Sql::Bind(const int index, const sqlite3_int64 value) {
460
  return this->BindInt64(index, value);
461
}
462
463
template <>
464
185
inline bool Sql::Bind(const int index, const std::string value) {
465
185
  return this->BindTextTransient(index, value);
466
}
467
468
template <>
469
4
inline bool Sql::Bind(const int index, const char *value) {
470
4
  return this->BindTextTransient(index, value, strlen(value));
471
}
472
473
template <>
474
210
inline bool Sql::Bind(const int index, const float value) {
475
210
  return this->BindDouble(index, value);
476
}
477
478
template <>
479
4
inline bool Sql::Bind(const int index, const double value) {
480
4
  return this->BindDouble(index, value);
481
}
482
483
484
template <>
485
175
inline int Sql::Retrieve(const int index) {
486
175
  return this->RetrieveInt64(index);
487
}
488
489
template <>
490
inline bool Sql::Retrieve(const int index) {
491
  return static_cast<bool>(this->RetrieveInt(index));
492
}
493
494
template <>
495
inline sqlite3_int64 Sql::Retrieve(const int index) {
496
  return this->RetrieveInt64(index);
497
}
498
499
template <>
500
84
inline uint64_t Sql::Retrieve(const int index) {
501
84
  return static_cast<uint64_t>(this->RetrieveInt64(index));
502
}
503
504
template <>
505
120
inline std::string Sql::Retrieve(const int index) {
506
120
  return RetrieveString(index);
507
}
508
509
template <>
510
4
inline float Sql::Retrieve(const int index) {
511
4
  return this->RetrieveDouble(index);
512
}
513
514
template <>
515
174
inline double Sql::Retrieve(const int index) {
516
174
  return this->RetrieveDouble(index);
517
}
518
519
}  // namespace sqlite
520
521
#endif  // CVMFS_SQL_IMPL_H_