CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
sqlitevfs.cc
Go to the documentation of this file.
1 
10 #ifndef __STDC_FORMAT_MACROS
11 #define __STDC_FORMAT_MACROS
12 #endif
13 
14 #include "cvmfs_config.h"
15 #include "sqlitevfs.h"
16 
17 #include <dlfcn.h>
18 #include <errno.h>
19 #include <fcntl.h>
20 #include <inttypes.h>
21 #include <stdint.h>
22 #include <sys/select.h>
23 #include <sys/time.h>
24 #include <sys/types.h>
25 #include <sys/uio.h>
26 #include <unistd.h>
27 
28 #include <cassert>
29 #include <climits>
30 #include <cstring>
31 #include <ctime>
32 #include <vector>
33 
34 #include "cache.h"
35 #include "duplex_sqlite3.h"
36 #include "logging.h"
37 #include "platform.h"
38 #include "smalloc.h"
39 #include "statistics.h"
40 #include "util/string.h"
41 
42 using namespace std; // NOLINT
43 
44 namespace sqlite {
45 
46 namespace {
47 
48 const char *kVfsName = "cvmfs-readonly";
49 
53 struct VfsRdOnly {
55  : cache_mgr(NULL)
56  , n_access(NULL)
57  , no_open(NULL)
58  , n_rand(NULL)
59  , sz_rand(NULL)
60  , n_read(NULL)
61  , sz_read(NULL)
62  , n_sleep(NULL)
63  , sz_sleep(NULL)
64  , n_time(NULL)
65  { }
76 };
77 
81 struct VfsRdOnlyFile {
82  sqlite3_file base; // Base class. Must be first.
84  int fd;
85  uint64_t size;
86 };
87 
91 std::vector<int> *fd_from_ = NULL;
92 std::vector<int> *fd_to_ = NULL;
93 
94 } // anonymous namespace
95 
96 static void ApplyFdMap(VfsRdOnlyFile *pFile) {
97  unsigned N = fd_from_->size();
98  for (unsigned i = 0; i < N; ++i) {
99  if (pFile->fd == (*fd_from_)[i]) {
100  LogCvmfs(kLogSql, kLogDebug, "map fd %d --> %d",
101  (*fd_from_)[i], (*fd_to_)[i]);
102  pFile->fd = (*fd_to_)[i];
103  fd_from_->erase(fd_from_->begin() + i);
104  fd_to_->erase(fd_to_->begin() + i);
105  return;
106  }
107  }
108 }
109 
110 
111 static int VfsRdOnlyClose(sqlite3_file *pFile) {
112  VfsRdOnlyFile *p = reinterpret_cast<VfsRdOnlyFile *>(pFile);
113  ApplyFdMap(p);
114  int retval = p->vfs_rdonly->cache_mgr->Close(p->fd);
115  if (retval == 0) {
116  perf::Dec(p->vfs_rdonly->no_open);
117  return SQLITE_OK;
118  }
119  return SQLITE_IOERR_CLOSE;
120 }
121 
122 
127 static int VfsRdOnlyRead(
128  sqlite3_file *pFile,
129  void *zBuf,
130  int iAmt,
131  sqlite_int64 iOfst
132 ) {
133  VfsRdOnlyFile *p = reinterpret_cast<VfsRdOnlyFile *>(pFile);
134  ApplyFdMap(p);
135  ssize_t got = p->vfs_rdonly->cache_mgr->Pread(p->fd, zBuf, iAmt, iOfst);
136  perf::Inc(p->vfs_rdonly->n_read);
137  if (got == iAmt) {
138  perf::Xadd(p->vfs_rdonly->sz_read, iAmt);
139  return SQLITE_OK;
140  } else if (got < 0) {
141  return SQLITE_IOERR_READ;
142  } else {
143  perf::Xadd(p->vfs_rdonly->sz_read, got);
144  memset(reinterpret_cast<char *>(zBuf) + got, 0, iAmt - got);
145  return SQLITE_IOERR_SHORT_READ;
146  }
147 }
148 
149 
150 static int VfsRdOnlyWrite(
151  sqlite3_file *pFile __attribute__((unused)),
152  const void *zBuf __attribute__((unused)),
153  int iAmt __attribute__((unused)),
154  sqlite_int64 iOfst __attribute__((unused)))
155 {
156  return SQLITE_READONLY;
157 }
158 
159 
160 static int VfsRdOnlyTruncate(
161  sqlite3_file *pFile __attribute__((unused)),
162  sqlite_int64 size __attribute__((unused)))
163 {
164  return SQLITE_READONLY;
165 }
166 
167 
168 static int VfsRdOnlySync(
169  sqlite3_file *pFile __attribute__((unused)),
170  int flags __attribute__((unused)))
171 {
172  return SQLITE_OK;
173 }
174 
175 
176 static int VfsRdOnlyFileSize(sqlite3_file *pFile, sqlite_int64 *pSize) {
177  VfsRdOnlyFile *p = reinterpret_cast<VfsRdOnlyFile *>(pFile);
178  *pSize = p->size;
179  return SQLITE_OK;
180 }
181 
182 
183 static int VfsRdOnlyLock(
184  sqlite3_file *p __attribute__((unused)),
185  int level __attribute__((unused))
186 ) {
187  return SQLITE_OK;
188 }
189 
190 
191 static int VfsRdOnlyUnlock(
192  sqlite3_file *p __attribute__((unused)),
193  int level __attribute__((unused))
194 ) {
195  return SQLITE_OK;
196 }
197 
198 
200  sqlite3_file *p __attribute__((unused)),
201  int *pResOut
202 ) {
203  *pResOut = 0;
204  return SQLITE_OK;
205 }
206 
207 
212  sqlite3_file *p __attribute__((unused)),
213  int op __attribute__((unused)),
214  void *pArg __attribute__((unused))
215 ) {
216  return SQLITE_NOTFOUND;
217 }
218 
219 
223 static int VfsRdOnlySectorSize(sqlite3_file *p __attribute__((unused))) {
224  return 4096;
225 }
226 
227 
232  sqlite3_file *p __attribute__((unused)))
233 {
234  return 0;
235 }
236 
237 
242 static int VfsRdOnlyOpen(
243  sqlite3_vfs *vfs,
244  const char *zName,
245  sqlite3_file *pFile,
246  int flags,
247  int *pOutFlags)
248 {
249  static const sqlite3_io_methods io_methods = {
250  1, // iVersion
263  };
264 
265  VfsRdOnlyFile *p = reinterpret_cast<VfsRdOnlyFile *>(pFile);
266  CacheManager *cache_mgr =
267  reinterpret_cast<VfsRdOnly *>(vfs->pAppData)->cache_mgr;
268  // Prevent xClose from being called in case of errors
269  p->base.pMethods = NULL;
270 
271  if (flags & SQLITE_OPEN_READWRITE)
272  return SQLITE_IOERR;
273  if (flags & SQLITE_OPEN_DELETEONCLOSE)
274  return SQLITE_IOERR;
275  if (flags & SQLITE_OPEN_EXCLUSIVE)
276  return SQLITE_IOERR;
277 
278  assert(zName && (zName[0] == '@'));
279  p->fd = String2Int64(string(&zName[1]));
280  if (p->fd < 0)
281  return SQLITE_IOERR;
282  int64_t size = cache_mgr->GetSize(p->fd);
283  if (size < 0) {
284  cache_mgr->Close(p->fd);
285  p->fd = -1;
286  return SQLITE_IOERR_FSTAT;
287  }
288  if (cache_mgr->Readahead(p->fd) != 0) {
289  cache_mgr->Close(p->fd);
290  p->fd = -1;
291  return SQLITE_IOERR;
292  }
293  p->size = static_cast<uint64_t>(size);
294  if (pOutFlags)
295  *pOutFlags = flags;
296  p->vfs_rdonly = reinterpret_cast<VfsRdOnly *>(vfs->pAppData);
297  p->base.pMethods = &io_methods;
298  perf::Inc(p->vfs_rdonly->no_open);
299  LogCvmfs(kLogSql, kLogDebug, "open sqlite3 catalog on fd %d, size %" PRIu64,
300  p->fd, p->size);
301  return SQLITE_OK;
302 }
303 
304 
305 static int VfsRdOnlyDelete(
306  sqlite3_vfs* __attribute__((unused)),
307  const char *zName __attribute__((unused)),
308  int syncDir __attribute__((unused)))
309 {
310  return SQLITE_IOERR_DELETE;
311 }
312 
313 
317 static int VfsRdOnlyAccess(
318  sqlite3_vfs *vfs,
319  const char *zPath,
320  int flags,
321  int *pResOut)
322 {
323  if (flags == SQLITE_ACCESS_READWRITE) {
324  *pResOut = 0;
325  return SQLITE_OK;
326  }
327  if (HasSuffix(zPath, "-wal", false) || HasSuffix(zPath, "-journal", false)) {
328  *pResOut = 0;
329  return SQLITE_OK;
330  }
331 
332  // This VFS deals with file descriptors, we know the files are there
333  *pResOut = 0;
334  perf::Inc(reinterpret_cast<VfsRdOnly *>(vfs->pAppData)->n_access);
335  return SQLITE_OK;
336 }
337 
338 
343  sqlite3_vfs *vfs __attribute__((unused)),
344  const char *zPath,
345  int nOut,
346  char *zOut)
347 {
348  zOut[nOut-1] = '\0';
349  sqlite3_snprintf(nOut, zOut, "%s", zPath);
350  return SQLITE_OK;
351 }
352 
353 
358  sqlite3_vfs *vfs,
359  int nBuf,
360  char *zBuf)
361 {
362  assert((size_t)nBuf >= (sizeof(time_t) + sizeof(int)));
363  perf::Inc(reinterpret_cast<VfsRdOnly *>(vfs->pAppData)->n_rand);
364  memset(zBuf, 0, nBuf);
365  pid_t randomnessPid = getpid();
366  int fd, got;
367  fd = open("/dev/urandom", O_RDONLY, 0);
368  if (fd < 0) {
369  time_t t;
370  time(&t);
371  memcpy(zBuf, &t, sizeof(t));
372  memcpy(&zBuf[sizeof(t)], &randomnessPid, sizeof(randomnessPid));
373  assert(sizeof(t) + sizeof(randomnessPid) <= (size_t)nBuf);
374  nBuf = sizeof(t) + sizeof(randomnessPid);
375  } else {
376  do {
377  got = read(fd, zBuf, nBuf);
378  } while (got < 0 && errno == EINTR);
379  close(fd);
380  }
381  perf::Xadd(reinterpret_cast<VfsRdOnly *>(vfs->pAppData)->sz_rand, nBuf);
382  return nBuf;
383 }
384 
385 
389 static int VfsRdOnlySleep(
390  sqlite3_vfs *vfs,
391  int microseconds)
392 {
393  struct timeval wait_for;
394  wait_for.tv_sec = microseconds / (1000*1000);
395  wait_for.tv_usec = microseconds % (1000 * 1000);
396  select(0, NULL, NULL, NULL, &wait_for);
397  perf::Inc(reinterpret_cast<VfsRdOnly *>(vfs->pAppData)->n_sleep);
398  perf::Xadd(reinterpret_cast<VfsRdOnly *>(vfs->pAppData)->sz_sleep,
399  microseconds);
400  return microseconds;
401 }
402 
403 
408  sqlite3_vfs *vfs,
409  sqlite3_int64 *piNow)
410 {
411  static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000;
412  int rc = SQLITE_OK;
413  struct timeval sNow;
414  if (gettimeofday(&sNow, 0) == 0) {
415  *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_usec/1000;
416  perf::Inc(reinterpret_cast<VfsRdOnly *>(vfs->pAppData)->n_time);
417  } else {
418  rc = SQLITE_ERROR;
419  }
420  return rc;
421 }
422 
423 
428  sqlite3_vfs *vfs,
429  double *prNow)
430 {
431  sqlite3_int64 i = 0;
432  int rc = VfsRdOnlyCurrentTimeInt64(vfs, &i);
433  *prNow = i/86400000.0;
434  return rc;
435 }
436 
437 
442  sqlite3_vfs *vfs __attribute__((unused)),
443  int not_used1 __attribute__((unused)),
444  char *not_used2 __attribute__((unused)))
445 {
446  return 0;
447 }
448 
449 
454  CacheManager *cache_mgr,
455  perf::Statistics *statistics,
456  const VfsOptions options)
457 {
458  fd_from_ = new std::vector<int>();
459  fd_to_ = new std::vector<int>();
460 
461  sqlite3_vfs *vfs = reinterpret_cast<sqlite3_vfs *>(
462  smalloc(sizeof(sqlite3_vfs)));
463  memset(vfs, 0, sizeof(sqlite3_vfs));
464  VfsRdOnly *vfs_rdonly = new VfsRdOnly();
465 
466  vfs->iVersion = 2;
467  vfs->szOsFile = sizeof(VfsRdOnlyFile);
468  vfs->mxPathname = PATH_MAX;
469  vfs->zName = kVfsName;
470  vfs->pAppData = vfs_rdonly;
471  vfs->xOpen = VfsRdOnlyOpen;
472  vfs->xDelete = VfsRdOnlyDelete;
473  vfs->xAccess = VfsRdOnlyAccess;
474  vfs->xFullPathname = VfsRdOnlyFullPathname;
475  vfs->xDlOpen = NULL;
476  vfs->xDlError = NULL;
477  vfs->xDlSym = NULL;
478  vfs->xDlClose = NULL;
479  vfs->xRandomness = VfsRdOnlyRandomness;
480  vfs->xSleep = VfsRdOnlySleep;
481  vfs->xCurrentTime = VfsRdOnlyCurrentTime;
482  vfs->xGetLastError = VfsRdOnlyGetLastError;
483  vfs->xCurrentTimeInt64 = VfsRdOnlyCurrentTimeInt64;
484  assert(vfs->zName);
485 
486  int retval = sqlite3_vfs_register(vfs, options == kVfsOptDefault);
487  if (retval != SQLITE_OK) {
488  free(const_cast<char *>(vfs->zName));
489  delete vfs_rdonly;
490  free(vfs);
491  return false;
492  }
493 
494  vfs_rdonly->cache_mgr = cache_mgr;
495  vfs_rdonly->n_access =
496  statistics->Register("sqlite.n_access", "overall number of access() calls");
497  vfs_rdonly->no_open =
498  statistics->Register("sqlite.no_open", "currently open sqlite files");
499  vfs_rdonly->n_rand =
500  statistics->Register("sqlite.n_rand", "overall number of random() calls");
501  vfs_rdonly->sz_rand =
502  statistics->Register("sqlite.sz_rand", "overall number of random bytes");
503  vfs_rdonly->n_read =
504  statistics->Register("sqlite.n_read", "overall number of read() calls");
505  vfs_rdonly->sz_read =
506  statistics->Register("sqlite.sz_read", "overall bytes read()");
507  vfs_rdonly->n_sleep =
508  statistics->Register("sqlite.n_sleep", "overall number of sleep() calls");
509  vfs_rdonly->sz_sleep =
510  statistics->Register("sqlite.sz_sleep", "overall microseconds slept");
511  vfs_rdonly->n_time =
512  statistics->Register("sqlite.n_time", "overall number of time() calls");
513 
514  return true;
515 }
516 
517 
523  sqlite3_vfs *vfs = sqlite3_vfs_find(kVfsName);
524  if (vfs == NULL)
525  return false;
526  int retval = sqlite3_vfs_unregister(vfs);
527  if (retval != SQLITE_OK)
528  return false;
529 
530  delete reinterpret_cast<VfsRdOnly *>(vfs->pAppData);
531  free(vfs);
532 
533  delete fd_from_;
534  delete fd_to_;
535  fd_from_ = NULL;
536  fd_to_ = NULL;
537 
538  return true;
539 }
540 
541 void RegisterFdMapping(int from, int to) {
542  fd_from_->push_back(from);
543  fd_to_->push_back(to);
544 }
545 
546 } // namespace sqlite
static int VfsRdOnlySync(sqlite3_file *pFile __attribute__((unused)), int flags __attribute__((unused)))
Definition: sqlitevfs.cc:168
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
void Dec(class Counter *counter)
Definition: statistics.h:49
Counter * Register(const std::string &name, const std::string &desc)
Definition: statistics.cc:139
void RegisterFdMapping(int from, int to)
Definition: sqlitevfs.cc:541
virtual int64_t GetSize(int fd)=0
int64_t Xadd(class Counter *counter, const int64_t delta)
Definition: statistics.h:51
VfsOptions
Definition: sqlitevfs.h:17
static int VfsRdOnlyOpen(sqlite3_vfs *vfs, const char *zName, sqlite3_file *pFile, int flags, int *pOutFlags)
Definition: sqlitevfs.cc:242
static int VfsRdOnlySleep(sqlite3_vfs *vfs, int microseconds)
Definition: sqlitevfs.cc:389
static void ApplyFdMap(VfsRdOnlyFile *pFile)
Definition: sqlitevfs.cc:96
static int VfsRdOnlyTruncate(sqlite3_file *pFile __attribute__((unused)), sqlite_int64 size __attribute__((unused)))
Definition: sqlitevfs.cc:160
static int VfsRdOnlyDeviceCharacteristics(sqlite3_file *p __attribute__((unused)))
Definition: sqlitevfs.cc:231
assert((mem||(size==0))&&"Out Of Memory")
bool UnregisterVfsRdOnly()
Definition: sqlitevfs.cc:522
struct cvmcache_object_info __attribute__
Definition: atomic.h:24
static int VfsRdOnlyWrite(sqlite3_file *pFile __attribute__((unused)), const void *zBuf __attribute__((unused)), int iAmt __attribute__((unused)), sqlite_int64 iOfst __attribute__((unused)))
Definition: sqlitevfs.cc:150
int64_t String2Int64(const string &value)
Definition: string.cc:221
static int VfsRdOnlyFileControl(sqlite3_file *p __attribute__((unused)), int op __attribute__((unused)), void *pArg __attribute__((unused)))
Definition: sqlitevfs.cc:211
static int VfsRdOnlyRead(sqlite3_file *pFile, void *zBuf, int iAmt, sqlite_int64 iOfst)
Definition: sqlitevfs.cc:127
bool RegisterVfsRdOnly(CacheManager *cache_mgr, perf::Statistics *statistics, const VfsOptions options)
Definition: sqlitevfs.cc:453
static int VfsRdOnlyDelete(sqlite3_vfs *__attribute__((unused)), const char *zName __attribute__((unused)), int syncDir __attribute__((unused)))
Definition: sqlitevfs.cc:305
bool HasSuffix(const std::string &str, const std::string &suffix, const bool ignore_case)
Definition: string.cc:278
static int VfsRdOnlyUnlock(sqlite3_file *p __attribute__((unused)), int level __attribute__((unused)))
Definition: sqlitevfs.cc:191
int VfsRdOnlyFullPathname(sqlite3_vfs *vfs __attribute__((unused)), const char *zPath, int nOut, char *zOut)
Definition: sqlitevfs.cc:342
static int VfsRdOnlyCheckReservedLock(sqlite3_file *p __attribute__((unused)), int *pResOut)
Definition: sqlitevfs.cc:199
static int VfsRdOnlyLock(sqlite3_file *p __attribute__((unused)), int level __attribute__((unused)))
Definition: sqlitevfs.cc:183
static int VfsRdOnlyClose(sqlite3_file *pFile)
Definition: sqlitevfs.cc:111
static int VfsRdOnlyRandomness(sqlite3_vfs *vfs, int nBuf, char *zBuf)
Definition: sqlitevfs.cc:357
static int VfsRdOnlySectorSize(sqlite3_file *p __attribute__((unused)))
Definition: sqlitevfs.cc:223
void Inc(class Counter *counter)
Definition: statistics.h:50
virtual int Close(int fd)=0
static int VfsRdOnlyGetLastError(sqlite3_vfs *vfs __attribute__((unused)), int not_used1 __attribute__((unused)), char *not_used2 __attribute__((unused)))
Definition: sqlitevfs.cc:441
static int VfsRdOnlyFileSize(sqlite3_file *pFile, sqlite_int64 *pSize)
Definition: sqlitevfs.cc:176
static int VfsRdOnlyCurrentTimeInt64(sqlite3_vfs *vfs, sqlite3_int64 *piNow)
Definition: sqlitevfs.cc:407
static int VfsRdOnlyCurrentTime(sqlite3_vfs *vfs, double *prNow)
Definition: sqlitevfs.cc:427
virtual int Readahead(int fd)=0
static int VfsRdOnlyAccess(sqlite3_vfs *vfs, const char *zPath, int flags, int *pResOut)
Definition: sqlitevfs.cc:317
static void size_t size
Definition: smalloc.h:47