sqlitevfs.cc

Go to the documentation of this file.

#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS
#endif


#include "sqlitevfs.h"

#include <dlfcn.h>
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <stdint.h>
#include <sys/select.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/uio.h>
#include <unistd.h>

#include <cassert>
#include <climits>
#include <cstring>
#include <ctime>
#include <vector>

#include "cache.h"
#include "duplex_sqlite3.h"
#include "statistics.h"
#include "util/logging.h"
#include "util/platform.h"
#include "util/smalloc.h"
#include "util/string.h"

using namespace std;  // NOLINT

namespace sqlite {

namespace {

const char *kVfsName = "cvmfs-readonly";

struct VfsRdOnly {
  VfsRdOnly()
      : cache_mgr(NULL)
      , n_access(NULL)
      , no_open(NULL)
      , n_rand(NULL)
      , sz_rand(NULL)
      , n_read(NULL)
      , sz_read(NULL)
      , n_sleep(NULL)
      , sz_sleep(NULL)
      , n_time(NULL) { }
  CacheManager *cache_mgr;
  perf::Counter *n_access;
  perf::Counter *no_open;
  perf::Counter *n_rand;
  perf::Counter *sz_rand;
  perf::Counter *n_read;
  perf::Counter *sz_read;
  perf::Counter *n_sleep;
  perf::Counter *sz_sleep;
  perf::Counter *n_time;
};

struct VfsRdOnlyFile {
  sqlite3_file base;  // Base class. Must be first.
  VfsRdOnly *vfs_rdonly;
  int fd;
  uint64_t size;
};

std::vector<int> *fd_from_ = NULL;
std::vector<int> *fd_to_ = NULL;

}  // anonymous namespace

static void ApplyFdMap(VfsRdOnlyFile *pFile) {
  const unsigned N = fd_from_->size();
  for (unsigned i = 0; i < N; ++i) {
    if (pFile->fd == (*fd_from_)[i]) {
      LogCvmfs(kLogSql, kLogDebug, "map fd %d --> %d", (*fd_from_)[i],
               (*fd_to_)[i]);
      pFile->fd = (*fd_to_)[i];
      fd_from_->erase(fd_from_->begin() + i);
      fd_to_->erase(fd_to_->begin() + i);
      return;
    }
  }
}


static int VfsRdOnlyClose(sqlite3_file *pFile) {
  VfsRdOnlyFile *p = reinterpret_cast<VfsRdOnlyFile *>(pFile);
  ApplyFdMap(p);
  const int retval = p->vfs_rdonly->cache_mgr->Close(p->fd);
  if (retval == 0) {
    perf::Dec(p->vfs_rdonly->no_open);
    return SQLITE_OK;
  }
  return SQLITE_IOERR_CLOSE;
}


static int VfsRdOnlyRead(sqlite3_file *pFile,
                         void *zBuf,
                         int iAmt,
                         sqlite_int64 iOfst) {
  VfsRdOnlyFile *p = reinterpret_cast<VfsRdOnlyFile *>(pFile);
  ApplyFdMap(p);
  const ssize_t got = p->vfs_rdonly->cache_mgr->Pread(p->fd, zBuf, iAmt, iOfst);
  perf::Inc(p->vfs_rdonly->n_read);
  if (got == iAmt) {
    perf::Xadd(p->vfs_rdonly->sz_read, iAmt);
    return SQLITE_OK;
  } else if (got < 0) {
    return SQLITE_IOERR_READ;
  } else {
    perf::Xadd(p->vfs_rdonly->sz_read, got);
    memset(reinterpret_cast<char *>(zBuf) + got, 0, iAmt - got);
    return SQLITE_IOERR_SHORT_READ;
  }
}


static int VfsRdOnlyWrite(sqlite3_file *pFile __attribute__((unused)),
                          const void *zBuf __attribute__((unused)),
                          int iAmt __attribute__((unused)),
                          sqlite_int64 iOfst __attribute__((unused))) {
  return SQLITE_READONLY;
}


static int VfsRdOnlyTruncate(sqlite3_file *pFile __attribute__((unused)),
                             sqlite_int64 size __attribute__((unused))) {
  return SQLITE_READONLY;
}


static int VfsRdOnlySync(sqlite3_file *pFile __attribute__((unused)),
                         int flags __attribute__((unused))) {
  return SQLITE_OK;
}


static int VfsRdOnlyFileSize(sqlite3_file *pFile, sqlite_int64 *pSize) {
  VfsRdOnlyFile *p = reinterpret_cast<VfsRdOnlyFile *>(pFile);
  *pSize = p->size;
  return SQLITE_OK;
}


static int VfsRdOnlyLock(sqlite3_file *p __attribute__((unused)),
                         int level __attribute__((unused))) {
  return SQLITE_OK;
}


static int VfsRdOnlyUnlock(sqlite3_file *p __attribute__((unused)),
                           int level __attribute__((unused))) {
  return SQLITE_OK;
}


static int VfsRdOnlyCheckReservedLock(sqlite3_file *p __attribute__((unused)),
                                      int *pResOut) {
  *pResOut = 0;
  return SQLITE_OK;
}


static int VfsRdOnlyFileControl(sqlite3_file *p __attribute__((unused)),
                                int op __attribute__((unused)),
                                void *pArg __attribute__((unused))) {
  return SQLITE_NOTFOUND;
}


static int VfsRdOnlySectorSize(sqlite3_file *p __attribute__((unused))) {
  return 4096;
}


static int VfsRdOnlyDeviceCharacteristics(sqlite3_file *p
                                          __attribute__((unused))) {
  return 0;
}


static int VfsRdOnlyOpen(sqlite3_vfs *vfs,
                         const char *zName,
                         sqlite3_file *pFile,
                         int flags,
                         int *pOutFlags) {
  static const sqlite3_io_methods io_methods = {1,  // iVersion
                                                VfsRdOnlyClose,
                                                VfsRdOnlyRead,
                                                VfsRdOnlyWrite,
                                                VfsRdOnlyTruncate,
                                                VfsRdOnlySync,
                                                VfsRdOnlyFileSize,
                                                VfsRdOnlyLock,
                                                VfsRdOnlyUnlock,
                                                VfsRdOnlyCheckReservedLock,
                                                VfsRdOnlyFileControl,
                                                VfsRdOnlySectorSize,
                                                VfsRdOnlyDeviceCharacteristics};

  VfsRdOnlyFile *p = reinterpret_cast<VfsRdOnlyFile *>(pFile);
  CacheManager *cache_mgr = reinterpret_cast<VfsRdOnly *>(vfs->pAppData)
                                ->cache_mgr;
  // Prevent xClose from being called in case of errors
  p->base.pMethods = NULL;

  if (flags & SQLITE_OPEN_READWRITE)
    return SQLITE_IOERR;
  if (flags & SQLITE_OPEN_DELETEONCLOSE)
    return SQLITE_IOERR;
  if (flags & SQLITE_OPEN_EXCLUSIVE)
    return SQLITE_IOERR;

  assert(zName && (zName[0] == '@'));
  p->fd = String2Int64(string(&zName[1]));
  if (p->fd < 0)
    return SQLITE_IOERR;
  const int64_t size = cache_mgr->GetSize(p->fd);
  if (size < 0) {
    cache_mgr->Close(p->fd);
    p->fd = -1;
    return SQLITE_IOERR_FSTAT;
  }
  if (cache_mgr->Readahead(p->fd) != 0) {
    cache_mgr->Close(p->fd);
    p->fd = -1;
    return SQLITE_IOERR;
  }
  p->size = static_cast<uint64_t>(size);
  if (pOutFlags)
    *pOutFlags = flags;
  p->vfs_rdonly = reinterpret_cast<VfsRdOnly *>(vfs->pAppData);
  p->base.pMethods = &io_methods;
  perf::Inc(p->vfs_rdonly->no_open);
  LogCvmfs(kLogSql, kLogDebug, "open sqlite3 catalog on fd %d, size %" PRIu64,
           p->fd, p->size);
  return SQLITE_OK;
}


static int VfsRdOnlyDelete(sqlite3_vfs * __attribute__((unused)),
                           const char *zName __attribute__((unused)),
                           int syncDir __attribute__((unused))) {
  return SQLITE_IOERR_DELETE;
}


static int VfsRdOnlyAccess(sqlite3_vfs *vfs,
                           const char *zPath,
                           int flags,
                           int *pResOut) {
  if (flags == SQLITE_ACCESS_READWRITE) {
    *pResOut = 0;
    return SQLITE_OK;
  }
  if (HasSuffix(zPath, "-wal", false) || HasSuffix(zPath, "-journal", false)) {
    *pResOut = 0;
    return SQLITE_OK;
  }

  // This VFS deals with file descriptors, we know the files are there
  *pResOut = 0;
  perf::Inc(reinterpret_cast<VfsRdOnly *>(vfs->pAppData)->n_access);
  return SQLITE_OK;
}


int VfsRdOnlyFullPathname(sqlite3_vfs *vfs __attribute__((unused)),
                          const char *zPath,
                          int nOut,
                          char *zOut) {
  zOut[nOut - 1] = '\0';
  sqlite3_snprintf(nOut, zOut, "%s", zPath);
  return SQLITE_OK;
}


static int VfsRdOnlyRandomness(sqlite3_vfs *vfs, int nBuf, char *zBuf) {
  assert(static_cast<size_t>(nBuf) >= (sizeof(time_t) + sizeof(int)));
  perf::Inc(reinterpret_cast<VfsRdOnly *>(vfs->pAppData)->n_rand);
  memset(zBuf, 0, nBuf);
  pid_t randomnessPid = getpid();
  int fd, got;
  fd = open("/dev/urandom", O_RDONLY, 0);
  if (fd < 0) {
    time_t t;
    time(&t);
    memcpy(zBuf, &t, sizeof(t));
    memcpy(&zBuf[sizeof(t)], &randomnessPid, sizeof(randomnessPid));
    assert(sizeof(t) + sizeof(randomnessPid) <= static_cast<size_t>(nBuf));
    nBuf = sizeof(t) + sizeof(randomnessPid);
  } else {
    do {
      got = read(fd, zBuf, nBuf);
    } while (got < 0 && errno == EINTR);
    close(fd);
  }
  perf::Xadd(reinterpret_cast<VfsRdOnly *>(vfs->pAppData)->sz_rand, nBuf);
  return nBuf;
}


static int VfsRdOnlySleep(sqlite3_vfs *vfs, int microseconds) {
  struct timeval wait_for;
  wait_for.tv_sec = microseconds / (1000 * 1000);
  wait_for.tv_usec = microseconds % (1000 * 1000);
  select(0, NULL, NULL, NULL, &wait_for);
  perf::Inc(reinterpret_cast<VfsRdOnly *>(vfs->pAppData)->n_sleep);
  perf::Xadd(reinterpret_cast<VfsRdOnly *>(vfs->pAppData)->sz_sleep,
             microseconds);
  return microseconds;
}


static int VfsRdOnlyCurrentTimeInt64(sqlite3_vfs *vfs, sqlite3_int64 *piNow) {
  static const sqlite3_int64 unixEpoch = 24405875 * (sqlite3_int64)8640000;
  int rc = SQLITE_OK;
  struct timeval sNow;
  if (gettimeofday(&sNow, 0) == 0) {
    *piNow = unixEpoch + 1000 * (sqlite3_int64)sNow.tv_sec
             + sNow.tv_usec / 1000;
    perf::Inc(reinterpret_cast<VfsRdOnly *>(vfs->pAppData)->n_time);
  } else {
    rc = SQLITE_ERROR;
  }
  return rc;
}


static int VfsRdOnlyCurrentTime(sqlite3_vfs *vfs, double *prNow) {
  sqlite3_int64 i = 0;
  const int rc = VfsRdOnlyCurrentTimeInt64(vfs, &i);
  *prNow = i / 86400000.0;
  return rc;
}


static int VfsRdOnlyGetLastError(sqlite3_vfs *vfs __attribute__((unused)),
                                 int not_used1 __attribute__((unused)),
                                 char *not_used2 __attribute__((unused))) {
  return 0;
}


bool RegisterVfsRdOnly(CacheManager *cache_mgr,
                       perf::Statistics *statistics,
                       const VfsOptions options) {
  fd_from_ = new std::vector<int>();
  fd_to_ = new std::vector<int>();

  sqlite3_vfs *vfs = reinterpret_cast<sqlite3_vfs *>(
      smalloc(sizeof(sqlite3_vfs)));
  memset(vfs, 0, sizeof(sqlite3_vfs));
  VfsRdOnly *vfs_rdonly = new VfsRdOnly();

  vfs->iVersion = 2;
  vfs->szOsFile = sizeof(VfsRdOnlyFile);
  vfs->mxPathname = PATH_MAX;
  vfs->zName = kVfsName;
  vfs->pAppData = vfs_rdonly;
  vfs->xOpen = VfsRdOnlyOpen;
  vfs->xDelete = VfsRdOnlyDelete;
  vfs->xAccess = VfsRdOnlyAccess;
  vfs->xFullPathname = VfsRdOnlyFullPathname;
  vfs->xDlOpen = NULL;
  vfs->xDlError = NULL;
  vfs->xDlSym = NULL;
  vfs->xDlClose = NULL;
  vfs->xRandomness = VfsRdOnlyRandomness;
  vfs->xSleep = VfsRdOnlySleep;
  vfs->xCurrentTime = VfsRdOnlyCurrentTime;
  vfs->xGetLastError = VfsRdOnlyGetLastError;
  vfs->xCurrentTimeInt64 = VfsRdOnlyCurrentTimeInt64;
  assert(vfs->zName);

  const int retval = sqlite3_vfs_register(vfs, options == kVfsOptDefault);
  if (retval != SQLITE_OK) {
    free(const_cast<char *>(vfs->zName));
    delete vfs_rdonly;
    free(vfs);
    return false;
  }

  vfs_rdonly->cache_mgr = cache_mgr;
  vfs_rdonly->n_access = statistics->Register(
      "sqlite.n_access", "overall number of access() calls");
  vfs_rdonly->no_open = statistics->Register("sqlite.no_open",
                                             "currently open sqlite files");
  vfs_rdonly->n_rand = statistics->Register("sqlite.n_rand",
                                            "overall number of random() calls");
  vfs_rdonly->sz_rand = statistics->Register("sqlite.sz_rand",
                                             "overall number of random bytes");
  vfs_rdonly->n_read = statistics->Register("sqlite.n_read",
                                            "overall number of read() calls");
  vfs_rdonly->sz_read = statistics->Register("sqlite.sz_read",
                                             "overall bytes read()");
  vfs_rdonly->n_sleep = statistics->Register("sqlite.n_sleep",
                                             "overall number of sleep() calls");
  vfs_rdonly->sz_sleep = statistics->Register("sqlite.sz_sleep",
                                              "overall microseconds slept");
  vfs_rdonly->n_time = statistics->Register("sqlite.n_time",
                                            "overall number of time() calls");

  return true;
}


bool UnregisterVfsRdOnly() {
  sqlite3_vfs *vfs = sqlite3_vfs_find(kVfsName);
  if (vfs == NULL)
    return false;
  const int retval = sqlite3_vfs_unregister(vfs);
  if (retval != SQLITE_OK)
    return false;

  delete reinterpret_cast<VfsRdOnly *>(vfs->pAppData);
  free(vfs);

  delete fd_from_;
  delete fd_to_;
  fd_from_ = NULL;
  fd_to_ = NULL;

  return true;
}

void RegisterFdMapping(int from, int to) {
  fd_from_->push_back(from);
  fd_to_->push_back(to);
}

void ReplaceCacheManager(CacheManager *new_cache_mgr) {
  sqlite3_vfs *vfs = sqlite3_vfs_find(kVfsName);
  if (vfs == NULL)
    return;
  static_cast<VfsRdOnly *>(vfs->pAppData)->cache_mgr = new_cache_mgr;
}

}  // namespace sqlite