catalog_sql.h

Go to the documentation of this file.

#ifndef CVMFS_CATALOG_SQL_H_
#define CVMFS_CATALOG_SQL_H_

#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS
#endif

#include <inttypes.h>

#include <string>

#include "compression/compression.h"
#include "crypto/hash.h"
#include "directory_entry.h"
#include "file_chunk.h"
#include "shortstring.h"
#include "sql.h"

class XattrList;

namespace catalog {

class Catalog;


class CatalogDatabase : public sqlite::Database<CatalogDatabase> {
 public:
  static const float kLatestSchema;
  static const float kLatestSupportedSchema;  // + 1.X catalogs (r/o)
  // Backwards-compatible schema changes
  static const unsigned kLatestSchemaRevision;

  bool CreateEmptyDatabase();
  bool InsertInitialValues(const std::string     &root_path,
                           const bool             volatile_content,
                           const std::string     &voms_authz,
                           const DirectoryEntry  &root_entry
                                             = DirectoryEntry(kDirentNegative));

  bool CheckSchemaCompatibility();
  bool LiveSchemaUpgradeIfNecessary();
  bool CompactDatabase() const;

  double GetRowIdWasteRatio() const;
  bool SetVOMSAuthz(const std::string&);

 protected:
  // TODO(rmeusel): C++11 - constructor inheritance
  friend class sqlite::Database<CatalogDatabase>;
  CatalogDatabase(const std::string  &filename,
                  const OpenMode      open_mode)
    : sqlite::Database<CatalogDatabase>(filename, open_mode) { }
};


//------------------------------------------------------------------------------


class SqlCatalog : public sqlite::Sql {
 public:
  SqlCatalog(const CatalogDatabase &database, const std::string &statement) {
    Init(database.sqlite_db(), statement);
  }

  inline shash::Md5 RetrieveMd5(const int idx_high, const int idx_low) const {
    return shash::Md5(RetrieveInt64(idx_high), RetrieveInt64(idx_low));
  }

  inline shash::Any RetrieveHashBlob(
    const int                idx_column,
    const shash::Algorithms  hash_algo,
    const char               hash_suffix = shash::kSuffixNone) const
  {
    // Note: SQLite documentation advises to first define the data type of BLOB
    //       by calling sqlite3_column_XXX() on the column and _afterwards_ get
    //       the number of bytes using sqlite3_column_bytes().
    //
    //  See: https://www.sqlite.org/c3ref/column_blob.html
    const unsigned char *buffer = static_cast<const unsigned char *>(
      RetrieveBlob(idx_column));
    const int byte_count = RetrieveBytes(idx_column);
    return (byte_count > 0) ? shash::Any(hash_algo, buffer, hash_suffix)
                            : shash::Any(hash_algo);
  }

  inline shash::Any RetrieveHashHex(
    const int  idx_column,
    const char hash_suffix = shash::kSuffixNone) const
  {
    const std::string hash_string = std::string(reinterpret_cast<const char *>(
      RetrieveText(idx_column)));
    return shash::MkFromHexPtr(shash::HexPtr(hash_string), hash_suffix);
  }

  inline bool BindMd5(const int idx_high, const int idx_low,
                      const shash::Md5 &hash)
  {
    uint64_t high, low;
    hash.ToIntPair(&high, &low);
    const bool retval = BindInt64(idx_high, static_cast<int64_t>(high)) &&
                        BindInt64(idx_low, static_cast<int64_t>(low));
    return retval;
  }

  inline bool BindHashBlob(const int idx_column, const shash::Any &hash) {
    if (hash.IsNull()) {
      return BindNull(idx_column);
    } else {
      return BindBlob(idx_column, hash.digest, hash.GetDigestSize());
    }
  }

 protected:
  SqlCatalog() : sqlite::Sql() {}
};


//------------------------------------------------------------------------------


class SqlDirent : public SqlCatalog {
 public:
  // Definition of bit positions for the flags field of a DirectoryEntry
  // All other bit positions are unused
  static const int kFlagDir                 = 1;
  // Link in the parent catalog
  static const int kFlagDirNestedMountpoint = 2;
  // Link in the child catalog
  static const int kFlagDirNestedRoot       = 32;
  static const int kFlagFile                = 4;
  static const int kFlagLink                = 8;
  static const int kFlagFileSpecial         = 16;
  static const int kFlagFileChunk           = 64;
  static const int kFlagFileExternal        = 128;
  // as of 2^8: 3 bit for hashes
  //   - 0: SHA-1
  //   - 1: RIPEMD-160
  //   - ...
  // Corresponds to shash::algorithms with offset in order to support future
  // hashes
  static const int kFlagPosHash             = 8;
  // Compression methods, 3 bits starting at 2^11
  // Corresponds to zlib::Algorithms
  static const int kFlagPosCompression      = 11;
  static const int kFlagDirBindMountpoint   = 0x4000;  // 2^14
  static const int kFlagHidden              = 0x8000;  // 2^15
  static const int kFlagDirectIo            = 0x10000;  // 2^16


 protected:
  unsigned CreateDatabaseFlags(const DirectoryEntry &entry) const;
  void StoreHashAlgorithm(const shash::Algorithms algo, unsigned *flags) const;
  shash::Algorithms RetrieveHashAlgorithm(const unsigned flags) const;
  zlib::Algorithms RetrieveCompressionAlgorithm(const unsigned flags) const;

  uint32_t Hardlinks2Linkcount(const uint64_t hardlinks) const;
  uint32_t Hardlinks2HardlinkGroup(const uint64_t hardlinks) const;
  uint64_t MakeHardlinks(const uint32_t hardlink_group,
                         const uint32_t linkcount) const;

  void ExpandSymlink(LinkString *raw_symlink) const;
};


//------------------------------------------------------------------------------


class SqlDirentWrite : public SqlDirent {
 public:
  virtual bool BindDirent(const DirectoryEntry &entry) = 0;

 protected:
  bool BindDirentFields(const int hash_idx,
                        const int hardlinks_idx,
                        const int size_idx,
                        const int mode_idx,
                        const int mtime_idx,
                        const int mtimens_idx,
                        const int flags_idx,
                        const int name_idx,
                        const int symlink_idx,
                        const int uid_idx,
                        const int gid_idx,
                        const DirectoryEntry &entry);
};


//------------------------------------------------------------------------------


class SqlListContentHashes : public SqlDirent {
 public:
  explicit SqlListContentHashes(const CatalogDatabase &database);
  shash::Any GetHash() const;
};


//------------------------------------------------------------------------------


class SqlLookup : public SqlDirent {
 public:
  DirectoryEntry GetDirent(const Catalog *catalog,
                           const bool expand_symlink = true) const;

  shash::Md5 GetPathHash() const;

  shash::Md5 GetParentPathHash() const;
};


//------------------------------------------------------------------------------


class SqlListing : public SqlLookup {
 public:
  explicit SqlListing(const CatalogDatabase &database);
  bool BindPathHash(const struct shash::Md5 &hash);
};


//------------------------------------------------------------------------------


class SqlLookupPathHash : public SqlLookup {
 public:
  explicit SqlLookupPathHash(const CatalogDatabase &database);
  bool BindPathHash(const struct shash::Md5 &hash);
};


//------------------------------------------------------------------------------


class SqlLookupInode : public SqlLookup {
 public:
  explicit SqlLookupInode(const CatalogDatabase &database);
  bool BindRowId(const uint64_t inode);
};


//------------------------------------------------------------------------------


class SqlLookupDanglingMountpoints : public catalog::SqlLookup {
 public:
  explicit SqlLookupDanglingMountpoints(const CatalogDatabase &database);
};


//------------------------------------------------------------------------------


class SqlDirentTouch : public SqlCatalog {
 public:
  explicit SqlDirentTouch(const CatalogDatabase &database);

  bool BindDirentBase(const DirectoryEntryBase &entry);
  bool BindPathHash(const shash::Md5 &hash);
  bool BindXattr(const XattrList &xattrs);
  bool BindXattrEmpty();
};


//------------------------------------------------------------------------------


class SqlNestedCatalogLookup : public SqlCatalog {
 public:
  explicit SqlNestedCatalogLookup(const CatalogDatabase &database);
  bool BindSearchPath(const PathString &path);
  shash::Any GetContentHash() const;
  uint64_t GetSize() const;
};


//------------------------------------------------------------------------------


class SqlNestedCatalogListing : public SqlCatalog {
 public:
  explicit SqlNestedCatalogListing(const CatalogDatabase &database);
  PathString GetPath() const;
  shash::Any GetContentHash() const;
  uint64_t GetSize() const;
};


//------------------------------------------------------------------------------


class SqlOwnNestedCatalogListing : public SqlCatalog {
 public:
  explicit SqlOwnNestedCatalogListing(const CatalogDatabase &database);
  PathString GetPath() const;
  shash::Any GetContentHash() const;
  uint64_t GetSize() const;
};


//------------------------------------------------------------------------------


class SqlDirentInsert : public SqlDirentWrite {
 public:
  explicit SqlDirentInsert(const CatalogDatabase &database);
  bool BindPathHash(const shash::Md5 &hash);
  bool BindParentPathHash(const shash::Md5 &hash);
  bool BindDirent(const DirectoryEntry &entry);
  bool BindXattr(const XattrList &xattrs);
  bool BindXattrEmpty();
};


//------------------------------------------------------------------------------


class SqlDirentUpdate : public SqlDirentWrite {
 public:
  explicit SqlDirentUpdate(const CatalogDatabase &database);
  bool BindPathHash(const shash::Md5 &hash);
  bool BindDirent(const DirectoryEntry &entry);
};


//------------------------------------------------------------------------------


class SqlDirentUnlink : public SqlCatalog {
 public:
  explicit SqlDirentUnlink(const CatalogDatabase &database);
  bool BindPathHash(const shash::Md5 &hash);
};


//------------------------------------------------------------------------------


class SqlIncLinkcount : public SqlCatalog {
 public:
  explicit SqlIncLinkcount(const CatalogDatabase &database);
  bool BindPathHash(const shash::Md5 &hash);
  bool BindDelta(const int delta);
};


//------------------------------------------------------------------------------


class SqlChunkInsert : public SqlCatalog {
 public:
  explicit SqlChunkInsert(const CatalogDatabase &database);
  bool BindPathHash(const shash::Md5 &hash);
  bool BindFileChunk(const FileChunk &chunk);
};


//------------------------------------------------------------------------------


class SqlChunksRemove : public SqlCatalog {
 public:
  explicit SqlChunksRemove(const CatalogDatabase &database);
  bool BindPathHash(const shash::Md5 &hash);
};


//------------------------------------------------------------------------------


class SqlChunksListing : public SqlCatalog {
 public:
  explicit SqlChunksListing(const CatalogDatabase &database);
  bool BindPathHash(const shash::Md5 &hash);
  FileChunk GetFileChunk(const shash::Algorithms interpret_hash_as) const;
};


//------------------------------------------------------------------------------


class SqlChunksCount : public SqlCatalog {
 public:
  explicit SqlChunksCount(const CatalogDatabase &database);
  bool BindPathHash(const shash::Md5 &hash);
  int GetChunkCount() const;
};


//------------------------------------------------------------------------------


class SqlMaxHardlinkGroup : public SqlCatalog {
 public:
  explicit SqlMaxHardlinkGroup(const CatalogDatabase &database);
  uint32_t GetMaxGroupId() const;
};


//------------------------------------------------------------------------------


class SqlGetCounter : public SqlCatalog {
 public:
  explicit SqlGetCounter(const CatalogDatabase &database);
  bool BindCounter(const std::string &counter);
  uint64_t GetCounter() const;
 private:
  bool compat_;
};


//------------------------------------------------------------------------------


class SqlUpdateCounter : public SqlCatalog {
 public:
  explicit SqlUpdateCounter(const CatalogDatabase &database);
  bool BindCounter(const std::string &counter);
  bool BindDelta(const int64_t delta);
};


//------------------------------------------------------------------------------


class SqlCreateCounter : public SqlCatalog {
 public:
  explicit SqlCreateCounter(const CatalogDatabase &database);
  bool BindCounter(const std::string &counter);
  bool BindInitialValue(const int64_t value);
};


//------------------------------------------------------------------------------


class SqlAllChunks : public SqlCatalog {
 public:
  explicit SqlAllChunks(const CatalogDatabase &database);
  bool Open();
  bool Next(shash::Any *hash, zlib::Algorithms *compression_alg);
  bool Close();
};


//------------------------------------------------------------------------------


class SqlLookupXattrs : public SqlCatalog {
 public:
  explicit SqlLookupXattrs(const CatalogDatabase &database);
  bool BindPathHash(const shash::Md5 &hash);
  XattrList GetXattrs();
};

}  // namespace catalog

#endif  // CVMFS_CATALOG_SQL_H_