catalog_virtual.cc

Go to the documentation of this file.

#include "catalog_virtual.h"

#include <algorithm>
#include <cassert>
#include <cstdlib>

#include "catalog_mgr_rw.h"
#include "compression/compression.h"
#include "history.h"
#include "swissknife_history.h"
#include "swissknife_sync.h"
#include "util/logging.h"
#include "util/pointer.h"
#include "util/posix.h"
#include "util/string.h"
#include "xattr.h"

using namespace std;  // NOLINT

namespace catalog {

const char *VirtualCatalog::kVirtualPath = ".cvmfs";
const char *VirtualCatalog::kSnapshotDirectory = "snapshots";
const int VirtualCatalog::kActionNone = 0x00;
const int VirtualCatalog::kActionGenerateSnapshots = 0x01;
const int VirtualCatalog::kActionRemove = 0x02;


void VirtualCatalog::CreateBaseDirectory() {
  // Add /.cvmfs as a nested catalog
  DirectoryEntryBase entry_dir;
  entry_dir.name_ = NameString(string(kVirtualPath));
  entry_dir.mode_ = S_IFDIR | S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH
                    | S_IXOTH;
  entry_dir.uid_ = 0;
  entry_dir.gid_ = 0;
  entry_dir.size_ = 97;
  entry_dir.mtime_ = time(NULL);
  catalog_mgr_->AddDirectory(entry_dir, XattrList(), "");
  WritableCatalog *parent_catalog = catalog_mgr_->GetHostingCatalog(
      kVirtualPath);
  catalog_mgr_->CreateNestedCatalog(kVirtualPath);
  WritableCatalog *virtual_catalog = catalog_mgr_->GetHostingCatalog(
      kVirtualPath);
  assert(parent_catalog != virtual_catalog);

  // Set hidden flag in parent catalog
  DirectoryEntry entry_parent;
  bool retval = parent_catalog->LookupPath(
      PathString("/" + string(kVirtualPath)), &entry_parent);
  assert(retval);
  entry_parent.set_is_hidden(true);
  parent_catalog->UpdateEntry(entry_parent, "/" + string(kVirtualPath));

  // Set hidden flag in nested catalog
  DirectoryEntry entry_virtual;
  retval = virtual_catalog->LookupPath(PathString("/" + string(kVirtualPath)),
                                       &entry_virtual);
  assert(retval);
  entry_virtual.set_is_hidden(true);
  virtual_catalog->UpdateEntry(entry_virtual, "/" + string(kVirtualPath));
}


void VirtualCatalog::CreateNestedCatalogMarker() {
  DirectoryEntryBase entry_marker;
  // Note that another entity needs to ensure that the object of an empty
  // file is in the repository!  It is currently done by the sync_mediator.
  const shash::Algorithms algorithm =
      catalog_mgr_->spooler_->GetHashAlgorithm();
  shash::Any file_hash(algorithm);
  void *empty_compressed;
  uint64_t sz_empty_compressed;
  const bool retval =
      zlib::CompressMem2Mem(NULL, 0, &empty_compressed, &sz_empty_compressed);
  assert(retval);
  shash::HashMem(static_cast<unsigned char *>(empty_compressed),
                 sz_empty_compressed, &file_hash);
  free(empty_compressed);
  entry_marker.name_ = NameString(".cvmfscatalog");
  entry_marker.mode_ = S_IFREG | S_IRUSR | S_IRGRP | S_IROTH;
  entry_marker.checksum_ = file_hash;
  entry_marker.mtime_ = time(NULL);
  entry_marker.uid_ = 0;
  entry_marker.gid_ = 0;
  const XattrList xattrs;
  catalog_mgr_->AddFile(entry_marker, xattrs, kVirtualPath);
}


void VirtualCatalog::CreateSnapshotDirectory() {
  DirectoryEntryBase entry_dir;
  entry_dir.name_ = NameString(string(kSnapshotDirectory));
  entry_dir.mode_ = S_IFDIR | S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH
                    | S_IXOTH;
  entry_dir.uid_ = 0;
  entry_dir.gid_ = 0;
  entry_dir.size_ = 97;
  entry_dir.mtime_ = time(NULL);
  catalog_mgr_->AddDirectory(entry_dir, XattrList(), kVirtualPath);
}


void VirtualCatalog::EnsurePresence() {
  DirectoryEntry e;
  const bool retval =
      catalog_mgr_->LookupPath("/" + string(kVirtualPath), kLookupDefault, &e);
  if (!retval) {
    LogCvmfs(kLogCatalog, kLogDebug, "creating new virtual catalog");
    CreateBaseDirectory();
    CreateNestedCatalogMarker();
    CreateSnapshotDirectory();
  }
  assert(catalog_mgr_->IsTransitionPoint(kVirtualPath));
}


void VirtualCatalog::Generate(int actions) {
  if (actions & kActionGenerateSnapshots) {
    GenerateSnapshots();
  }
  if (actions & kActionRemove) {
    Remove();
  }
}


void VirtualCatalog::GenerateSnapshots() {
  LogCvmfs(kLogCvmfs, kLogStdout, "Creating virtual snapshots");
  EnsurePresence();

  vector<TagId> tags_history;
  vector<TagId> tags_catalog;
  GetSortedTagsFromHistory(&tags_history);
  GetSortedTagsFromCatalog(&tags_catalog);
  // Add artificial end markers to both lists
  string tag_name_end = "";
  if (!tags_history.empty())
    tag_name_end = std::max(tag_name_end, tags_history.rbegin()->name);
  if (!tags_catalog.empty())
    tag_name_end = std::max(tag_name_end, tags_catalog.rbegin()->name);
  tag_name_end += "X";
  tags_history.push_back(TagId(tag_name_end, shash::Any()));
  tags_catalog.push_back(TagId(tag_name_end, shash::Any()));

  // Walk through both sorted lists concurrently and determine change set
  unsigned i_history = 0, i_catalog = 0;
  const unsigned last_history = tags_history.size() - 1;
  const unsigned last_catalog = tags_catalog.size() - 1;
  while ((i_history < last_history) || (i_catalog < last_catalog)) {
    const TagId t_history = tags_history[i_history];
    const TagId t_catalog = tags_catalog[i_catalog];

    // Both the same, nothing to do
    if (t_history == t_catalog) {
      i_history++;
      i_catalog++;
      continue;
    }

    // Same tag name for different hash, re-insert
    if (t_history.name == t_catalog.name) {
      RemoveSnapshot(t_catalog);
      InsertSnapshot(t_history);
      i_history++;
      i_catalog++;
      continue;
    }

    // New tag that's missing
    if (t_history.name < t_catalog.name) {
      InsertSnapshot(t_history);
      i_history++;
      continue;
    }

    // A tag was removed but it is still present in the catalog
    assert(t_history.name > t_catalog.name);
    RemoveSnapshot(t_catalog);
    i_catalog++;
  }
}


bool VirtualCatalog::ParseActions(const string &action_desc, int *actions) {
  *actions = kActionNone;
  if (action_desc.empty())
    return true;

  vector<string> action_tokens = SplitString(action_desc, ',');
  for (unsigned i = 0; i < action_tokens.size(); ++i) {
    if (action_tokens[i] == "snapshots") {
      *actions |= kActionGenerateSnapshots;
    } else if (action_tokens[i] == "remove") {
      *actions |= kActionRemove;
    } else {
      return false;
    }
  }
  return true;
}


void VirtualCatalog::GetSortedTagsFromHistory(vector<TagId> *tags) {
  const UniquePtr<history::History> history(
      assistant_.GetHistory(swissknife::Assistant::kOpenReadOnly));
  vector<history::History::Tag> tags_history;
  const bool retval = history->List(&tags_history);
  assert(retval);
  for (unsigned i = 0, l = tags_history.size(); i < l; ++i) {
    if ((tags_history[i].name == swissknife::CommandTag::kHeadTag)
        || (tags_history[i].name == swissknife::CommandTag::kPreviousHeadTag)) {
      continue;
    }
    tags->push_back(TagId(tags_history[i].name, tags_history[i].root_hash));
  }
  std::sort(tags->begin(), tags->end());
}


void VirtualCatalog::GetSortedTagsFromCatalog(vector<TagId> *tags) {
  WritableCatalog *virtual_catalog = catalog_mgr_->GetHostingCatalog(
      kVirtualPath);
  assert(virtual_catalog != NULL);
  Catalog::NestedCatalogList nested_catalogs = virtual_catalog
                                                   ->ListNestedCatalogs();
  for (unsigned i = 0, l = nested_catalogs.size(); i < l; ++i) {
    tags->push_back(TagId(GetFileName(nested_catalogs[i].mountpoint).ToString(),
                          nested_catalogs[i].hash));
  }
  std::sort(tags->begin(), tags->end());
}


void VirtualCatalog::InsertSnapshot(TagId tag) {
  LogCvmfs(kLogCatalog, kLogDebug, "add snapshot %s (%s) to virtual catalog",
           tag.name.c_str(), tag.hash.ToString().c_str());
  const UniquePtr<Catalog> catalog(
      assistant_.GetCatalog(tag.hash, swissknife::Assistant::kOpenReadOnly));
  assert(catalog.IsValid());
  assert(catalog->root_prefix().IsEmpty());
  DirectoryEntry entry_root;
  const bool retval = catalog->LookupPath(PathString(""), &entry_root);
  assert(retval);

  // Add directory entry
  DirectoryEntryBase entry_dir = entry_root;
  entry_dir.name_ = NameString(tag.name);
  catalog_mgr_->AddDirectory(
      entry_dir, XattrList(),
      string(kVirtualPath) + "/" + string(kSnapshotDirectory));

  // Set "bind mount" flag
  WritableCatalog *virtual_catalog = catalog_mgr_->GetHostingCatalog(
      kVirtualPath);
  assert(virtual_catalog != NULL);
  const string mountpoint = "/" + string(kVirtualPath) + "/" +
                            string(kSnapshotDirectory) + "/" + tag.name;
  DirectoryEntry entry_bind_mountpoint(entry_dir);
  entry_bind_mountpoint.set_is_bind_mountpoint(true);
  virtual_catalog->UpdateEntry(entry_bind_mountpoint, mountpoint);

  // Register nested catalog
  const uint64_t catalog_size = GetFileSize(catalog->database_path());
  assert(catalog_size > 0);
  virtual_catalog->InsertBindMountpoint(mountpoint, tag.hash, catalog_size);
}


void VirtualCatalog::Remove() {
  LogCvmfs(kLogCvmfs, kLogStdout, "Removing .cvmfs virtual catalog");

  // Safety check, make sure we don't remove the entire repository
  WritableCatalog *virtual_catalog = catalog_mgr_->GetHostingCatalog(
      kVirtualPath);
  assert(!virtual_catalog->IsRoot());
  DirectoryEntry entry_virtual;
  const bool retval = catalog_mgr_->LookupPath(
      PathString("/" + string(kVirtualPath)), kLookupDefault, &entry_virtual);
  assert(retval);
  assert(entry_virtual.IsHidden());

  RemoveRecursively(kVirtualPath);
  catalog_mgr_->RemoveNestedCatalog(kVirtualPath);
  catalog_mgr_->RemoveDirectory(kVirtualPath);
}


void VirtualCatalog::RemoveRecursively(const string &directory) {
  DirectoryEntryList listing;
  const bool retval =
      catalog_mgr_->Listing(PathString("/" + directory), &listing);
  assert(retval);
  for (unsigned i = 0; i < listing.size(); ++i) {
    const string this_path = directory + "/" + listing[i].name().ToString();
    if (listing[i].IsDirectory()) {
      if (!listing[i].IsBindMountpoint())
        RemoveRecursively(this_path);
      catalog_mgr_->RemoveDirectory(this_path);
    } else if (listing[i].IsRegular()) {
      assert(listing[i].name().ToString() == ".cvmfscatalog");
      catalog_mgr_->RemoveFile(this_path);
    } else {
      abort();
    }
  }
}


void VirtualCatalog::RemoveSnapshot(TagId tag) {
  LogCvmfs(kLogCatalog, kLogDebug,
           "remove snapshot %s (%s) from virtual catalog", tag.name.c_str(),
           tag.hash.ToString().c_str());
  const string tag_dir =
      string(kVirtualPath) + "/" + string(kSnapshotDirectory) + "/" + tag.name;
  catalog_mgr_->RemoveDirectory(tag_dir);

  WritableCatalog *virtual_catalog = catalog_mgr_->GetHostingCatalog(
      kVirtualPath);
  assert(virtual_catalog != NULL);
  virtual_catalog->RemoveBindMountpoint("/" + tag_dir);
}


VirtualCatalog::VirtualCatalog(manifest::Manifest *m,
                               download::DownloadManager *d,
                               catalog::WritableCatalogManager *c,
                               SyncParameters *p)
    : catalog_mgr_(c), assistant_(d, m, p->stratum0, p->dir_temp) { }

}  // namespace catalog