CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
catalog_virtual.cc
Go to the documentation of this file.
1 
4 #include "cvmfs_config.h"
5 #include "catalog_virtual.h"
6 
7 #include <algorithm>
8 #include <cassert>
9 #include <cstdlib>
10 
11 #include "catalog_mgr_rw.h"
12 #include "compression.h"
13 #include "history.h"
14 #include "logging.h"
15 #include "swissknife_history.h"
16 #include "swissknife_sync.h"
17 #include "util/pointer.h"
18 #include "util/posix.h"
19 #include "util/string.h"
20 #include "xattr.h"
21 
22 using namespace std; // NOLINT
23 
24 namespace catalog {
25 
26 const char *VirtualCatalog::kVirtualPath = ".cvmfs";
27 const char *VirtualCatalog::kSnapshotDirectory = "snapshots";
28 const unsigned VirtualCatalog::kActionNone = 0x00;
29 const unsigned VirtualCatalog::kActionGenerateSnapshots = 0x01;
30 const unsigned VirtualCatalog::kActionRemove = 0x02;
31 
32 
33 void VirtualCatalog::CreateBaseDirectory() {
34  // Add /.cvmfs as a nested catalog
35  DirectoryEntryBase entry_dir;
36  entry_dir.name_ = NameString(string(kVirtualPath));
37  entry_dir.mode_ = S_IFDIR |
38  S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH;
39  entry_dir.uid_ = 0;
40  entry_dir.gid_ = 0;
41  entry_dir.size_ = 97;
42  entry_dir.mtime_ = time(NULL);
43  catalog_mgr_->AddDirectory(entry_dir, XattrList(), "");
44  WritableCatalog *parent_catalog =
45  catalog_mgr_->GetHostingCatalog(kVirtualPath);
46  catalog_mgr_->CreateNestedCatalog(kVirtualPath);
47  WritableCatalog *virtual_catalog =
48  catalog_mgr_->GetHostingCatalog(kVirtualPath);
49  assert(parent_catalog != virtual_catalog);
50 
51  // Set hidden flag in parent catalog
52  DirectoryEntry entry_parent;
53  bool retval = parent_catalog->LookupPath(
54  PathString("/" + string(kVirtualPath)), &entry_parent);
55  assert(retval);
56  entry_parent.set_is_hidden(true);
57  parent_catalog->UpdateEntry(entry_parent, "/" + string(kVirtualPath));
58 
59  // Set hidden flag in nested catalog
60  DirectoryEntry entry_virtual;
61  retval = virtual_catalog->LookupPath(PathString("/" + string(kVirtualPath)),
62  &entry_virtual);
63  assert(retval);
64  entry_virtual.set_is_hidden(true);
65  virtual_catalog->UpdateEntry(entry_virtual, "/" + string(kVirtualPath));
66 }
67 
68 
69 void VirtualCatalog::CreateNestedCatalogMarker() {
70  DirectoryEntryBase entry_marker;
71  // Note that another entity needs to ensure that the object of an empty
72  // file is in the repository! It is currently done by the sync_mediator.
73  shash::Algorithms algorithm = catalog_mgr_->spooler_->GetHashAlgorithm();
74  shash::Any file_hash(algorithm);
75  void *empty_compressed;
76  uint64_t sz_empty_compressed;
77  bool retval = zlib::CompressMem2Mem(
78  NULL, 0, &empty_compressed, &sz_empty_compressed);
79  assert(retval);
80  shash::HashMem(static_cast<unsigned char *>(empty_compressed),
81  sz_empty_compressed, &file_hash);
82  free(empty_compressed);
83  entry_marker.name_ = NameString(".cvmfscatalog");
84  entry_marker.mode_ = S_IFREG | S_IRUSR | S_IRGRP | S_IROTH;
85  entry_marker.checksum_ = file_hash;
86  entry_marker.mtime_ = time(NULL);
87  entry_marker.uid_ = 0;
88  entry_marker.gid_ = 0;
89  XattrList xattrs;
90  catalog_mgr_->AddFile(entry_marker, xattrs, kVirtualPath);
91 }
92 
93 
94 void VirtualCatalog::CreateSnapshotDirectory() {
95  DirectoryEntryBase entry_dir;
96  entry_dir.name_ = NameString(string(kSnapshotDirectory));
97  entry_dir.mode_ = S_IFDIR |
98  S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH;
99  entry_dir.uid_ = 0;
100  entry_dir.gid_ = 0;
101  entry_dir.size_ = 97;
102  entry_dir.mtime_ = time(NULL);
103  catalog_mgr_->AddDirectory(entry_dir, XattrList(), kVirtualPath);
104 }
105 
106 
111 void VirtualCatalog::EnsurePresence() {
112  DirectoryEntry e;
113  bool retval = catalog_mgr_->LookupPath("/" + string(kVirtualPath),
114  kLookupSole, &e);
115  if (!retval) {
116  LogCvmfs(kLogCatalog, kLogDebug, "creating new virtual catalog");
117  CreateBaseDirectory();
118  CreateNestedCatalogMarker();
119  CreateSnapshotDirectory();
120  }
121  assert(catalog_mgr_->IsTransitionPoint(kVirtualPath));
122 }
123 
124 
125 void VirtualCatalog::Generate(int actions) {
126  if (actions & kActionGenerateSnapshots) {
127  GenerateSnapshots();
128  }
129  if (actions & kActionRemove) {
130  Remove();
131  }
132 }
133 
134 
135 void VirtualCatalog::GenerateSnapshots() {
136  LogCvmfs(kLogCvmfs, kLogStdout, "Creating virtual snapshots");
137  EnsurePresence();
138 
139  vector<TagId> tags_history;
140  vector<TagId> tags_catalog;
141  GetSortedTagsFromHistory(&tags_history);
142  GetSortedTagsFromCatalog(&tags_catalog);
143  // Add artifical end markers to both lists
144  string tag_name_end = "";
145  if (!tags_history.empty())
146  tag_name_end = std::max(tag_name_end, tags_history.rbegin()->name);
147  if (!tags_catalog.empty())
148  tag_name_end = std::max(tag_name_end, tags_catalog.rbegin()->name);
149  tag_name_end += "X";
150  tags_history.push_back(TagId(tag_name_end, shash::Any()));
151  tags_catalog.push_back(TagId(tag_name_end, shash::Any()));
152 
153  // Walk through both sorted lists concurrently and determine change set
154  unsigned i_history = 0, i_catalog = 0;
155  unsigned last_history = tags_history.size() - 1;
156  unsigned last_catalog = tags_catalog.size() - 1;
157  while ((i_history < last_history) || (i_catalog < last_catalog)) {
158  TagId t_history = tags_history[i_history];
159  TagId t_catalog = tags_catalog[i_catalog];
160 
161  // Both the same, nothing to do
162  if (t_history == t_catalog) {
163  i_history++;
164  i_catalog++;
165  continue;
166  }
167 
168  // Same tag name for different hash, re-insert
169  if (t_history.name == t_catalog.name) {
170  RemoveSnapshot(t_catalog);
171  InsertSnapshot(t_history);
172  i_history++;
173  i_catalog++;
174  continue;
175  }
176 
177  // New tag that's missing
178  if (t_history.name < t_catalog.name) {
179  InsertSnapshot(t_history);
180  i_history++;
181  continue;
182  }
183 
184  // A tag was removed but it is still present in the catalog
185  assert(t_history.name > t_catalog.name);
186  RemoveSnapshot(t_catalog);
187  i_catalog++;
188  }
189 }
190 
191 
192 bool VirtualCatalog::ParseActions(
193  const string &action_desc,
194  unsigned *actions)
195 {
196  *actions = kActionNone;
197  if (action_desc.empty())
198  return true;
199 
200  vector<string> action_tokens = SplitString(action_desc, ',');
201  for (unsigned i = 0; i < action_tokens.size(); ++i) {
202  if (action_tokens[i] == "snapshots") {
203  *actions |= kActionGenerateSnapshots;
204  } else if (action_tokens[i] == "remove") {
205  *actions |= kActionRemove;
206  } else {
207  return false;
208  }
209  }
210  return true;
211 }
212 
213 
214 void VirtualCatalog::GetSortedTagsFromHistory(vector<TagId> *tags) {
216  assistant_.GetHistory(swissknife::Assistant::kOpenReadOnly));
217  vector<history::History::Tag> tags_history;
218  bool retval = history->List(&tags_history);
219  assert(retval);
220  for (unsigned i = 0, l = tags_history.size(); i < l; ++i) {
221  if ((tags_history[i].name == swissknife::CommandTag::kHeadTag) ||
222  (tags_history[i].name == swissknife::CommandTag::kPreviousHeadTag))
223  {
224  continue;
225  }
226  tags->push_back(TagId(tags_history[i].name, tags_history[i].root_hash));
227  }
228  std::sort(tags->begin(), tags->end());
229 }
230 
231 
232 void VirtualCatalog::GetSortedTagsFromCatalog(vector<TagId> *tags) {
233  WritableCatalog *virtual_catalog =
234  catalog_mgr_->GetHostingCatalog(kVirtualPath);
235  assert(virtual_catalog != NULL);
236  Catalog::NestedCatalogList nested_catalogs =
237  virtual_catalog->ListNestedCatalogs();
238  for (unsigned i = 0, l = nested_catalogs.size(); i < l; ++i) {
239  tags->push_back(TagId(GetFileName(nested_catalogs[i].mountpoint).ToString(),
240  nested_catalogs[i].hash));
241  }
242  std::sort(tags->begin(), tags->end());
243 }
244 
245 
246 void VirtualCatalog::InsertSnapshot(TagId tag) {
247  LogCvmfs(kLogCatalog, kLogDebug, "add snapshot %s (%s) to virtual catalog",
248  tag.name.c_str(), tag.hash.ToString().c_str());
249  UniquePtr<Catalog> catalog(assistant_.GetCatalog(tag.hash,
251  assert(catalog.IsValid());
252  assert(catalog->root_prefix().IsEmpty());
253  DirectoryEntry entry_root;
254  bool retval = catalog->LookupPath(PathString(""), &entry_root);
255  assert(retval);
256 
257  // Add directory entry
258  DirectoryEntryBase entry_dir = entry_root;
259  entry_dir.name_ = NameString(tag.name);
260  catalog_mgr_->AddDirectory(entry_dir, XattrList(),
261  string(kVirtualPath) + "/" + string(kSnapshotDirectory));
262 
263  // Set "bind mount" flag
264  WritableCatalog *virtual_catalog =
265  catalog_mgr_->GetHostingCatalog(kVirtualPath);
266  assert(virtual_catalog != NULL);
267  string mountpoint =
268  "/" + string(kVirtualPath) + "/" + string(kSnapshotDirectory) + "/" +
269  tag.name;
270  DirectoryEntry entry_bind_mountpoint(entry_dir);
271  entry_bind_mountpoint.set_is_bind_mountpoint(true);
272  virtual_catalog->UpdateEntry(entry_bind_mountpoint, mountpoint);
273 
274  // Register nested catalog
275  uint64_t catalog_size = GetFileSize(catalog->database_path());
276  assert(catalog_size > 0);
277  virtual_catalog->InsertBindMountpoint(mountpoint, tag.hash, catalog_size);
278 }
279 
280 
281 void VirtualCatalog::Remove() {
282  LogCvmfs(kLogCvmfs, kLogStdout, "Removing .cvmfs virtual catalog");
283 
284  // Safety check, make sure we don't remove the entire repository
285  WritableCatalog *virtual_catalog =
286  catalog_mgr_->GetHostingCatalog(kVirtualPath);
287  assert(!virtual_catalog->IsRoot());
288  DirectoryEntry entry_virtual;
289  bool retval = catalog_mgr_->LookupPath(
290  PathString("/" + string(kVirtualPath)), kLookupSole, &entry_virtual);
291  assert(retval);
292  assert(entry_virtual.IsHidden());
293 
294  RemoveRecursively(kVirtualPath);
295  catalog_mgr_->RemoveNestedCatalog(kVirtualPath);
296  catalog_mgr_->RemoveDirectory(kVirtualPath);
297 }
298 
299 
300 void VirtualCatalog::RemoveRecursively(const string &directory) {
301  DirectoryEntryList listing;
302  bool retval = catalog_mgr_->Listing(PathString("/" + directory), &listing);
303  assert(retval);
304  for (unsigned i = 0; i < listing.size(); ++i) {
305  string this_path = directory + "/" + listing[i].name().ToString();
306  if (listing[i].IsDirectory()) {
307  if (!listing[i].IsBindMountpoint())
308  RemoveRecursively(this_path);
309  catalog_mgr_->RemoveDirectory(this_path);
310  } else if (listing[i].IsRegular()) {
311  assert(listing[i].name().ToString() == ".cvmfscatalog");
312  catalog_mgr_->RemoveFile(this_path);
313  } else {
314  abort();
315  }
316  }
317 }
318 
319 
320 void VirtualCatalog::RemoveSnapshot(TagId tag) {
322  "remove snapshot %s (%s) from virtual catalog",
323  tag.name.c_str(), tag.hash.ToString().c_str());
324  string tag_dir =
325  string(kVirtualPath) + "/" + string(kSnapshotDirectory) + "/" + tag.name;
326  catalog_mgr_->RemoveDirectory(tag_dir);
327 
328  WritableCatalog *virtual_catalog =
329  catalog_mgr_->GetHostingCatalog(kVirtualPath);
330  assert(virtual_catalog != NULL);
331  virtual_catalog->RemoveBindMountpoint("/" + tag_dir);
332 }
333 
334 
335 VirtualCatalog::VirtualCatalog(
339  SyncParameters *p)
340  : catalog_mgr_(c)
341  , assistant_(d, m, p->stratum0, p->dir_temp)
342 { }
343 
344 } // namespace catalog
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
void set_is_bind_mountpoint(const bool val)
bool IsRoot() const
Definition: catalog.h:193
ShortString< kDefaultMaxName, 1 > NameString
Definition: shortstring.h:191
void AddDirectory(const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &parent_directory)
std::string GetFileName(const std::string &path)
Definition: posix.cc:162
vector< string > SplitString(const string &str, const char delim, const unsigned max_chunks)
Definition: string.cc:288
static const std::string kPreviousHeadTag
Definition: repository.cc:41
void RemoveDirectory(const std::string &directory_path)
std::string ToString(const bool with_suffix=false) const
Definition: hash.h:245
WritableCatalog * GetHostingCatalog(const std::string &path)
void InsertBindMountpoint(const std::string &mountpoint, const shash::Any content_hash, const uint64_t size)
Definition: catalog_rw.cc:535
const history::History * history() const
gid_t gid_
bool LookupPath(const PathString &path, DirectoryEntry *dirent) const
Definition: catalog.h:124
void RemoveBindMountpoint(const std::string &mountpoint)
Definition: catalog_rw.cc:597
assert((mem||(size==0))&&"Out Of Memory")
bool LookupPath(const PathString &path, const LookupOptions options, DirectoryEntry *entry)
const NestedCatalogList & ListNestedCatalogs() const
Definition: catalog.cc:630
uint64_t size_
catalog::WritableCatalogManager * catalog_mgr_
Definition: repository.h:379
char algorithm
bool IsTransitionPoint(const std::string &mountpoint)
Algorithms
Definition: hash.h:39
bool Listing(const PathString &path, DirectoryEntryList *listing, const bool expand_symlink)
NameString name_
std::vector< DirectoryEntry > DirectoryEntryList
virtual bool List(std::vector< Tag > *tags) const =0
static const std::string kHeadTag
Definition: repository.cc:40
time_t mtime_
void RemoveFile(const std::string &file_path)
void set_is_hidden(const bool val)
void RemoveNestedCatalog(const std::string &mountpoint, const bool merge=true)
void HashMem(const unsigned char *buffer, const unsigned buffer_size, Any *any_digest)
Definition: hash.cc:255
std::vector< NestedCatalog > NestedCatalogList
Definition: catalog.h:208
unsigned int mode_
ShortString< kDefaultMaxPath, 0 > PathString
Definition: shortstring.h:190
int64_t GetFileSize(const std::string &path)
Definition: posix.cc:826
bool CompressMem2Mem(const void *buf, const int64_t size, void **out_buf, uint64_t *out_size)
Definition: compression.cc:739
shash::Any checksum_
void AddFile(const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &parent_directory)
void CreateNestedCatalog(const std::string &mountpoint)
void UpdateEntry(const DirectoryEntry &entry, const shash::Md5 &path_hash)
Definition: catalog_rw.cc:252
uid_t uid_