GCC Code Coverage Report
Directory: cvmfs/ Exec Total Coverage
File: cvmfs/catalog_virtual.cc Lines: 12 184 6.5 %
Date: 2019-02-03 02:48:13 Branches: 8 92 8.7 %

Line Branch Exec Source
1
/**
2
 * This file is part of the CernVM File System.
3
 */
4
#include "cvmfs_config.h"
5
#include "catalog_virtual.h"
6
7
#include <algorithm>
8
#include <cassert>
9
#include <cstdlib>
10
11
#include "catalog_mgr_rw.h"
12
#include "compression.h"
13
#include "history.h"
14
#include "logging.h"
15
#include "swissknife_history.h"
16
#include "swissknife_sync.h"
17
#include "util/pointer.h"
18
#include "util/posix.h"
19
#include "util/string.h"
20
#include "xattr.h"
21
22
using namespace std;  // NOLINT
23
24
namespace catalog {
25
26
const char *VirtualCatalog::kVirtualPath = ".cvmfs";
27
const char *VirtualCatalog::kSnapshotDirectory = "snapshots";
28
const unsigned VirtualCatalog::kActionNone              = 0x00;
29
const unsigned VirtualCatalog::kActionGenerateSnapshots = 0x01;
30
const unsigned VirtualCatalog::kActionRemove            = 0x02;
31
32
33
void VirtualCatalog::CreateBaseDirectory() {
34
  // Add /.cvmfs as a nested catalog
35
  DirectoryEntryBase entry_dir;
36
  entry_dir.name_ = NameString(string(kVirtualPath));
37
  entry_dir.mode_ = S_IFDIR |
38
                    S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH;
39
  entry_dir.uid_ = 0;
40
  entry_dir.gid_ = 0;
41
  entry_dir.size_ = 97;
42
  entry_dir.mtime_ = time(NULL);
43
  catalog_mgr_->AddDirectory(entry_dir, "");
44
  WritableCatalog *parent_catalog =
45
    catalog_mgr_->GetHostingCatalog(kVirtualPath);
46
  catalog_mgr_->CreateNestedCatalog(kVirtualPath);
47
  WritableCatalog *virtual_catalog =
48
    catalog_mgr_->GetHostingCatalog(kVirtualPath);
49
  assert(parent_catalog != virtual_catalog);
50
51
  // Set hidden flag in parent catalog
52
  DirectoryEntry entry_parent;
53
  bool retval = parent_catalog->LookupPath(
54
    PathString("/" + string(kVirtualPath)), &entry_parent);
55
  assert(retval);
56
  entry_parent.set_is_hidden(true);
57
  parent_catalog->UpdateEntry(entry_parent, "/" + string(kVirtualPath));
58
59
  // Set hidden flag in nested catalog
60
  DirectoryEntry entry_virtual;
61
  retval = virtual_catalog->LookupPath(PathString("/" + string(kVirtualPath)),
62
                                       &entry_virtual);
63
  assert(retval);
64
  entry_virtual.set_is_hidden(true);
65
  virtual_catalog->UpdateEntry(entry_virtual, "/" + string(kVirtualPath));
66
}
67
68
69
void VirtualCatalog::CreateNestedCatalogMarker() {
70
  DirectoryEntryBase entry_marker;
71
  // Note that another entity needs to ensure that the object of an empty
72
  // file is in the repository!  It is currently done by the sync_mediator.
73
  shash::Algorithms algorithm = catalog_mgr_->spooler_->GetHashAlgorithm();
74
  shash::Any file_hash(algorithm);
75
  void *empty_compressed;
76
  uint64_t sz_empty_compressed;
77
  bool retval = zlib::CompressMem2Mem(
78
    NULL, 0, &empty_compressed, &sz_empty_compressed);
79
  assert(retval);
80
  shash::HashMem(static_cast<unsigned char *>(empty_compressed),
81
                 sz_empty_compressed, &file_hash);
82
  free(empty_compressed);
83
  entry_marker.name_ = NameString(".cvmfscatalog");
84
  entry_marker.mode_ = S_IFREG | S_IRUSR | S_IRGRP | S_IROTH;
85
  entry_marker.checksum_ = file_hash;
86
  entry_marker.mtime_ = time(NULL);
87
  entry_marker.uid_ = 0;
88
  entry_marker.gid_ = 0;
89
  XattrList xattrs;
90
  catalog_mgr_->AddFile(entry_marker, xattrs, kVirtualPath);
91
}
92
93
94
void VirtualCatalog::CreateSnapshotDirectory() {
95
  DirectoryEntryBase entry_dir;
96
  entry_dir.name_ = NameString(string(kSnapshotDirectory));
97
  entry_dir.mode_ = S_IFDIR |
98
                    S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH;
99
  entry_dir.uid_ = 0;
100
  entry_dir.gid_ = 0;
101
  entry_dir.size_ = 97;
102
  entry_dir.mtime_ = time(NULL);
103
  catalog_mgr_->AddDirectory(entry_dir, kVirtualPath);
104
}
105
106
107
/**
108
 * Checks for the top-level /.cvmfs directory and creates it as a nested catalog
109
 * if necessary.
110
 */
111
void VirtualCatalog::EnsurePresence() {
112
  DirectoryEntry e;
113
  bool retval = catalog_mgr_->LookupPath("/" + string(kVirtualPath),
114
                                        kLookupSole, &e);
115
  if (!retval) {
116
    LogCvmfs(kLogCatalog, kLogDebug, "creating new virtual catalog");
117
    CreateBaseDirectory();
118
    CreateNestedCatalogMarker();
119
    CreateSnapshotDirectory();
120
  }
121
  assert(catalog_mgr_->IsTransitionPoint(kVirtualPath));
122
}
123
124
125
void VirtualCatalog::Generate(int actions) {
126
  if (actions & kActionGenerateSnapshots) {
127
    GenerateSnapshots();
128
  }
129
  if (actions & kActionRemove) {
130
    Remove();
131
  }
132
}
133
134
135
void VirtualCatalog::GenerateSnapshots() {
136
  LogCvmfs(kLogCvmfs, kLogStdout, "Creating virtual snapshots");
137
  EnsurePresence();
138
139
  vector<TagId> tags_history;
140
  vector<TagId> tags_catalog;
141
  GetSortedTagsFromHistory(&tags_history);
142
  GetSortedTagsFromCatalog(&tags_catalog);
143
  // Add artifical end markers to both lists
144
  string tag_name_end = "";
145
  if (!tags_history.empty())
146
    tag_name_end = std::max(tag_name_end, tags_history.rbegin()->name);
147
  if (!tags_catalog.empty())
148
    tag_name_end = std::max(tag_name_end, tags_catalog.rbegin()->name);
149
  tag_name_end += "X";
150
  tags_history.push_back(TagId(tag_name_end, shash::Any()));
151
  tags_catalog.push_back(TagId(tag_name_end, shash::Any()));
152
153
  // Walk through both sorted lists concurrently and determine change set
154
  unsigned i_history = 0, i_catalog = 0;
155
  unsigned last_history = tags_history.size() - 1;
156
  unsigned last_catalog = tags_catalog.size() - 1;
157
  while ((i_history < last_history) || (i_catalog < last_catalog)) {
158
    TagId t_history = tags_history[i_history];
159
    TagId t_catalog = tags_catalog[i_catalog];
160
161
    // Both the same, nothing to do
162
    if (t_history == t_catalog) {
163
      i_history++;
164
      i_catalog++;
165
      continue;
166
    }
167
168
    // Same tag name for different hash, re-insert
169
    if (t_history.name == t_catalog.name) {
170
      RemoveSnapshot(t_catalog);
171
      InsertSnapshot(t_history);
172
      i_history++;
173
      i_catalog++;
174
      continue;
175
    }
176
177
    // New tag that's missing
178
    if (t_history.name < t_catalog.name) {
179
      InsertSnapshot(t_history);
180
      i_history++;
181
      continue;
182
    }
183
184
    // A tag was removed but it is still present in the catalog
185
    assert(t_history.name > t_catalog.name);
186
    RemoveSnapshot(t_catalog);
187
    i_catalog++;
188
  }
189
}
190
191
192
7
bool VirtualCatalog::ParseActions(
193
  const string &action_desc,
194
  unsigned *actions)
195
{
196
7
  *actions = kActionNone;
197
7
  if (action_desc.empty())
198
1
    return true;
199
200
6
  vector<string> action_tokens = SplitString(action_desc, ',');
201
14
  for (unsigned i = 0; i < action_tokens.size(); ++i) {
202
10
    if (action_tokens[i] == "snapshots") {
203
4
      *actions |= kActionGenerateSnapshots;
204
6
    } else if (action_tokens[i] == "remove") {
205
4
      *actions |= kActionRemove;
206
    } else {
207
2
      return false;
208
    }
209
  }
210
4
  return true;
211
}
212
213
214
void VirtualCatalog::GetSortedTagsFromHistory(vector<TagId> *tags) {
215
  UniquePtr<history::History> history(
216
    assistant_.GetHistory(swissknife::Assistant::kOpenReadOnly));
217
  vector<history::History::Tag> tags_history;
218
  bool retval = history->List(&tags_history);
219
  assert(retval);
220
  for (unsigned i = 0, l = tags_history.size(); i < l; ++i) {
221
    if ((tags_history[i].name == swissknife::CommandTag::kHeadTag) ||
222
        (tags_history[i].name == swissknife::CommandTag::kPreviousHeadTag))
223
    {
224
      continue;
225
    }
226
    tags->push_back(TagId(tags_history[i].name, tags_history[i].root_hash));
227
  }
228
  std::sort(tags->begin(), tags->end());
229
}
230
231
232
void VirtualCatalog::GetSortedTagsFromCatalog(vector<TagId> *tags) {
233
  WritableCatalog *virtual_catalog =
234
    catalog_mgr_->GetHostingCatalog(kVirtualPath);
235
  assert(virtual_catalog != NULL);
236
  Catalog::NestedCatalogList nested_catalogs =
237
    virtual_catalog->ListNestedCatalogs();
238
  for (unsigned i = 0, l = nested_catalogs.size(); i < l; ++i) {
239
    tags->push_back(TagId(GetFileName(nested_catalogs[i].mountpoint).ToString(),
240
                          nested_catalogs[i].hash));
241
  }
242
  std::sort(tags->begin(), tags->end());
243
}
244
245
246
void VirtualCatalog::InsertSnapshot(TagId tag) {
247
  LogCvmfs(kLogCatalog, kLogDebug, "add snapshot %s (%s) to virtual catalog",
248
           tag.name.c_str(), tag.hash.ToString().c_str());
249
  UniquePtr<Catalog> catalog(assistant_.GetCatalog(tag.hash,
250
                             swissknife::Assistant::kOpenReadOnly));
251
  assert(catalog.IsValid());
252
  assert(catalog->root_prefix().IsEmpty());
253
  DirectoryEntry entry_root;
254
  bool retval = catalog->LookupPath(PathString(""), &entry_root);
255
  assert(retval);
256
257
  // Add directory entry
258
  DirectoryEntryBase entry_dir = entry_root;
259
  entry_dir.name_ = NameString(tag.name);
260
  catalog_mgr_->AddDirectory(
261
    entry_dir, string(kVirtualPath) + "/" + string(kSnapshotDirectory));
262
263
  // Set "bind mount" flag
264
  WritableCatalog *virtual_catalog =
265
    catalog_mgr_->GetHostingCatalog(kVirtualPath);
266
  assert(virtual_catalog != NULL);
267
  string mountpoint =
268
    "/" + string(kVirtualPath) + "/" + string(kSnapshotDirectory) + "/" +
269
    tag.name;
270
  DirectoryEntry entry_bind_mountpoint(entry_dir);
271
  entry_bind_mountpoint.set_is_bind_mountpoint(true);
272
  virtual_catalog->UpdateEntry(entry_bind_mountpoint, mountpoint);
273
274
  // Register nested catalog
275
  uint64_t catalog_size = GetFileSize(catalog->database_path());
276
  assert(catalog_size > 0);
277
  virtual_catalog->InsertBindMountpoint(mountpoint, tag.hash, catalog_size);
278
}
279
280
281
void VirtualCatalog::Remove() {
282
  LogCvmfs(kLogCvmfs, kLogStdout, "Removing .cvmfs virtual catalog");
283
284
  // Safety check, make sure we don't remove the entire repository
285
  WritableCatalog *virtual_catalog =
286
    catalog_mgr_->GetHostingCatalog(kVirtualPath);
287
  assert(!virtual_catalog->IsRoot());
288
  DirectoryEntry entry_virtual;
289
  bool retval = catalog_mgr_->LookupPath(
290
    PathString("/" + string(kVirtualPath)), kLookupSole, &entry_virtual);
291
  assert(retval);
292
  assert(entry_virtual.IsHidden());
293
294
  RemoveRecursively(kVirtualPath);
295
  catalog_mgr_->RemoveNestedCatalog(kVirtualPath);
296
  catalog_mgr_->RemoveDirectory(kVirtualPath);
297
}
298
299
300
void VirtualCatalog::RemoveRecursively(const string &directory) {
301
  DirectoryEntryList listing;
302
  bool retval = catalog_mgr_->Listing(PathString("/" + directory), &listing);
303
  assert(retval);
304
  for (unsigned i = 0; i < listing.size(); ++i) {
305
    string this_path = directory + "/" + listing[i].name().ToString();
306
    if (listing[i].IsDirectory()) {
307
      if (!listing[i].IsBindMountpoint())
308
        RemoveRecursively(this_path);
309
      catalog_mgr_->RemoveDirectory(this_path);
310
    } else if (listing[i].IsRegular()) {
311
      assert(listing[i].name().ToString() == ".cvmfscatalog");
312
      catalog_mgr_->RemoveFile(this_path);
313
    } else {
314
      abort();
315
    }
316
  }
317
}
318
319
320
void VirtualCatalog::RemoveSnapshot(TagId tag) {
321
  LogCvmfs(kLogCatalog, kLogDebug,
322
           "remove snapshot %s (%s) from virtual catalog",
323
           tag.name.c_str(), tag.hash.ToString().c_str());
324
  string tag_dir =
325
    string(kVirtualPath) + "/" + string(kSnapshotDirectory) + "/" + tag.name;
326
  catalog_mgr_->RemoveDirectory(tag_dir);
327
328
  WritableCatalog *virtual_catalog =
329
    catalog_mgr_->GetHostingCatalog(kVirtualPath);
330
  assert(virtual_catalog != NULL);
331
  virtual_catalog->RemoveBindMountpoint("/" + tag_dir);
332
}
333
334
335
VirtualCatalog::VirtualCatalog(
336
  manifest::Manifest *m,
337
  download::DownloadManager *d,
338
  catalog::WritableCatalogManager *c,
339
  SyncParameters *p)
340
  : catalog_mgr_(c)
341
  , assistant_(d, m, p->stratum0, p->dir_temp)
342
{ }
343
344
}  // namespace catalog