GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/catalog_mgr_rw.cc
Date: 2026-08-30 02:40:36
Exec Total Coverage
Lines: 477 880 54.2%
Branches: 361 1283 28.1%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM file system.
3 */
4
5 #include "catalog_mgr_rw.h"
6
7 #include <inttypes.h>
8 #include <unistd.h>
9
10 #include <cassert>
11 #include <cstdio>
12 #include <cstdlib>
13 #include <string>
14
15 #include "catalog_balancer.h"
16 #include "catalog_rw.h"
17 #include "manifest.h"
18 #include "statistics.h"
19 #include "upload.h"
20 #include "util/exception.h"
21 #include "util/logging.h"
22 #include "util/posix.h"
23 #include "util/smalloc.h"
24
25 using namespace std; // NOLINT
26
27 namespace catalog {
28
29 1607 WritableCatalogManager::WritableCatalogManager(
30 const shash::Any &base_hash,
31 const std::string &stratum0,
32 const string &dir_temp,
33 upload::Spooler *spooler,
34 download::DownloadManager *download_manager,
35 bool enforce_limits,
36 const unsigned nested_kcatalog_limit,
37 const unsigned root_kcatalog_limit,
38 const unsigned file_mbyte_limit,
39 perf::Statistics *statistics,
40 bool is_balanceable,
41 unsigned max_weight,
42 unsigned min_weight,
43 1607 const std::string &dir_cache)
44 : SimpleCatalogManager(base_hash, stratum0, dir_temp, download_manager,
45 statistics, false, dir_cache,
46 true /* copy to tmpdir */)
47 1607 , spooler_(spooler)
48 1607 , enforce_limits_(enforce_limits)
49 1607 , nested_kcatalog_limit_(nested_kcatalog_limit)
50 1607 , root_kcatalog_limit_(root_kcatalog_limit)
51 1607 , file_mbyte_limit_(file_mbyte_limit)
52 1607 , is_balanceable_(is_balanceable)
53 1607 , max_weight_(max_weight)
54 1607 , min_weight_(min_weight)
55 1607 , balance_weight_(max_weight / 2) {
56 1607 sync_lock_ = reinterpret_cast<pthread_mutex_t *>(
57 1607 smalloc(sizeof(pthread_mutex_t)));
58 1607 int retval = pthread_mutex_init(sync_lock_, NULL);
59
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1607 times.
 1607 assert(retval == 0);
60 1607 catalog_processing_lock_ = reinterpret_cast<pthread_mutex_t *>(
61 1607 smalloc(sizeof(pthread_mutex_t)));
62 1607 retval = pthread_mutex_init(catalog_processing_lock_, NULL);
63
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1607 times.
 1607 assert(retval == 0);
64 1607 }
65
66
67 6424 WritableCatalogManager::~WritableCatalogManager() {
68 3212 pthread_mutex_destroy(sync_lock_);
69 3212 free(sync_lock_);
70 3212 pthread_mutex_destroy(catalog_processing_lock_);
71 3212 free(catalog_processing_lock_);
72 6424 }
73
74
75 /**
76 * This method is virtual in AbstractCatalogManager. It returns a new catalog
77 * structure in the form the different CatalogManagers need it.
78 * In this case it returns a stub for a WritableCatalog.
79 * @param mountpoint the mount point of the catalog stub to create
80 * @param catalog_hash the content hash of the catalog to create
81 * @param parent_catalog the parent of the catalog stub to create
82 * @return a pointer to the catalog stub structure created
83 */
84 3010 Catalog *WritableCatalogManager::CreateCatalog(const PathString &mountpoint,
85 const shash::Any &catalog_hash,
86 Catalog *parent_catalog) {
87 6020 return new WritableCatalog(mountpoint.ToString(), catalog_hash,
88
2/4
✓ Branch 1 taken 3010 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3010 times.
✗ Branch 5 not taken.
 6020 parent_catalog);
89 }
90
91
92 3010 void WritableCatalogManager::ActivateCatalog(Catalog *catalog) {
93 3010 catalog->TakeDatabaseFileOwnership();
94 3010 }
95
96
97 /**
98 * This method is invoked if we create a completely new repository.
99 * The new root catalog will already contain a root entry.
100 * It is uploaded by a Forklift to the upstream storage.
101 * @return true on success, false otherwise
102 */
103 992 manifest::Manifest *WritableCatalogManager::CreateRepository(
104 const string &dir_temp,
105 const bool volatile_content,
106 const std::string &voms_authz,
107 upload::Spooler *spooler) {
108 // Create a new root catalog at file_path
109
1/2
✓ Branch 1 taken 992 times.
✗ Branch 2 not taken.
 992 const string file_path = dir_temp + "/new_root_catalog";
110
111 992 const shash::Algorithms hash_algorithm = spooler->GetHashAlgorithm();
112
113 // A newly created catalog always needs a root entry
114 // we create and configure this here
115
1/2
✓ Branch 1 taken 992 times.
✗ Branch 2 not taken.
 992 DirectoryEntry root_entry;
116 992 root_entry.inode_ = DirectoryEntry::kInvalidInode;
117 992 root_entry.mode_ = 16877;
118 992 root_entry.size_ = 4096;
119 992 root_entry.mtime_ = time(NULL);
120 992 root_entry.uid_ = getuid();
121 992 root_entry.gid_ = getgid();
122
1/2
✓ Branch 1 taken 992 times.
✗ Branch 2 not taken.
 992 root_entry.checksum_ = shash::Any(hash_algorithm);
123 992 root_entry.linkcount_ = 2;
124
1/2
✓ Branch 2 taken 992 times.
✗ Branch 3 not taken.
 992 const string root_path = "";
125
126 // Create the database schema and the initial root entry
127 {
128 const std::unique_ptr<CatalogDatabase> new_clg_db(
129
1/2
✓ Branch 1 taken 992 times.
✗ Branch 2 not taken.
 992 CatalogDatabase::Create(file_path));
130 992 if (new_clg_db.get() == nullptr
131
4/8
✓ Branch 0 taken 992 times.
✗ Branch 1 not taken.
✓ Branch 4 taken 992 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 992 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 992 times.
 992 || !new_clg_db->InsertInitialValues(root_path, volatile_content,
132 voms_authz, root_entry)) {
133 ✗ LogCvmfs(kLogCatalog, kLogStderr, "creation of catalog '%s' failed",
134 file_path.c_str());
135 ✗ return NULL;
136 }
137
1/2
✓ Branch 1 taken 992 times.
✗ Branch 2 not taken.
 992 }
138
139 // Compress root catalog;
140
1/2
✓ Branch 1 taken 992 times.
✗ Branch 2 not taken.
 992 const int64_t catalog_size = GetFileSize(file_path);
141
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 992 times.
 992 if (catalog_size < 0) {
142 ✗ unlink(file_path.c_str());
143 ✗ return NULL;
144 }
145
1/2
✓ Branch 1 taken 992 times.
✗ Branch 2 not taken.
 992 const string file_path_compressed = file_path + ".compressed";
146
1/2
✓ Branch 1 taken 992 times.
✗ Branch 2 not taken.
 992 shash::Any hash_catalog(hash_algorithm, shash::kSuffixCatalog);
147
1/2
✓ Branch 1 taken 992 times.
✗ Branch 2 not taken.
 992 const bool retval = zlib::CompressPath2Path(file_path, file_path_compressed,
148 &hash_catalog);
149
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 992 times.
 992 if (!retval) {
150 ✗ LogCvmfs(kLogCatalog, kLogStderr, "compression of catalog '%s' failed",
151 file_path.c_str());
152 ✗ unlink(file_path.c_str());
153 ✗ return NULL;
154 }
155 992 unlink(file_path.c_str());
156
157 // Create manifest
158
1/2
✓ Branch 1 taken 992 times.
✗ Branch 2 not taken.
 992 const string manifest_path = dir_temp + "/manifest";
159 manifest::Manifest *manifest = new manifest::Manifest(hash_catalog,
160
3/6
✓ Branch 2 taken 992 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 992 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 992 times.
✗ Branch 9 not taken.
 992 catalog_size, "");
161
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 992 times.
 992 if (!voms_authz.empty()) {
162 ✗ manifest->set_has_alt_catalog_path(true);
163 }
164
165 // Upload catalog
166
3/6
✓ Branch 1 taken 992 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 992 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 992 times.
✗ Branch 8 not taken.
 992 spooler->Upload(file_path_compressed, "data/" + hash_catalog.MakePath());
167
1/2
✓ Branch 1 taken 992 times.
✗ Branch 2 not taken.
 992 spooler->WaitForUpload();
168 992 unlink(file_path_compressed.c_str());
169
2/4
✓ Branch 1 taken 992 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 992 times.
 992 if (spooler->GetNumberOfErrors() > 0) {
170 ✗ LogCvmfs(kLogCatalog, kLogStderr, "failed to commit catalog %s",
171 file_path_compressed.c_str());
172 ✗ delete manifest;
173 ✗ return NULL;
174 }
175
176 992 return manifest;
177 992 }
178
179
180 /**
181 * Retrieve the catalog containing the given path.
182 * Other than AbstractCatalogManager::FindCatalog() this mounts nested
183 * catalogs if necessary and returns WritableCatalog objects.
184 * Furthermore it optionally returns the looked-up DirectoryEntry.
185 *
186 * @param path the path to look for
187 * @param result the retrieved catalog (as a pointer)
188 * @param dirent is set to looked up DirectoryEntry for 'path' if non-NULL
189 * @return true if catalog was found
190 */
191 11792 bool WritableCatalogManager::FindCatalog(const string &path,
192 WritableCatalog **result,
193 DirectoryEntry *dirent) {
194
1/2
✓ Branch 1 taken 11792 times.
✗ Branch 2 not taken.
 11792 const PathString ps_path(path);
195
196
1/2
✓ Branch 1 taken 11792 times.
✗ Branch 2 not taken.
 11792 Catalog *best_fit = AbstractCatalogManager<Catalog>::FindCatalog(ps_path);
197
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11792 times.
 11792 assert(best_fit != NULL);
198 11792 Catalog *catalog = NULL;
199
1/2
✓ Branch 1 taken 11792 times.
✗ Branch 2 not taken.
 11792 const bool retval = MountSubtree(ps_path, best_fit, true /* is_listable */,
200 &catalog);
201
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11792 times.
 11792 if (!retval)
202 ✗ return false;
203
204 11792 *result = static_cast<WritableCatalog *>(catalog);
205
206
1/2
✓ Branch 1 taken 11792 times.
✗ Branch 2 not taken.
 11792 catalog::DirectoryEntry dummy;
207
2/2
✓ Branch 0 taken 5736 times.
✓ Branch 1 taken 6056 times.
 11792 if (NULL == dirent) {
208 5736 dirent = &dummy;
209 }
210
1/2
✓ Branch 1 taken 11792 times.
✗ Branch 2 not taken.
 11792 const bool found = catalog->LookupPath(ps_path, dirent);
211
6/8
✓ Branch 0 taken 11763 times.
✓ Branch 1 taken 29 times.
✓ Branch 3 taken 11763 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 11763 times.
✓ Branch 7 taken 29 times.
✓ Branch 8 taken 11763 times.
 11792 if (!found || !catalog->IsWritable())
212 29 return false;
213
214 11763 return true;
215 11792 }
216
217
218 ✗ WritableCatalog *WritableCatalogManager::GetHostingCatalog(
219 const std::string &path) {
220 ✗ WritableCatalog *result = NULL;
221 ✗ const bool retval = FindCatalog(MakeRelativePath(path), &result, NULL);
222 ✗ if (!retval)
223 ✗ return NULL;
224 ✗ return result;
225 }
226
227
228 /**
229 * Remove the given file from the catalogs.
230 * @param file_path the full path to the file to be removed
231 * @return true on success, false otherwise
232 */
233 176 void WritableCatalogManager::RemoveFile(const std::string &path) {
234
1/2
✓ Branch 1 taken 176 times.
✗ Branch 2 not taken.
 176 const string file_path = MakeRelativePath(path);
235
236 176 SyncLock();
237 176 WritableCatalog *catalog = NULL;
238
1/2
✓ Branch 1 taken 176 times.
✗ Branch 2 not taken.
 176 DirectoryEntry entry;
239
2/4
✓ Branch 1 taken 176 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 176 times.
✗ Branch 4 not taken.
 176 if (FindCatalog(file_path, &catalog, &entry)) {
240
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 176 times.
 176 if (entry.IsBundleTrigger()) {
241 ✗ catalog->RemoveEntry(GetParentPath(file_path) + "/.cvmfsbundle-"
242 ✗ + GetFileName(file_path));
243 }
244
1/2
✓ Branch 1 taken 176 times.
✗ Branch 2 not taken.
 176 catalog->RemoveEntry(file_path);
245 }
246 176 SyncUnlock();
247
248
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 176 times.
 176 if (catalog == NULL) {
249 ✗ PANIC(kLogStderr, "catalog for file '%s' cannot be found",
250 file_path.c_str());
251 }
252 176 }
253
254
255 /**
256 * Remove the given directory from the catalogs.
257 * @param directory_path the full path to the directory to be removed
258 * @return true on success, false otherwise
259 */
260 188 void WritableCatalogManager::RemoveDirectory(const std::string &path) {
261
1/2
✓ Branch 1 taken 188 times.
✗ Branch 2 not taken.
 188 const string directory_path = MakeRelativePath(path);
262
1/2
✓ Branch 1 taken 188 times.
✗ Branch 2 not taken.
 188 const string parent_path = GetParentPath(directory_path);
263
264 188 SyncLock();
265 WritableCatalog *catalog;
266
1/2
✓ Branch 1 taken 188 times.
✗ Branch 2 not taken.
 188 DirectoryEntry parent_entry;
267
2/4
✓ Branch 1 taken 188 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 188 times.
 188 if (!FindCatalog(parent_path, &catalog, &parent_entry)) {
268 ✗ PANIC(kLogStderr, "catalog for directory '%s' cannot be found",
269 directory_path.c_str());
270 }
271
272 188 parent_entry.set_linkcount(parent_entry.linkcount() - 1);
273
274
1/2
✓ Branch 1 taken 188 times.
✗ Branch 2 not taken.
 188 catalog->RemoveEntry(directory_path);
275
1/2
✓ Branch 1 taken 188 times.
✗ Branch 2 not taken.
 188 catalog->UpdateEntry(parent_entry, parent_path);
276
2/2
✓ Branch 1 taken 60 times.
✓ Branch 2 taken 128 times.
 188 if (parent_entry.IsNestedCatalogRoot()) {
277
1/2
✓ Branch 2 taken 60 times.
✗ Branch 3 not taken.
 60 LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating transition point %s",
278 parent_path.c_str());
279 WritableCatalog *parent_catalog = reinterpret_cast<WritableCatalog *>(
280 60 catalog->parent());
281 60 parent_entry.set_is_nested_catalog_mountpoint(true);
282 60 parent_entry.set_is_nested_catalog_root(false);
283
1/2
✓ Branch 1 taken 60 times.
✗ Branch 2 not taken.
 60 parent_catalog->UpdateEntry(parent_entry, parent_path);
284 }
285 188 SyncUnlock();
286 188 }
287
288 /**
289 * Clone the file called `source` changing its name into `destination`, the
290 * source file is keep intact.
291 * @params destination, the name of the new file, complete path
292 * @params source, the name of the file to clone, which must be already in the
293 * repository
294 * @params fail_if_source_missing, when true (default) abort if the source is
295 * not in the catalog; when false, skip the clone and return false instead.
296 * This is used by the tarball ingestion engine to tolerate hardlinks whose
297 * target is not part of the same archive (e.g. cross-layer hardlinks in OCI
298 * image layers).
299 * @return true if the file was cloned, false if the source was missing and
300 * fail_if_source_missing was false
301 */
302 ✗ bool WritableCatalogManager::Clone(const std::string destination,
303 const std::string source,
304 const bool fail_if_source_missing) {
305 ✗ const std::string relative_source = MakeRelativePath(source);
306
307 ✗ DirectoryEntry source_dirent;
308 ✗ if (!LookupPath(relative_source, kLookupDefault, &source_dirent)) {
309 ✗ if (fail_if_source_missing) {
310 ✗ PANIC(kLogStderr, "catalog for file '%s' cannot be found, aborting",
311 source.c_str());
312 }
313 // The caller opted out of aborting and is expected to handle the missing
314 // source (e.g. by materializing a replacement file), so only log at debug
315 // level here to avoid duplicate user-facing warnings.
316 ✗ LogCvmfs(kLogCatalog, kLogDebug,
317 "catalog for clone source '%s' cannot be found, not cloning "
318 "to '%s'",
319 source.c_str(), destination.c_str());
320 ✗ return false;
321 }
322 ✗ if (source_dirent.IsDirectory()) {
323 ✗ PANIC(kLogStderr, "Trying to clone a directory: '%s', aborting",
324 source.c_str());
325 }
326
327 // if the file is already there we remove it and we add it back
328 ✗ DirectoryEntry check_dirent;
329 ✗ const bool destination_already_present = LookupPath(
330 ✗ MakeRelativePath(destination), kLookupDefault, &check_dirent);
331 ✗ if (destination_already_present) {
332 ✗ this->RemoveFile(destination);
333 }
334
335 ✗ DirectoryEntry destination_dirent(source_dirent);
336 ✗ std::string destination_dirname;
337 ✗ std::string destination_filename;
338 ✗ SplitPath(destination, &destination_dirname, &destination_filename);
339
340 ✗ destination_dirent.name_.Assign(
341 ✗ NameString(destination_filename.c_str(), destination_filename.length()));
342
343 // TODO(jblomer): clone is used by tarball engine and should eventually
344 // support extended attributes
345 ✗ this->AddFile(destination_dirent, empty_xattrs, destination_dirname);
346 ✗ return true;
347 }
348
349
350 /**
351 * Copies an entire directory tree from the existing from_dir to the
352 * non-existing to_dir. The destination's parent directory must exist. On the
353 * catalog level, the new entries will be identical to the old ones except
354 * for their path hash fields.
355 */
356 290 void WritableCatalogManager::CloneTree(const std::string &from_dir,
357 const std::string &to_dir) {
358 // Sanitize input paths
359
6/6
✓ Branch 1 taken 232 times.
✓ Branch 2 taken 58 times.
✓ Branch 4 taken 29 times.
✓ Branch 5 taken 203 times.
✓ Branch 6 taken 87 times.
✓ Branch 7 taken 203 times.
 290 if (from_dir.empty() || to_dir.empty())
360 87 PANIC(kLogStderr, "clone tree from or to root impossible");
361
362
1/2
✓ Branch 1 taken 203 times.
✗ Branch 2 not taken.
 203 const std::string relative_source = MakeRelativePath(from_dir);
363
1/2
✓ Branch 1 taken 203 times.
✗ Branch 2 not taken.
 203 const std::string relative_dest = MakeRelativePath(to_dir);
364
365
2/2
✓ Branch 1 taken 29 times.
✓ Branch 2 taken 174 times.
 203 if (relative_source == relative_dest) {
366 29 PANIC(kLogStderr, "cannot clone tree into itself ('%s')", to_dir.c_str());
367 }
368
4/7
✓ Branch 1 taken 174 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 174 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 29 times.
✓ Branch 8 taken 145 times.
 174 if (HasPrefix(relative_dest, relative_source + "/", false /*ignore_case*/)) {
369 29 PANIC(kLogStderr,
370 "cannot clone tree into sub directory of source '%s' --> '%s'",
371 from_dir.c_str(), to_dir.c_str());
372 }
373
374
1/2
✓ Branch 1 taken 145 times.
✗ Branch 2 not taken.
 145 DirectoryEntry source_dirent;
375
3/4
✓ Branch 1 taken 145 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 29 times.
✓ Branch 4 taken 116 times.
 145 if (!LookupPath(relative_source, kLookupDefault, &source_dirent)) {
376 29 PANIC(kLogStderr, "path '%s' cannot be found, aborting", from_dir.c_str());
377 }
378
2/2
✓ Branch 1 taken 29 times.
✓ Branch 2 taken 87 times.
 116 if (!source_dirent.IsDirectory()) {
379 29 PANIC(kLogStderr, "CloneTree: source '%s' not a directory, aborting",
380 from_dir.c_str());
381 }
382
383
1/2
✓ Branch 1 taken 87 times.
✗ Branch 2 not taken.
 87 DirectoryEntry dest_dirent;
384
3/4
✓ Branch 1 taken 87 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 29 times.
✓ Branch 4 taken 58 times.
 87 if (LookupPath(relative_dest, kLookupDefault, &dest_dirent)) {
385 29 PANIC(kLogStderr, "destination '%s' exists, aborting", to_dir.c_str());
386 }
387
388
1/2
✓ Branch 1 taken 58 times.
✗ Branch 2 not taken.
 58 const std::string dest_parent = GetParentPath(relative_dest);
389
1/2
✓ Branch 1 taken 58 times.
✗ Branch 2 not taken.
 58 DirectoryEntry dest_parent_dirent;
390
3/4
✓ Branch 1 taken 58 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 29 times.
✓ Branch 4 taken 29 times.
 58 if (!LookupPath(dest_parent, kLookupDefault, &dest_parent_dirent)) {
391 29 PANIC(kLogStderr, "destination '%s' not on a known path, aborting",
392 to_dir.c_str());
393 }
394
395
2/4
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 29 times.
✗ Branch 5 not taken.
 29 CloneTreeImpl(PathString(from_dir),
396
1/2
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
 58 GetParentPath(to_dir),
397
2/4
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 29 times.
✗ Branch 5 not taken.
 58 NameString(GetFileName(to_dir)));
398 609 }
399
400
401 /**
402 * Called from CloneTree(), assumes that from_dir and to_dir are sufficiently
403 * sanitized
404 */
405 232 void WritableCatalogManager::CloneTreeImpl(const PathString &source_dir,
406 const std::string &dest_parent_dir,
407 const NameString &dest_name) {
408
1/2
✓ Branch 4 taken 232 times.
✗ Branch 5 not taken.
 232 LogCvmfs(kLogCatalog, kLogDebug, "cloning %s --> %s/%s", source_dir.c_str(),
409
1/2
✓ Branch 1 taken 232 times.
✗ Branch 2 not taken.
 464 dest_parent_dir.c_str(), dest_name.ToString().c_str());
410
3/6
✓ Branch 1 taken 232 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 232 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 232 times.
✗ Branch 8 not taken.
 464 const PathString relative_source(MakeRelativePath(source_dir.ToString()));
411
412
1/2
✓ Branch 1 taken 232 times.
✗ Branch 2 not taken.
 232 DirectoryEntry source_dirent;
413
1/2
✓ Branch 1 taken 232 times.
✗ Branch 2 not taken.
 232 bool retval = LookupPath(relative_source, kLookupDefault, &source_dirent);
414
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 232 times.
 232 assert(retval);
415
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 232 times.
 232 assert(!source_dirent.IsBindMountpoint());
416
417
1/2
✓ Branch 1 taken 232 times.
✗ Branch 2 not taken.
 232 DirectoryEntry dest_dirent(source_dirent);
418
1/2
✓ Branch 1 taken 232 times.
✗ Branch 2 not taken.
 232 dest_dirent.name_.Assign(dest_name);
419 // Just in case, reset the nested catalog markers
420 232 dest_dirent.set_is_nested_catalog_mountpoint(false);
421 232 dest_dirent.set_is_nested_catalog_root(false);
422
423 232 XattrList xattrs;
424
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 232 times.
 232 if (source_dirent.HasXattrs()) {
425 ✗ retval = LookupXattrs(relative_source, &xattrs);
426 ✗ assert(retval);
427 }
428
1/2
✓ Branch 1 taken 232 times.
✗ Branch 2 not taken.
 232 AddDirectory(dest_dirent, xattrs, dest_parent_dir);
429
430
1/2
✓ Branch 1 taken 232 times.
✗ Branch 2 not taken.
 232 std::string dest_dir = dest_parent_dir;
431
2/2
✓ Branch 1 taken 203 times.
✓ Branch 2 taken 29 times.
 232 if (!dest_dir.empty())
432
1/2
✓ Branch 1 taken 203 times.
✗ Branch 2 not taken.
 203 dest_dir.push_back('/');
433
2/4
✓ Branch 1 taken 232 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 232 times.
✗ Branch 5 not taken.
 232 dest_dir += dest_name.ToString();
434 232 if (source_dirent.IsNestedCatalogRoot()
435
5/6
✓ Branch 0 taken 145 times.
✓ Branch 1 taken 87 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 145 times.
✓ Branch 5 taken 87 times.
✓ Branch 6 taken 145 times.
 232 || source_dirent.IsNestedCatalogMountpoint()) {
436
1/2
✓ Branch 1 taken 87 times.
✗ Branch 2 not taken.
 87 CreateNestedCatalog(dest_dir);
437 }
438
439 232 DirectoryEntryList ls;
440
1/2
✓ Branch 1 taken 232 times.
✗ Branch 2 not taken.
 232 retval = Listing(relative_source, &ls, false /* expand_symlink */);
441
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 232 times.
 232 assert(retval);
442
2/2
✓ Branch 1 taken 493 times.
✓ Branch 2 taken 232 times.
 725 for (unsigned i = 0; i < ls.size(); ++i) {
443
1/2
✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
 493 PathString sub_path(source_dir);
444
2/4
✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 493 times.
 493 assert(!sub_path.IsEmpty());
445
1/2
✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
 493 sub_path.Append("/", 1);
446
3/6
✓ Branch 2 taken 493 times.
✗ Branch 3 not taken.
✓ Branch 7 taken 493 times.
✗ Branch 8 not taken.
✓ Branch 11 taken 493 times.
✗ Branch 12 not taken.
 493 sub_path.Append(ls[i].name().GetChars(), ls[i].name().GetLength());
447
448
2/2
✓ Branch 2 taken 203 times.
✓ Branch 3 taken 290 times.
 493 if (ls[i].IsDirectory()) {
449
2/4
✓ Branch 2 taken 203 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 203 times.
✗ Branch 6 not taken.
 203 CloneTreeImpl(sub_path, dest_dir, ls[i].name());
450 203 continue;
451 }
452
453 // We break hard-links during cloning
454 290 ls[i].set_hardlink_group(0);
455 290 ls[i].set_linkcount(1);
456
457 290 xattrs.Clear();
458
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 290 times.
 290 if (ls[i].HasXattrs()) {
459 ✗ retval = LookupXattrs(sub_path, &xattrs);
460 ✗ assert(retval);
461 }
462
463
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 290 times.
 290 if (ls[i].IsChunkedFile()) {
464 ✗ FileChunkList chunks;
465 const std::string relative_sub_path = MakeRelativePath(
466 ✗ sub_path.ToString());
467 ✗ retval = ListFileChunks(PathString(relative_sub_path),
468 ✗ ls[i].hash_algorithm(), &chunks);
469 ✗ assert(retval);
470 ✗ AddChunkedFile(ls[i], xattrs, dest_dir, chunks);
471 ✗ } else {
472
1/2
✓ Branch 2 taken 290 times.
✗ Branch 3 not taken.
 290 AddFile(ls[i], xattrs, dest_dir);
473 }
474
2/2
✓ Branch 1 taken 290 times.
✓ Branch 2 taken 203 times.
 493 }
475 232 }
476
477
478 /**
479 * Add a new directory to the catalogs.
480 * @param entry a DirectoryEntry structure describing the new directory
481 * @param parent_directory the absolute path of the directory containing the
482 * directory to be created
483 * @return true on success, false otherwise
484 */
485 4575 void WritableCatalogManager::AddDirectory(const DirectoryEntryBase &entry,
486 const XattrList &xattrs,
487 const std::string &parent_directory) {
488
1/2
✓ Branch 1 taken 4575 times.
✗ Branch 2 not taken.
 4575 const string parent_path = MakeRelativePath(parent_directory);
489
1/2
✓ Branch 1 taken 4575 times.
✗ Branch 2 not taken.
 4575 string directory_path = parent_path + "/";
490
3/6
✓ Branch 1 taken 4575 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 4575 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 4575 times.
✗ Branch 10 not taken.
 4575 directory_path.append(entry.name().GetChars(), entry.name().GetLength());
491
492 4575 SyncLock();
493 WritableCatalog *catalog;
494
1/2
✓ Branch 1 taken 4575 times.
✗ Branch 2 not taken.
 4575 DirectoryEntry parent_entry;
495
2/4
✓ Branch 1 taken 4575 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 4575 times.
 4575 if (!FindCatalog(parent_path, &catalog, &parent_entry)) {
496 ✗ PANIC(kLogStderr, "catalog for directory '%s' cannot be found",
497 directory_path.c_str());
498 }
499
500
1/2
✓ Branch 1 taken 4575 times.
✗ Branch 2 not taken.
 4575 DirectoryEntry fixed_hardlink_count(entry);
501 4575 fixed_hardlink_count.set_linkcount(2);
502
1/2
✓ Branch 1 taken 4575 times.
✗ Branch 2 not taken.
 4575 catalog->AddEntry(fixed_hardlink_count, xattrs, directory_path, parent_path);
503
504 4575 parent_entry.set_linkcount(parent_entry.linkcount() + 1);
505
1/2
✓ Branch 1 taken 4575 times.
✗ Branch 2 not taken.
 4575 catalog->UpdateEntry(parent_entry, parent_path);
506
2/2
✓ Branch 1 taken 87 times.
✓ Branch 2 taken 4488 times.
 4575 if (parent_entry.IsNestedCatalogRoot()) {
507
1/2
✓ Branch 2 taken 87 times.
✗ Branch 3 not taken.
 87 LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating transition point %s",
508 parent_path.c_str());
509 WritableCatalog *parent_catalog = reinterpret_cast<WritableCatalog *>(
510 87 catalog->parent());
511 87 parent_entry.set_is_nested_catalog_mountpoint(true);
512 87 parent_entry.set_is_nested_catalog_root(false);
513
1/2
✓ Branch 1 taken 87 times.
✗ Branch 2 not taken.
 87 parent_catalog->UpdateEntry(parent_entry, parent_path);
514 }
515 4575 SyncUnlock();
516 4575 }
517
518 /**
519 * Add a new file to the catalogs.
520 * @param entry a DirectoryEntry structure describing the new file
521 * @param parent_directory the absolute path of the directory containing the
522 * file to be created
523 * @return true on success, false otherwise
524 */
525 5372 void WritableCatalogManager::AddFile(const DirectoryEntry &entry,
526 const XattrList &xattrs,
527 const std::string &parent_directory) {
528
1/2
✓ Branch 1 taken 5372 times.
✗ Branch 2 not taken.
 5372 const string parent_path = MakeRelativePath(parent_directory);
529
1/2
✓ Branch 1 taken 5372 times.
✗ Branch 2 not taken.
 5372 const string file_path = entry.GetFullPath(parent_path);
530
531 5372 SyncLock();
532 WritableCatalog *catalog;
533
2/4
✓ Branch 1 taken 5372 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 5372 times.
 5372 if (!FindCatalog(parent_path, &catalog)) {
534 ✗ PANIC(kLogStderr, "catalog for file '%s' cannot be found",
535 file_path.c_str());
536 }
537
538
4/6
✓ Branch 1 taken 5244 times.
✓ Branch 2 taken 128 times.
✓ Branch 4 taken 5244 times.
✗ Branch 5 not taken.
✗ Branch 8 not taken.
✓ Branch 9 taken 5244 times.
 5372 assert(!entry.IsRegular() || entry.IsChunkedFile()
539 || !entry.checksum().IsNull());
540
3/4
✓ Branch 1 taken 128 times.
✓ Branch 2 taken 5244 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 128 times.
 5372 assert(entry.IsRegular() || !entry.IsExternalFile());
541
542 // check if file is too big
543
1/2
✓ Branch 1 taken 5372 times.
✗ Branch 2 not taken.
 5372 const unsigned mbytes = entry.size() / (1024 * 1024);
544
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 5372 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 5372 if ((file_mbyte_limit_ > 0) && (mbytes > file_mbyte_limit_)) {
545 ✗ LogCvmfs(kLogCatalog, kLogStderr,
546 "%s: file at %s is larger than %u megabytes (%u). "
547 "CernVM-FS works best with small files. "
548 "Please remove the file or increase the limit.",
549 ✗ enforce_limits_ ? "FATAL" : "WARNING", file_path.c_str(),
550 file_mbyte_limit_, mbytes);
551 ✗ if (enforce_limits_)
552 ✗ PANIC(kLogStderr, "file at %s is larger than %u megabytes (%u).",
553 file_path.c_str(), file_mbyte_limit_, mbytes);
554 }
555
556
1/2
✓ Branch 1 taken 5372 times.
✗ Branch 2 not taken.
 5372 catalog->AddEntry(entry, xattrs, file_path, parent_path);
557 5372 SyncUnlock();
558 5372 }
559
560
561 ✗ void WritableCatalogManager::AddChunkedFile(const DirectoryEntryBase &entry,
562 const XattrList &xattrs,
563 const std::string &parent_directory,
564 const FileChunkList &file_chunks) {
565 ✗ assert(file_chunks.size() > 0);
566
567 ✗ DirectoryEntry full_entry(entry);
568 ✗ full_entry.set_is_chunked_file(true);
569
570 ✗ AddFile(full_entry, xattrs, parent_directory);
571
572 ✗ const string parent_path = MakeRelativePath(parent_directory);
573 ✗ const string file_path = entry.GetFullPath(parent_path);
574
575 ✗ SyncLock();
576 WritableCatalog *catalog;
577 ✗ if (!FindCatalog(parent_path, &catalog)) {
578 ✗ PANIC(kLogStderr, "catalog for file '%s' cannot be found",
579 file_path.c_str());
580 }
581
582 ✗ for (unsigned i = 0; i < file_chunks.size(); ++i) {
583 ✗ catalog->AddFileChunk(file_path, *file_chunks.AtPtr(i));
584 }
585 ✗ SyncUnlock();
586 }
587
588 /**
589 * Reads the entry given by file_path and sets the bundle trigger flag
590 * accordingly. If new_value is true, the regular file at file_path must exist.
591 * If new_value is false, a missing file_path is ignored (otherwise it would
592 * be hard to get rid of dangling bundle trigger markers).
593 */
594 ✗ void WritableCatalogManager::UpdateBundleTrigger(const std::string &file_path,
595 bool new_value) {
596 ✗ SyncLock();
597
598 ✗ WritableCatalog *catalog = NULL;
599 ✗ DirectoryEntry entry;
600 ✗ if (!FindCatalog(file_path, &catalog, &entry)) {
601 ✗ SyncUnlock();
602
603 ✗ if (new_value) {
604 ✗ PANIC(kLogStderr, "failed to find catalog of %s", file_path.c_str());
605 }
606
607 ✗ LogCvmfs(kLogCatalog, kLogDebug, "dangling bundle trigger %s",
608 file_path.c_str());
609 ✗ return;
610 }
611
612 ✗ if (new_value && !entry.IsRegular()) {
613 ✗ SyncUnlock();
614 ✗ PANIC(kLogStderr, "failed to set bundle trigger on non-regular file %s",
615 file_path.c_str());
616 }
617
618 ✗ entry.set_is_bundle_trigger(new_value);
619 ✗ catalog->UpdateEntry(entry, file_path);
620
621 ✗ SyncUnlock();
622 }
623
624
625 /**
626 * Add a hardlink group to the catalogs.
627 * @param entries a list of DirectoryEntries describing the new files
628 * @param parent_directory the absolute path of the directory containing the
629 * files to be created
630 * @return true on success, false otherwise
631 */
632 ✗ void WritableCatalogManager::AddHardlinkGroup(
633 const DirectoryEntryBaseList &entries,
634 const XattrList &xattrs,
635 const std::string &parent_directory,
636 const FileChunkList &file_chunks) {
637 ✗ assert(entries.size() >= 1);
638 ✗ assert(file_chunks.IsEmpty() || entries[0].IsRegular());
639 ✗ if (entries.size() == 1) {
640 ✗ DirectoryEntry fix_linkcount(entries[0]);
641 ✗ fix_linkcount.set_linkcount(1);
642 ✗ if (file_chunks.IsEmpty())
643 ✗ return AddFile(fix_linkcount, xattrs, parent_directory);
644 ✗ return AddChunkedFile(fix_linkcount, xattrs, parent_directory, file_chunks);
645 }
646
647 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "adding hardlink group %s/%s",
648 ✗ parent_directory.c_str(), entries[0].name().c_str());
649
650 // Hardlink groups have to reside in the same directory.
651 // Therefore we only have one parent directory here
652 ✗ const string parent_path = MakeRelativePath(parent_directory);
653
654 // check if hard link is too big
655 ✗ const unsigned mbytes = entries[0].size() / (1024 * 1024);
656 ✗ if ((file_mbyte_limit_ > 0) && (mbytes > file_mbyte_limit_)) {
657 ✗ LogCvmfs(kLogCatalog, kLogStderr,
658 "%s: hard link at %s is larger than %u megabytes (%u). "
659 "CernVM-FS works best with small files. "
660 "Please remove the file or increase the limit.",
661 ✗ enforce_limits_ ? "FATAL" : "WARNING",
662 ✗ (parent_path + entries[0].name().ToString()).c_str(),
663 file_mbyte_limit_, mbytes);
664 ✗ if (enforce_limits_)
665 ✗ PANIC(kLogStderr, "hard link at %s is larger than %u megabytes (%u)",
666 (parent_path + entries[0].name().ToString()).c_str(),
667 file_mbyte_limit_, mbytes);
668 }
669
670 ✗ SyncLock();
671 WritableCatalog *catalog;
672 ✗ if (!FindCatalog(parent_path, &catalog)) {
673 ✗ PANIC(kLogStderr,
674 "catalog for hardlink group containing '%s' cannot be found",
675 parent_path.c_str());
676 }
677
678 // Get a valid hardlink group id for the catalog the group will end up in
679 // TODO(unknown): Compaction
680 ✗ const uint32_t new_group_id = catalog->GetMaxLinkId() + 1;
681 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "hardlink group id %u issued",
682 new_group_id);
683 ✗ assert(new_group_id > 0);
684
685 // Add the file entries to the catalog
686 ✗ for (DirectoryEntryBaseList::const_iterator i = entries.begin(),
687 ✗ iEnd = entries.end();
688 ✗ i != iEnd;
689 ✗ ++i) {
690 ✗ string file_path = parent_path + "/";
691 ✗ file_path.append(i->name().GetChars(), i->name().GetLength());
692
693 // create a fully fledged DirectoryEntry to add the hardlink group to it
694 // which is CVMFS specific meta data.
695 ✗ DirectoryEntry hardlink(*i);
696 ✗ hardlink.set_hardlink_group(new_group_id);
697 ✗ hardlink.set_linkcount(entries.size());
698 ✗ hardlink.set_is_chunked_file(!file_chunks.IsEmpty());
699
700 ✗ catalog->AddEntry(hardlink, xattrs, file_path, parent_path);
701 ✗ if (hardlink.IsChunkedFile()) {
702 ✗ for (unsigned i = 0; i < file_chunks.size(); ++i) {
703 ✗ catalog->AddFileChunk(file_path, *file_chunks.AtPtr(i));
704 }
705 }
706 }
707 ✗ SyncUnlock();
708 }
709
710
711 ✗ void WritableCatalogManager::ShrinkHardlinkGroup(const string &remove_path) {
712 ✗ const string relative_path = MakeRelativePath(remove_path);
713
714 ✗ SyncLock();
715 WritableCatalog *catalog;
716 ✗ if (!FindCatalog(relative_path, &catalog)) {
717 ✗ PANIC(kLogStderr,
718 "catalog for hardlink group containing '%s' cannot be found",
719 remove_path.c_str());
720 }
721
722 ✗ catalog->IncLinkcount(relative_path, -1);
723 ✗ SyncUnlock();
724 }
725
726
727 /**
728 * Update entry meta data (mode, owner, ...).
729 * CVMFS specific meta data (i.e. nested catalog transition points) are NOT
730 * changed by this method, although transition points intrinsics are taken into
731 * account, to keep nested catalogs consistent.
732 * @param entry the directory entry to be touched
733 * @param path the path of the directory entry to be touched
734 */
735 49 void WritableCatalogManager::TouchDirectory(const DirectoryEntryBase &entry,
736 const XattrList &xattrs,
737 const std::string &directory_path) {
738
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 49 times.
 49 assert(entry.IsDirectory());
739
740
1/2
✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
 49 const string entry_path = MakeRelativePath(directory_path);
741
1/2
✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
 49 const string parent_path = GetParentPath(entry_path);
742
743 49 SyncLock();
744 // find the catalog to be updated
745 WritableCatalog *catalog;
746
2/4
✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 49 times.
 49 if (!FindCatalog(parent_path, &catalog)) {
747 ✗ PANIC(kLogStderr, "catalog for entry '%s' cannot be found",
748 entry_path.c_str());
749 }
750
751
1/2
✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
 49 catalog->TouchEntry(entry, xattrs, entry_path);
752
753 // since we deal with a directory here, we might just touch a
754 // nested catalog transition point. If this is the case we would need to
755 // update two catalog entries:
756 // * the nested catalog MOUNTPOINT in the parent catalog
757 // * the nested catalog ROOT in the nested catalog
758
759 // first check if we really have a nested catalog transition point
760
1/2
✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
 49 catalog::DirectoryEntry potential_transition_point;
761
1/2
✓ Branch 3 taken 49 times.
✗ Branch 4 not taken.
 49 const PathString transition_path(entry_path.data(), entry_path.length());
762
1/2
✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
 49 bool retval = catalog->LookupPath(transition_path,
763 &potential_transition_point);
764
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
 49 assert(retval);
765
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 49 times.
 49 if (potential_transition_point.IsNestedCatalogMountpoint()) {
766 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating transition point at %s",
767 entry_path.c_str());
768
769 // find and mount nested catalog associated to this transition point
770 ✗ shash::Any nested_hash;
771 uint64_t nested_size;
772 ✗ retval = catalog->FindNested(transition_path, &nested_hash, &nested_size);
773 ✗ assert(retval);
774 Catalog *nested_catalog;
775 ✗ nested_catalog = MountCatalog(transition_path, nested_hash, catalog);
776 ✗ assert(nested_catalog != NULL);
777
778 // update nested catalog root in the child catalog
779 reinterpret_cast<WritableCatalog *>(nested_catalog)
780 ✗ ->TouchEntry(entry, xattrs, entry_path);
781 }
782
783 49 SyncUnlock();
784 49 }
785
786
787 /**
788 * Create a new nested catalog. Includes moving all entries belonging there
789 * from it's parent catalog.
790 * @param mountpoint the path of the directory to become a nested root
791 * @return true on success, false otherwise
792 */
793 1057 void WritableCatalogManager::CreateNestedCatalog(
794 const std::string &mountpoint) {
795
1/2
✓ Branch 1 taken 1057 times.
✗ Branch 2 not taken.
 1057 const string nested_root_path = MakeRelativePath(mountpoint);
796
1/2
✓ Branch 1 taken 1057 times.
✗ Branch 2 not taken.
 1057 const PathString ps_nested_root_path(nested_root_path);
797
798 1057 SyncLock();
799 // Find the catalog currently containing the directory structure, which
800 // will be represented as a new nested catalog and its root-entry/mountpoint
801 // along the way
802 1057 WritableCatalog *old_catalog = NULL;
803
1/2
✓ Branch 1 taken 1057 times.
✗ Branch 2 not taken.
 1057 DirectoryEntry new_root_entry;
804
2/4
✓ Branch 1 taken 1057 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 1057 times.
 1057 if (!FindCatalog(nested_root_path, &old_catalog, &new_root_entry)) {
805 ✗ PANIC(kLogStderr,
806 "failed to create nested catalog '%s': "
807 "mountpoint was not found in current catalog structure",
808 nested_root_path.c_str());
809 }
810
811 // Create the database schema and the initial root entry
812 // for the new nested catalog
813
1/2
✓ Branch 2 taken 1057 times.
✗ Branch 3 not taken.
 1057 const string database_file_path = CreateTempPath(dir_temp() + "/catalog",
814
1/2
✓ Branch 1 taken 1057 times.
✗ Branch 2 not taken.
 1057 0666);
815 1057 const bool volatile_content = false;
816
1/2
✓ Branch 1 taken 1057 times.
✗ Branch 2 not taken.
 1057 CatalogDatabase *new_catalog_db = CatalogDatabase::Create(database_file_path);
817
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1057 times.
 1057 assert(NULL != new_catalog_db);
818 // Note we do not set the external_data bit for nested catalogs
819
2/4
✓ Branch 2 taken 1057 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1057 times.
✗ Branch 6 not taken.
 1057 bool retval = new_catalog_db->InsertInitialValues(
820 nested_root_path,
821 volatile_content,
822 "", // At this point, only root
823 // catalog gets VOMS authz
824 new_root_entry);
825
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1057 times.
 1057 assert(retval);
826 // TODO(rmeusel): we need a way to attach a catalog directly from an open
827 // database to remove this indirection
828
1/2
✓ Branch 0 taken 1057 times.
✗ Branch 1 not taken.
 1057 delete new_catalog_db;
829 1057 new_catalog_db = NULL;
830
831 // Attach the just created nested catalog
832
2/4
✓ Branch 1 taken 1057 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1057 times.
✗ Branch 5 not taken.
 1057 Catalog *new_catalog = CreateCatalog(ps_nested_root_path, shash::Any(),
833 old_catalog);
834
1/2
✓ Branch 1 taken 1057 times.
✗ Branch 2 not taken.
 1057 retval = AttachCatalog(database_file_path, new_catalog);
835
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1057 times.
 1057 assert(retval);
836
837
2/4
✓ Branch 1 taken 1057 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 1057 times.
 1057 assert(new_catalog->IsWritable());
838 1057 WritableCatalog *wr_new_catalog = static_cast<WritableCatalog *>(new_catalog);
839
840
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1057 times.
 1057 if (new_root_entry.HasXattrs()) {
841 ✗ XattrList xattrs;
842 ✗ retval = old_catalog->LookupXattrsPath(ps_nested_root_path, &xattrs);
843 ✗ assert(retval);
844 ✗ wr_new_catalog->TouchEntry(new_root_entry, xattrs, nested_root_path);
845 }
846
847 // From now on, there are two catalogs, spanning the same directory structure
848 // we have to split the overlapping directory entries from the old catalog
849 // to the new catalog to re-gain a valid catalog structure
850
1/2
✓ Branch 1 taken 1057 times.
✗ Branch 2 not taken.
 1057 old_catalog->Partition(wr_new_catalog);
851
852 // Add the newly created nested catalog to the references of the containing
853 // catalog
854
4/8
✓ Branch 1 taken 1057 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1057 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 1057 times.
✗ Branch 8 not taken.
✓ Branch 10 taken 1057 times.
✗ Branch 11 not taken.
 1057 old_catalog->InsertNestedCatalog(new_catalog->mountpoint().ToString(), NULL,
855 1057 shash::Any(spooler_->GetHashAlgorithm()), 0);
856
857 // Fix subtree counters in new nested catalogs: subtree is the sum of all
858 // entries of all "grand-nested" catalogs
859 // Note: taking a copy of the nested catalog list here
860 const Catalog::NestedCatalogList
861
1/2
✓ Branch 1 taken 1057 times.
✗ Branch 2 not taken.
 1057 &grand_nested = wr_new_catalog->ListOwnNestedCatalogs();
862
1/2
✓ Branch 1 taken 1057 times.
✗ Branch 2 not taken.
 1057 DeltaCounters fix_subtree_counters;
863 2114 for (Catalog::NestedCatalogList::const_iterator i = grand_nested.begin(),
864 1057 iEnd = grand_nested.end();
865
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1057 times.
 1057 i != iEnd;
866 ✗ ++i) {
867 WritableCatalog *grand_catalog;
868 ✗ retval = FindCatalog(i->mountpoint.ToString(), &grand_catalog);
869 ✗ assert(retval);
870 ✗ const Counters &grand_counters = grand_catalog->GetCounters();
871 ✗ grand_counters.AddAsSubtree(&fix_subtree_counters);
872 }
873 1057 const DeltaCounters save_counters = wr_new_catalog->delta_counters_;
874 1057 wr_new_catalog->delta_counters_ = fix_subtree_counters;
875
1/2
✓ Branch 1 taken 1057 times.
✗ Branch 2 not taken.
 1057 wr_new_catalog->UpdateCounters();
876 1057 wr_new_catalog->delta_counters_ = save_counters;
877
878 1057 SyncUnlock();
879 1057 }
880
881
882 /**
883 * Remove a nested catalog
884 *
885 * If the merged parameter is true, when you remove a nested catalog
886 * all entries currently held by it will be merged into its parent
887 * catalog.
888 * @param mountpoint - the path of the nested catalog to be removed
889 * @param merge - merge the subtree associated with the nested catalog
890 * into its parent catalog
891 * @return - true on success, false otherwise
892 */
893 109 void WritableCatalogManager::RemoveNestedCatalog(const string &mountpoint,
894 const bool merge) {
895
1/2
✓ Branch 1 taken 109 times.
✗ Branch 2 not taken.
 109 const string nested_root_path = MakeRelativePath(mountpoint);
896
897 109 SyncLock();
898 // Find the catalog which should be removed
899 109 WritableCatalog *nested_catalog = NULL;
900
2/4
✓ Branch 1 taken 109 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 109 times.
 109 if (!FindCatalog(nested_root_path, &nested_catalog)) {
901 ✗ PANIC(kLogStderr,
902 "failed to remove nested catalog '%s': "
903 "mountpoint was not found in current catalog structure",
904 nested_root_path.c_str());
905 }
906
907 // Check if the found catalog is really the nested catalog to be deleted
908
6/19
✓ Branch 1 taken 109 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 109 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 109 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 109 times.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 109 times.
✗ Branch 14 not taken.
✓ Branch 16 taken 109 times.
✗ Branch 17 not taken.
✗ Branch 19 not taken.
✗ Branch 20 not taken.
✗ Branch 22 not taken.
✗ Branch 23 not taken.
 218 assert(!nested_catalog->IsRoot()
909 && (nested_catalog->mountpoint().ToString() == nested_root_path));
910
911
1/2
✓ Branch 0 taken 109 times.
✗ Branch 1 not taken.
 109 if (merge) {
912 // Merge all data from the nested catalog into it's parent
913
1/2
✓ Branch 1 taken 109 times.
✗ Branch 2 not taken.
 109 nested_catalog->MergeIntoParent();
914 } else {
915 ✗ nested_catalog->RemoveFromParent();
916 }
917
918 // Delete the catalog database file from the working copy
919
2/6
✓ Branch 1 taken 109 times.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 109 times.
 109 if (unlink(nested_catalog->database_path().c_str()) != 0) {
920 ✗ PANIC(kLogStderr,
921 "unable to delete the removed nested catalog database file '%s'",
922 nested_catalog->database_path().c_str());
923 }
924
925 // Remove the catalog from internal data structures.
926 // On merge the children have been re-parented into this catalog's parent by
927 // MergeIntoParent (CopyCatalogsToParent), so only this catalog is detached.
928 // On removal (fast delete) the whole subtree is discarded together with the
929 // catalog, so any attached children must be detached as well - otherwise they
930 // would be left in the manager with a dangling parent pointer.
931
1/2
✓ Branch 0 taken 109 times.
✗ Branch 1 not taken.
 109 if (merge) {
932
1/2
✓ Branch 1 taken 109 times.
✗ Branch 2 not taken.
 109 DetachCatalog(nested_catalog);
933 } else {
934 ✗ DetachSubtree(nested_catalog);
935 }
936 109 SyncUnlock();
937 109 }
938
939
940 /**
941 * Swap in a new nested catalog
942 *
943 * The old nested catalog must not have been already attached to the
944 * catalog tree. This method will not attach the new nested catalog
945 * to the catalog tree.
946 *
947 * @param mountpoint - the path of the nested catalog to be removed
948 * @param new_hash - the hash of the new nested catalog
949 * @param new_size - the size of the new nested catalog
950 */
951 206 void WritableCatalogManager::SwapNestedCatalog(const string &mountpoint,
952 const shash::Any &new_hash,
953 const uint64_t new_size) {
954
1/2
✓ Branch 1 taken 206 times.
✗ Branch 2 not taken.
 206 const string nested_root_path = MakeRelativePath(mountpoint);
955
1/2
✓ Branch 1 taken 206 times.
✗ Branch 2 not taken.
 206 const string parent_path = GetParentPath(nested_root_path);
956
1/2
✓ Branch 1 taken 206 times.
✗ Branch 2 not taken.
 206 const PathString nested_root_ps = PathString(nested_root_path);
957
958 206 SyncLock();
959
960 // Find the immediate parent catalog
961 206 WritableCatalog *parent = NULL;
962
3/4
✓ Branch 1 taken 206 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 29 times.
✓ Branch 4 taken 177 times.
 206 if (!FindCatalog(parent_path, &parent)) {
963 29 SyncUnlock(); // this is needed for the unittest. otherwise they get stuck
964 29 PANIC(kLogStderr,
965 "failed to swap nested catalog '%s': could not find parent '%s'",
966 nested_root_path.c_str(), parent_path.c_str());
967 }
968
969 // Get old nested catalog counters
970
1/2
✓ Branch 1 taken 177 times.
✗ Branch 2 not taken.
 177 Catalog *old_attached_catalog = parent->FindChild(nested_root_ps);
971
1/2
✓ Branch 1 taken 177 times.
✗ Branch 2 not taken.
 177 Counters old_counters;
972
2/2
✓ Branch 0 taken 29 times.
✓ Branch 1 taken 148 times.
 177 if (old_attached_catalog) {
973 // Old catalog was already attached (e.g. as a child catalog
974 // attached by a prior call to CreateNestedCatalog()). Ensure
975 // that it has not been modified, get counters, and detach it.
976 29 WritableCatalogList list;
977
2/4
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 29 times.
✗ Branch 4 not taken.
 29 if (GetModifiedCatalogLeafsRecursively(old_attached_catalog, &list)) {
978 29 SyncUnlock();
979 29 PANIC(kLogStderr,
980 "failed to swap nested catalog '%s': already modified",
981 nested_root_path.c_str());
982 }
983 ✗ old_counters = old_attached_catalog->GetCounters();
984 ✗ DetachSubtree(old_attached_catalog);
985
986 29 } else {
987 // Old catalog was not attached. Download a freely attached
988 // version and get counters.
989
1/2
✓ Branch 1 taken 148 times.
✗ Branch 2 not taken.
 148 shash::Any old_hash;
990 uint64_t old_size;
991
1/2
✓ Branch 1 taken 148 times.
✗ Branch 2 not taken.
 148 const bool old_found = parent->FindNested(nested_root_ps, &old_hash,
992 &old_size);
993
2/2
✓ Branch 0 taken 29 times.
✓ Branch 1 taken 119 times.
 148 if (!old_found) {
994 29 SyncUnlock();
995 29 PANIC(kLogStderr,
996 "failed to swap nested catalog '%s': not found in parent",
997 nested_root_path.c_str());
998 }
999 const std::unique_ptr<Catalog> old_free_catalog(
1000
1/2
✓ Branch 1 taken 119 times.
✗ Branch 2 not taken.
 119 LoadFreeCatalog(nested_root_ps, old_hash));
1001
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 119 times.
 119 if (old_free_catalog.get() == nullptr) {
1002 ✗ SyncUnlock();
1003 ✗ PANIC(kLogStderr,
1004 "failed to swap nested catalog '%s': failed to load old catalog",
1005 nested_root_path.c_str());
1006 }
1007 119 old_counters = old_free_catalog->GetCounters();
1008 119 }
1009
1010 // Load freely attached new catalog
1011 const std::unique_ptr<Catalog> new_catalog(
1012
2/2
✓ Branch 1 taken 90 times.
✓ Branch 2 taken 29 times.
 119 LoadFreeCatalog(nested_root_ps, new_hash));
1013
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 90 times.
 90 if (new_catalog.get() == nullptr) {
1014 ✗ SyncUnlock();
1015 ✗ PANIC(kLogStderr,
1016 "failed to swap nested catalog '%s': failed to load new catalog",
1017 nested_root_path.c_str());
1018 }
1019
1020 // Get new catalog root directory entry
1021
1/2
✓ Branch 1 taken 90 times.
✗ Branch 2 not taken.
 90 DirectoryEntry dirent;
1022 90 XattrList xattrs;
1023
1/2
✓ Branch 2 taken 90 times.
✗ Branch 3 not taken.
 90 const bool dirent_found = new_catalog->LookupPath(nested_root_ps, &dirent);
1024
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 90 times.
 90 if (!dirent_found) {
1025 ✗ SyncUnlock();
1026 ✗ PANIC(kLogStderr,
1027 "failed to swap nested catalog '%s': missing dirent in new catalog",
1028 nested_root_path.c_str());
1029 }
1030
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 90 times.
 90 if (dirent.HasXattrs()) {
1031 ✗ const bool xattrs_found = new_catalog->LookupXattrsPath(nested_root_ps,
1032 &xattrs);
1033 ✗ if (!xattrs_found) {
1034 ✗ SyncUnlock();
1035 ✗ PANIC(kLogStderr,
1036 "failed to swap nested catalog '%s': missing xattrs in new catalog",
1037 nested_root_path.c_str());
1038 }
1039 }
1040
1041 // Swap catalogs
1042
1/2
✓ Branch 1 taken 90 times.
✗ Branch 2 not taken.
 90 parent->RemoveNestedCatalog(nested_root_path, NULL);
1043
1/2
✓ Branch 1 taken 90 times.
✗ Branch 2 not taken.
 90 parent->InsertNestedCatalog(nested_root_path, NULL, new_hash, new_size);
1044
1045 // Update parent directory entry
1046 90 dirent.set_is_nested_catalog_mountpoint(true);
1047 90 dirent.set_is_nested_catalog_root(false);
1048
1/2
✓ Branch 1 taken 90 times.
✗ Branch 2 not taken.
 90 parent->UpdateEntry(dirent, nested_root_path);
1049
1/2
✓ Branch 1 taken 90 times.
✗ Branch 2 not taken.
 90 parent->TouchEntry(dirent, xattrs, nested_root_path);
1050
1051 // Update counters
1052
1/2
✓ Branch 3 taken 90 times.
✗ Branch 4 not taken.
 90 const DeltaCounters delta = Counters::Diff(old_counters,
1053 new_catalog->GetCounters());
1054
1/2
✓ Branch 1 taken 90 times.
✗ Branch 2 not taken.
 90 delta.PopulateToParent(&parent->delta_counters_);
1055
1056 90 SyncUnlock();
1057 438 }
1058
1059 /**
1060 * Install a nested catalog (catalog hierarchy) at a new, empty directory
1061 *
1062 * The mountpoint directory must not yet exist. Its parent directory, however
1063 * must exist. This method combines functionality from AddDirectory(),
1064 * CreateNestedCatalog() and SwapNestedCatalog().
1065 * The new nested catalog won't get attached.
1066 *
1067 * @param mountpoint - the path where the nested catalog should be installed
1068 * @param new_hash - the hash of the new nested catalog
1069 * @param new_size - the size of the new nested catalog
1070 */
1071 90 void WritableCatalogManager::GraftNestedCatalog(const string &mountpoint,
1072 const shash::Any &new_hash,
1073 const uint64_t new_size) {
1074
2/2
✓ Branch 1 taken 60 times.
✓ Branch 2 taken 30 times.
 90 if (!TryGraftNestedCatalog(mountpoint, new_hash, new_size)) {
1075 60 PANIC(kLogStderr, "failed to graft nested catalog '%s'",
1076 mountpoint.c_str());
1077 }
1078 30 }
1079
1080 90 bool WritableCatalogManager::TryGraftNestedCatalog(const string &mountpoint,
1081 const shash::Any &new_hash,
1082 const uint64_t new_size) {
1083
1/2
✓ Branch 1 taken 90 times.
✗ Branch 2 not taken.
 90 const string nested_root_path = MakeRelativePath(mountpoint);
1084
1/2
✓ Branch 1 taken 90 times.
✗ Branch 2 not taken.
 90 const string parent_path = GetParentPath(nested_root_path);
1085
1/2
✓ Branch 1 taken 90 times.
✗ Branch 2 not taken.
 90 const PathString nested_root_ps = PathString(nested_root_path);
1086
1087
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 90 times.
 90 if (nested_root_path.empty()) {
1088 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1089 "failed to graft nested catalog: empty mountpoint");
1090 ✗ return false;
1091 }
1092
1093 // Load freely attached new catalog
1094 const std::unique_ptr<Catalog> new_catalog(
1095
1/2
✓ Branch 1 taken 90 times.
✗ Branch 2 not taken.
 90 LoadFreeCatalog(nested_root_ps, new_hash));
1096
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 90 times.
 90 if (new_catalog.get() == nullptr) {
1097 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1098 "failed to graft nested catalog '%s': failed to load new catalog",
1099 nested_root_path.c_str());
1100 ✗ return false;
1101 }
1102
4/7
✓ Branch 2 taken 90 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 90 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 30 times.
✓ Branch 9 taken 60 times.
 90 if (new_catalog->root_prefix() != nested_root_ps) {
1103
1/7
✗ Branch 2 not taken.
✓ Branch 3 taken 30 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
 30 LogCvmfs(kLogCatalog, kLogStderr,
1104 "invalid nested catalog for grafting at '%s': catalog rooted at "
1105 "'%s'",
1106 nested_root_path.c_str(),
1107
2/4
✓ Branch 2 taken 30 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 30 times.
✗ Branch 6 not taken.
 60 new_catalog->root_prefix().ToString().c_str());
1108 30 return false;
1109 }
1110
1111 // Get new catalog root directory entry
1112
1/2
✓ Branch 1 taken 60 times.
✗ Branch 2 not taken.
 60 DirectoryEntry dirent;
1113 60 XattrList xattrs;
1114
1/2
✓ Branch 2 taken 60 times.
✗ Branch 3 not taken.
 60 const bool dirent_found = new_catalog->LookupPath(nested_root_ps, &dirent);
1115
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 60 times.
 60 if (!dirent_found) {
1116 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1117 "failed to graft nested catalog '%s': missing dirent in new "
1118 "catalog",
1119 nested_root_path.c_str());
1120 ✗ return false;
1121 }
1122
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 60 times.
 60 if (!dirent.IsDirectory()) {
1123 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1124 "failed to graft nested catalog '%s': root entry is not a "
1125 "directory",
1126 nested_root_path.c_str());
1127 ✗ return false;
1128 }
1129
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 60 times.
 60 if (dirent.HasXattrs()) {
1130 ✗ const bool xattrs_found = new_catalog->LookupXattrsPath(nested_root_ps,
1131 &xattrs);
1132 ✗ if (!xattrs_found) {
1133 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1134 "failed to graft nested catalog '%s': missing xattrs in new "
1135 "catalog",
1136 nested_root_path.c_str());
1137 ✗ return false;
1138 }
1139 }
1140 // Transform the nested catalog root into a transition point to be inserted
1141 // in the parent catalog
1142 60 dirent.set_is_nested_catalog_root(false);
1143 60 dirent.set_is_nested_catalog_mountpoint(true);
1144
1145 // Add directory and nested catalog
1146
1147 60 SyncLock();
1148 WritableCatalog *parent_catalog;
1149
1/2
✓ Branch 1 taken 60 times.
✗ Branch 2 not taken.
 60 DirectoryEntry parent_entry;
1150
2/4
✓ Branch 1 taken 60 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 60 times.
 60 if (!FindCatalog(parent_path, &parent_catalog, &parent_entry)) {
1151 ✗ SyncUnlock();
1152 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1153 "catalog for directory '%s' cannot be found", parent_path.c_str());
1154 ✗ return false;
1155 }
1156
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 60 times.
 60 if (!parent_entry.IsDirectory()) {
1157 ✗ SyncUnlock();
1158 ✗ LogCvmfs(kLogCatalog, kLogStderr, "parent path '%s' is not a directory",
1159 parent_path.c_str());
1160 ✗ return false;
1161 }
1162
3/4
✓ Branch 1 taken 60 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 30 times.
✓ Branch 4 taken 30 times.
 60 if (parent_catalog->LookupPath(nested_root_ps, NULL)) {
1163 30 SyncUnlock();
1164
1/2
✓ Branch 2 taken 30 times.
✗ Branch 3 not taken.
 30 LogCvmfs(kLogCatalog, kLogStderr,
1165 "invalid attempt to graft nested catalog into existing directory "
1166 "'%s'",
1167 nested_root_path.c_str());
1168 30 return false;
1169 }
1170
1/2
✓ Branch 1 taken 30 times.
✗ Branch 2 not taken.
 30 parent_catalog->AddEntry(dirent, xattrs, nested_root_path, parent_path);
1171 30 parent_entry.set_linkcount(parent_entry.linkcount() + 1);
1172
1/2
✓ Branch 1 taken 30 times.
✗ Branch 2 not taken.
 30 parent_catalog->UpdateEntry(parent_entry, parent_path);
1173
1/2
✓ Branch 1 taken 30 times.
✗ Branch 2 not taken.
 30 if (parent_entry.IsNestedCatalogRoot()) {
1174 WritableCatalog *grand_parent_catalog = reinterpret_cast<WritableCatalog *>(
1175 30 parent_catalog->parent());
1176 30 parent_entry.set_is_nested_catalog_root(false);
1177 30 parent_entry.set_is_nested_catalog_mountpoint(true);
1178
1/2
✓ Branch 1 taken 30 times.
✗ Branch 2 not taken.
 30 grand_parent_catalog->UpdateEntry(parent_entry, parent_path);
1179 }
1180
1181
1/2
✓ Branch 1 taken 30 times.
✗ Branch 2 not taken.
 30 parent_catalog->InsertNestedCatalog(nested_root_path, NULL, new_hash,
1182 new_size);
1183
1184 // Fix-up counters
1185
1/2
✓ Branch 1 taken 30 times.
✗ Branch 2 not taken.
 30 const Counters counters;
1186
1/2
✓ Branch 3 taken 30 times.
✗ Branch 4 not taken.
 30 const DeltaCounters delta = Counters::Diff(counters,
1187 new_catalog->GetCounters());
1188
1/2
✓ Branch 1 taken 30 times.
✗ Branch 2 not taken.
 30 delta.PopulateToParent(&parent_catalog->delta_counters_);
1189
1190 30 SyncUnlock();
1191 30 return true;
1192 90 }
1193
1194 /**
1195 * Checks if a nested catalog starts at this path. The path must be valid.
1196 */
1197 ✗ bool WritableCatalogManager::IsTransitionPoint(const string &mountpoint) {
1198 ✗ const string path = MakeRelativePath(mountpoint);
1199
1200 ✗ SyncLock();
1201 WritableCatalog *catalog;
1202 ✗ DirectoryEntry entry;
1203 ✗ if (!FindCatalog(path, &catalog, &entry)) {
1204 ✗ PANIC(kLogStderr, "catalog for directory '%s' cannot be found",
1205 path.c_str());
1206 }
1207 ✗ const bool result = entry.IsNestedCatalogRoot();
1208 ✗ SyncUnlock();
1209 ✗ return result;
1210 }
1211
1212
1213 ✗ void WritableCatalogManager::PrecalculateListings() {
1214 // TODO(jblomer): meant for micro catalogs
1215 }
1216
1217
1218 ✗ void WritableCatalogManager::SetTTL(const uint64_t new_ttl) {
1219 ✗ SyncLock();
1220 ✗ reinterpret_cast<WritableCatalog *>(GetRootCatalog())->SetTTL(new_ttl);
1221 ✗ SyncUnlock();
1222 }
1223
1224
1225 ✗ bool WritableCatalogManager::SetVOMSAuthz(const std::string &voms_authz) {
1226 bool result;
1227 ✗ SyncLock();
1228 ✗ result = reinterpret_cast<WritableCatalog *>(GetRootCatalog())
1229 ✗ ->SetVOMSAuthz(voms_authz);
1230 ✗ SyncUnlock();
1231 ✗ return result;
1232 }
1233
1234
1235 1163 bool WritableCatalogManager::Commit(const bool stop_for_tweaks,
1236 const uint64_t manual_revision,
1237 manifest::Manifest *manifest) {
1238 WritableCatalog *root_catalog = reinterpret_cast<WritableCatalog *>(
1239 1163 GetRootCatalog());
1240
1/2
✓ Branch 1 taken 1163 times.
✗ Branch 2 not taken.
 1163 root_catalog->SetDirty();
1241
1242 // set root catalog revision to manually provided number if available
1243
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1163 times.
 1163 if (manual_revision > 0) {
1244 ✗ const uint64_t revision = root_catalog->GetRevision();
1245 ✗ if (revision >= manual_revision) {
1246 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1247 "Manual revision (%" PRIu64 ") must not be "
1248 "smaller than the current root catalog's (%" PRIu64
1249 "). Skipped!",
1250 manual_revision, revision);
1251 } else {
1252 // Gets incremented by FinalizeCatalog() afterwards!
1253 ✗ root_catalog->SetRevision(manual_revision - 1);
1254 }
1255 }
1256
1257 // do the actual catalog snapshotting and upload
1258
1/2
✓ Branch 1 taken 1163 times.
✗ Branch 2 not taken.
 1163 CatalogInfo root_catalog_info;
1259
1/2
✓ Branch 1 taken 1163 times.
✗ Branch 2 not taken.
 1163 if (getenv("_CVMFS_SERIALIZED_CATALOG_PROCESSING_") == NULL)
1260
1/2
✓ Branch 1 taken 1163 times.
✗ Branch 2 not taken.
 1163 root_catalog_info = SnapshotCatalogs(stop_for_tweaks);
1261 else
1262 ✗ root_catalog_info = SnapshotCatalogsSerialized(stop_for_tweaks);
1263
2/4
✓ Branch 1 taken 1163 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 1163 times.
 1163 if (spooler_->GetNumberOfErrors() > 0) {
1264 ✗ LogCvmfs(kLogCatalog, kLogStderr, "failed to commit catalogs");
1265 ✗ return false;
1266 }
1267
1268 // .cvmfspublished export
1269
1/2
✓ Branch 1 taken 1163 times.
✗ Branch 2 not taken.
 1163 LogCvmfs(kLogCatalog, kLogVerboseMsg, "Committing repository manifest");
1270 1163 set_base_hash(root_catalog_info.content_hash);
1271
1272 1163 manifest->set_catalog_hash(root_catalog_info.content_hash);
1273 1163 manifest->set_catalog_size(root_catalog_info.size);
1274
2/4
✓ Branch 2 taken 1163 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1163 times.
✗ Branch 6 not taken.
 1163 manifest->set_root_path("");
1275 1163 manifest->set_ttl(root_catalog_info.ttl);
1276 1163 manifest->set_revision(root_catalog_info.revision);
1277
1278 1163 return true;
1279 }
1280
1281
1282 /**
1283 * Handles the snapshotting of dirty (i.e. modified) catalogs while trying to
1284 * parallelize the compression and upload as much as possible. We use a parallel
1285 * depth first post order tree traversal based on 'continuations'.
1286 *
1287 * The idea is as follows:
1288 * 1. find all leaf-catalogs (i.e. dirty catalogs with no dirty children)
1289 * --> these can be processed and uploaded immediately and independently
1290 * see WritableCatalogManager::GetModifiedCatalogLeafs()
1291 * 2. annotate non-leaf catalogs with their number of dirty children
1292 * --> a finished child will notify it's parent and decrement this number
1293 * see WritableCatalogManager::CatalogUploadCallback()
1294 * 3. if a non-leaf catalog's dirty children number reaches 0, it is scheduled
1295 * for processing as well (continuation)
1296 * --> the parallel processing walks bottom-up through the catalog tree
1297 * see WritableCatalogManager::CatalogUploadCallback()
1298 * 4. when the root catalog is reached, we notify the main thread and return
1299 * --> done through a Future<> in WritableCatalogManager::SnapshotCatalogs
1300 *
1301 * Note: The catalog finalisation (see WritableCatalogManager::FinalizeCatalog)
1302 * happens in a worker thread (i.e. the callback method) for non-leaf
1303 * catalogs.
1304 *
1305 * TODO(rmeusel): since all leaf catalogs are finalized in the main thread, we
1306 * sacrifice some potential concurrency for simplicity.
1307 */
1308 1163 WritableCatalogManager::CatalogInfo WritableCatalogManager::SnapshotCatalogs(
1309 const bool stop_for_tweaks) {
1310 // prepare environment for parallel processing
1311
1/2
✓ Branch 1 taken 1163 times.
✗ Branch 2 not taken.
 1163 Future<CatalogInfo> root_catalog_info_future;
1312 CatalogUploadContext upload_context;
1313 1163 upload_context.root_catalog_info = &root_catalog_info_future;
1314 1163 upload_context.stop_for_tweaks = stop_for_tweaks;
1315
1316
1/2
✓ Branch 1 taken 1163 times.
✗ Branch 2 not taken.
 1163 spooler_->RegisterListener(&WritableCatalogManager::CatalogUploadCallback,
1317 this, upload_context);
1318
1319 // find dirty leaf catalogs and annotate non-leaf catalogs (dirty child count)
1320 // post-condition: the entire catalog tree is ready for concurrent processing
1321 1163 WritableCatalogList leafs_to_snapshot;
1322
1/2
✓ Branch 1 taken 1163 times.
✗ Branch 2 not taken.
 1163 GetModifiedCatalogLeafs(&leafs_to_snapshot);
1323
1324 // finalize and schedule the catalog processing
1325 1163 WritableCatalogList::const_iterator i = leafs_to_snapshot.begin();
1326 1163 const WritableCatalogList::const_iterator iend = leafs_to_snapshot.end();
1327
2/2
✓ Branch 2 taken 1490 times.
✓ Branch 3 taken 1163 times.
 2653 for (; i != iend; ++i) {
1328
1/2
✓ Branch 2 taken 1490 times.
✗ Branch 3 not taken.
 1490 FinalizeCatalog(*i, stop_for_tweaks);
1329
1/2
✓ Branch 2 taken 1490 times.
✗ Branch 3 not taken.
 1490 ScheduleCatalogProcessing(*i);
1330 }
1331
1332
1/2
✓ Branch 1 taken 1163 times.
✗ Branch 2 not taken.
 1163 LogCvmfs(kLogCatalog, kLogVerboseMsg, "waiting for upload of catalogs");
1333
1/2
✓ Branch 1 taken 1163 times.
✗ Branch 2 not taken.
 1163 const CatalogInfo &root_catalog_info = root_catalog_info_future.Get();
1334
1/2
✓ Branch 1 taken 1163 times.
✗ Branch 2 not taken.
 1163 spooler_->WaitForUpload();
1335
1336
1/2
✓ Branch 1 taken 1163 times.
✗ Branch 2 not taken.
 1163 spooler_->UnregisterListeners();
1337 1163 return root_catalog_info;
1338 1163 }
1339
1340
1341 2133 void WritableCatalogManager::FinalizeCatalog(WritableCatalog *catalog,
1342 const bool stop_for_tweaks) {
1343 // update meta information of this catalog
1344
1/3
✓ Branch 2 taken 2133 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 2133 LogCvmfs(kLogCatalog, kLogVerboseMsg, "creating snapshot of catalog '%s'",
1345 4266 catalog->mountpoint().c_str());
1346
1347 2133 catalog->UpdateCounters();
1348 2133 catalog->UpdateLastModified();
1349 2133 catalog->IncrementRevision();
1350
1351 // update the previous catalog revision pointer
1352
2/2
✓ Branch 1 taken 1163 times.
✓ Branch 2 taken 970 times.
 2133 if (catalog->IsRoot()) {
1353
1/4
✓ Branch 2 taken 1163 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 1163 LogCvmfs(kLogCatalog, kLogVerboseMsg,
1354 "setting '%s' as previous revision "
1355 "for root catalog",
1356 2326 base_hash().ToStringWithSuffix().c_str());
1357 1163 catalog->SetPreviousRevision(base_hash());
1358 } else {
1359 // Multiple catalogs might query the parent concurrently
1360 970 SyncLock();
1361
1/2
✓ Branch 1 taken 970 times.
✗ Branch 2 not taken.
 970 shash::Any hash_previous;
1362 uint64_t size_previous;
1363
1/2
✓ Branch 2 taken 970 times.
✗ Branch 3 not taken.
 1940 const bool retval = catalog->parent()->FindNested(
1364
1/2
✓ Branch 1 taken 970 times.
✗ Branch 2 not taken.
 1940 catalog->mountpoint(), &hash_previous, &size_previous);
1365
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 970 times.
 970 assert(retval);
1366 970 SyncUnlock();
1367
1368
1/8
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 970 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
 1940 LogCvmfs(kLogCatalog, kLogVerboseMsg,
1369 "found '%s' as previous revision "
1370 "for nested catalog '%s'",
1371
1/2
✓ Branch 1 taken 970 times.
✗ Branch 2 not taken.
 1940 hash_previous.ToStringWithSuffix().c_str(),
1372
1/2
✓ Branch 1 taken 970 times.
✗ Branch 2 not taken.
 1940 catalog->mountpoint().c_str());
1373
1/2
✓ Branch 1 taken 970 times.
✗ Branch 2 not taken.
 970 catalog->SetPreviousRevision(hash_previous);
1374 }
1375 2133 catalog->Commit();
1376
1377 // check if catalog has too many entries
1378 const uint64_t catalog_limit = uint64_t(1000)
1379
2/2
✓ Branch 1 taken 1163 times.
✓ Branch 2 taken 970 times.
 2133 * uint64_t((catalog->IsRoot()
1380 1163 ? root_kcatalog_limit_
1381 970 : nested_kcatalog_limit_));
1382 2133 if ((catalog_limit > 0)
1383
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 2133 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 2133 times.
 2133 && (catalog->GetCounters().GetSelfEntries() > catalog_limit)) {
1384 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1385 "%s: catalog at %s has more than %lu entries (%lu). "
1386 "Large catalogs stress the CernVM-FS transport infrastructure. "
1387 "Please split it into nested catalogs or increase the limit.",
1388 ✗ enforce_limits_ ? "FATAL" : "WARNING",
1389 ✗ (catalog->IsRoot() ? "/" : catalog->mountpoint().c_str()),
1390 ✗ catalog_limit, catalog->GetCounters().GetSelfEntries());
1391 ✗ if (enforce_limits_)
1392 ✗ PANIC(kLogStderr, "catalog at %s has more than %u entries (%u). ",
1393 (catalog->IsRoot() ? "/" : catalog->mountpoint().c_str()),
1394 catalog_limit, catalog->GetCounters().GetSelfEntries());
1395 }
1396
1397 // allow for manual adjustments in the catalog
1398
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2133 times.
 2133 if (stop_for_tweaks) {
1399 ✗ LogCvmfs(kLogCatalog, kLogStdout,
1400 "Allowing for tweaks in %s at %s "
1401 "(hit return to continue)",
1402 ✗ catalog->database_path().c_str(), catalog->mountpoint().c_str());
1403 ✗ const int read_char = getchar();
1404 ✗ assert(read_char != EOF);
1405 }
1406
1407 // compaction of bloated catalogs (usually after high database churn)
1408 2133 catalog->VacuumDatabaseIfNecessary();
1409 2133 }
1410
1411
1412 2133 void WritableCatalogManager::ScheduleCatalogProcessing(
1413 WritableCatalog *catalog) {
1414 {
1415 2133 const MutexLockGuard guard(catalog_processing_lock_);
1416 // register catalog object for WritableCatalogManager::CatalogUploadCallback
1417
2/4
✓ Branch 1 taken 2133 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2133 times.
✗ Branch 5 not taken.
 2133 catalog_processing_map_[catalog->database_path()] = catalog;
1418 2133 }
1419
1/2
✓ Branch 2 taken 2133 times.
✗ Branch 3 not taken.
 2133 spooler_->ProcessCatalog(catalog->database_path());
1420 2133 }
1421
1422 /**
1423 * Copy catalog to local cache.server
1424 * Must be an atomic write into the cache_dir
1425 * As such: create a temporary copy in cache_dir/txn and then do a
1426 * `rename` (which is atomic) to the actual cache path
1427 *
1428 * @returns true on success, otherwise false
1429 */
1430 ✗ bool WritableCatalogManager::CopyCatalogToLocalCache(
1431 const upload::SpoolerResult &result) {
1432 ✗ std::string tmp_catalog_path;
1433 ✗ const std::string cache_catalog_path = dir_cache_ + "/"
1434 + result.content_hash
1435 ✗ .MakePathWithoutSuffix();
1436 ✗ FILE *fcatalog = CreateTempFile(dir_cache_ + "/txn/catalog", 0666, "w",
1437 &tmp_catalog_path);
1438 ✗ if (!fcatalog) {
1439 ✗ PANIC(kLogDebug | kLogStderr,
1440 "Creating file for temporary catalog failed: %s",
1441 tmp_catalog_path.c_str());
1442 }
1443 ✗ CopyPath2File(result.local_path.c_str(), fcatalog);
1444 ✗ (void)fclose(fcatalog);
1445
1446 ✗ if (rename(tmp_catalog_path.c_str(), cache_catalog_path.c_str()) != 0) {
1447 ✗ PANIC(kLogDebug | kLogStderr, "Failed to copy catalog from %s to cache %s",
1448 result.local_path.c_str(), cache_catalog_path.c_str());
1449 }
1450 ✗ return true;
1451 }
1452
1453 2133 void WritableCatalogManager::CatalogUploadCallback(
1454 const upload::SpoolerResult &result,
1455 const CatalogUploadContext catalog_upload_context) {
1456
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2133 times.
 2133 if (result.return_code != 0) {
1457 ✗ PANIC(kLogStderr, "failed to upload '%s' (retval: %d)",
1458 result.local_path.c_str(), result.return_code);
1459 }
1460
1461 // retrieve the catalog object based on the callback information
1462 // see WritableCatalogManager::ScheduleCatalogProcessing()
1463 2133 WritableCatalog *catalog = NULL;
1464 {
1465 2133 const MutexLockGuard guard(catalog_processing_lock_);
1466 const std::map<std::string, WritableCatalog *>::iterator
1467
1/2
✓ Branch 1 taken 2133 times.
✗ Branch 2 not taken.
 2133 c = catalog_processing_map_.find(result.local_path);
1468
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 2133 times.
 2133 assert(c != catalog_processing_map_.end());
1469 2133 catalog = c->second;
1470 2133 }
1471
1472 2133 const uint64_t catalog_size = GetFileSize(result.local_path);
1473
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2133 times.
 2133 assert(catalog_size > 0);
1474
1475
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 2133 times.
 2133 if (UseLocalCache()) {
1476 ✗ CopyCatalogToLocalCache(result);
1477 }
1478
1479 2133 SyncLock();
1480
2/2
✓ Branch 1 taken 970 times.
✓ Branch 2 taken 1163 times.
 2133 if (catalog->HasParent()) {
1481 // finalized nested catalogs will update their parent's pointer and schedule
1482 // them for processing (continuation) if the 'dirty children count' == 0
1483 970 LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating nested catalog link");
1484 970 WritableCatalog *parent = catalog->GetWritableParent();
1485
1486
2/4
✓ Branch 1 taken 970 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 970 times.
✗ Branch 5 not taken.
 970 parent->UpdateNestedCatalog(catalog->mountpoint().ToString(),
1487 970 result.content_hash,
1488 catalog_size,
1489 970 catalog->delta_counters_);
1490 970 catalog->delta_counters_.SetZero();
1491
1492 const int remaining_dirty_children = catalog->GetWritableParent()
1493 970 ->DecrementDirtyChildren();
1494
1495 970 SyncUnlock();
1496
1497 // continuation of the dirty catalog tree traversal
1498 // see WritableCatalogManager::SnapshotCatalogs()
1499
2/2
✓ Branch 0 taken 643 times.
✓ Branch 1 taken 327 times.
 970 if (remaining_dirty_children == 0) {
1500 643 FinalizeCatalog(parent, catalog_upload_context.stop_for_tweaks);
1501 643 ScheduleCatalogProcessing(parent);
1502 }
1503
1504
1/2
✓ Branch 1 taken 1163 times.
✗ Branch 2 not taken.
 1163 } else if (catalog->IsRoot()) {
1505 // once the root catalog is reached, we are done with processing and report
1506 // back to the main via a Future<> and provide the necessary information
1507
1/2
✓ Branch 1 taken 1163 times.
✗ Branch 2 not taken.
 1163 CatalogInfo root_catalog_info;
1508 1163 root_catalog_info.size = catalog_size;
1509
1/2
✓ Branch 1 taken 1163 times.
✗ Branch 2 not taken.
 1163 root_catalog_info.ttl = catalog->GetTTL();
1510 1163 root_catalog_info.content_hash = result.content_hash;
1511
1/2
✓ Branch 1 taken 1163 times.
✗ Branch 2 not taken.
 1163 root_catalog_info.revision = catalog->GetRevision();
1512
1/2
✓ Branch 1 taken 1163 times.
✗ Branch 2 not taken.
 1163 catalog_upload_context.root_catalog_info->Set(root_catalog_info);
1513 1163 SyncUnlock();
1514 } else {
1515 ✗ PANIC(kLogStderr, "inconsistent state detected");
1516 }
1517 2133 }
1518
1519
1520 /**
1521 * Finds dirty catalogs that can be snapshot right away and annotates all the
1522 * other catalogs with their number of dirty descendants.
1523 * Note that there is a convenience wrapper to start the recursion:
1524 * WritableCatalogManager::GetModifiedCatalogLeafs()
1525 *
1526 * @param catalog the catalog for this recursion step
1527 * @param result the result list to be appended to
1528 * @return true if 'catalog' is dirty
1529 */
1530 2162 bool WritableCatalogManager::GetModifiedCatalogLeafsRecursively(
1531 Catalog *catalog, WritableCatalogList *result) const {
1532 2162 WritableCatalog *wr_catalog = static_cast<WritableCatalog *>(catalog);
1533
1534 // Look for dirty catalogs in the descendants of *catalog
1535 2162 int dirty_children = 0;
1536
1/2
✓ Branch 1 taken 2162 times.
✗ Branch 2 not taken.
 2162 CatalogList children = wr_catalog->GetChildren();
1537 2162 CatalogList::const_iterator i = children.begin();
1538 2162 const CatalogList::const_iterator iend = children.end();
1539
2/2
✓ Branch 2 taken 970 times.
✓ Branch 3 taken 2162 times.
 3132 for (; i != iend; ++i) {
1540
2/4
✓ Branch 2 taken 970 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 970 times.
✗ Branch 5 not taken.
 970 if (GetModifiedCatalogLeafsRecursively(*i, result)) {
1541 970 ++dirty_children;
1542 }
1543 }
1544
1545 // a catalog is dirty if itself or one of its children has changed
1546 // a leaf catalog doesn't have any dirty children
1547 2162 wr_catalog->set_dirty_children(dirty_children);
1548
1/4
✗ Branch 1 not taken.
✓ Branch 2 taken 2162 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 2162 const bool is_dirty = wr_catalog->IsDirty() || dirty_children > 0;
1549 2162 const bool is_leaf = dirty_children == 0;
1550
3/4
✓ Branch 0 taken 2162 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1519 times.
✓ Branch 3 taken 643 times.
 2162 if (is_dirty && is_leaf) {
1551
1/2
✓ Branch 1 taken 1519 times.
✗ Branch 2 not taken.
 1519 result->push_back(const_cast<WritableCatalog *>(wr_catalog));
1552 }
1553
1554 2162 return is_dirty;
1555 2162 }
1556
1557
1558 ✗ void WritableCatalogManager::DoBalance() {
1559 ✗ CatalogList catalog_list = GetCatalogs();
1560 ✗ reverse(catalog_list.begin(), catalog_list.end());
1561 ✗ for (unsigned i = 0; i < catalog_list.size(); ++i) {
1562 ✗ FixWeight(static_cast<WritableCatalog *>(catalog_list[i]));
1563 }
1564 }
1565
1566 ✗ void WritableCatalogManager::FixWeight(WritableCatalog *catalog) {
1567 // firstly check underflow because they can provoke overflows
1568 ✗ if (catalog->GetNumEntries() < min_weight_ && !catalog->IsRoot()
1569 ✗ && catalog->IsAutogenerated()) {
1570 ✗ LogCvmfs(kLogCatalog, kLogStdout,
1571 "Deleting an autogenerated catalog in '%s'",
1572 ✗ catalog->mountpoint().c_str());
1573 // Remove the .cvmfscatalog and .cvmfsautocatalog files first
1574 ✗ const string path = catalog->mountpoint().ToString();
1575 ✗ catalog->RemoveEntry(path + "/.cvmfscatalog");
1576 ✗ catalog->RemoveEntry(path + "/.cvmfsautocatalog");
1577 // Remove the actual catalog
1578 ✗ const string catalog_path = catalog->mountpoint().ToString().substr(1);
1579 ✗ RemoveNestedCatalog(catalog_path);
1580 ✗ } else if (catalog->GetNumEntries() > max_weight_) {
1581 ✗ CatalogBalancer<WritableCatalogManager> catalog_balancer(this);
1582 ✗ catalog_balancer.Balance(catalog);
1583 }
1584 }
1585
1586
1587 //****************************************************************************
1588 // Workaround -- Serialized Catalog Committing
1589
1590 ✗ int WritableCatalogManager::GetModifiedCatalogsRecursively(
1591 const Catalog *catalog, WritableCatalogList *result) const {
1592 // A catalog must be snapshot, if itself or one of it's descendants is dirty.
1593 // So we traverse the catalog tree recursively and look for dirty catalogs
1594 // on the way.
1595 ✗ const WritableCatalog *wr_catalog = static_cast<const WritableCatalog *>(
1596 catalog);
1597 // This variable will contain the number of dirty catalogs in the sub tree
1598 // with *catalog as it's root.
1599 ✗ int dirty_catalogs = (wr_catalog->IsDirty()) ? 1 : 0;
1600
1601 // Look for dirty catalogs in the descendants of *catalog
1602 ✗ CatalogList children = wr_catalog->GetChildren();
1603 ✗ for (CatalogList::const_iterator i = children.begin(), iEnd = children.end();
1604 ✗ i != iEnd; ++i) {
1605 ✗ dirty_catalogs += GetModifiedCatalogsRecursively(*i, result);
1606 }
1607
1608 // If we found a dirty catalog in the checked sub tree, the root (*catalog)
1609 // must be snapshot and ends up in the result list
1610 ✗ if (dirty_catalogs > 0)
1611 ✗ result->push_back(const_cast<WritableCatalog *>(wr_catalog));
1612
1613 // tell the upper layer about number of catalogs
1614 ✗ return dirty_catalogs;
1615 }
1616
1617
1618 ✗ void WritableCatalogManager::CatalogUploadSerializedCallback(
1619 const upload::SpoolerResult &result, const CatalogUploadContext unused) {
1620 ✗ if (result.return_code != 0) {
1621 ✗ PANIC(kLogStderr, "failed to upload '%s' (retval: %d)",
1622 result.local_path.c_str(), result.return_code);
1623 }
1624
1625 ✗ if (UseLocalCache()) {
1626 ✗ CopyCatalogToLocalCache(result);
1627 }
1628
1629 ✗ unlink(result.local_path.c_str());
1630 }
1631
1632
1633 WritableCatalogManager::CatalogInfo
1634 ✗ WritableCatalogManager::SnapshotCatalogsSerialized(const bool stop_for_tweaks) {
1635 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Serialized committing of file catalogs...");
1636 ✗ reinterpret_cast<WritableCatalog *>(GetRootCatalog())->SetDirty();
1637 ✗ WritableCatalogList catalogs_to_snapshot;
1638 ✗ GetModifiedCatalogs(&catalogs_to_snapshot);
1639 CatalogUploadContext unused;
1640 ✗ unused.root_catalog_info = NULL;
1641 ✗ unused.stop_for_tweaks = false;
1642 ✗ spooler_->RegisterListener(
1643 &WritableCatalogManager::CatalogUploadSerializedCallback, this, unused);
1644
1645 ✗ CatalogInfo root_catalog_info;
1646 ✗ WritableCatalogList::const_iterator i = catalogs_to_snapshot.begin();
1647 ✗ const WritableCatalogList::const_iterator iend = catalogs_to_snapshot.end();
1648 ✗ for (; i != iend; ++i) {
1649 ✗ FinalizeCatalog(*i, stop_for_tweaks);
1650
1651 // Compress and upload catalog
1652 ✗ shash::Any hash_catalog(spooler_->GetHashAlgorithm(),
1653 ✗ shash::kSuffixCatalog);
1654 ✗ if (!zlib::CompressPath2Null((*i)->database_path(), &hash_catalog)) {
1655 ✗ PANIC(kLogStderr, "could not compress catalog %s",
1656 (*i)->mountpoint().ToString().c_str());
1657 }
1658
1659 ✗ const int64_t catalog_size = GetFileSize((*i)->database_path());
1660 ✗ assert(catalog_size > 0);
1661
1662 ✗ if ((*i)->HasParent()) {
1663 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating nested catalog link");
1664 ✗ WritableCatalog *parent = (*i)->GetWritableParent();
1665 ✗ parent->UpdateNestedCatalog((*i)->mountpoint().ToString(), hash_catalog,
1666 ✗ catalog_size, (*i)->delta_counters_);
1667 ✗ (*i)->delta_counters_.SetZero();
1668 ✗ } else if ((*i)->IsRoot()) {
1669 ✗ root_catalog_info.size = catalog_size;
1670 ✗ root_catalog_info.ttl = (*i)->GetTTL();
1671 ✗ root_catalog_info.content_hash = hash_catalog;
1672 ✗ root_catalog_info.revision = (*i)->GetRevision();
1673 } else {
1674 ✗ PANIC(kLogStderr, "inconsistent state detected");
1675 }
1676
1677 ✗ spooler_->ProcessCatalog((*i)->database_path());
1678 }
1679 ✗ spooler_->WaitForUpload();
1680
1681 ✗ spooler_->UnregisterListeners();
1682 ✗ return root_catalog_info;
1683 }
1684
1685 ✗ void WritableCatalogManager::SetupSingleCatalogUploadCallback() {
1686 ✗ spooler_->RegisterListener(
1687 &WritableCatalogManager::SingleCatalogUploadCallback, this);
1688 }
1689
1690 ✗ void WritableCatalogManager::RemoveSingleCatalogUploadCallback() {
1691 ✗ spooler_->WaitForUpload(); // wait for all outstanding jobs to finish before
1692 // tearing it down
1693 ✗ spooler_->UnregisterListeners();
1694 ✗ pending_catalogs_ =
1695 ✗ {}; // whatever we couldn't process, leave it to the Commit
1696 }
1697
1698 ✗ void WritableCatalogManager::AddCatalogToQueue(const std::string &path) {
1699 ✗ SyncLock();
1700 ✗ WritableCatalog *catalog = NULL;
1701 ✗ bool const retval = FindCatalog(MakeRelativePath(path), &catalog, NULL);
1702 ✗ assert(retval);
1703 ✗ assert(catalog);
1704 ✗ catalog->SetDirty(); // ensure it's dirty so its parent will wait for it
1705 ✗ SyncUnlock();
1706 ✗ pending_catalogs_.push_back(catalog);
1707 }
1708
1709 ✗ void WritableCatalogManager::ScheduleReadyCatalogs() {
1710 // best effort to schedule as many catalogs for upload as possible
1711 ✗ for (auto it = pending_catalogs_.begin(); it != pending_catalogs_.end();) {
1712 ✗ if ((*it)->dirty_children() == 0) {
1713 ✗ FinalizeCatalog(*it, false /* stop_for_tweaks */);
1714 ✗ ScheduleCatalogProcessing(*it);
1715 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "scheduled %s for processing",
1716 ✗ (*it)->mountpoint().c_str());
1717 ✗ it = pending_catalogs_.erase(it);
1718 } else {
1719 ✗ ++it;
1720 }
1721 }
1722 }
1723
1724 // Callback for uploading a single catalog, similar to CatalogUploadCallback.
1725 // The main difference is that this callback would not trigger processing of the
1726 // parent
1727 ✗ void WritableCatalogManager::SingleCatalogUploadCallback(
1728 const upload::SpoolerResult &result) {
1729 ✗ if (result.return_code != 0) {
1730 ✗ PANIC(kLogStderr, "failed to upload '%s' (retval: %d)",
1731 result.local_path.c_str(), result.return_code);
1732 }
1733
1734 // retrieve the catalog object based on the callback information
1735 // see WritableCatalogManager::ScheduleCatalogProcessing()
1736 ✗ WritableCatalog *catalog = NULL;
1737 {
1738 ✗ MutexLockGuard const guard(catalog_processing_lock_);
1739 std::map<std::string, WritableCatalog *>::iterator const
1740 ✗ c = catalog_processing_map_.find(result.local_path);
1741 ✗ assert(c != catalog_processing_map_.end());
1742 ✗ catalog = c->second;
1743 }
1744
1745 ✗ uint64_t const catalog_size = GetFileSize(result.local_path);
1746 ✗ assert(catalog_size > 0);
1747
1748 ✗ SyncLock();
1749 ✗ if (catalog->HasParent()) {
1750 // finalized nested catalogs will update their parent's pointer
1751 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating nested catalog link");
1752 ✗ WritableCatalog *parent = catalog->GetWritableParent();
1753
1754 ✗ parent->UpdateNestedCatalog(catalog->mountpoint().ToString(),
1755 ✗ result.content_hash,
1756 catalog_size,
1757 ✗ catalog->delta_counters_);
1758 ✗ parent->DecrementDirtyChildren();
1759 ✗ catalog->delta_counters_.SetZero();
1760 }
1761 // JUMP: detach the catalog after uploading to free sqlite related resources
1762 ✗ DetachCatalog(catalog);
1763 ✗ SyncUnlock();
1764 }
1765 // using the given list of dirs, fetch all relevant catalogs
1766 ✗ void WritableCatalogManager::LoadCatalogs(
1767 const std::string &base_path, const std::unordered_set<std::string> &dirs) {
1768 // mount everything up to "base_path" first (this would be our lease_path
1769 // typically)
1770 Catalog *base_catalog;
1771 ✗ MountSubtree(PathString(base_path), NULL /* entry_point */,
1772 true /* is_listable */, &base_catalog);
1773
1774 // start up the downloader
1775 CatalogDownloadContext context;
1776 ✗ context.dirs = &dirs;
1777 ✗ catalog_download_pipeline_ = new CatalogDownloadPipeline(
1778 ✗ static_cast<SimpleCatalogManager *>(this));
1779 ✗ catalog_download_pipeline_->RegisterListener(
1780 &WritableCatalogManager::CatalogDownloadCallback, this, context);
1781 ✗ catalog_download_pipeline_->Spawn();
1782
1783 Catalog::NestedCatalogList nested_catalogs = base_catalog
1784 ✗ ->ListNestedCatalogs();
1785 ✗ for (auto it = nested_catalogs.begin(); it != nested_catalogs.end(); ++it) {
1786 // schedule relevant child nested catalogs for download
1787 ✗ std::string const mountpoint = it->mountpoint.ToString();
1788 ✗ if (dirs.find(mountpoint) != dirs.end()) {
1789 ✗ Catalog *catalog = CreateCatalog(it->mountpoint, it->hash,
1790 ✗ NULL /* parent */);
1791 {
1792 ✗ MutexLockGuard const guard(catalog_download_lock_);
1793 ✗ catalog_download_map_.insert(
1794 ✗ std::make_pair(it->hash.ToString(), catalog));
1795 }
1796 ✗ catalog_download_pipeline_->Process(it->hash);
1797 }
1798 }
1799
1800 ✗ catalog_download_pipeline_->WaitFor();
1801 ✗ delete catalog_download_pipeline_; // terminate all the threads
1802 }
1803
1804 ✗ bool WritableCatalogManager::LookupDirEntry(const string &path,
1805 const LookupOptions options,
1806 DirectoryEntry *dirent) {
1807 ✗ SyncLock();
1808 ✗ bool const exists = LookupPath(path, options, dirent);
1809 ✗ SyncUnlock();
1810 ✗ return exists;
1811 }
1812
1813 ✗ void WritableCatalogManager::CatalogHashSerializedCallback(
1814 const CompressHashResult &result) {
1815 ✗ MutexLockGuard const guard(catalog_hash_lock_);
1816 ✗ catalog_hash_map_[result.path] = result.hash;
1817 }
1818
1819 ✗ void WritableCatalogManager::CatalogDownloadCallback(
1820 const CatalogDownloadResult &result, CatalogDownloadContext context) {
1821 Catalog *downloaded_catalog;
1822 {
1823 ✗ MutexLockGuard const guard(catalog_download_lock_);
1824 ✗ auto it = catalog_download_map_.find(result.hash);
1825 ✗ assert(it != catalog_download_map_.end());
1826 ✗ downloaded_catalog = it->second;
1827 }
1828
1829 ✗ if (!downloaded_catalog->OpenDatabase(result.db_path)) {
1830 ✗ LogCvmfs(kLogCvmfs, kLogDebug, "failed to initialize catalog");
1831 ✗ delete downloaded_catalog;
1832 ✗ return;
1833 }
1834
1835 Catalog::NestedCatalogList nested_catalogs = downloaded_catalog
1836 ✗ ->ListNestedCatalogs();
1837 ✗ for (auto it = nested_catalogs.begin(); it != nested_catalogs.end(); ++it) {
1838 // schedule relevant child nested catalogs for download
1839 ✗ if (context.dirs->find(it->mountpoint.ToString()) != context.dirs->end()) {
1840 ✗ Catalog *child_catalog = CreateCatalog(it->mountpoint, it->hash,
1841 ✗ NULL /* parent */);
1842 {
1843 ✗ MutexLockGuard const guard(catalog_download_lock_);
1844 ✗ catalog_download_map_.insert(
1845 ✗ std::make_pair(it->hash.ToString(), child_catalog));
1846 }
1847 ✗ catalog_download_pipeline_->Process(it->hash);
1848 }
1849 }
1850 ✗ delete downloaded_catalog;
1851 }
1852
1853
1854 } // namespace catalog
1855