GCC Code Coverage Report


Directory: cvmfs/
File: cvmfs/catalog_mgr_rw.cc
Date: 2026-11-08 02:41:18
Exec Total Coverage
Lines: 489 897 54.5%
Branches: 363 1293 28.1%

Line Branch Exec Source
1 /**
2 * This file is part of the CernVM file system.
3 */
4
5 #include "catalog_mgr_rw.h"
6
7 #include <inttypes.h>
8 #include <unistd.h>
9
10 #include <cassert>
11 #include <cstdio>
12 #include <cstdlib>
13 #include <string>
14
15 #include "catalog_balancer.h"
16 #include "catalog_rw.h"
17 #include "manifest.h"
18 #include "statistics.h"
19 #include "upload.h"
20 #include "util/exception.h"
21 #include "util/logging.h"
22 #include "util/posix.h"
23 #include "util/smalloc.h"
24
25 using namespace std; // NOLINT
26
27 namespace catalog {
28
29 442 WritableCatalogManager::WritableCatalogManager(
30 const shash::Any &base_hash,
31 const std::string &stratum0,
32 const string &dir_temp,
33 upload::Spooler *spooler,
34 download::DownloadManager *download_manager,
35 bool enforce_limits,
36 const unsigned nested_kcatalog_limit,
37 const unsigned root_kcatalog_limit,
38 const unsigned file_mbyte_limit,
39 perf::Statistics *statistics,
40 bool is_balanceable,
41 unsigned max_weight,
42 unsigned min_weight,
43 442 const std::string &dir_cache)
44 : SimpleCatalogManager(base_hash, stratum0, dir_temp, download_manager,
45 statistics, false, dir_cache,
46 true /* copy to tmpdir */)
47 442 , spooler_(spooler)
48 442 , enforce_limits_(enforce_limits)
49 442 , nested_kcatalog_limit_(nested_kcatalog_limit)
50 442 , root_kcatalog_limit_(root_kcatalog_limit)
51 442 , file_mbyte_limit_(file_mbyte_limit)
52 442 , is_balanceable_(is_balanceable)
53 442 , max_weight_(max_weight)
54 442 , min_weight_(min_weight)
55 442 , balance_weight_(max_weight / 2) {
56 442 sync_lock_ = reinterpret_cast<pthread_mutex_t *>(
57 442 smalloc(sizeof(pthread_mutex_t)));
58 442 int retval = pthread_mutex_init(sync_lock_, NULL);
59
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 442 times.
442 assert(retval == 0);
60 442 catalog_processing_lock_ = reinterpret_cast<pthread_mutex_t *>(
61 442 smalloc(sizeof(pthread_mutex_t)));
62 442 retval = pthread_mutex_init(catalog_processing_lock_, NULL);
63
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 442 times.
442 assert(retval == 0);
64 442 catalog_download_lock_ = reinterpret_cast<pthread_mutex_t *>(
65 442 smalloc(sizeof(pthread_mutex_t)));
66 442 retval = pthread_mutex_init(catalog_download_lock_, NULL);
67
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 442 times.
442 assert(retval == 0);
68 442 catalog_hash_lock_ = reinterpret_cast<pthread_mutex_t *>(
69 442 smalloc(sizeof(pthread_mutex_t)));
70 442 retval = pthread_mutex_init(catalog_hash_lock_, NULL);
71
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 442 times.
442 assert(retval == 0);
72 442 }
73
74
75 1764 WritableCatalogManager::~WritableCatalogManager() {
76 882 pthread_mutex_destroy(sync_lock_);
77 882 free(sync_lock_);
78 882 pthread_mutex_destroy(catalog_processing_lock_);
79 882 free(catalog_processing_lock_);
80 882 pthread_mutex_destroy(catalog_download_lock_);
81 882 free(catalog_download_lock_);
82 882 pthread_mutex_destroy(catalog_hash_lock_);
83 882 free(catalog_hash_lock_);
84 1764 }
85
86
87 /**
88 * This method is virtual in AbstractCatalogManager. It returns a new catalog
89 * structure in the form the different CatalogManagers need it.
90 * In this case it returns a stub for a WritableCatalog.
91 * @param mountpoint the mount point of the catalog stub to create
92 * @param catalog_hash the content hash of the catalog to create
93 * @param parent_catalog the parent of the catalog stub to create
94 * @return a pointer to the catalog stub structure created
95 */
96 1071 Catalog *WritableCatalogManager::CreateCatalog(const PathString &mountpoint,
97 const shash::Any &catalog_hash,
98 Catalog *parent_catalog) {
99 2142 return new WritableCatalog(mountpoint.ToString(), catalog_hash,
100
2/4
✓ Branch 1 taken 1071 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1071 times.
✗ Branch 5 not taken.
2142 parent_catalog);
101 }
102
103
104 1071 void WritableCatalogManager::ActivateCatalog(Catalog *catalog) {
105 1071 catalog->TakeDatabaseFileOwnership();
106 1071 }
107
108
109 /**
110 * This method is invoked if we create a completely new repository.
111 * The new root catalog will already contain a root entry.
112 * It is uploaded by a Forklift to the upstream storage.
113 * @return true on success, false otherwise
114 */
115 374 manifest::Manifest *WritableCatalogManager::CreateRepository(
116 const string &dir_temp,
117 const bool volatile_content,
118 const std::string &voms_authz,
119 upload::Spooler *spooler) {
120 // Create a new root catalog at file_path
121
1/2
✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
374 const string file_path = dir_temp + "/new_root_catalog";
122
123 374 const shash::Algorithms hash_algorithm = spooler->GetHashAlgorithm();
124
125 // A newly created catalog always needs a root entry
126 // we create and configure this here
127
1/2
✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
374 DirectoryEntry root_entry;
128 374 root_entry.inode_ = DirectoryEntry::kInvalidInode;
129 374 root_entry.mode_ = 16877;
130 374 root_entry.size_ = 4096;
131 374 root_entry.mtime_ = time(NULL);
132 374 root_entry.uid_ = getuid();
133 374 root_entry.gid_ = getgid();
134
1/2
✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
374 root_entry.checksum_ = shash::Any(hash_algorithm);
135 374 root_entry.linkcount_ = 2;
136
1/2
✓ Branch 2 taken 374 times.
✗ Branch 3 not taken.
374 const string root_path = "";
137
138 // Create the database schema and the initial root entry
139 {
140 const std::unique_ptr<CatalogDatabase> new_clg_db(
141
1/2
✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
374 CatalogDatabase::Create(file_path));
142 374 if (new_clg_db.get() == nullptr
143
4/8
✓ Branch 0 taken 374 times.
✗ Branch 1 not taken.
✓ Branch 4 taken 374 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 374 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 374 times.
374 || !new_clg_db->InsertInitialValues(root_path, volatile_content,
144 voms_authz, root_entry)) {
145 ✗ LogCvmfs(kLogCatalog, kLogStderr, "creation of catalog '%s' failed",
146 file_path.c_str());
147 ✗ return NULL;
148 }
149
1/2
✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
374 }
150
151 // Compress root catalog;
152
1/2
✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
374 const int64_t catalog_size = GetFileSize(file_path);
153
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 374 times.
374 if (catalog_size < 0) {
154 ✗ unlink(file_path.c_str());
155 ✗ return NULL;
156 }
157
1/2
✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
374 const string file_path_compressed = file_path + ".compressed";
158
1/2
✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
374 shash::Any hash_catalog(hash_algorithm, shash::kSuffixCatalog);
159
1/2
✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
374 const bool retval = zlib::CompressPath2Path(file_path, file_path_compressed,
160 &hash_catalog);
161
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 374 times.
374 if (!retval) {
162 ✗ LogCvmfs(kLogCatalog, kLogStderr, "compression of catalog '%s' failed",
163 file_path.c_str());
164 ✗ unlink(file_path.c_str());
165 ✗ return NULL;
166 }
167 374 unlink(file_path.c_str());
168
169 // Create manifest
170
1/2
✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
374 const string manifest_path = dir_temp + "/manifest";
171 manifest::Manifest *manifest = new manifest::Manifest(hash_catalog,
172
3/6
✓ Branch 2 taken 374 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 374 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 374 times.
✗ Branch 9 not taken.
374 catalog_size, "");
173
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 374 times.
374 if (!voms_authz.empty()) {
174 ✗ manifest->set_has_alt_catalog_path(true);
175 }
176
177 // Upload catalog
178
3/6
✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 374 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 374 times.
✗ Branch 8 not taken.
374 spooler->Upload(file_path_compressed, "data/" + hash_catalog.MakePath());
179
1/2
✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
374 spooler->WaitForUpload();
180 374 unlink(file_path_compressed.c_str());
181
2/4
✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 374 times.
374 if (spooler->GetNumberOfErrors() > 0) {
182 ✗ LogCvmfs(kLogCatalog, kLogStderr, "failed to commit catalog %s",
183 file_path_compressed.c_str());
184 ✗ delete manifest;
185 ✗ return NULL;
186 }
187
188 374 return manifest;
189 374 }
190
191
192 /**
193 * Retrieve the catalog containing the given path.
194 * Other than AbstractCatalogManager::FindCatalog() this mounts nested
195 * catalogs if necessary and returns WritableCatalog objects.
196 * Furthermore it optionally returns the looked-up DirectoryEntry.
197 *
198 * @param path the path to look for
199 * @param result the retrieved catalog (as a pointer)
200 * @param dirent is set to looked up DirectoryEntry for 'path' if non-NULL
201 * @return true if catalog was found
202 */
203 4967 bool WritableCatalogManager::FindCatalog(const string &path,
204 WritableCatalog **result,
205 DirectoryEntry *dirent) {
206
1/2
✓ Branch 1 taken 4967 times.
✗ Branch 2 not taken.
4967 const PathString ps_path(path);
207
208
1/2
✓ Branch 1 taken 4967 times.
✗ Branch 2 not taken.
4967 Catalog *best_fit = AbstractCatalogManager<Catalog>::FindCatalog(ps_path);
209
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4967 times.
4967 assert(best_fit != NULL);
210 4967 Catalog *catalog = NULL;
211
1/2
✓ Branch 1 taken 4967 times.
✗ Branch 2 not taken.
4967 const bool retval = MountSubtree(ps_path, best_fit, true /* is_listable */,
212 &catalog);
213
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4967 times.
4967 if (!retval)
214 ✗ return false;
215
216 4967 *result = static_cast<WritableCatalog *>(catalog);
217
218
1/2
✓ Branch 1 taken 4967 times.
✗ Branch 2 not taken.
4967 catalog::DirectoryEntry dummy;
219
2/2
✓ Branch 0 taken 2521 times.
✓ Branch 1 taken 2446 times.
4967 if (NULL == dirent) {
220 2521 dirent = &dummy;
221 }
222
1/2
✓ Branch 1 taken 4967 times.
✗ Branch 2 not taken.
4967 const bool found = catalog->LookupPath(ps_path, dirent);
223
6/8
✓ Branch 0 taken 4955 times.
✓ Branch 1 taken 12 times.
✓ Branch 3 taken 4955 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 4955 times.
✓ Branch 7 taken 12 times.
✓ Branch 8 taken 4955 times.
4967 if (!found || !catalog->IsWritable())
224 12 return false;
225
226 4955 return true;
227 4967 }
228
229
230 ✗ WritableCatalog *WritableCatalogManager::GetHostingCatalog(
231 const std::string &path) {
232 ✗ WritableCatalog *result = NULL;
233 ✗ const bool retval = FindCatalog(MakeRelativePath(path), &result, NULL);
234 ✗ if (!retval)
235 ✗ return NULL;
236 ✗ return result;
237 }
238
239
240 /**
241 * Remove the given file from the catalogs.
242 * @param file_path the full path to the file to be removed
243 * @return true on success, false otherwise
244 */
245 30 void WritableCatalogManager::RemoveFile(const std::string &path) {
246
1/2
✓ Branch 1 taken 30 times.
✗ Branch 2 not taken.
30 const string file_path = MakeRelativePath(path);
247
248 30 SyncLock();
249 30 WritableCatalog *catalog = NULL;
250
1/2
✓ Branch 1 taken 30 times.
✗ Branch 2 not taken.
30 DirectoryEntry entry;
251
2/4
✓ Branch 1 taken 30 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 30 times.
✗ Branch 4 not taken.
30 if (FindCatalog(file_path, &catalog, &entry)) {
252
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 30 times.
30 if (entry.IsBundleTrigger()) {
253 ✗ catalog->RemoveEntry(GetParentPath(file_path) + "/.cvmfsbundle-"
254 ✗ + GetFileName(file_path));
255 }
256
1/2
✓ Branch 1 taken 30 times.
✗ Branch 2 not taken.
30 catalog->RemoveEntry(file_path);
257 }
258 30 SyncUnlock();
259
260
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 30 times.
30 if (catalog == NULL) {
261 ✗ PANIC(kLogStderr, "catalog for file '%s' cannot be found",
262 file_path.c_str());
263 }
264 30 }
265
266
267 /**
268 * Remove the given directory from the catalogs.
269 * @param directory_path the full path to the directory to be removed
270 * @return true on success, false otherwise
271 */
272 51 void WritableCatalogManager::RemoveDirectory(const std::string &path) {
273
1/2
✓ Branch 1 taken 51 times.
✗ Branch 2 not taken.
51 const string directory_path = MakeRelativePath(path);
274
1/2
✓ Branch 1 taken 51 times.
✗ Branch 2 not taken.
51 const string parent_path = GetParentPath(directory_path);
275
276 51 SyncLock();
277 WritableCatalog *catalog;
278
1/2
✓ Branch 1 taken 51 times.
✗ Branch 2 not taken.
51 DirectoryEntry parent_entry;
279
2/4
✓ Branch 1 taken 51 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 51 times.
51 if (!FindCatalog(parent_path, &catalog, &parent_entry)) {
280 ✗ PANIC(kLogStderr, "catalog for directory '%s' cannot be found",
281 directory_path.c_str());
282 }
283
284 51 parent_entry.set_linkcount(parent_entry.linkcount() - 1);
285
286
1/2
✓ Branch 1 taken 51 times.
✗ Branch 2 not taken.
51 catalog->RemoveEntry(directory_path);
287
1/2
✓ Branch 1 taken 51 times.
✗ Branch 2 not taken.
51 catalog->UpdateEntry(parent_entry, parent_path);
288
2/2
✓ Branch 1 taken 26 times.
✓ Branch 2 taken 25 times.
51 if (parent_entry.IsNestedCatalogRoot()) {
289
1/2
✓ Branch 2 taken 26 times.
✗ Branch 3 not taken.
26 LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating transition point %s",
290 parent_path.c_str());
291 WritableCatalog *parent_catalog = reinterpret_cast<WritableCatalog *>(
292 26 catalog->parent());
293 26 parent_entry.set_is_nested_catalog_mountpoint(true);
294 26 parent_entry.set_is_nested_catalog_root(false);
295
1/2
✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
26 parent_catalog->UpdateEntry(parent_entry, parent_path);
296 }
297 51 SyncUnlock();
298 51 }
299
300 /**
301 * Clone the file called `source` changing its name into `destination`, the
302 * source file is keep intact.
303 * @params destination, the name of the new file, complete path
304 * @params source, the name of the file to clone, which must be already in the
305 * repository
306 * @params fail_if_source_missing, when true (default) abort if the source is
307 * not in the catalog; when false, skip the clone and return false instead.
308 * This is used by the tarball ingestion engine to tolerate hardlinks whose
309 * target is not part of the same archive (e.g. cross-layer hardlinks in OCI
310 * image layers).
311 * @return true if the file was cloned, false if the source was missing and
312 * fail_if_source_missing was false
313 */
314 ✗ bool WritableCatalogManager::Clone(const std::string destination,
315 const std::string source,
316 const bool fail_if_source_missing) {
317 ✗ const std::string relative_source = MakeRelativePath(source);
318
319 ✗ DirectoryEntry source_dirent;
320 ✗ if (!LookupPath(relative_source, kLookupDefault, &source_dirent)) {
321 ✗ if (fail_if_source_missing) {
322 ✗ PANIC(kLogStderr, "catalog for file '%s' cannot be found, aborting",
323 source.c_str());
324 }
325 // The caller opted out of aborting and is expected to handle the missing
326 // source (e.g. by materializing a replacement file), so only log at debug
327 // level here to avoid duplicate user-facing warnings.
328 ✗ LogCvmfs(kLogCatalog, kLogDebug,
329 "catalog for clone source '%s' cannot be found, not cloning "
330 "to '%s'",
331 source.c_str(), destination.c_str());
332 ✗ return false;
333 }
334 ✗ if (source_dirent.IsDirectory()) {
335 ✗ PANIC(kLogStderr, "Trying to clone a directory: '%s', aborting",
336 source.c_str());
337 }
338
339 // if the file is already there we remove it and we add it back
340 ✗ DirectoryEntry check_dirent;
341 ✗ const bool destination_already_present = LookupPath(
342 ✗ MakeRelativePath(destination), kLookupDefault, &check_dirent);
343 ✗ if (destination_already_present) {
344 ✗ this->RemoveFile(destination);
345 }
346
347 ✗ DirectoryEntry destination_dirent(source_dirent);
348 ✗ std::string destination_dirname;
349 ✗ std::string destination_filename;
350 ✗ SplitPath(destination, &destination_dirname, &destination_filename);
351
352 ✗ destination_dirent.name_.Assign(
353 ✗ NameString(destination_filename.c_str(), destination_filename.length()));
354
355 // TODO(jblomer): clone is used by tarball engine and should eventually
356 // support extended attributes
357 ✗ this->AddFile(destination_dirent, empty_xattrs, destination_dirname);
358 ✗ return true;
359 }
360
361
362 /**
363 * Copies an entire directory tree from the existing from_dir to the
364 * non-existing to_dir. The destination's parent directory must exist. On the
365 * catalog level, the new entries will be identical to the old ones except
366 * for their path hash fields.
367 */
368 121 void WritableCatalogManager::CloneTree(const std::string &from_dir,
369 const std::string &to_dir) {
370 // Sanitize input paths
371
6/6
✓ Branch 1 taken 97 times.
✓ Branch 2 taken 24 times.
✓ Branch 4 taken 12 times.
✓ Branch 5 taken 85 times.
✓ Branch 6 taken 36 times.
✓ Branch 7 taken 85 times.
121 if (from_dir.empty() || to_dir.empty())
372 36 PANIC(kLogStderr, "clone tree from or to root impossible");
373
374
1/2
✓ Branch 1 taken 85 times.
✗ Branch 2 not taken.
85 const std::string relative_source = MakeRelativePath(from_dir);
375
1/2
✓ Branch 1 taken 85 times.
✗ Branch 2 not taken.
85 const std::string relative_dest = MakeRelativePath(to_dir);
376
377
2/2
✓ Branch 1 taken 12 times.
✓ Branch 2 taken 73 times.
85 if (relative_source == relative_dest) {
378 12 PANIC(kLogStderr, "cannot clone tree into itself ('%s')", to_dir.c_str());
379 }
380
4/7
✓ Branch 1 taken 73 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 73 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 12 times.
✓ Branch 8 taken 61 times.
73 if (HasPrefix(relative_dest, relative_source + "/", false /*ignore_case*/)) {
381 12 PANIC(kLogStderr,
382 "cannot clone tree into sub directory of source '%s' --> '%s'",
383 from_dir.c_str(), to_dir.c_str());
384 }
385
386
1/2
✓ Branch 1 taken 61 times.
✗ Branch 2 not taken.
61 DirectoryEntry source_dirent;
387
3/4
✓ Branch 1 taken 61 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 12 times.
✓ Branch 4 taken 49 times.
61 if (!LookupPath(relative_source, kLookupDefault, &source_dirent)) {
388 12 PANIC(kLogStderr, "path '%s' cannot be found, aborting", from_dir.c_str());
389 }
390
2/2
✓ Branch 1 taken 12 times.
✓ Branch 2 taken 37 times.
49 if (!source_dirent.IsDirectory()) {
391 12 PANIC(kLogStderr, "CloneTree: source '%s' not a directory, aborting",
392 from_dir.c_str());
393 }
394
395
1/2
✓ Branch 1 taken 37 times.
✗ Branch 2 not taken.
37 DirectoryEntry dest_dirent;
396
3/4
✓ Branch 1 taken 37 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 12 times.
✓ Branch 4 taken 25 times.
37 if (LookupPath(relative_dest, kLookupDefault, &dest_dirent)) {
397 12 PANIC(kLogStderr, "destination '%s' exists, aborting", to_dir.c_str());
398 }
399
400
1/2
✓ Branch 1 taken 25 times.
✗ Branch 2 not taken.
25 const std::string dest_parent = GetParentPath(relative_dest);
401
1/2
✓ Branch 1 taken 25 times.
✗ Branch 2 not taken.
25 DirectoryEntry dest_parent_dirent;
402
3/4
✓ Branch 1 taken 25 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 12 times.
✓ Branch 4 taken 13 times.
25 if (!LookupPath(dest_parent, kLookupDefault, &dest_parent_dirent)) {
403 12 PANIC(kLogStderr, "destination '%s' not on a known path, aborting",
404 to_dir.c_str());
405 }
406
407
2/4
✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 13 times.
✗ Branch 5 not taken.
13 CloneTreeImpl(PathString(from_dir),
408
1/2
✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
26 GetParentPath(to_dir),
409
2/4
✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 13 times.
✗ Branch 5 not taken.
26 NameString(GetFileName(to_dir)));
410 253 }
411
412
413 /**
414 * Called from CloneTree(), assumes that from_dir and to_dir are sufficiently
415 * sanitized
416 */
417 104 void WritableCatalogManager::CloneTreeImpl(const PathString &source_dir,
418 const std::string &dest_parent_dir,
419 const NameString &dest_name) {
420
1/2
✓ Branch 4 taken 104 times.
✗ Branch 5 not taken.
104 LogCvmfs(kLogCatalog, kLogDebug, "cloning %s --> %s/%s", source_dir.c_str(),
421
1/2
✓ Branch 1 taken 104 times.
✗ Branch 2 not taken.
208 dest_parent_dir.c_str(), dest_name.ToString().c_str());
422
3/6
✓ Branch 1 taken 104 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 104 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 104 times.
✗ Branch 8 not taken.
208 const PathString relative_source(MakeRelativePath(source_dir.ToString()));
423
424
1/2
✓ Branch 1 taken 104 times.
✗ Branch 2 not taken.
104 DirectoryEntry source_dirent;
425
1/2
✓ Branch 1 taken 104 times.
✗ Branch 2 not taken.
104 bool retval = LookupPath(relative_source, kLookupDefault, &source_dirent);
426
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 104 times.
104 assert(retval);
427
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 104 times.
104 assert(!source_dirent.IsBindMountpoint());
428
429
1/2
✓ Branch 1 taken 104 times.
✗ Branch 2 not taken.
104 DirectoryEntry dest_dirent(source_dirent);
430
1/2
✓ Branch 1 taken 104 times.
✗ Branch 2 not taken.
104 dest_dirent.name_.Assign(dest_name);
431 // Just in case, reset the nested catalog markers
432 104 dest_dirent.set_is_nested_catalog_mountpoint(false);
433 104 dest_dirent.set_is_nested_catalog_root(false);
434
435 104 XattrList xattrs;
436
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 104 times.
104 if (source_dirent.HasXattrs()) {
437 ✗ retval = LookupXattrs(relative_source, &xattrs);
438 ✗ assert(retval);
439 }
440
1/2
✓ Branch 1 taken 104 times.
✗ Branch 2 not taken.
104 AddDirectory(dest_dirent, xattrs, dest_parent_dir);
441
442
1/2
✓ Branch 1 taken 104 times.
✗ Branch 2 not taken.
104 std::string dest_dir = dest_parent_dir;
443
2/2
✓ Branch 1 taken 91 times.
✓ Branch 2 taken 13 times.
104 if (!dest_dir.empty())
444
1/2
✓ Branch 1 taken 91 times.
✗ Branch 2 not taken.
91 dest_dir.push_back('/');
445
2/4
✓ Branch 1 taken 104 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 104 times.
✗ Branch 5 not taken.
104 dest_dir += dest_name.ToString();
446 104 if (source_dirent.IsNestedCatalogRoot()
447
5/6
✓ Branch 0 taken 65 times.
✓ Branch 1 taken 39 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 65 times.
✓ Branch 5 taken 39 times.
✓ Branch 6 taken 65 times.
104 || source_dirent.IsNestedCatalogMountpoint()) {
448
1/2
✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
39 CreateNestedCatalog(dest_dir);
449 }
450
451 104 DirectoryEntryList ls;
452
1/2
✓ Branch 1 taken 104 times.
✗ Branch 2 not taken.
104 retval = Listing(relative_source, &ls, false /* expand_symlink */);
453
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 104 times.
104 assert(retval);
454
2/2
✓ Branch 1 taken 221 times.
✓ Branch 2 taken 104 times.
325 for (unsigned i = 0; i < ls.size(); ++i) {
455
1/2
✓ Branch 1 taken 221 times.
✗ Branch 2 not taken.
221 PathString sub_path(source_dir);
456
2/4
✓ Branch 1 taken 221 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 221 times.
221 assert(!sub_path.IsEmpty());
457
1/2
✓ Branch 1 taken 221 times.
✗ Branch 2 not taken.
221 sub_path.Append("/", 1);
458
3/6
✓ Branch 2 taken 221 times.
✗ Branch 3 not taken.
✓ Branch 7 taken 221 times.
✗ Branch 8 not taken.
✓ Branch 11 taken 221 times.
✗ Branch 12 not taken.
221 sub_path.Append(ls[i].name().GetChars(), ls[i].name().GetLength());
459
460
2/2
✓ Branch 2 taken 91 times.
✓ Branch 3 taken 130 times.
221 if (ls[i].IsDirectory()) {
461
2/4
✓ Branch 2 taken 91 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 91 times.
✗ Branch 6 not taken.
91 CloneTreeImpl(sub_path, dest_dir, ls[i].name());
462 91 continue;
463 }
464
465 // We break hard-links during cloning
466 130 ls[i].set_hardlink_group(0);
467 130 ls[i].set_linkcount(1);
468
469 130 xattrs.Clear();
470
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 130 times.
130 if (ls[i].HasXattrs()) {
471 ✗ retval = LookupXattrs(sub_path, &xattrs);
472 ✗ assert(retval);
473 }
474
475
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 130 times.
130 if (ls[i].IsChunkedFile()) {
476 ✗ FileChunkList chunks;
477 const std::string relative_sub_path = MakeRelativePath(
478 ✗ sub_path.ToString());
479 ✗ retval = ListFileChunks(PathString(relative_sub_path),
480 ✗ ls[i].hash_algorithm(), &chunks);
481 ✗ assert(retval);
482 ✗ AddChunkedFile(ls[i], xattrs, dest_dir, chunks);
483 ✗ } else {
484
1/2
✓ Branch 2 taken 130 times.
✗ Branch 3 not taken.
130 AddFile(ls[i], xattrs, dest_dir);
485 }
486
2/2
✓ Branch 1 taken 130 times.
✓ Branch 2 taken 91 times.
221 }
487 104 }
488
489
490 /**
491 * Add a new directory to the catalogs.
492 * @param entry a DirectoryEntry structure describing the new directory
493 * @param parent_directory the absolute path of the directory containing the
494 * directory to be created
495 * @return true on success, false otherwise
496 */
497 1846 void WritableCatalogManager::AddDirectory(const DirectoryEntryBase &entry,
498 const XattrList &xattrs,
499 const std::string &parent_directory) {
500
1/2
✓ Branch 1 taken 1846 times.
✗ Branch 2 not taken.
1846 const string parent_path = MakeRelativePath(parent_directory);
501
1/2
✓ Branch 1 taken 1846 times.
✗ Branch 2 not taken.
1846 string directory_path = parent_path + "/";
502
3/6
✓ Branch 1 taken 1846 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 1846 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 1846 times.
✗ Branch 10 not taken.
1846 directory_path.append(entry.name().GetChars(), entry.name().GetLength());
503
504 1846 SyncLock();
505 WritableCatalog *catalog;
506
1/2
✓ Branch 1 taken 1846 times.
✗ Branch 2 not taken.
1846 DirectoryEntry parent_entry;
507
2/4
✓ Branch 1 taken 1846 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 1846 times.
1846 if (!FindCatalog(parent_path, &catalog, &parent_entry)) {
508 ✗ PANIC(kLogStderr, "catalog for directory '%s' cannot be found",
509 directory_path.c_str());
510 }
511
512
1/2
✓ Branch 1 taken 1846 times.
✗ Branch 2 not taken.
1846 DirectoryEntry fixed_hardlink_count(entry);
513 1846 fixed_hardlink_count.set_linkcount(2);
514
1/2
✓ Branch 1 taken 1846 times.
✗ Branch 2 not taken.
1846 catalog->AddEntry(fixed_hardlink_count, xattrs, directory_path, parent_path);
515
516 1846 parent_entry.set_linkcount(parent_entry.linkcount() + 1);
517
1/2
✓ Branch 1 taken 1846 times.
✗ Branch 2 not taken.
1846 catalog->UpdateEntry(parent_entry, parent_path);
518
2/2
✓ Branch 1 taken 39 times.
✓ Branch 2 taken 1807 times.
1846 if (parent_entry.IsNestedCatalogRoot()) {
519
1/2
✓ Branch 2 taken 39 times.
✗ Branch 3 not taken.
39 LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating transition point %s",
520 parent_path.c_str());
521 WritableCatalog *parent_catalog = reinterpret_cast<WritableCatalog *>(
522 39 catalog->parent());
523 39 parent_entry.set_is_nested_catalog_mountpoint(true);
524 39 parent_entry.set_is_nested_catalog_root(false);
525
1/2
✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
39 parent_catalog->UpdateEntry(parent_entry, parent_path);
526 }
527 1846 SyncUnlock();
528 1846 }
529
530 /**
531 * Add a new file to the catalogs.
532 * @param entry a DirectoryEntry structure describing the new file
533 * @param parent_directory the absolute path of the directory containing the
534 * file to be created
535 * @return true on success, false otherwise
536 */
537 2396 void WritableCatalogManager::AddFile(const DirectoryEntry &entry,
538 const XattrList &xattrs,
539 const std::string &parent_directory) {
540
1/2
✓ Branch 1 taken 2396 times.
✗ Branch 2 not taken.
2396 const string parent_path = MakeRelativePath(parent_directory);
541
1/2
✓ Branch 1 taken 2396 times.
✗ Branch 2 not taken.
2396 const string file_path = entry.GetFullPath(parent_path);
542
543 2396 SyncLock();
544 WritableCatalog *catalog;
545
2/4
✓ Branch 1 taken 2396 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 2396 times.
2396 if (!FindCatalog(parent_path, &catalog)) {
546 ✗ PANIC(kLogStderr, "catalog for file '%s' cannot be found",
547 file_path.c_str());
548 }
549
550
4/6
✓ Branch 1 taken 2368 times.
✓ Branch 2 taken 28 times.
✓ Branch 4 taken 2368 times.
✗ Branch 5 not taken.
✗ Branch 8 not taken.
✓ Branch 9 taken 2368 times.
2396 assert(!entry.IsRegular() || entry.IsChunkedFile()
551 || !entry.checksum().IsNull());
552
3/4
✓ Branch 1 taken 28 times.
✓ Branch 2 taken 2368 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 28 times.
2396 assert(entry.IsRegular() || !entry.IsExternalFile());
553
554 // check if file is too big
555
1/2
✓ Branch 1 taken 2396 times.
✗ Branch 2 not taken.
2396 const unsigned mbytes = entry.size() / (1024 * 1024);
556
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2396 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
2396 if ((file_mbyte_limit_ > 0) && (mbytes > file_mbyte_limit_)) {
557 ✗ LogCvmfs(kLogCatalog, kLogStderr,
558 "%s: file at %s is larger than %u megabytes (%u). "
559 "CernVM-FS works best with small files. "
560 "Please remove the file or increase the limit.",
561 ✗ enforce_limits_ ? "FATAL" : "WARNING", file_path.c_str(),
562 file_mbyte_limit_, mbytes);
563 ✗ if (enforce_limits_)
564 ✗ PANIC(kLogStderr, "file at %s is larger than %u megabytes (%u).",
565 file_path.c_str(), file_mbyte_limit_, mbytes);
566 }
567
568
1/2
✓ Branch 1 taken 2396 times.
✗ Branch 2 not taken.
2396 catalog->AddEntry(entry, xattrs, file_path, parent_path);
569 2396 SyncUnlock();
570 2396 }
571
572
573 ✗ void WritableCatalogManager::AddChunkedFile(const DirectoryEntryBase &entry,
574 const XattrList &xattrs,
575 const std::string &parent_directory,
576 const FileChunkList &file_chunks) {
577 ✗ assert(file_chunks.size() > 0);
578
579 ✗ DirectoryEntry full_entry(entry);
580 ✗ full_entry.set_is_chunked_file(true);
581
582 ✗ AddFile(full_entry, xattrs, parent_directory);
583
584 ✗ const string parent_path = MakeRelativePath(parent_directory);
585 ✗ const string file_path = entry.GetFullPath(parent_path);
586
587 ✗ SyncLock();
588 WritableCatalog *catalog;
589 ✗ if (!FindCatalog(parent_path, &catalog)) {
590 ✗ PANIC(kLogStderr, "catalog for file '%s' cannot be found",
591 file_path.c_str());
592 }
593
594 ✗ for (unsigned i = 0; i < file_chunks.size(); ++i) {
595 ✗ catalog->AddFileChunk(file_path, *file_chunks.AtPtr(i));
596 }
597 ✗ SyncUnlock();
598 }
599
600 /**
601 * Reads the entry given by file_path and sets the bundle trigger flag
602 * accordingly. If new_value is true, the regular file at file_path must exist.
603 * If new_value is false, a missing file_path is ignored (otherwise it would
604 * be hard to get rid of dangling bundle trigger markers).
605 */
606 ✗ void WritableCatalogManager::UpdateBundleTrigger(const std::string &file_path,
607 bool new_value) {
608 ✗ SyncLock();
609
610 ✗ WritableCatalog *catalog = NULL;
611 ✗ DirectoryEntry entry;
612 ✗ if (!FindCatalog(file_path, &catalog, &entry)) {
613 ✗ SyncUnlock();
614
615 ✗ if (new_value) {
616 ✗ PANIC(kLogStderr, "failed to find catalog of %s", file_path.c_str());
617 }
618
619 ✗ LogCvmfs(kLogCatalog, kLogDebug, "dangling bundle trigger %s",
620 file_path.c_str());
621 ✗ return;
622 }
623
624 ✗ if (new_value && !entry.IsRegular()) {
625 ✗ SyncUnlock();
626 ✗ PANIC(kLogStderr, "failed to set bundle trigger on non-regular file %s",
627 file_path.c_str());
628 }
629
630 ✗ entry.set_is_bundle_trigger(new_value);
631 ✗ catalog->UpdateEntry(entry, file_path);
632
633 ✗ SyncUnlock();
634 }
635
636
637 /**
638 * Add a hardlink group to the catalogs.
639 * @param entries a list of DirectoryEntries describing the new files
640 * @param parent_directory the absolute path of the directory containing the
641 * files to be created
642 * @return true on success, false otherwise
643 */
644 ✗ void WritableCatalogManager::AddHardlinkGroup(
645 const DirectoryEntryBaseList &entries,
646 const XattrList &xattrs,
647 const std::string &parent_directory,
648 const FileChunkList &file_chunks) {
649 ✗ assert(entries.size() >= 1);
650 ✗ assert(file_chunks.IsEmpty() || entries[0].IsRegular());
651 ✗ if (entries.size() == 1) {
652 ✗ DirectoryEntry fix_linkcount(entries[0]);
653 ✗ fix_linkcount.set_linkcount(1);
654 ✗ if (file_chunks.IsEmpty())
655 ✗ return AddFile(fix_linkcount, xattrs, parent_directory);
656 ✗ return AddChunkedFile(fix_linkcount, xattrs, parent_directory, file_chunks);
657 }
658
659 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "adding hardlink group %s/%s",
660 ✗ parent_directory.c_str(), entries[0].name().c_str());
661
662 // Hardlink groups have to reside in the same directory.
663 // Therefore we only have one parent directory here
664 ✗ const string parent_path = MakeRelativePath(parent_directory);
665
666 // check if hard link is too big
667 ✗ const unsigned mbytes = entries[0].size() / (1024 * 1024);
668 ✗ if ((file_mbyte_limit_ > 0) && (mbytes > file_mbyte_limit_)) {
669 ✗ LogCvmfs(kLogCatalog, kLogStderr,
670 "%s: hard link at %s is larger than %u megabytes (%u). "
671 "CernVM-FS works best with small files. "
672 "Please remove the file or increase the limit.",
673 ✗ enforce_limits_ ? "FATAL" : "WARNING",
674 ✗ (parent_path + entries[0].name().ToString()).c_str(),
675 file_mbyte_limit_, mbytes);
676 ✗ if (enforce_limits_)
677 ✗ PANIC(kLogStderr, "hard link at %s is larger than %u megabytes (%u)",
678 (parent_path + entries[0].name().ToString()).c_str(),
679 file_mbyte_limit_, mbytes);
680 }
681
682 ✗ SyncLock();
683 WritableCatalog *catalog;
684 ✗ if (!FindCatalog(parent_path, &catalog)) {
685 ✗ PANIC(kLogStderr,
686 "catalog for hardlink group containing '%s' cannot be found",
687 parent_path.c_str());
688 }
689
690 // Get a valid hardlink group id for the catalog the group will end up in
691 // TODO(unknown): Compaction
692 ✗ const uint32_t new_group_id = catalog->GetMaxLinkId() + 1;
693 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "hardlink group id %u issued",
694 new_group_id);
695 ✗ assert(new_group_id > 0);
696
697 // Add the file entries to the catalog
698 ✗ for (DirectoryEntryBaseList::const_iterator i = entries.begin(),
699 ✗ iEnd = entries.end();
700 ✗ i != iEnd;
701 ✗ ++i) {
702 ✗ string file_path = parent_path + "/";
703 ✗ file_path.append(i->name().GetChars(), i->name().GetLength());
704
705 // create a fully fledged DirectoryEntry to add the hardlink group to it
706 // which is CVMFS specific meta data.
707 ✗ DirectoryEntry hardlink(*i);
708 ✗ hardlink.set_hardlink_group(new_group_id);
709 ✗ hardlink.set_linkcount(entries.size());
710 ✗ hardlink.set_is_chunked_file(!file_chunks.IsEmpty());
711
712 ✗ catalog->AddEntry(hardlink, xattrs, file_path, parent_path);
713 ✗ if (hardlink.IsChunkedFile()) {
714 ✗ for (unsigned i = 0; i < file_chunks.size(); ++i) {
715 ✗ catalog->AddFileChunk(file_path, *file_chunks.AtPtr(i));
716 }
717 }
718 }
719 ✗ SyncUnlock();
720 }
721
722
723 ✗ void WritableCatalogManager::ShrinkHardlinkGroup(const string &remove_path) {
724 ✗ const string relative_path = MakeRelativePath(remove_path);
725
726 ✗ SyncLock();
727 WritableCatalog *catalog;
728 ✗ if (!FindCatalog(relative_path, &catalog)) {
729 ✗ PANIC(kLogStderr,
730 "catalog for hardlink group containing '%s' cannot be found",
731 remove_path.c_str());
732 }
733
734 ✗ catalog->IncLinkcount(relative_path, -1);
735 ✗ SyncUnlock();
736 }
737
738
739 /**
740 * Update entry meta data (mode, owner, ...).
741 * CVMFS specific meta data (i.e. nested catalog transition points) are NOT
742 * changed by this method, although transition points intrinsics are taken into
743 * account, to keep nested catalogs consistent.
744 * @param entry the directory entry to be touched
745 * @param path the path of the directory entry to be touched
746 */
747 6 void WritableCatalogManager::TouchDirectory(const DirectoryEntryBase &entry,
748 const XattrList &xattrs,
749 const std::string &directory_path) {
750
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6 times.
6 assert(entry.IsDirectory());
751
752
1/2
✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
6 const string entry_path = MakeRelativePath(directory_path);
753
1/2
✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
6 const string parent_path = GetParentPath(entry_path);
754
755 6 SyncLock();
756 // find the catalog to be updated
757 WritableCatalog *catalog;
758
2/4
✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 6 times.
6 if (!FindCatalog(parent_path, &catalog)) {
759 ✗ PANIC(kLogStderr, "catalog for entry '%s' cannot be found",
760 entry_path.c_str());
761 }
762
763
1/2
✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
6 catalog->TouchEntry(entry, xattrs, entry_path);
764
765 // since we deal with a directory here, we might just touch a
766 // nested catalog transition point. If this is the case we would need to
767 // update two catalog entries:
768 // * the nested catalog MOUNTPOINT in the parent catalog
769 // * the nested catalog ROOT in the nested catalog
770
771 // first check if we really have a nested catalog transition point
772
1/2
✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
6 catalog::DirectoryEntry potential_transition_point;
773
1/2
✓ Branch 3 taken 6 times.
✗ Branch 4 not taken.
6 const PathString transition_path(entry_path.data(), entry_path.length());
774
1/2
✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
6 bool retval = catalog->LookupPath(transition_path,
775 &potential_transition_point);
776
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6 times.
6 assert(retval);
777
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6 times.
6 if (potential_transition_point.IsNestedCatalogMountpoint()) {
778 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating transition point at %s",
779 entry_path.c_str());
780
781 // find and mount nested catalog associated to this transition point
782 ✗ shash::Any nested_hash;
783 uint64_t nested_size;
784 ✗ retval = catalog->FindNested(transition_path, &nested_hash, &nested_size);
785 ✗ assert(retval);
786 Catalog *nested_catalog;
787 ✗ nested_catalog = MountCatalog(transition_path, nested_hash, catalog);
788 ✗ assert(nested_catalog != NULL);
789
790 // update nested catalog root in the child catalog
791 reinterpret_cast<WritableCatalog *>(nested_catalog)
792 ✗ ->TouchEntry(entry, xattrs, entry_path);
793 }
794
795 6 SyncUnlock();
796 6 }
797
798
799 /**
800 * Create a new nested catalog. Includes moving all entries belonging there
801 * from it's parent catalog.
802 * @param mountpoint the path of the directory to become a nested root
803 * @return true on success, false otherwise
804 */
805 493 void WritableCatalogManager::CreateNestedCatalog(
806 const std::string &mountpoint) {
807
1/2
✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
493 const string nested_root_path = MakeRelativePath(mountpoint);
808
1/2
✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
493 const PathString ps_nested_root_path(nested_root_path);
809
810 493 SyncLock();
811 // Find the catalog currently containing the directory structure, which
812 // will be represented as a new nested catalog and its root-entry/mountpoint
813 // along the way
814 493 WritableCatalog *old_catalog = NULL;
815
1/2
✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
493 DirectoryEntry new_root_entry;
816
2/4
✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 493 times.
493 if (!FindCatalog(nested_root_path, &old_catalog, &new_root_entry)) {
817 ✗ PANIC(kLogStderr,
818 "failed to create nested catalog '%s': "
819 "mountpoint was not found in current catalog structure",
820 nested_root_path.c_str());
821 }
822
823 // Create the database schema and the initial root entry
824 // for the new nested catalog
825
1/2
✓ Branch 2 taken 493 times.
✗ Branch 3 not taken.
493 const string database_file_path = CreateTempPath(dir_temp() + "/catalog",
826
1/2
✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
493 0666);
827 493 const bool volatile_content = false;
828
1/2
✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
493 CatalogDatabase *new_catalog_db = CatalogDatabase::Create(database_file_path);
829
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 493 times.
493 assert(NULL != new_catalog_db);
830 // Note we do not set the external_data bit for nested catalogs
831
2/4
✓ Branch 2 taken 493 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 493 times.
✗ Branch 6 not taken.
493 bool retval = new_catalog_db->InsertInitialValues(
832 nested_root_path,
833 volatile_content,
834 "", // At this point, only root
835 // catalog gets VOMS authz
836 new_root_entry);
837
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 493 times.
493 assert(retval);
838 // TODO(rmeusel): we need a way to attach a catalog directly from an open
839 // database to remove this indirection
840
1/2
✓ Branch 0 taken 493 times.
✗ Branch 1 not taken.
493 delete new_catalog_db;
841 493 new_catalog_db = NULL;
842
843 // Attach the just created nested catalog
844
2/4
✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 493 times.
✗ Branch 5 not taken.
493 Catalog *new_catalog = CreateCatalog(ps_nested_root_path, shash::Any(),
845 old_catalog);
846
1/2
✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
493 retval = AttachCatalog(database_file_path, new_catalog);
847
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 493 times.
493 assert(retval);
848
849
2/4
✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 493 times.
493 assert(new_catalog->IsWritable());
850 493 WritableCatalog *wr_new_catalog = static_cast<WritableCatalog *>(new_catalog);
851
852
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 493 times.
493 if (new_root_entry.HasXattrs()) {
853 ✗ XattrList xattrs;
854 ✗ retval = old_catalog->LookupXattrsPath(ps_nested_root_path, &xattrs);
855 ✗ assert(retval);
856 ✗ wr_new_catalog->TouchEntry(new_root_entry, xattrs, nested_root_path);
857 }
858
859 // From now on, there are two catalogs, spanning the same directory structure
860 // we have to split the overlapping directory entries from the old catalog
861 // to the new catalog to re-gain a valid catalog structure
862
1/2
✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
493 old_catalog->Partition(wr_new_catalog);
863
864 // Add the newly created nested catalog to the references of the containing
865 // catalog
866
4/8
✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 493 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 493 times.
✗ Branch 8 not taken.
✓ Branch 10 taken 493 times.
✗ Branch 11 not taken.
493 old_catalog->InsertNestedCatalog(new_catalog->mountpoint().ToString(), NULL,
867 493 shash::Any(spooler_->GetHashAlgorithm()), 0);
868
869 // Fix subtree counters in new nested catalogs: subtree is the sum of all
870 // entries of all "grand-nested" catalogs
871 // Note: taking a copy of the nested catalog list here
872 const Catalog::NestedCatalogList
873
1/2
✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
493 &grand_nested = wr_new_catalog->ListOwnNestedCatalogs();
874
1/2
✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
493 DeltaCounters fix_subtree_counters;
875 986 for (Catalog::NestedCatalogList::const_iterator i = grand_nested.begin(),
876 493 iEnd = grand_nested.end();
877
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 493 times.
493 i != iEnd;
878 ✗ ++i) {
879 WritableCatalog *grand_catalog;
880 ✗ retval = FindCatalog(i->mountpoint.ToString(), &grand_catalog);
881 ✗ assert(retval);
882 ✗ const Counters &grand_counters = grand_catalog->GetCounters();
883 ✗ grand_counters.AddAsSubtree(&fix_subtree_counters);
884 }
885 493 const DeltaCounters save_counters = wr_new_catalog->delta_counters_;
886 493 wr_new_catalog->delta_counters_ = fix_subtree_counters;
887
1/2
✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
493 wr_new_catalog->UpdateCounters();
888 493 wr_new_catalog->delta_counters_ = save_counters;
889
890 493 SyncUnlock();
891 493 }
892
893
894 /**
895 * Remove a nested catalog
896 *
897 * If the merged parameter is true, when you remove a nested catalog
898 * all entries currently held by it will be merged into its parent
899 * catalog.
900 * @param mountpoint - the path of the nested catalog to be removed
901 * @param merge - merge the subtree associated with the nested catalog
902 * into its parent catalog
903 * @return - true on success, false otherwise
904 */
905 32 void WritableCatalogManager::RemoveNestedCatalog(const string &mountpoint,
906 const bool merge) {
907
1/2
✓ Branch 1 taken 32 times.
✗ Branch 2 not taken.
32 const string nested_root_path = MakeRelativePath(mountpoint);
908
909 32 SyncLock();
910 // Find the catalog which should be removed
911 32 WritableCatalog *nested_catalog = NULL;
912
2/4
✓ Branch 1 taken 32 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 32 times.
32 if (!FindCatalog(nested_root_path, &nested_catalog)) {
913 ✗ PANIC(kLogStderr,
914 "failed to remove nested catalog '%s': "
915 "mountpoint was not found in current catalog structure",
916 nested_root_path.c_str());
917 }
918
919 // Check if the found catalog is really the nested catalog to be deleted
920
6/19
✓ Branch 1 taken 32 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 32 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 32 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 32 times.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 32 times.
✗ Branch 14 not taken.
✓ Branch 16 taken 32 times.
✗ Branch 17 not taken.
✗ Branch 19 not taken.
✗ Branch 20 not taken.
✗ Branch 22 not taken.
✗ Branch 23 not taken.
64 assert(!nested_catalog->IsRoot()
921 && (nested_catalog->mountpoint().ToString() == nested_root_path));
922
923
1/2
✓ Branch 0 taken 32 times.
✗ Branch 1 not taken.
32 if (merge) {
924 // Merge all data from the nested catalog into it's parent
925
1/2
✓ Branch 1 taken 32 times.
✗ Branch 2 not taken.
32 nested_catalog->MergeIntoParent();
926 } else {
927 ✗ nested_catalog->RemoveFromParent();
928 }
929
930 // Delete the catalog database file from the working copy
931
2/6
✓ Branch 1 taken 32 times.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 32 times.
32 if (unlink(nested_catalog->database_path().c_str()) != 0) {
932 ✗ PANIC(kLogStderr,
933 "unable to delete the removed nested catalog database file '%s'",
934 nested_catalog->database_path().c_str());
935 }
936
937 // Remove the catalog from internal data structures.
938 // On merge the children have been re-parented into this catalog's parent by
939 // MergeIntoParent (CopyCatalogsToParent), so only this catalog is detached.
940 // On removal (fast delete) the whole subtree is discarded together with the
941 // catalog, so any attached children must be detached as well - otherwise they
942 // would be left in the manager with a dangling parent pointer.
943
1/2
✓ Branch 0 taken 32 times.
✗ Branch 1 not taken.
32 if (merge) {
944
1/2
✓ Branch 1 taken 32 times.
✗ Branch 2 not taken.
32 DetachCatalog(nested_catalog);
945 } else {
946 ✗ DetachSubtree(nested_catalog);
947 }
948 32 SyncUnlock();
949 32 }
950
951
952 /**
953 * Swap in a new nested catalog
954 *
955 * The old nested catalog must not have been already attached to the
956 * catalog tree. This method will not attach the new nested catalog
957 * to the catalog tree.
958 *
959 * @param mountpoint - the path of the nested catalog to be removed
960 * @param new_hash - the hash of the new nested catalog
961 * @param new_size - the size of the new nested catalog
962 */
963 87 void WritableCatalogManager::SwapNestedCatalog(const string &mountpoint,
964 const shash::Any &new_hash,
965 const uint64_t new_size) {
966
1/2
✓ Branch 1 taken 87 times.
✗ Branch 2 not taken.
87 const string nested_root_path = MakeRelativePath(mountpoint);
967
1/2
✓ Branch 1 taken 87 times.
✗ Branch 2 not taken.
87 const string parent_path = GetParentPath(nested_root_path);
968
1/2
✓ Branch 1 taken 87 times.
✗ Branch 2 not taken.
87 const PathString nested_root_ps = PathString(nested_root_path);
969
970 87 SyncLock();
971
972 // Find the immediate parent catalog
973 87 WritableCatalog *parent = NULL;
974
3/4
✓ Branch 1 taken 87 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 12 times.
✓ Branch 4 taken 75 times.
87 if (!FindCatalog(parent_path, &parent)) {
975 12 SyncUnlock(); // this is needed for the unittest. otherwise they get stuck
976 12 PANIC(kLogStderr,
977 "failed to swap nested catalog '%s': could not find parent '%s'",
978 nested_root_path.c_str(), parent_path.c_str());
979 }
980
981 // Get old nested catalog counters
982
1/2
✓ Branch 1 taken 75 times.
✗ Branch 2 not taken.
75 Catalog *old_attached_catalog = parent->FindChild(nested_root_ps);
983
1/2
✓ Branch 1 taken 75 times.
✗ Branch 2 not taken.
75 Counters old_counters;
984
2/2
✓ Branch 0 taken 12 times.
✓ Branch 1 taken 63 times.
75 if (old_attached_catalog) {
985 // Old catalog was already attached (e.g. as a child catalog
986 // attached by a prior call to CreateNestedCatalog()). Ensure
987 // that it has not been modified, get counters, and detach it.
988 12 WritableCatalogList list;
989
2/4
✓ Branch 1 taken 12 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 12 times.
✗ Branch 4 not taken.
12 if (GetModifiedCatalogLeafsRecursively(old_attached_catalog, &list)) {
990 12 SyncUnlock();
991 12 PANIC(kLogStderr,
992 "failed to swap nested catalog '%s': already modified",
993 nested_root_path.c_str());
994 }
995 ✗ old_counters = old_attached_catalog->GetCounters();
996 ✗ DetachSubtree(old_attached_catalog);
997
998 12 } else {
999 // Old catalog was not attached. Download a freely attached
1000 // version and get counters.
1001
1/2
✓ Branch 1 taken 63 times.
✗ Branch 2 not taken.
63 shash::Any old_hash;
1002 uint64_t old_size;
1003
1/2
✓ Branch 1 taken 63 times.
✗ Branch 2 not taken.
63 const bool old_found = parent->FindNested(nested_root_ps, &old_hash,
1004 &old_size);
1005
2/2
✓ Branch 0 taken 12 times.
✓ Branch 1 taken 51 times.
63 if (!old_found) {
1006 12 SyncUnlock();
1007 12 PANIC(kLogStderr,
1008 "failed to swap nested catalog '%s': not found in parent",
1009 nested_root_path.c_str());
1010 }
1011 const std::unique_ptr<Catalog> old_free_catalog(
1012
1/2
✓ Branch 1 taken 51 times.
✗ Branch 2 not taken.
51 LoadFreeCatalog(nested_root_ps, old_hash));
1013
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 51 times.
51 if (old_free_catalog.get() == nullptr) {
1014 ✗ SyncUnlock();
1015 ✗ PANIC(kLogStderr,
1016 "failed to swap nested catalog '%s': failed to load old catalog",
1017 nested_root_path.c_str());
1018 }
1019 51 old_counters = old_free_catalog->GetCounters();
1020 51 }
1021
1022 // Load freely attached new catalog
1023 const std::unique_ptr<Catalog> new_catalog(
1024
2/2
✓ Branch 1 taken 39 times.
✓ Branch 2 taken 12 times.
51 LoadFreeCatalog(nested_root_ps, new_hash));
1025
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 39 times.
39 if (new_catalog.get() == nullptr) {
1026 ✗ SyncUnlock();
1027 ✗ PANIC(kLogStderr,
1028 "failed to swap nested catalog '%s': failed to load new catalog",
1029 nested_root_path.c_str());
1030 }
1031
1032 // Get new catalog root directory entry
1033
1/2
✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
39 DirectoryEntry dirent;
1034 39 XattrList xattrs;
1035
1/2
✓ Branch 2 taken 39 times.
✗ Branch 3 not taken.
39 const bool dirent_found = new_catalog->LookupPath(nested_root_ps, &dirent);
1036
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 39 times.
39 if (!dirent_found) {
1037 ✗ SyncUnlock();
1038 ✗ PANIC(kLogStderr,
1039 "failed to swap nested catalog '%s': missing dirent in new catalog",
1040 nested_root_path.c_str());
1041 }
1042
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 39 times.
39 if (dirent.HasXattrs()) {
1043 ✗ const bool xattrs_found = new_catalog->LookupXattrsPath(nested_root_ps,
1044 &xattrs);
1045 ✗ if (!xattrs_found) {
1046 ✗ SyncUnlock();
1047 ✗ PANIC(kLogStderr,
1048 "failed to swap nested catalog '%s': missing xattrs in new catalog",
1049 nested_root_path.c_str());
1050 }
1051 }
1052
1053 // Swap catalogs
1054
1/2
✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
39 parent->RemoveNestedCatalog(nested_root_path, NULL);
1055
1/2
✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
39 parent->InsertNestedCatalog(nested_root_path, NULL, new_hash, new_size);
1056
1057 // Update parent directory entry
1058 39 dirent.set_is_nested_catalog_mountpoint(true);
1059 39 dirent.set_is_nested_catalog_root(false);
1060
1/2
✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
39 parent->UpdateEntry(dirent, nested_root_path);
1061
1/2
✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
39 parent->TouchEntry(dirent, xattrs, nested_root_path);
1062
1063 // Update counters
1064
1/2
✓ Branch 3 taken 39 times.
✗ Branch 4 not taken.
39 const DeltaCounters delta = Counters::Diff(old_counters,
1065 new_catalog->GetCounters());
1066
1/2
✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
39 delta.PopulateToParent(&parent->delta_counters_);
1067
1068 39 SyncUnlock();
1069 183 }
1070
1071 /**
1072 * Install a nested catalog (catalog hierarchy) at a new, empty directory
1073 *
1074 * The mountpoint directory must not yet exist. Its parent directory, however
1075 * must exist. This method combines functionality from AddDirectory(),
1076 * CreateNestedCatalog() and SwapNestedCatalog().
1077 * The new nested catalog won't get attached.
1078 *
1079 * @param mountpoint - the path where the nested catalog should be installed
1080 * @param new_hash - the hash of the new nested catalog
1081 * @param new_size - the size of the new nested catalog
1082 */
1083 39 void WritableCatalogManager::GraftNestedCatalog(const string &mountpoint,
1084 const shash::Any &new_hash,
1085 const uint64_t new_size) {
1086
2/2
✓ Branch 1 taken 26 times.
✓ Branch 2 taken 13 times.
39 if (!TryGraftNestedCatalog(mountpoint, new_hash, new_size)) {
1087 26 PANIC(kLogStderr, "failed to graft nested catalog '%s'",
1088 mountpoint.c_str());
1089 }
1090 13 }
1091
1092 39 bool WritableCatalogManager::TryGraftNestedCatalog(const string &mountpoint,
1093 const shash::Any &new_hash,
1094 const uint64_t new_size) {
1095
1/2
✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
39 const string nested_root_path = MakeRelativePath(mountpoint);
1096
1/2
✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
39 const string parent_path = GetParentPath(nested_root_path);
1097
1/2
✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
39 const PathString nested_root_ps = PathString(nested_root_path);
1098
1099
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 39 times.
39 if (nested_root_path.empty()) {
1100 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1101 "failed to graft nested catalog: empty mountpoint");
1102 ✗ return false;
1103 }
1104
1105 // Load freely attached new catalog
1106 const std::unique_ptr<Catalog> new_catalog(
1107
1/2
✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
39 LoadFreeCatalog(nested_root_ps, new_hash));
1108
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 39 times.
39 if (new_catalog.get() == nullptr) {
1109 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1110 "failed to graft nested catalog '%s': failed to load new catalog",
1111 nested_root_path.c_str());
1112 ✗ return false;
1113 }
1114
4/7
✓ Branch 2 taken 39 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 39 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 13 times.
✓ Branch 9 taken 26 times.
39 if (new_catalog->root_prefix() != nested_root_ps) {
1115
1/7
✗ Branch 2 not taken.
✓ Branch 3 taken 13 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
13 LogCvmfs(kLogCatalog, kLogStderr,
1116 "invalid nested catalog for grafting at '%s': catalog rooted at "
1117 "'%s'",
1118 nested_root_path.c_str(),
1119
2/4
✓ Branch 2 taken 13 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 13 times.
✗ Branch 6 not taken.
26 new_catalog->root_prefix().ToString().c_str());
1120 13 return false;
1121 }
1122
1123 // Get new catalog root directory entry
1124
1/2
✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
26 DirectoryEntry dirent;
1125 26 XattrList xattrs;
1126
1/2
✓ Branch 2 taken 26 times.
✗ Branch 3 not taken.
26 const bool dirent_found = new_catalog->LookupPath(nested_root_ps, &dirent);
1127
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 26 times.
26 if (!dirent_found) {
1128 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1129 "failed to graft nested catalog '%s': missing dirent in new "
1130 "catalog",
1131 nested_root_path.c_str());
1132 ✗ return false;
1133 }
1134
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 26 times.
26 if (!dirent.IsDirectory()) {
1135 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1136 "failed to graft nested catalog '%s': root entry is not a "
1137 "directory",
1138 nested_root_path.c_str());
1139 ✗ return false;
1140 }
1141
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 26 times.
26 if (dirent.HasXattrs()) {
1142 ✗ const bool xattrs_found = new_catalog->LookupXattrsPath(nested_root_ps,
1143 &xattrs);
1144 ✗ if (!xattrs_found) {
1145 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1146 "failed to graft nested catalog '%s': missing xattrs in new "
1147 "catalog",
1148 nested_root_path.c_str());
1149 ✗ return false;
1150 }
1151 }
1152 // Transform the nested catalog root into a transition point to be inserted
1153 // in the parent catalog
1154 26 dirent.set_is_nested_catalog_root(false);
1155 26 dirent.set_is_nested_catalog_mountpoint(true);
1156
1157 // Add directory and nested catalog
1158
1159 26 SyncLock();
1160 WritableCatalog *parent_catalog;
1161
1/2
✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
26 DirectoryEntry parent_entry;
1162
2/4
✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 26 times.
26 if (!FindCatalog(parent_path, &parent_catalog, &parent_entry)) {
1163 ✗ SyncUnlock();
1164 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1165 "catalog for directory '%s' cannot be found", parent_path.c_str());
1166 ✗ return false;
1167 }
1168
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 26 times.
26 if (!parent_entry.IsDirectory()) {
1169 ✗ SyncUnlock();
1170 ✗ LogCvmfs(kLogCatalog, kLogStderr, "parent path '%s' is not a directory",
1171 parent_path.c_str());
1172 ✗ return false;
1173 }
1174
3/4
✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 13 times.
✓ Branch 4 taken 13 times.
26 if (parent_catalog->LookupPath(nested_root_ps, NULL)) {
1175 13 SyncUnlock();
1176
1/2
✓ Branch 2 taken 13 times.
✗ Branch 3 not taken.
13 LogCvmfs(kLogCatalog, kLogStderr,
1177 "invalid attempt to graft nested catalog into existing directory "
1178 "'%s'",
1179 nested_root_path.c_str());
1180 13 return false;
1181 }
1182
1/2
✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
13 parent_catalog->AddEntry(dirent, xattrs, nested_root_path, parent_path);
1183 13 parent_entry.set_linkcount(parent_entry.linkcount() + 1);
1184
1/2
✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
13 parent_catalog->UpdateEntry(parent_entry, parent_path);
1185
1/2
✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
13 if (parent_entry.IsNestedCatalogRoot()) {
1186 WritableCatalog *grand_parent_catalog = reinterpret_cast<WritableCatalog *>(
1187 13 parent_catalog->parent());
1188 13 parent_entry.set_is_nested_catalog_root(false);
1189 13 parent_entry.set_is_nested_catalog_mountpoint(true);
1190
1/2
✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
13 grand_parent_catalog->UpdateEntry(parent_entry, parent_path);
1191 }
1192
1193
1/2
✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
13 parent_catalog->InsertNestedCatalog(nested_root_path, NULL, new_hash,
1194 new_size);
1195
1196 // Fix-up counters
1197
1/2
✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
13 const Counters counters;
1198
1/2
✓ Branch 3 taken 13 times.
✗ Branch 4 not taken.
13 const DeltaCounters delta = Counters::Diff(counters,
1199 new_catalog->GetCounters());
1200
1/2
✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
13 delta.PopulateToParent(&parent_catalog->delta_counters_);
1201
1202 13 SyncUnlock();
1203 13 return true;
1204 39 }
1205
1206 /**
1207 * Checks if a nested catalog starts at this path. The path must be valid.
1208 */
1209 ✗ bool WritableCatalogManager::IsTransitionPoint(const string &mountpoint) {
1210 ✗ const string path = MakeRelativePath(mountpoint);
1211
1212 ✗ SyncLock();
1213 WritableCatalog *catalog;
1214 ✗ DirectoryEntry entry;
1215 ✗ if (!FindCatalog(path, &catalog, &entry)) {
1216 ✗ PANIC(kLogStderr, "catalog for directory '%s' cannot be found",
1217 path.c_str());
1218 }
1219 ✗ const bool result = entry.IsNestedCatalogRoot();
1220 ✗ SyncUnlock();
1221 ✗ return result;
1222 }
1223
1224
1225 ✗ void WritableCatalogManager::PrecalculateListings() {
1226 // TODO(jblomer): meant for micro catalogs
1227 }
1228
1229
1230 ✗ void WritableCatalogManager::SetTTL(const uint64_t new_ttl) {
1231 ✗ SyncLock();
1232 ✗ reinterpret_cast<WritableCatalog *>(GetRootCatalog())->SetTTL(new_ttl);
1233 ✗ SyncUnlock();
1234 }
1235
1236
1237 ✗ bool WritableCatalogManager::SetVOMSAuthz(const std::string &voms_authz) {
1238 bool result;
1239 ✗ SyncLock();
1240 ✗ result = reinterpret_cast<WritableCatalog *>(GetRootCatalog())
1241 ✗ ->SetVOMSAuthz(voms_authz);
1242 ✗ SyncUnlock();
1243 ✗ return result;
1244 }
1245
1246
1247 326 bool WritableCatalogManager::Commit(const bool stop_for_tweaks,
1248 const uint64_t manual_revision,
1249 manifest::Manifest *manifest) {
1250 WritableCatalog *root_catalog = reinterpret_cast<WritableCatalog *>(
1251 326 GetRootCatalog());
1252
1/2
✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
326 root_catalog->SetDirty();
1253
1254 // set root catalog revision to manually provided number if available
1255
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 326 times.
326 if (manual_revision > 0) {
1256 ✗ const uint64_t revision = root_catalog->GetRevision();
1257 ✗ if (revision >= manual_revision) {
1258 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1259 "Manual revision (%" PRIu64 ") must not be "
1260 "smaller than the current root catalog's (%" PRIu64
1261 "). Skipped!",
1262 manual_revision, revision);
1263 } else {
1264 // Gets incremented by FinalizeCatalog() afterwards!
1265 ✗ root_catalog->SetRevision(manual_revision - 1);
1266 }
1267 }
1268
1269 // do the actual catalog snapshotting and upload
1270
1/2
✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
326 CatalogInfo root_catalog_info;
1271
1/2
✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
326 if (getenv("_CVMFS_SERIALIZED_CATALOG_PROCESSING_") == NULL)
1272
1/2
✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
326 root_catalog_info = SnapshotCatalogs(stop_for_tweaks);
1273 else
1274 ✗ root_catalog_info = SnapshotCatalogsSerialized(stop_for_tweaks);
1275
2/4
✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 326 times.
326 if (spooler_->GetNumberOfErrors() > 0) {
1276 ✗ LogCvmfs(kLogCatalog, kLogStderr, "failed to commit catalogs");
1277 ✗ return false;
1278 }
1279
1280 // .cvmfspublished export
1281
1/2
✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
326 LogCvmfs(kLogCatalog, kLogVerboseMsg, "Committing repository manifest");
1282 326 set_base_hash(root_catalog_info.content_hash);
1283
1284 326 manifest->set_catalog_hash(root_catalog_info.content_hash);
1285 326 manifest->set_catalog_size(root_catalog_info.size);
1286
2/4
✓ Branch 2 taken 326 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 326 times.
✗ Branch 6 not taken.
326 manifest->set_root_path("");
1287 326 manifest->set_ttl(root_catalog_info.ttl);
1288 326 manifest->set_revision(root_catalog_info.revision);
1289
1290 326 return true;
1291 }
1292
1293
1294 /**
1295 * Handles the snapshotting of dirty (i.e. modified) catalogs while trying to
1296 * parallelize the compression and upload as much as possible. We use a parallel
1297 * depth first post order tree traversal based on 'continuations'.
1298 *
1299 * The idea is as follows:
1300 * 1. find all leaf-catalogs (i.e. dirty catalogs with no dirty children)
1301 * --> these can be processed and uploaded immediately and independently
1302 * see WritableCatalogManager::GetModifiedCatalogLeafs()
1303 * 2. annotate non-leaf catalogs with their number of dirty children
1304 * --> a finished child will notify it's parent and decrement this number
1305 * see WritableCatalogManager::CatalogUploadCallback()
1306 * 3. if a non-leaf catalog's dirty children number reaches 0, it is scheduled
1307 * for processing as well (continuation)
1308 * --> the parallel processing walks bottom-up through the catalog tree
1309 * see WritableCatalogManager::CatalogUploadCallback()
1310 * 4. when the root catalog is reached, we notify the main thread and return
1311 * --> done through a Future<> in WritableCatalogManager::SnapshotCatalogs
1312 *
1313 * Note: The catalog finalisation (see WritableCatalogManager::FinalizeCatalog)
1314 * happens in a worker thread (i.e. the callback method) for non-leaf
1315 * catalogs.
1316 *
1317 * TODO(rmeusel): since all leaf catalogs are finalized in the main thread, we
1318 * sacrifice some potential concurrency for simplicity.
1319 */
1320 326 WritableCatalogManager::CatalogInfo WritableCatalogManager::SnapshotCatalogs(
1321 const bool stop_for_tweaks) {
1322 // prepare environment for parallel processing
1323
1/2
✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
326 Future<CatalogInfo> root_catalog_info_future;
1324 CatalogUploadContext upload_context;
1325 326 upload_context.root_catalog_info = &root_catalog_info_future;
1326 326 upload_context.stop_for_tweaks = stop_for_tweaks;
1327
1328
1/2
✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
326 spooler_->RegisterListener(&WritableCatalogManager::CatalogUploadCallback,
1329 this, upload_context);
1330
1331 // find dirty leaf catalogs and annotate non-leaf catalogs (dirty child count)
1332 // post-condition: the entire catalog tree is ready for concurrent processing
1333 326 WritableCatalogList leafs_to_snapshot;
1334
1/2
✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
326 GetModifiedCatalogLeafs(&leafs_to_snapshot);
1335
1336 // finalize and schedule the catalog processing
1337 326 WritableCatalogList::const_iterator i = leafs_to_snapshot.begin();
1338 326 const WritableCatalogList::const_iterator iend = leafs_to_snapshot.end();
1339
2/2
✓ Branch 2 taken 488 times.
✓ Branch 3 taken 326 times.
814 for (; i != iend; ++i) {
1340
1/2
✓ Branch 2 taken 488 times.
✗ Branch 3 not taken.
488 FinalizeCatalog(*i, stop_for_tweaks);
1341
1/2
✓ Branch 2 taken 488 times.
✗ Branch 3 not taken.
488 ScheduleCatalogProcessing(*i);
1342 }
1343
1344
1/2
✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
326 LogCvmfs(kLogCatalog, kLogVerboseMsg, "waiting for upload of catalogs");
1345
1/2
✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
326 const CatalogInfo &root_catalog_info = root_catalog_info_future.Get();
1346
1/2
✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
326 spooler_->WaitForUpload();
1347
1348
1/2
✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
326 spooler_->UnregisterListeners();
1349 326 return root_catalog_info;
1350 326 }
1351
1352
1353 780 void WritableCatalogManager::FinalizeCatalog(WritableCatalog *catalog,
1354 const bool stop_for_tweaks) {
1355 // update meta information of this catalog
1356
1/3
✓ Branch 2 taken 780 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
780 LogCvmfs(kLogCatalog, kLogVerboseMsg, "creating snapshot of catalog '%s'",
1357 1560 catalog->mountpoint().c_str());
1358
1359 780 catalog->UpdateCounters();
1360 780 catalog->UpdateLastModified();
1361 780 catalog->IncrementRevision();
1362
1363 // update the previous catalog revision pointer
1364
2/2
✓ Branch 1 taken 326 times.
✓ Branch 2 taken 454 times.
780 if (catalog->IsRoot()) {
1365
1/4
✓ Branch 2 taken 326 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
326 LogCvmfs(kLogCatalog, kLogVerboseMsg,
1366 "setting '%s' as previous revision "
1367 "for root catalog",
1368 652 base_hash().ToStringWithSuffix().c_str());
1369 326 catalog->SetPreviousRevision(base_hash());
1370 } else {
1371 // Multiple catalogs might query the parent concurrently
1372 454 SyncLock();
1373
1/2
✓ Branch 1 taken 454 times.
✗ Branch 2 not taken.
454 shash::Any hash_previous;
1374 uint64_t size_previous;
1375
1/2
✓ Branch 2 taken 454 times.
✗ Branch 3 not taken.
908 const bool retval = catalog->parent()->FindNested(
1376
1/2
✓ Branch 1 taken 454 times.
✗ Branch 2 not taken.
908 catalog->mountpoint(), &hash_previous, &size_previous);
1377
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 454 times.
454 assert(retval);
1378 454 SyncUnlock();
1379
1380
1/8
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 454 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
908 LogCvmfs(kLogCatalog, kLogVerboseMsg,
1381 "found '%s' as previous revision "
1382 "for nested catalog '%s'",
1383
1/2
✓ Branch 1 taken 454 times.
✗ Branch 2 not taken.
908 hash_previous.ToStringWithSuffix().c_str(),
1384
1/2
✓ Branch 1 taken 454 times.
✗ Branch 2 not taken.
908 catalog->mountpoint().c_str());
1385
1/2
✓ Branch 1 taken 454 times.
✗ Branch 2 not taken.
454 catalog->SetPreviousRevision(hash_previous);
1386 }
1387 780 catalog->Commit();
1388
1389 // check if catalog has too many entries
1390 const uint64_t catalog_limit = uint64_t(1000)
1391
2/2
✓ Branch 1 taken 326 times.
✓ Branch 2 taken 454 times.
780 * uint64_t((catalog->IsRoot()
1392 326 ? root_kcatalog_limit_
1393 454 : nested_kcatalog_limit_));
1394 780 if ((catalog_limit > 0)
1395
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 780 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 780 times.
780 && (catalog->GetCounters().GetSelfEntries() > catalog_limit)) {
1396 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1397 "%s: catalog at %s has more than %lu entries (%lu). "
1398 "Large catalogs stress the CernVM-FS transport infrastructure. "
1399 "Please split it into nested catalogs or increase the limit.",
1400 ✗ enforce_limits_ ? "FATAL" : "WARNING",
1401 ✗ (catalog->IsRoot() ? "/" : catalog->mountpoint().c_str()),
1402 ✗ catalog_limit, catalog->GetCounters().GetSelfEntries());
1403 ✗ if (enforce_limits_)
1404 ✗ PANIC(kLogStderr, "catalog at %s has more than %u entries (%u). ",
1405 (catalog->IsRoot() ? "/" : catalog->mountpoint().c_str()),
1406 catalog_limit, catalog->GetCounters().GetSelfEntries());
1407 }
1408
1409 // allow for manual adjustments in the catalog
1410
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 780 times.
780 if (stop_for_tweaks) {
1411 ✗ LogCvmfs(kLogCatalog, kLogStdout,
1412 "Allowing for tweaks in %s at %s "
1413 "(hit return to continue)",
1414 ✗ catalog->database_path().c_str(), catalog->mountpoint().c_str());
1415 ✗ const int read_char = getchar();
1416 ✗ assert(read_char != EOF);
1417 }
1418
1419 // compaction of bloated catalogs (usually after high database churn)
1420 780 catalog->VacuumDatabaseIfNecessary();
1421 780 }
1422
1423
1424 780 void WritableCatalogManager::ScheduleCatalogProcessing(
1425 WritableCatalog *catalog) {
1426 {
1427 780 const MutexLockGuard guard(catalog_processing_lock_);
1428 // register catalog object for WritableCatalogManager::CatalogUploadCallback
1429
2/4
✓ Branch 1 taken 780 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 780 times.
✗ Branch 5 not taken.
780 catalog_processing_map_[catalog->database_path()] = catalog;
1430 780 }
1431
1/2
✓ Branch 2 taken 780 times.
✗ Branch 3 not taken.
780 spooler_->ProcessCatalog(catalog->database_path());
1432 780 }
1433
1434 /**
1435 * Copy catalog to local cache.server
1436 * Must be an atomic write into the cache_dir
1437 * As such: create a temporary copy in cache_dir/txn and then do a
1438 * `rename` (which is atomic) to the actual cache path
1439 *
1440 * @returns true on success, otherwise false
1441 */
1442 ✗ bool WritableCatalogManager::CopyCatalogToLocalCache(
1443 const upload::SpoolerResult &result) {
1444 ✗ std::string tmp_catalog_path;
1445 ✗ const std::string cache_catalog_path = dir_cache_ + "/"
1446 + result.content_hash
1447 ✗ .MakePathWithoutSuffix();
1448 ✗ FILE *fcatalog = CreateTempFile(dir_cache_ + "/txn/catalog", 0666, "w",
1449 &tmp_catalog_path);
1450 ✗ if (!fcatalog) {
1451 ✗ PANIC(kLogDebug | kLogStderr,
1452 "Creating file for temporary catalog failed: %s",
1453 tmp_catalog_path.c_str());
1454 }
1455 ✗ CopyPath2File(result.local_path.c_str(), fcatalog);
1456 ✗ (void)fclose(fcatalog);
1457
1458 ✗ if (rename(tmp_catalog_path.c_str(), cache_catalog_path.c_str()) != 0) {
1459 ✗ PANIC(kLogDebug | kLogStderr, "Failed to copy catalog from %s to cache %s",
1460 result.local_path.c_str(), cache_catalog_path.c_str());
1461 }
1462 ✗ return true;
1463 }
1464
1465 780 void WritableCatalogManager::CatalogUploadCallback(
1466 const upload::SpoolerResult &result,
1467 const CatalogUploadContext catalog_upload_context) {
1468
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 780 times.
780 if (result.return_code != 0) {
1469 ✗ PANIC(kLogStderr, "failed to upload '%s' (retval: %d)",
1470 result.local_path.c_str(), result.return_code);
1471 }
1472
1473 // retrieve the catalog object based on the callback information
1474 // see WritableCatalogManager::ScheduleCatalogProcessing()
1475 780 WritableCatalog *catalog = NULL;
1476 {
1477 780 const MutexLockGuard guard(catalog_processing_lock_);
1478 const std::map<std::string, WritableCatalog *>::iterator
1479
1/2
✓ Branch 1 taken 780 times.
✗ Branch 2 not taken.
780 c = catalog_processing_map_.find(result.local_path);
1480
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 780 times.
780 assert(c != catalog_processing_map_.end());
1481 780 catalog = c->second;
1482 780 }
1483
1484 780 const uint64_t catalog_size = GetFileSize(result.local_path);
1485
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 780 times.
780 assert(catalog_size > 0);
1486
1487
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 780 times.
780 if (UseLocalCache()) {
1488 ✗ CopyCatalogToLocalCache(result);
1489 }
1490
1491 780 SyncLock();
1492
2/2
✓ Branch 1 taken 454 times.
✓ Branch 2 taken 326 times.
780 if (catalog->HasParent()) {
1493 // finalized nested catalogs will update their parent's pointer and schedule
1494 // them for processing (continuation) if the 'dirty children count' == 0
1495 454 LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating nested catalog link");
1496 454 WritableCatalog *parent = catalog->GetWritableParent();
1497
1498
2/4
✓ Branch 1 taken 454 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 454 times.
✗ Branch 5 not taken.
454 parent->UpdateNestedCatalog(catalog->mountpoint().ToString(),
1499 454 result.content_hash,
1500 catalog_size,
1501 454 catalog->delta_counters_);
1502 454 catalog->delta_counters_.SetZero();
1503
1504 const int remaining_dirty_children = catalog->GetWritableParent()
1505 454 ->DecrementDirtyChildren();
1506
1507 454 SyncUnlock();
1508
1509 // continuation of the dirty catalog tree traversal
1510 // see WritableCatalogManager::SnapshotCatalogs()
1511
2/2
✓ Branch 0 taken 292 times.
✓ Branch 1 taken 162 times.
454 if (remaining_dirty_children == 0) {
1512 292 FinalizeCatalog(parent, catalog_upload_context.stop_for_tweaks);
1513 292 ScheduleCatalogProcessing(parent);
1514 }
1515
1516
1/2
✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
326 } else if (catalog->IsRoot()) {
1517 // once the root catalog is reached, we are done with processing and report
1518 // back to the main via a Future<> and provide the necessary information
1519
1/2
✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
326 CatalogInfo root_catalog_info;
1520 326 root_catalog_info.size = catalog_size;
1521
1/2
✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
326 root_catalog_info.ttl = catalog->GetTTL();
1522 326 root_catalog_info.content_hash = result.content_hash;
1523
1/2
✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
326 root_catalog_info.revision = catalog->GetRevision();
1524
1/2
✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
326 catalog_upload_context.root_catalog_info->Set(root_catalog_info);
1525 326 SyncUnlock();
1526 } else {
1527 ✗ PANIC(kLogStderr, "inconsistent state detected");
1528 }
1529 780 }
1530
1531
1532 /**
1533 * Finds dirty catalogs that can be snapshot right away and annotates all the
1534 * other catalogs with their number of dirty descendants.
1535 * Note that there is a convenience wrapper to start the recursion:
1536 * WritableCatalogManager::GetModifiedCatalogLeafs()
1537 *
1538 * @param catalog the catalog for this recursion step
1539 * @param result the result list to be appended to
1540 * @return true if 'catalog' is dirty
1541 */
1542 792 bool WritableCatalogManager::GetModifiedCatalogLeafsRecursively(
1543 Catalog *catalog, WritableCatalogList *result) const {
1544 792 WritableCatalog *wr_catalog = static_cast<WritableCatalog *>(catalog);
1545
1546 // Look for dirty catalogs in the descendants of *catalog
1547 792 int dirty_children = 0;
1548
1/2
✓ Branch 1 taken 792 times.
✗ Branch 2 not taken.
792 CatalogList children = wr_catalog->GetChildren();
1549 792 CatalogList::const_iterator i = children.begin();
1550 792 const CatalogList::const_iterator iend = children.end();
1551
2/2
✓ Branch 2 taken 454 times.
✓ Branch 3 taken 792 times.
1246 for (; i != iend; ++i) {
1552
2/4
✓ Branch 2 taken 454 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 454 times.
✗ Branch 5 not taken.
454 if (GetModifiedCatalogLeafsRecursively(*i, result)) {
1553 454 ++dirty_children;
1554 }
1555 }
1556
1557 // a catalog is dirty if itself or one of its children has changed
1558 // a leaf catalog doesn't have any dirty children
1559 792 wr_catalog->set_dirty_children(dirty_children);
1560
1/4
✗ Branch 1 not taken.
✓ Branch 2 taken 792 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
792 const bool is_dirty = wr_catalog->IsDirty() || dirty_children > 0;
1561 792 const bool is_leaf = dirty_children == 0;
1562
3/4
✓ Branch 0 taken 792 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 500 times.
✓ Branch 3 taken 292 times.
792 if (is_dirty && is_leaf) {
1563
1/2
✓ Branch 1 taken 500 times.
✗ Branch 2 not taken.
500 result->push_back(const_cast<WritableCatalog *>(wr_catalog));
1564 }
1565
1566 792 return is_dirty;
1567 792 }
1568
1569
1570 ✗ void WritableCatalogManager::DoBalance() {
1571 ✗ CatalogList catalog_list = GetCatalogs();
1572 ✗ reverse(catalog_list.begin(), catalog_list.end());
1573 ✗ for (unsigned i = 0; i < catalog_list.size(); ++i) {
1574 ✗ FixWeight(static_cast<WritableCatalog *>(catalog_list[i]));
1575 }
1576 }
1577
1578 ✗ void WritableCatalogManager::FixWeight(WritableCatalog *catalog) {
1579 // firstly check underflow because they can provoke overflows
1580 ✗ if (catalog->GetNumEntries() < min_weight_ && !catalog->IsRoot()
1581 ✗ && catalog->IsAutogenerated()) {
1582 ✗ LogCvmfs(kLogCatalog, kLogStdout,
1583 "Deleting an autogenerated catalog in '%s'",
1584 ✗ catalog->mountpoint().c_str());
1585 // Remove the .cvmfscatalog and .cvmfsautocatalog files first
1586 ✗ const string path = catalog->mountpoint().ToString();
1587 ✗ catalog->RemoveEntry(path + "/.cvmfscatalog");
1588 ✗ catalog->RemoveEntry(path + "/.cvmfsautocatalog");
1589 // Remove the actual catalog
1590 ✗ const string catalog_path = catalog->mountpoint().ToString().substr(1);
1591 ✗ RemoveNestedCatalog(catalog_path);
1592 ✗ } else if (catalog->GetNumEntries() > max_weight_) {
1593 ✗ CatalogBalancer<WritableCatalogManager> catalog_balancer(this);
1594 ✗ catalog_balancer.Balance(catalog);
1595 }
1596 }
1597
1598
1599 //****************************************************************************
1600 // Workaround -- Serialized Catalog Committing
1601
1602 ✗ int WritableCatalogManager::GetModifiedCatalogsRecursively(
1603 const Catalog *catalog, WritableCatalogList *result) const {
1604 // A catalog must be snapshot, if itself or one of it's descendants is dirty.
1605 // So we traverse the catalog tree recursively and look for dirty catalogs
1606 // on the way.
1607 ✗ const WritableCatalog *wr_catalog = static_cast<const WritableCatalog *>(
1608 catalog);
1609 // This variable will contain the number of dirty catalogs in the sub tree
1610 // with *catalog as it's root.
1611 ✗ int dirty_catalogs = (wr_catalog->IsDirty()) ? 1 : 0;
1612
1613 // Look for dirty catalogs in the descendants of *catalog
1614 ✗ CatalogList children = wr_catalog->GetChildren();
1615 ✗ for (CatalogList::const_iterator i = children.begin(), iEnd = children.end();
1616 ✗ i != iEnd; ++i) {
1617 ✗ dirty_catalogs += GetModifiedCatalogsRecursively(*i, result);
1618 }
1619
1620 // If we found a dirty catalog in the checked sub tree, the root (*catalog)
1621 // must be snapshot and ends up in the result list
1622 ✗ if (dirty_catalogs > 0)
1623 ✗ result->push_back(const_cast<WritableCatalog *>(wr_catalog));
1624
1625 // tell the upper layer about number of catalogs
1626 ✗ return dirty_catalogs;
1627 }
1628
1629
1630 ✗ void WritableCatalogManager::CatalogUploadSerializedCallback(
1631 const upload::SpoolerResult &result, const CatalogUploadContext unused) {
1632 ✗ if (result.return_code != 0) {
1633 ✗ PANIC(kLogStderr, "failed to upload '%s' (retval: %d)",
1634 result.local_path.c_str(), result.return_code);
1635 }
1636
1637 ✗ if (UseLocalCache()) {
1638 ✗ CopyCatalogToLocalCache(result);
1639 }
1640
1641 ✗ unlink(result.local_path.c_str());
1642 }
1643
1644
1645 WritableCatalogManager::CatalogInfo
1646 ✗ WritableCatalogManager::SnapshotCatalogsSerialized(const bool stop_for_tweaks) {
1647 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Serialized committing of file catalogs...");
1648 ✗ reinterpret_cast<WritableCatalog *>(GetRootCatalog())->SetDirty();
1649 ✗ WritableCatalogList catalogs_to_snapshot;
1650 ✗ GetModifiedCatalogs(&catalogs_to_snapshot);
1651 CatalogUploadContext unused;
1652 ✗ unused.root_catalog_info = NULL;
1653 ✗ unused.stop_for_tweaks = false;
1654 ✗ spooler_->RegisterListener(
1655 &WritableCatalogManager::CatalogUploadSerializedCallback, this, unused);
1656
1657 ✗ CatalogInfo root_catalog_info;
1658 ✗ WritableCatalogList::const_iterator i = catalogs_to_snapshot.begin();
1659 ✗ const WritableCatalogList::const_iterator iend = catalogs_to_snapshot.end();
1660 ✗ for (; i != iend; ++i) {
1661 ✗ FinalizeCatalog(*i, stop_for_tweaks);
1662
1663 // Compress and upload catalog
1664 ✗ shash::Any hash_catalog(spooler_->GetHashAlgorithm(),
1665 ✗ shash::kSuffixCatalog);
1666 ✗ if (!zlib::CompressPath2Null((*i)->database_path(), &hash_catalog)) {
1667 ✗ PANIC(kLogStderr, "could not compress catalog %s",
1668 (*i)->mountpoint().ToString().c_str());
1669 }
1670
1671 ✗ const int64_t catalog_size = GetFileSize((*i)->database_path());
1672 ✗ assert(catalog_size > 0);
1673
1674 ✗ if ((*i)->HasParent()) {
1675 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating nested catalog link");
1676 ✗ WritableCatalog *parent = (*i)->GetWritableParent();
1677 ✗ parent->UpdateNestedCatalog((*i)->mountpoint().ToString(), hash_catalog,
1678 ✗ catalog_size, (*i)->delta_counters_);
1679 ✗ (*i)->delta_counters_.SetZero();
1680 ✗ } else if ((*i)->IsRoot()) {
1681 ✗ root_catalog_info.size = catalog_size;
1682 ✗ root_catalog_info.ttl = (*i)->GetTTL();
1683 ✗ root_catalog_info.content_hash = hash_catalog;
1684 ✗ root_catalog_info.revision = (*i)->GetRevision();
1685 } else {
1686 ✗ PANIC(kLogStderr, "inconsistent state detected");
1687 }
1688
1689 ✗ spooler_->ProcessCatalog((*i)->database_path());
1690 }
1691 ✗ spooler_->WaitForUpload();
1692
1693 ✗ spooler_->UnregisterListeners();
1694 ✗ return root_catalog_info;
1695 }
1696
1697 ✗ void WritableCatalogManager::SetupSingleCatalogUploadCallback() {
1698 ✗ spooler_->RegisterListener(
1699 &WritableCatalogManager::SingleCatalogUploadCallback, this);
1700 }
1701
1702 ✗ void WritableCatalogManager::RemoveSingleCatalogUploadCallback() {
1703 ✗ spooler_->WaitForUpload(); // wait for all outstanding jobs to finish before
1704 // tearing it down
1705 // parents that were waiting on children can be uploaded now
1706 ✗ ScheduleReadyCatalogs();
1707 ✗ spooler_->WaitForUpload();
1708 ✗ spooler_->UnregisterListeners();
1709 ✗ pending_catalogs_ =
1710 ✗ {}; // whatever we couldn't process, leave it to the Commit
1711 }
1712
1713 ✗ void WritableCatalogManager::AddCatalogToQueue(const std::string &path) {
1714 ✗ SyncLock();
1715 ✗ WritableCatalog *catalog = NULL;
1716 ✗ bool const retval = FindCatalog(MakeRelativePath(path), &catalog, NULL);
1717 ✗ assert(retval);
1718 ✗ assert(catalog);
1719 ✗ catalog->SetDirty(); // ensure it's dirty so its parent will wait for it
1720 // the parent must not be finalized before this catalog's upload callback
1721 // has written the new nested catalog link into it
1722 ✗ if (catalog->HasParent())
1723 ✗ catalog->GetWritableParent()->IncrementDirtyChildren();
1724 ✗ SyncUnlock();
1725 ✗ pending_catalogs_.push_back(catalog);
1726 }
1727
1728 ✗ void WritableCatalogManager::ScheduleReadyCatalogs() {
1729 // best effort to schedule as many catalogs for upload as possible
1730 ✗ for (auto it = pending_catalogs_.begin(); it != pending_catalogs_.end();) {
1731 ✗ if ((*it)->dirty_children() == 0) {
1732 ✗ FinalizeCatalog(*it, false /* stop_for_tweaks */);
1733 ✗ ScheduleCatalogProcessing(*it);
1734 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "scheduled %s for processing",
1735 ✗ (*it)->mountpoint().c_str());
1736 ✗ it = pending_catalogs_.erase(it);
1737 } else {
1738 ✗ ++it;
1739 }
1740 }
1741 }
1742
1743 // Callback for uploading a single catalog, similar to CatalogUploadCallback.
1744 // The main difference is that this callback would not trigger processing of the
1745 // parent
1746 ✗ void WritableCatalogManager::SingleCatalogUploadCallback(
1747 const upload::SpoolerResult &result) {
1748 ✗ if (result.return_code != 0) {
1749 ✗ PANIC(kLogStderr, "failed to upload '%s' (retval: %d)",
1750 result.local_path.c_str(), result.return_code);
1751 }
1752
1753 // retrieve the catalog object based on the callback information
1754 // see WritableCatalogManager::ScheduleCatalogProcessing()
1755 ✗ WritableCatalog *catalog = NULL;
1756 {
1757 ✗ MutexLockGuard const guard(catalog_processing_lock_);
1758 std::map<std::string, WritableCatalog *>::iterator const
1759 ✗ c = catalog_processing_map_.find(result.local_path);
1760 ✗ assert(c != catalog_processing_map_.end());
1761 ✗ catalog = c->second;
1762 }
1763
1764 ✗ uint64_t const catalog_size = GetFileSize(result.local_path);
1765 ✗ assert(catalog_size > 0);
1766
1767 ✗ SyncLock();
1768 ✗ if (catalog->HasParent()) {
1769 // finalized nested catalogs will update their parent's pointer
1770 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating nested catalog link");
1771 ✗ WritableCatalog *parent = catalog->GetWritableParent();
1772
1773 ✗ parent->UpdateNestedCatalog(catalog->mountpoint().ToString(),
1774 ✗ result.content_hash,
1775 catalog_size,
1776 ✗ catalog->delta_counters_);
1777 ✗ parent->DecrementDirtyChildren();
1778 ✗ catalog->delta_counters_.SetZero();
1779 }
1780 // JUMP: detach the catalog after uploading to free sqlite related resources
1781 ✗ DetachCatalog(catalog);
1782 ✗ SyncUnlock();
1783 }
1784 // using the given list of dirs, fetch all relevant catalogs
1785 ✗ void WritableCatalogManager::LoadCatalogs(
1786 const std::string &base_path, const std::unordered_set<std::string> &dirs) {
1787 // mount everything up to "base_path" first (this would be our lease_path
1788 // typically)
1789 Catalog *base_catalog;
1790 ✗ MountSubtree(PathString(base_path), NULL /* entry_point */,
1791 true /* is_listable */, &base_catalog);
1792
1793 // start up the downloader
1794 CatalogDownloadContext context;
1795 ✗ context.dirs = &dirs;
1796 ✗ catalog_download_pipeline_ = new CatalogDownloadPipeline(
1797 ✗ static_cast<SimpleCatalogManager *>(this));
1798 ✗ catalog_download_pipeline_->RegisterListener(
1799 &WritableCatalogManager::CatalogDownloadCallback, this, context);
1800 ✗ catalog_download_pipeline_->Spawn();
1801
1802 Catalog::NestedCatalogList nested_catalogs = base_catalog
1803 ✗ ->ListNestedCatalogs();
1804 ✗ for (auto it = nested_catalogs.begin(); it != nested_catalogs.end(); ++it) {
1805 // schedule relevant child nested catalogs for download
1806 ✗ std::string const mountpoint = it->mountpoint.ToString();
1807 ✗ if (dirs.find(mountpoint) != dirs.end()) {
1808 ✗ Catalog *catalog = CreateCatalog(it->mountpoint, it->hash,
1809 ✗ NULL /* parent */);
1810 {
1811 ✗ MutexLockGuard const guard(catalog_download_lock_);
1812 ✗ catalog_download_map_.insert(
1813 ✗ std::make_pair(it->hash.ToString(), catalog));
1814 }
1815 ✗ catalog_download_pipeline_->Process(it->hash);
1816 }
1817 }
1818
1819 ✗ catalog_download_pipeline_->WaitFor();
1820 ✗ delete catalog_download_pipeline_; // terminate all the threads
1821 }
1822
1823 ✗ bool WritableCatalogManager::LookupDirEntry(const string &path,
1824 const LookupOptions options,
1825 DirectoryEntry *dirent) {
1826 ✗ SyncLock();
1827 ✗ bool const exists = LookupPath(path, options, dirent);
1828 ✗ SyncUnlock();
1829 ✗ return exists;
1830 }
1831
1832 ✗ void WritableCatalogManager::CatalogHashSerializedCallback(
1833 const CompressHashResult &result) {
1834 ✗ MutexLockGuard const guard(catalog_hash_lock_);
1835 ✗ catalog_hash_map_[result.path] = result.hash;
1836 }
1837
1838 ✗ void WritableCatalogManager::CatalogDownloadCallback(
1839 const CatalogDownloadResult &result, CatalogDownloadContext context) {
1840 Catalog *downloaded_catalog;
1841 {
1842 ✗ MutexLockGuard const guard(catalog_download_lock_);
1843 ✗ auto it = catalog_download_map_.find(result.hash);
1844 ✗ assert(it != catalog_download_map_.end());
1845 ✗ downloaded_catalog = it->second;
1846 }
1847
1848 ✗ if (!downloaded_catalog->OpenDatabase(result.db_path)) {
1849 ✗ LogCvmfs(kLogCvmfs, kLogDebug, "failed to initialize catalog");
1850 ✗ delete downloaded_catalog;
1851 ✗ return;
1852 }
1853 // the temp copy is only used to find nested catalogs; the download itself
1854 // warmed the local cache, so drop the copy together with the object
1855 ✗ downloaded_catalog->TakeDatabaseFileOwnership();
1856
1857 Catalog::NestedCatalogList nested_catalogs = downloaded_catalog
1858 ✗ ->ListNestedCatalogs();
1859 ✗ for (auto it = nested_catalogs.begin(); it != nested_catalogs.end(); ++it) {
1860 // schedule relevant child nested catalogs for download
1861 ✗ if (context.dirs->find(it->mountpoint.ToString()) != context.dirs->end()) {
1862 ✗ Catalog *child_catalog = CreateCatalog(it->mountpoint, it->hash,
1863 ✗ NULL /* parent */);
1864 {
1865 ✗ MutexLockGuard const guard(catalog_download_lock_);
1866 ✗ catalog_download_map_.insert(
1867 ✗ std::make_pair(it->hash.ToString(), child_catalog));
1868 }
1869 ✗ catalog_download_pipeline_->Process(it->hash);
1870 }
1871 }
1872 ✗ delete downloaded_catalog;
1873 }
1874
1875
1876 } // namespace catalog
1877