GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/catalog_mgr_rw.cc
Date: 2026-07-19 02:35:15
Exec Total Coverage
Lines: 477 880 54.2%
Branches: 365 1291 28.3%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM file system.
3 */
4
5 #include "catalog_mgr_rw.h"
6
7 #include <inttypes.h>
8 #include <unistd.h>
9
10 #include <cassert>
11 #include <cstdio>
12 #include <cstdlib>
13 #include <string>
14
15 #include "catalog_balancer.h"
16 #include "catalog_rw.h"
17 #include "manifest.h"
18 #include "statistics.h"
19 #include "upload.h"
20 #include "util/exception.h"
21 #include "util/logging.h"
22 #include "util/posix.h"
23 #include "util/smalloc.h"
24
25 using namespace std; // NOLINT
26
27 namespace catalog {
28
29 904 WritableCatalogManager::WritableCatalogManager(
30 const shash::Any &base_hash,
31 const std::string &stratum0,
32 const string &dir_temp,
33 upload::Spooler *spooler,
34 download::DownloadManager *download_manager,
35 bool enforce_limits,
36 const unsigned nested_kcatalog_limit,
37 const unsigned root_kcatalog_limit,
38 const unsigned file_mbyte_limit,
39 perf::Statistics *statistics,
40 bool is_balanceable,
41 unsigned max_weight,
42 unsigned min_weight,
43 904 const std::string &dir_cache)
44 : SimpleCatalogManager(base_hash, stratum0, dir_temp, download_manager,
45 statistics, false, dir_cache,
46 true /* copy to tmpdir */)
47 904 , spooler_(spooler)
48 904 , enforce_limits_(enforce_limits)
49 904 , nested_kcatalog_limit_(nested_kcatalog_limit)
50 904 , root_kcatalog_limit_(root_kcatalog_limit)
51 904 , file_mbyte_limit_(file_mbyte_limit)
52 904 , is_balanceable_(is_balanceable)
53 904 , max_weight_(max_weight)
54 904 , min_weight_(min_weight)
55 904 , balance_weight_(max_weight / 2) {
56 904 sync_lock_ = reinterpret_cast<pthread_mutex_t *>(
57 904 smalloc(sizeof(pthread_mutex_t)));
58 904 int retval = pthread_mutex_init(sync_lock_, NULL);
59
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 904 times.
 904 assert(retval == 0);
60 904 catalog_processing_lock_ = reinterpret_cast<pthread_mutex_t *>(
61 904 smalloc(sizeof(pthread_mutex_t)));
62 904 retval = pthread_mutex_init(catalog_processing_lock_, NULL);
63
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 904 times.
 904 assert(retval == 0);
64 904 }
65
66
67 3612 WritableCatalogManager::~WritableCatalogManager() {
68 1806 pthread_mutex_destroy(sync_lock_);
69 1806 free(sync_lock_);
70 1806 pthread_mutex_destroy(catalog_processing_lock_);
71 1806 free(catalog_processing_lock_);
72 3612 }
73
74
75 /**
76 * This method is virtual in AbstractCatalogManager. It returns a new catalog
77 * structure in the form the different CatalogManagers need it.
78 * In this case it returns a stub for a WritableCatalog.
79 * @param mountpoint the mount point of the catalog stub to create
80 * @param catalog_hash the content hash of the catalog to create
81 * @param parent_catalog the parent of the catalog stub to create
82 * @return a pointer to the catalog stub structure created
83 */
84 2382 Catalog *WritableCatalogManager::CreateCatalog(const PathString &mountpoint,
85 const shash::Any &catalog_hash,
86 Catalog *parent_catalog) {
87 4764 return new WritableCatalog(mountpoint.ToString(), catalog_hash,
88
2/4
✓ Branch 1 taken 2382 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2382 times.
✗ Branch 5 not taken.
 4764 parent_catalog);
89 }
90
91
92 2382 void WritableCatalogManager::ActivateCatalog(Catalog *catalog) {
93 2382 catalog->TakeDatabaseFileOwnership();
94 2382 }
95
96
97 /**
98 * This method is invoked if we create a completely new repository.
99 * The new root catalog will already contain a root entry.
100 * It is uploaded by a Forklift to the upstream storage.
101 * @return true on success, false otherwise
102 */
103 678 manifest::Manifest *WritableCatalogManager::CreateRepository(
104 const string &dir_temp,
105 const bool volatile_content,
106 const std::string &voms_authz,
107 upload::Spooler *spooler) {
108 // Create a new root catalog at file_path
109
1/2
✓ Branch 1 taken 678 times.
✗ Branch 2 not taken.
 678 const string file_path = dir_temp + "/new_root_catalog";
110
111 678 const shash::Algorithms hash_algorithm = spooler->GetHashAlgorithm();
112
113 // A newly created catalog always needs a root entry
114 // we create and configure this here
115
1/2
✓ Branch 1 taken 678 times.
✗ Branch 2 not taken.
 678 DirectoryEntry root_entry;
116 678 root_entry.inode_ = DirectoryEntry::kInvalidInode;
117 678 root_entry.mode_ = 16877;
118 678 root_entry.size_ = 4096;
119 678 root_entry.mtime_ = time(NULL);
120 678 root_entry.uid_ = getuid();
121 678 root_entry.gid_ = getgid();
122
1/2
✓ Branch 1 taken 678 times.
✗ Branch 2 not taken.
 678 root_entry.checksum_ = shash::Any(hash_algorithm);
123 678 root_entry.linkcount_ = 2;
124
1/2
✓ Branch 2 taken 678 times.
✗ Branch 3 not taken.
 678 const string root_path = "";
125
126 // Create the database schema and the initial root entry
127 {
128 const UniquePtr<CatalogDatabase> new_clg_db(
129
2/4
✓ Branch 1 taken 678 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 678 times.
✗ Branch 5 not taken.
 678 CatalogDatabase::Create(file_path));
130 678 if (!new_clg_db.IsValid()
131
4/8
✓ Branch 0 taken 678 times.
✗ Branch 1 not taken.
✓ Branch 4 taken 678 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 678 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 678 times.
 678 || !new_clg_db->InsertInitialValues(root_path, volatile_content,
132 voms_authz, root_entry)) {
133 ✗ LogCvmfs(kLogCatalog, kLogStderr, "creation of catalog '%s' failed",
134 file_path.c_str());
135 ✗ return NULL;
136 }
137
1/2
✓ Branch 1 taken 678 times.
✗ Branch 2 not taken.
 678 }
138
139 // Compress root catalog;
140
1/2
✓ Branch 1 taken 678 times.
✗ Branch 2 not taken.
 678 const int64_t catalog_size = GetFileSize(file_path);
141
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 678 times.
 678 if (catalog_size < 0) {
142 ✗ unlink(file_path.c_str());
143 ✗ return NULL;
144 }
145
1/2
✓ Branch 1 taken 678 times.
✗ Branch 2 not taken.
 678 const string file_path_compressed = file_path + ".compressed";
146
1/2
✓ Branch 1 taken 678 times.
✗ Branch 2 not taken.
 678 shash::Any hash_catalog(hash_algorithm, shash::kSuffixCatalog);
147
1/2
✓ Branch 1 taken 678 times.
✗ Branch 2 not taken.
 678 const bool retval = zlib::CompressPath2Path(file_path, file_path_compressed,
148 &hash_catalog);
149
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 678 times.
 678 if (!retval) {
150 ✗ LogCvmfs(kLogCatalog, kLogStderr, "compression of catalog '%s' failed",
151 file_path.c_str());
152 ✗ unlink(file_path.c_str());
153 ✗ return NULL;
154 }
155 678 unlink(file_path.c_str());
156
157 // Create manifest
158
1/2
✓ Branch 1 taken 678 times.
✗ Branch 2 not taken.
 678 const string manifest_path = dir_temp + "/manifest";
159 manifest::Manifest *manifest = new manifest::Manifest(hash_catalog,
160
3/6
✓ Branch 2 taken 678 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 678 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 678 times.
✗ Branch 9 not taken.
 678 catalog_size, "");
161
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 678 times.
 678 if (!voms_authz.empty()) {
162 ✗ manifest->set_has_alt_catalog_path(true);
163 }
164
165 // Upload catalog
166
3/6
✓ Branch 1 taken 678 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 678 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 678 times.
✗ Branch 8 not taken.
 678 spooler->Upload(file_path_compressed, "data/" + hash_catalog.MakePath());
167
1/2
✓ Branch 1 taken 678 times.
✗ Branch 2 not taken.
 678 spooler->WaitForUpload();
168 678 unlink(file_path_compressed.c_str());
169
2/4
✓ Branch 1 taken 678 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 678 times.
 678 if (spooler->GetNumberOfErrors() > 0) {
170 ✗ LogCvmfs(kLogCatalog, kLogStderr, "failed to commit catalog %s",
171 file_path_compressed.c_str());
172 ✗ delete manifest;
173 ✗ return NULL;
174 }
175
176 678 return manifest;
177 678 }
178
179
180 /**
181 * Retrieve the catalog containing the given path.
182 * Other than AbstractCatalogManager::FindCatalog() this mounts nested
183 * catalogs if necessary and returns WritableCatalog objects.
184 * Furthermore it optionally returns the looked-up DirectoryEntry.
185 *
186 * @param path the path to look for
187 * @param result the retrieved catalog (as a pointer)
188 * @param dirent is set to looked up DirectoryEntry for 'path' if non-NULL
189 * @return true if catalog was found
190 */
191 11922 bool WritableCatalogManager::FindCatalog(const string &path,
192 WritableCatalog **result,
193 DirectoryEntry *dirent) {
194
1/2
✓ Branch 1 taken 11922 times.
✗ Branch 2 not taken.
 11922 const PathString ps_path(path);
195
196
1/2
✓ Branch 1 taken 11922 times.
✗ Branch 2 not taken.
 11922 Catalog *best_fit = AbstractCatalogManager<Catalog>::FindCatalog(ps_path);
197
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11922 times.
 11922 assert(best_fit != NULL);
198 11922 Catalog *catalog = NULL;
199
1/2
✓ Branch 1 taken 11922 times.
✗ Branch 2 not taken.
 11922 const bool retval = MountSubtree(ps_path, best_fit, true /* is_listable */,
200 &catalog);
201
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11922 times.
 11922 if (!retval)
202 ✗ return false;
203
204 11922 *result = static_cast<WritableCatalog *>(catalog);
205
206
1/2
✓ Branch 1 taken 11922 times.
✗ Branch 2 not taken.
 11922 catalog::DirectoryEntry dummy;
207
2/2
✓ Branch 0 taken 6156 times.
✓ Branch 1 taken 5766 times.
 11922 if (NULL == dirent) {
208 6156 dirent = &dummy;
209 }
210
1/2
✓ Branch 1 taken 11922 times.
✗ Branch 2 not taken.
 11922 const bool found = catalog->LookupPath(ps_path, dirent);
211
6/8
✓ Branch 0 taken 11896 times.
✓ Branch 1 taken 26 times.
✓ Branch 3 taken 11896 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 11896 times.
✓ Branch 7 taken 26 times.
✓ Branch 8 taken 11896 times.
 11922 if (!found || !catalog->IsWritable())
212 26 return false;
213
214 11896 return true;
215 11922 }
216
217
218 ✗ WritableCatalog *WritableCatalogManager::GetHostingCatalog(
219 const std::string &path) {
220 ✗ WritableCatalog *result = NULL;
221 ✗ const bool retval = FindCatalog(MakeRelativePath(path), &result, NULL);
222 ✗ if (!retval)
223 ✗ return NULL;
224 ✗ return result;
225 }
226
227
228 /**
229 * Remove the given file from the catalogs.
230 * @param file_path the full path to the file to be removed
231 * @return true on success, false otherwise
232 */
233 32 void WritableCatalogManager::RemoveFile(const std::string &path) {
234
1/2
✓ Branch 1 taken 32 times.
✗ Branch 2 not taken.
 32 const string file_path = MakeRelativePath(path);
235
236 32 SyncLock();
237 32 WritableCatalog *catalog = NULL;
238
1/2
✓ Branch 1 taken 32 times.
✗ Branch 2 not taken.
 32 DirectoryEntry entry;
239
2/4
✓ Branch 1 taken 32 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 32 times.
✗ Branch 4 not taken.
 32 if (FindCatalog(file_path, &catalog, &entry)) {
240
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 32 times.
 32 if (entry.IsBundleTrigger()) {
241 ✗ catalog->RemoveEntry(
242 ✗ GetParentPath(file_path) + "/.cvmfsbundle-" + GetFileName(file_path));
243 }
244
1/2
✓ Branch 1 taken 32 times.
✗ Branch 2 not taken.
 32 catalog->RemoveEntry(file_path);
245 }
246 32 SyncUnlock();
247
248
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 32 times.
 32 if (catalog == NULL) {
249 ✗ PANIC(kLogStderr, "catalog for file '%s' cannot be found",
250 file_path.c_str());
251 }
252 32 }
253
254
255 /**
256 * Remove the given directory from the catalogs.
257 * @param directory_path the full path to the directory to be removed
258 * @return true on success, false otherwise
259 */
260 84 void WritableCatalogManager::RemoveDirectory(const std::string &path) {
261
1/2
✓ Branch 1 taken 84 times.
✗ Branch 2 not taken.
 84 const string directory_path = MakeRelativePath(path);
262
1/2
✓ Branch 1 taken 84 times.
✗ Branch 2 not taken.
 84 const string parent_path = GetParentPath(directory_path);
263
264 84 SyncLock();
265 WritableCatalog *catalog;
266
1/2
✓ Branch 1 taken 84 times.
✗ Branch 2 not taken.
 84 DirectoryEntry parent_entry;
267
2/4
✓ Branch 1 taken 84 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 84 times.
 84 if (!FindCatalog(parent_path, &catalog, &parent_entry)) {
268 ✗ PANIC(kLogStderr, "catalog for directory '%s' cannot be found",
269 directory_path.c_str());
270 }
271
272 84 parent_entry.set_linkcount(parent_entry.linkcount() - 1);
273
274
1/2
✓ Branch 1 taken 84 times.
✗ Branch 2 not taken.
 84 catalog->RemoveEntry(directory_path);
275
1/2
✓ Branch 1 taken 84 times.
✗ Branch 2 not taken.
 84 catalog->UpdateEntry(parent_entry, parent_path);
276
2/2
✓ Branch 1 taken 53 times.
✓ Branch 2 taken 31 times.
 84 if (parent_entry.IsNestedCatalogRoot()) {
277
1/2
✓ Branch 2 taken 53 times.
✗ Branch 3 not taken.
 53 LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating transition point %s",
278 parent_path.c_str());
279 WritableCatalog *parent_catalog = reinterpret_cast<WritableCatalog *>(
280 53 catalog->parent());
281 53 parent_entry.set_is_nested_catalog_mountpoint(true);
282 53 parent_entry.set_is_nested_catalog_root(false);
283
1/2
✓ Branch 1 taken 53 times.
✗ Branch 2 not taken.
 53 parent_catalog->UpdateEntry(parent_entry, parent_path);
284 }
285 84 SyncUnlock();
286 84 }
287
288 /**
289 * Clone the file called `source` changing its name into `destination`, the
290 * source file is keep intact.
291 * @params destination, the name of the new file, complete path
292 * @params source, the name of the file to clone, which must be already in the
293 * repository
294 * @params fail_if_source_missing, when true (default) abort if the source is
295 * not in the catalog; when false, skip the clone and return false instead.
296 * This is used by the tarball ingestion engine to tolerate hardlinks whose
297 * target is not part of the same archive (e.g. cross-layer hardlinks in OCI
298 * image layers).
299 * @return true if the file was cloned, false if the source was missing and
300 * fail_if_source_missing was false
301 */
302 ✗ bool WritableCatalogManager::Clone(const std::string destination,
303 const std::string source,
304 const bool fail_if_source_missing) {
305 ✗ const std::string relative_source = MakeRelativePath(source);
306
307 ✗ DirectoryEntry source_dirent;
308 ✗ if (!LookupPath(relative_source, kLookupDefault, &source_dirent)) {
309 ✗ if (fail_if_source_missing) {
310 ✗ PANIC(kLogStderr, "catalog for file '%s' cannot be found, aborting",
311 source.c_str());
312 }
313 // The caller opted out of aborting and is expected to handle the missing
314 // source (e.g. by materializing a replacement file), so only log at debug
315 // level here to avoid duplicate user-facing warnings.
316 ✗ LogCvmfs(kLogCatalog, kLogDebug,
317 "catalog for clone source '%s' cannot be found, not cloning "
318 "to '%s'",
319 source.c_str(), destination.c_str());
320 ✗ return false;
321 }
322 ✗ if (source_dirent.IsDirectory()) {
323 ✗ PANIC(kLogStderr, "Trying to clone a directory: '%s', aborting",
324 source.c_str());
325 }
326
327 // if the file is already there we remove it and we add it back
328 ✗ DirectoryEntry check_dirent;
329 ✗ const bool destination_already_present = LookupPath(
330 ✗ MakeRelativePath(destination), kLookupDefault, &check_dirent);
331 ✗ if (destination_already_present) {
332 ✗ this->RemoveFile(destination);
333 }
334
335 ✗ DirectoryEntry destination_dirent(source_dirent);
336 ✗ std::string destination_dirname;
337 ✗ std::string destination_filename;
338 ✗ SplitPath(destination, &destination_dirname, &destination_filename);
339
340 ✗ destination_dirent.name_.Assign(
341 ✗ NameString(destination_filename.c_str(), destination_filename.length()));
342
343 // TODO(jblomer): clone is used by tarball engine and should eventually
344 // support extended attributes
345 ✗ this->AddFile(destination_dirent, empty_xattrs, destination_dirname);
346 ✗ return true;
347 }
348
349
350 /**
351 * Copies an entire directory tree from the existing from_dir to the
352 * non-existing to_dir. The destination's parent directory must exist. On the
353 * catalog level, the new entries will be identical to the old ones except
354 * for their path hash fields.
355 */
356 260 void WritableCatalogManager::CloneTree(const std::string &from_dir,
357 const std::string &to_dir) {
358 // Sanitize input paths
359
6/6
✓ Branch 1 taken 208 times.
✓ Branch 2 taken 52 times.
✓ Branch 4 taken 26 times.
✓ Branch 5 taken 182 times.
✓ Branch 6 taken 78 times.
✓ Branch 7 taken 182 times.
 260 if (from_dir.empty() || to_dir.empty())
360 78 PANIC(kLogStderr, "clone tree from or to root impossible");
361
362
1/2
✓ Branch 1 taken 182 times.
✗ Branch 2 not taken.
 182 const std::string relative_source = MakeRelativePath(from_dir);
363
1/2
✓ Branch 1 taken 182 times.
✗ Branch 2 not taken.
 182 const std::string relative_dest = MakeRelativePath(to_dir);
364
365
2/2
✓ Branch 1 taken 26 times.
✓ Branch 2 taken 156 times.
 182 if (relative_source == relative_dest) {
366 26 PANIC(kLogStderr, "cannot clone tree into itself ('%s')", to_dir.c_str());
367 }
368
4/7
✓ Branch 1 taken 156 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 156 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 26 times.
✓ Branch 8 taken 130 times.
 156 if (HasPrefix(relative_dest, relative_source + "/", false /*ignore_case*/)) {
369 26 PANIC(kLogStderr,
370 "cannot clone tree into sub directory of source '%s' --> '%s'",
371 from_dir.c_str(), to_dir.c_str());
372 }
373
374
1/2
✓ Branch 1 taken 130 times.
✗ Branch 2 not taken.
 130 DirectoryEntry source_dirent;
375
3/4
✓ Branch 1 taken 130 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 26 times.
✓ Branch 4 taken 104 times.
 130 if (!LookupPath(relative_source, kLookupDefault, &source_dirent)) {
376 26 PANIC(kLogStderr, "path '%s' cannot be found, aborting", from_dir.c_str());
377 }
378
2/2
✓ Branch 1 taken 26 times.
✓ Branch 2 taken 78 times.
 104 if (!source_dirent.IsDirectory()) {
379 26 PANIC(kLogStderr, "CloneTree: source '%s' not a directory, aborting",
380 from_dir.c_str());
381 }
382
383
1/2
✓ Branch 1 taken 78 times.
✗ Branch 2 not taken.
 78 DirectoryEntry dest_dirent;
384
3/4
✓ Branch 1 taken 78 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 26 times.
✓ Branch 4 taken 52 times.
 78 if (LookupPath(relative_dest, kLookupDefault, &dest_dirent)) {
385 26 PANIC(kLogStderr, "destination '%s' exists, aborting", to_dir.c_str());
386 }
387
388
1/2
✓ Branch 1 taken 52 times.
✗ Branch 2 not taken.
 52 const std::string dest_parent = GetParentPath(relative_dest);
389
1/2
✓ Branch 1 taken 52 times.
✗ Branch 2 not taken.
 52 DirectoryEntry dest_parent_dirent;
390
3/4
✓ Branch 1 taken 52 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 26 times.
✓ Branch 4 taken 26 times.
 52 if (!LookupPath(dest_parent, kLookupDefault, &dest_parent_dirent)) {
391 26 PANIC(kLogStderr, "destination '%s' not on a known path, aborting",
392 to_dir.c_str());
393 }
394
395
2/4
✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 26 times.
✗ Branch 5 not taken.
 26 CloneTreeImpl(PathString(from_dir),
396
1/2
✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
 52 GetParentPath(to_dir),
397
2/4
✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 26 times.
✗ Branch 5 not taken.
 52 NameString(GetFileName(to_dir)));
398 546 }
399
400
401 /**
402 * Called from CloneTree(), assumes that from_dir and to_dir are sufficiently
403 * sanitized
404 */
405 208 void WritableCatalogManager::CloneTreeImpl(const PathString &source_dir,
406 const std::string &dest_parent_dir,
407 const NameString &dest_name) {
408
1/2
✓ Branch 4 taken 208 times.
✗ Branch 5 not taken.
 208 LogCvmfs(kLogCatalog, kLogDebug, "cloning %s --> %s/%s", source_dir.c_str(),
409
1/2
✓ Branch 1 taken 208 times.
✗ Branch 2 not taken.
 416 dest_parent_dir.c_str(), dest_name.ToString().c_str());
410
3/6
✓ Branch 1 taken 208 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 208 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 208 times.
✗ Branch 8 not taken.
 416 const PathString relative_source(MakeRelativePath(source_dir.ToString()));
411
412
1/2
✓ Branch 1 taken 208 times.
✗ Branch 2 not taken.
 208 DirectoryEntry source_dirent;
413
1/2
✓ Branch 1 taken 208 times.
✗ Branch 2 not taken.
 208 bool retval = LookupPath(relative_source, kLookupDefault, &source_dirent);
414
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 208 times.
 208 assert(retval);
415
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 208 times.
 208 assert(!source_dirent.IsBindMountpoint());
416
417
1/2
✓ Branch 1 taken 208 times.
✗ Branch 2 not taken.
 208 DirectoryEntry dest_dirent(source_dirent);
418
1/2
✓ Branch 1 taken 208 times.
✗ Branch 2 not taken.
 208 dest_dirent.name_.Assign(dest_name);
419 // Just in case, reset the nested catalog markers
420 208 dest_dirent.set_is_nested_catalog_mountpoint(false);
421 208 dest_dirent.set_is_nested_catalog_root(false);
422
423 208 XattrList xattrs;
424
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 208 times.
 208 if (source_dirent.HasXattrs()) {
425 ✗ retval = LookupXattrs(relative_source, &xattrs);
426 ✗ assert(retval);
427 }
428
1/2
✓ Branch 1 taken 208 times.
✗ Branch 2 not taken.
 208 AddDirectory(dest_dirent, xattrs, dest_parent_dir);
429
430
1/2
✓ Branch 1 taken 208 times.
✗ Branch 2 not taken.
 208 std::string dest_dir = dest_parent_dir;
431
2/2
✓ Branch 1 taken 182 times.
✓ Branch 2 taken 26 times.
 208 if (!dest_dir.empty())
432
1/2
✓ Branch 1 taken 182 times.
✗ Branch 2 not taken.
 182 dest_dir.push_back('/');
433
2/4
✓ Branch 1 taken 208 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 208 times.
✗ Branch 5 not taken.
 208 dest_dir += dest_name.ToString();
434 208 if (source_dirent.IsNestedCatalogRoot()
435
5/6
✓ Branch 0 taken 130 times.
✓ Branch 1 taken 78 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 130 times.
✓ Branch 5 taken 78 times.
✓ Branch 6 taken 130 times.
 208 || source_dirent.IsNestedCatalogMountpoint()) {
436
1/2
✓ Branch 1 taken 78 times.
✗ Branch 2 not taken.
 78 CreateNestedCatalog(dest_dir);
437 }
438
439 208 DirectoryEntryList ls;
440
1/2
✓ Branch 1 taken 208 times.
✗ Branch 2 not taken.
 208 retval = Listing(relative_source, &ls, false /* expand_symlink */);
441
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 208 times.
 208 assert(retval);
442
2/2
✓ Branch 1 taken 442 times.
✓ Branch 2 taken 208 times.
 650 for (unsigned i = 0; i < ls.size(); ++i) {
443
1/2
✓ Branch 1 taken 442 times.
✗ Branch 2 not taken.
 442 PathString sub_path(source_dir);
444
2/4
✓ Branch 1 taken 442 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 442 times.
 442 assert(!sub_path.IsEmpty());
445
1/2
✓ Branch 1 taken 442 times.
✗ Branch 2 not taken.
 442 sub_path.Append("/", 1);
446
3/6
✓ Branch 2 taken 442 times.
✗ Branch 3 not taken.
✓ Branch 7 taken 442 times.
✗ Branch 8 not taken.
✓ Branch 11 taken 442 times.
✗ Branch 12 not taken.
 442 sub_path.Append(ls[i].name().GetChars(), ls[i].name().GetLength());
447
448
2/2
✓ Branch 2 taken 182 times.
✓ Branch 3 taken 260 times.
 442 if (ls[i].IsDirectory()) {
449
2/4
✓ Branch 2 taken 182 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 182 times.
✗ Branch 6 not taken.
 182 CloneTreeImpl(sub_path, dest_dir, ls[i].name());
450 182 continue;
451 }
452
453 // We break hard-links during cloning
454 260 ls[i].set_hardlink_group(0);
455 260 ls[i].set_linkcount(1);
456
457 260 xattrs.Clear();
458
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 260 times.
 260 if (ls[i].HasXattrs()) {
459 ✗ retval = LookupXattrs(sub_path, &xattrs);
460 ✗ assert(retval);
461 }
462
463
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 260 times.
 260 if (ls[i].IsChunkedFile()) {
464 ✗ FileChunkList chunks;
465 const std::string relative_sub_path = MakeRelativePath(
466 ✗ sub_path.ToString());
467 ✗ retval = ListFileChunks(PathString(relative_sub_path),
468 ✗ ls[i].hash_algorithm(), &chunks);
469 ✗ assert(retval);
470 ✗ AddChunkedFile(ls[i], xattrs, dest_dir, chunks);
471 ✗ } else {
472
1/2
✓ Branch 2 taken 260 times.
✗ Branch 3 not taken.
 260 AddFile(ls[i], xattrs, dest_dir);
473 }
474
2/2
✓ Branch 1 taken 260 times.
✓ Branch 2 taken 182 times.
 442 }
475 208 }
476
477
478 /**
479 * Add a new directory to the catalogs.
480 * @param entry a DirectoryEntry structure describing the new directory
481 * @param parent_directory the absolute path of the directory containing the
482 * directory to be created
483 * @return true on success, false otherwise
484 */
485 4408 void WritableCatalogManager::AddDirectory(const DirectoryEntryBase &entry,
486 const XattrList &xattrs,
487 const std::string &parent_directory) {
488
1/2
✓ Branch 1 taken 4408 times.
✗ Branch 2 not taken.
 4408 const string parent_path = MakeRelativePath(parent_directory);
489
1/2
✓ Branch 1 taken 4408 times.
✗ Branch 2 not taken.
 4408 string directory_path = parent_path + "/";
490
3/6
✓ Branch 1 taken 4408 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 4408 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 4408 times.
✗ Branch 10 not taken.
 4408 directory_path.append(entry.name().GetChars(), entry.name().GetLength());
491
492 4408 SyncLock();
493 WritableCatalog *catalog;
494
1/2
✓ Branch 1 taken 4408 times.
✗ Branch 2 not taken.
 4408 DirectoryEntry parent_entry;
495
2/4
✓ Branch 1 taken 4408 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 4408 times.
 4408 if (!FindCatalog(parent_path, &catalog, &parent_entry)) {
496 ✗ PANIC(kLogStderr, "catalog for directory '%s' cannot be found",
497 directory_path.c_str());
498 }
499
500
1/2
✓ Branch 1 taken 4408 times.
✗ Branch 2 not taken.
 4408 DirectoryEntry fixed_hardlink_count(entry);
501 4408 fixed_hardlink_count.set_linkcount(2);
502
1/2
✓ Branch 1 taken 4408 times.
✗ Branch 2 not taken.
 4408 catalog->AddEntry(fixed_hardlink_count, xattrs, directory_path, parent_path);
503
504 4408 parent_entry.set_linkcount(parent_entry.linkcount() + 1);
505
1/2
✓ Branch 1 taken 4408 times.
✗ Branch 2 not taken.
 4408 catalog->UpdateEntry(parent_entry, parent_path);
506
2/2
✓ Branch 1 taken 78 times.
✓ Branch 2 taken 4330 times.
 4408 if (parent_entry.IsNestedCatalogRoot()) {
507
1/2
✓ Branch 2 taken 78 times.
✗ Branch 3 not taken.
 78 LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating transition point %s",
508 parent_path.c_str());
509 WritableCatalog *parent_catalog = reinterpret_cast<WritableCatalog *>(
510 78 catalog->parent());
511 78 parent_entry.set_is_nested_catalog_mountpoint(true);
512 78 parent_entry.set_is_nested_catalog_root(false);
513
1/2
✓ Branch 1 taken 78 times.
✗ Branch 2 not taken.
 78 parent_catalog->UpdateEntry(parent_entry, parent_path);
514 }
515 4408 SyncUnlock();
516 4408 }
517
518 /**
519 * Add a new file to the catalogs.
520 * @param entry a DirectoryEntry structure describing the new file
521 * @param parent_directory the absolute path of the directory containing the
522 * file to be created
523 * @return true on success, false otherwise
524 */
525 5914 void WritableCatalogManager::AddFile(const DirectoryEntry &entry,
526 const XattrList &xattrs,
527 const std::string &parent_directory) {
528
1/2
✓ Branch 1 taken 5914 times.
✗ Branch 2 not taken.
 5914 const string parent_path = MakeRelativePath(parent_directory);
529
1/2
✓ Branch 1 taken 5914 times.
✗ Branch 2 not taken.
 5914 const string file_path = entry.GetFullPath(parent_path);
530
531 5914 SyncLock();
532 WritableCatalog *catalog;
533
2/4
✓ Branch 1 taken 5914 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 5914 times.
 5914 if (!FindCatalog(parent_path, &catalog)) {
534 ✗ PANIC(kLogStderr, "catalog for file '%s' cannot be found",
535 file_path.c_str());
536 }
537
538
4/6
✓ Branch 1 taken 5861 times.
✓ Branch 2 taken 53 times.
✓ Branch 4 taken 5861 times.
✗ Branch 5 not taken.
✗ Branch 8 not taken.
✓ Branch 9 taken 5861 times.
 5914 assert(!entry.IsRegular() || entry.IsChunkedFile()
539 || !entry.checksum().IsNull());
540
3/4
✓ Branch 1 taken 53 times.
✓ Branch 2 taken 5861 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 53 times.
 5914 assert(entry.IsRegular() || !entry.IsExternalFile());
541
542 // check if file is too big
543
1/2
✓ Branch 1 taken 5914 times.
✗ Branch 2 not taken.
 5914 const unsigned mbytes = entry.size() / (1024 * 1024);
544
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 5914 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 5914 if ((file_mbyte_limit_ > 0) && (mbytes > file_mbyte_limit_)) {
545 ✗ LogCvmfs(kLogCatalog, kLogStderr,
546 "%s: file at %s is larger than %u megabytes (%u). "
547 "CernVM-FS works best with small files. "
548 "Please remove the file or increase the limit.",
549 ✗ enforce_limits_ ? "FATAL" : "WARNING", file_path.c_str(),
550 file_mbyte_limit_, mbytes);
551 ✗ if (enforce_limits_)
552 ✗ PANIC(kLogStderr, "file at %s is larger than %u megabytes (%u).",
553 file_path.c_str(), file_mbyte_limit_, mbytes);
554 }
555
556
1/2
✓ Branch 1 taken 5914 times.
✗ Branch 2 not taken.
 5914 catalog->AddEntry(entry, xattrs, file_path, parent_path);
557 5914 SyncUnlock();
558 5914 }
559
560
561 ✗ void WritableCatalogManager::AddChunkedFile(const DirectoryEntryBase &entry,
562 const XattrList &xattrs,
563 const std::string &parent_directory,
564 const FileChunkList &file_chunks) {
565 ✗ assert(file_chunks.size() > 0);
566
567 ✗ DirectoryEntry full_entry(entry);
568 ✗ full_entry.set_is_chunked_file(true);
569
570 ✗ AddFile(full_entry, xattrs, parent_directory);
571
572 ✗ const string parent_path = MakeRelativePath(parent_directory);
573 ✗ const string file_path = entry.GetFullPath(parent_path);
574
575 ✗ SyncLock();
576 WritableCatalog *catalog;
577 ✗ if (!FindCatalog(parent_path, &catalog)) {
578 ✗ PANIC(kLogStderr, "catalog for file '%s' cannot be found",
579 file_path.c_str());
580 }
581
582 ✗ for (unsigned i = 0; i < file_chunks.size(); ++i) {
583 ✗ catalog->AddFileChunk(file_path, *file_chunks.AtPtr(i));
584 }
585 ✗ SyncUnlock();
586 }
587
588 /**
589 * Reads the entry given by file_path and sets the bundle trigger flag
590 * accordingly. If new_value is true, the regular file at file_path must exist.
591 * If new_value is false, a missing file_path is ignored (otherwise it would
592 * be hard to get rid of dangling bundle trigger markers).
593 */
594 ✗ void WritableCatalogManager::UpdateBundleTrigger(const std::string &file_path,
595 bool new_value)
596 {
597 ✗ SyncLock();
598
599 ✗ WritableCatalog *catalog = NULL;
600 ✗ DirectoryEntry entry;
601 ✗ if (!FindCatalog(file_path, &catalog, &entry)) {
602 ✗ SyncUnlock();
603
604 ✗ if (new_value) {
605 ✗ PANIC(kLogStderr, "failed to find catalog of %s", file_path.c_str());
606 }
607
608 ✗ LogCvmfs(kLogCatalog, kLogDebug, "dangling bundle trigger %s",
609 file_path.c_str());
610 ✗ return;
611 }
612
613 ✗ if (new_value && !entry.IsRegular()) {
614 ✗ SyncUnlock();
615 ✗ PANIC(kLogStderr, "failed to set bundle trigger on non-regular file %s",
616 file_path.c_str());
617 }
618
619 ✗ entry.set_is_bundle_trigger(new_value);
620 ✗ catalog->UpdateEntry(entry, file_path);
621
622 ✗ SyncUnlock();
623 }
624
625
626 /**
627 * Add a hardlink group to the catalogs.
628 * @param entries a list of DirectoryEntries describing the new files
629 * @param parent_directory the absolute path of the directory containing the
630 * files to be created
631 * @return true on success, false otherwise
632 */
633 ✗ void WritableCatalogManager::AddHardlinkGroup(
634 const DirectoryEntryBaseList &entries,
635 const XattrList &xattrs,
636 const std::string &parent_directory,
637 const FileChunkList &file_chunks) {
638 ✗ assert(entries.size() >= 1);
639 ✗ assert(file_chunks.IsEmpty() || entries[0].IsRegular());
640 ✗ if (entries.size() == 1) {
641 ✗ DirectoryEntry fix_linkcount(entries[0]);
642 ✗ fix_linkcount.set_linkcount(1);
643 ✗ if (file_chunks.IsEmpty())
644 ✗ return AddFile(fix_linkcount, xattrs, parent_directory);
645 ✗ return AddChunkedFile(fix_linkcount, xattrs, parent_directory, file_chunks);
646 }
647
648 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "adding hardlink group %s/%s",
649 ✗ parent_directory.c_str(), entries[0].name().c_str());
650
651 // Hardlink groups have to reside in the same directory.
652 // Therefore we only have one parent directory here
653 ✗ const string parent_path = MakeRelativePath(parent_directory);
654
655 // check if hard link is too big
656 ✗ const unsigned mbytes = entries[0].size() / (1024 * 1024);
657 ✗ if ((file_mbyte_limit_ > 0) && (mbytes > file_mbyte_limit_)) {
658 ✗ LogCvmfs(kLogCatalog, kLogStderr,
659 "%s: hard link at %s is larger than %u megabytes (%u). "
660 "CernVM-FS works best with small files. "
661 "Please remove the file or increase the limit.",
662 ✗ enforce_limits_ ? "FATAL" : "WARNING",
663 ✗ (parent_path + entries[0].name().ToString()).c_str(),
664 file_mbyte_limit_, mbytes);
665 ✗ if (enforce_limits_)
666 ✗ PANIC(kLogStderr, "hard link at %s is larger than %u megabytes (%u)",
667 (parent_path + entries[0].name().ToString()).c_str(),
668 file_mbyte_limit_, mbytes);
669 }
670
671 ✗ SyncLock();
672 WritableCatalog *catalog;
673 ✗ if (!FindCatalog(parent_path, &catalog)) {
674 ✗ PANIC(kLogStderr,
675 "catalog for hardlink group containing '%s' cannot be found",
676 parent_path.c_str());
677 }
678
679 // Get a valid hardlink group id for the catalog the group will end up in
680 // TODO(unknown): Compaction
681 ✗ const uint32_t new_group_id = catalog->GetMaxLinkId() + 1;
682 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "hardlink group id %u issued",
683 new_group_id);
684 ✗ assert(new_group_id > 0);
685
686 // Add the file entries to the catalog
687 ✗ for (DirectoryEntryBaseList::const_iterator i = entries.begin(),
688 ✗ iEnd = entries.end();
689 ✗ i != iEnd;
690 ✗ ++i) {
691 ✗ string file_path = parent_path + "/";
692 ✗ file_path.append(i->name().GetChars(), i->name().GetLength());
693
694 // create a fully fledged DirectoryEntry to add the hardlink group to it
695 // which is CVMFS specific meta data.
696 ✗ DirectoryEntry hardlink(*i);
697 ✗ hardlink.set_hardlink_group(new_group_id);
698 ✗ hardlink.set_linkcount(entries.size());
699 ✗ hardlink.set_is_chunked_file(!file_chunks.IsEmpty());
700
701 ✗ catalog->AddEntry(hardlink, xattrs, file_path, parent_path);
702 ✗ if (hardlink.IsChunkedFile()) {
703 ✗ for (unsigned i = 0; i < file_chunks.size(); ++i) {
704 ✗ catalog->AddFileChunk(file_path, *file_chunks.AtPtr(i));
705 }
706 }
707 }
708 ✗ SyncUnlock();
709 }
710
711
712 ✗ void WritableCatalogManager::ShrinkHardlinkGroup(const string &remove_path) {
713 ✗ const string relative_path = MakeRelativePath(remove_path);
714
715 ✗ SyncLock();
716 WritableCatalog *catalog;
717 ✗ if (!FindCatalog(relative_path, &catalog)) {
718 ✗ PANIC(kLogStderr,
719 "catalog for hardlink group containing '%s' cannot be found",
720 remove_path.c_str());
721 }
722
723 ✗ catalog->IncLinkcount(relative_path, -1);
724 ✗ SyncUnlock();
725 }
726
727
728 /**
729 * Update entry meta data (mode, owner, ...).
730 * CVMFS specific meta data (i.e. nested catalog transition points) are NOT
731 * changed by this method, although transition points intrinsics are taken into
732 * account, to keep nested catalogs consistent.
733 * @param entry the directory entry to be touched
734 * @param path the path of the directory entry to be touched
735 */
736 2 void WritableCatalogManager::TouchDirectory(const DirectoryEntryBase &entry,
737 const XattrList &xattrs,
738 const std::string &directory_path) {
739
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 2 times.
 2 assert(entry.IsDirectory());
740
741
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 const string entry_path = MakeRelativePath(directory_path);
742
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 const string parent_path = GetParentPath(entry_path);
743
744 2 SyncLock();
745 // find the catalog to be updated
746 WritableCatalog *catalog;
747
2/4
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 2 times.
 2 if (!FindCatalog(parent_path, &catalog)) {
748 ✗ PANIC(kLogStderr, "catalog for entry '%s' cannot be found",
749 entry_path.c_str());
750 }
751
752
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 catalog->TouchEntry(entry, xattrs, entry_path);
753
754 // since we deal with a directory here, we might just touch a
755 // nested catalog transition point. If this is the case we would need to
756 // update two catalog entries:
757 // * the nested catalog MOUNTPOINT in the parent catalog
758 // * the nested catalog ROOT in the nested catalog
759
760 // first check if we really have a nested catalog transition point
761
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 catalog::DirectoryEntry potential_transition_point;
762
1/2
✓ Branch 3 taken 2 times.
✗ Branch 4 not taken.
 2 const PathString transition_path(entry_path.data(), entry_path.length());
763
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 bool retval = catalog->LookupPath(transition_path,
764 &potential_transition_point);
765
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
 2 assert(retval);
766
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 2 times.
 2 if (potential_transition_point.IsNestedCatalogMountpoint()) {
767 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating transition point at %s",
768 entry_path.c_str());
769
770 // find and mount nested catalog associated to this transition point
771 ✗ shash::Any nested_hash;
772 uint64_t nested_size;
773 ✗ retval = catalog->FindNested(transition_path, &nested_hash, &nested_size);
774 ✗ assert(retval);
775 Catalog *nested_catalog;
776 ✗ nested_catalog = MountCatalog(transition_path, nested_hash, catalog);
777 ✗ assert(nested_catalog != NULL);
778
779 // update nested catalog root in the child catalog
780 reinterpret_cast<WritableCatalog *>(nested_catalog)
781 ✗ ->TouchEntry(entry, xattrs, entry_path);
782 }
783
784 2 SyncUnlock();
785 2 }
786
787
788 /**
789 * Create a new nested catalog. Includes moving all entries belonging there
790 * from it's parent catalog.
791 * @param mountpoint the path of the directory to become a nested root
792 * @return true on success, false otherwise
793 */
794 1189 void WritableCatalogManager::CreateNestedCatalog(
795 const std::string &mountpoint) {
796
1/2
✓ Branch 1 taken 1189 times.
✗ Branch 2 not taken.
 1189 const string nested_root_path = MakeRelativePath(mountpoint);
797
1/2
✓ Branch 1 taken 1189 times.
✗ Branch 2 not taken.
 1189 const PathString ps_nested_root_path(nested_root_path);
798
799 1189 SyncLock();
800 // Find the catalog currently containing the directory structure, which
801 // will be represented as a new nested catalog and its root-entry/mountpoint
802 // along the way
803 1189 WritableCatalog *old_catalog = NULL;
804
1/2
✓ Branch 1 taken 1189 times.
✗ Branch 2 not taken.
 1189 DirectoryEntry new_root_entry;
805
2/4
✓ Branch 1 taken 1189 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 1189 times.
 1189 if (!FindCatalog(nested_root_path, &old_catalog, &new_root_entry)) {
806 ✗ PANIC(kLogStderr,
807 "failed to create nested catalog '%s': "
808 "mountpoint was not found in current catalog structure",
809 nested_root_path.c_str());
810 }
811
812 // Create the database schema and the initial root entry
813 // for the new nested catalog
814
1/2
✓ Branch 2 taken 1189 times.
✗ Branch 3 not taken.
 1189 const string database_file_path = CreateTempPath(dir_temp() + "/catalog",
815
1/2
✓ Branch 1 taken 1189 times.
✗ Branch 2 not taken.
 1189 0666);
816 1189 const bool volatile_content = false;
817
1/2
✓ Branch 1 taken 1189 times.
✗ Branch 2 not taken.
 1189 CatalogDatabase *new_catalog_db = CatalogDatabase::Create(database_file_path);
818
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1189 times.
 1189 assert(NULL != new_catalog_db);
819 // Note we do not set the external_data bit for nested catalogs
820
2/4
✓ Branch 2 taken 1189 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1189 times.
✗ Branch 6 not taken.
 1189 bool retval = new_catalog_db->InsertInitialValues(
821 nested_root_path,
822 volatile_content,
823 "", // At this point, only root
824 // catalog gets VOMS authz
825 new_root_entry);
826
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1189 times.
 1189 assert(retval);
827 // TODO(rmeusel): we need a way to attach a catalog directly from an open
828 // database to remove this indirection
829
1/2
✓ Branch 0 taken 1189 times.
✗ Branch 1 not taken.
 1189 delete new_catalog_db;
830 1189 new_catalog_db = NULL;
831
832 // Attach the just created nested catalog
833
2/4
✓ Branch 1 taken 1189 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1189 times.
✗ Branch 5 not taken.
 1189 Catalog *new_catalog = CreateCatalog(ps_nested_root_path, shash::Any(),
834 old_catalog);
835
1/2
✓ Branch 1 taken 1189 times.
✗ Branch 2 not taken.
 1189 retval = AttachCatalog(database_file_path, new_catalog);
836
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1189 times.
 1189 assert(retval);
837
838
2/4
✓ Branch 1 taken 1189 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 1189 times.
 1189 assert(new_catalog->IsWritable());
839 1189 WritableCatalog *wr_new_catalog = static_cast<WritableCatalog *>(new_catalog);
840
841
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1189 times.
 1189 if (new_root_entry.HasXattrs()) {
842 ✗ XattrList xattrs;
843 ✗ retval = old_catalog->LookupXattrsPath(ps_nested_root_path, &xattrs);
844 ✗ assert(retval);
845 ✗ wr_new_catalog->TouchEntry(new_root_entry, xattrs, nested_root_path);
846 }
847
848 // From now on, there are two catalogs, spanning the same directory structure
849 // we have to split the overlapping directory entries from the old catalog
850 // to the new catalog to re-gain a valid catalog structure
851
1/2
✓ Branch 1 taken 1189 times.
✗ Branch 2 not taken.
 1189 old_catalog->Partition(wr_new_catalog);
852
853 // Add the newly created nested catalog to the references of the containing
854 // catalog
855
4/8
✓ Branch 1 taken 1189 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1189 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 1189 times.
✗ Branch 8 not taken.
✓ Branch 10 taken 1189 times.
✗ Branch 11 not taken.
 1189 old_catalog->InsertNestedCatalog(new_catalog->mountpoint().ToString(), NULL,
856 1189 shash::Any(spooler_->GetHashAlgorithm()), 0);
857
858 // Fix subtree counters in new nested catalogs: subtree is the sum of all
859 // entries of all "grand-nested" catalogs
860 // Note: taking a copy of the nested catalog list here
861 const Catalog::NestedCatalogList
862
1/2
✓ Branch 1 taken 1189 times.
✗ Branch 2 not taken.
 1189 &grand_nested = wr_new_catalog->ListOwnNestedCatalogs();
863
1/2
✓ Branch 1 taken 1189 times.
✗ Branch 2 not taken.
 1189 DeltaCounters fix_subtree_counters;
864 2378 for (Catalog::NestedCatalogList::const_iterator i = grand_nested.begin(),
865 1189 iEnd = grand_nested.end();
866
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1189 times.
 1189 i != iEnd;
867 ✗ ++i) {
868 WritableCatalog *grand_catalog;
869 ✗ retval = FindCatalog(i->mountpoint.ToString(), &grand_catalog);
870 ✗ assert(retval);
871 ✗ const Counters &grand_counters = grand_catalog->GetCounters();
872 ✗ grand_counters.AddAsSubtree(&fix_subtree_counters);
873 }
874 1189 const DeltaCounters save_counters = wr_new_catalog->delta_counters_;
875 1189 wr_new_catalog->delta_counters_ = fix_subtree_counters;
876
1/2
✓ Branch 1 taken 1189 times.
✗ Branch 2 not taken.
 1189 wr_new_catalog->UpdateCounters();
877 1189 wr_new_catalog->delta_counters_ = save_counters;
878
879 1189 SyncUnlock();
880 1189 }
881
882
883 /**
884 * Remove a nested catalog
885 *
886 * If the merged parameter is true, when you remove a nested catalog
887 * all entries currently held by it will be merged into its parent
888 * catalog.
889 * @param mountpoint - the path of the nested catalog to be removed
890 * @param merge - merge the subtree associated with the nested catalog
891 * into its parent catalog
892 * @return - true on success, false otherwise
893 */
894 55 void WritableCatalogManager::RemoveNestedCatalog(const string &mountpoint,
895 const bool merge) {
896
1/2
✓ Branch 1 taken 55 times.
✗ Branch 2 not taken.
 55 const string nested_root_path = MakeRelativePath(mountpoint);
897
898 55 SyncLock();
899 // Find the catalog which should be removed
900 55 WritableCatalog *nested_catalog = NULL;
901
2/4
✓ Branch 1 taken 55 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 55 times.
 55 if (!FindCatalog(nested_root_path, &nested_catalog)) {
902 ✗ PANIC(kLogStderr,
903 "failed to remove nested catalog '%s': "
904 "mountpoint was not found in current catalog structure",
905 nested_root_path.c_str());
906 }
907
908 // Check if the found catalog is really the nested catalog to be deleted
909
6/19
✓ Branch 1 taken 55 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 55 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 55 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 55 times.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 55 times.
✗ Branch 14 not taken.
✓ Branch 16 taken 55 times.
✗ Branch 17 not taken.
✗ Branch 19 not taken.
✗ Branch 20 not taken.
✗ Branch 22 not taken.
✗ Branch 23 not taken.
 110 assert(!nested_catalog->IsRoot()
910 && (nested_catalog->mountpoint().ToString() == nested_root_path));
911
912
1/2
✓ Branch 0 taken 55 times.
✗ Branch 1 not taken.
 55 if (merge) {
913 // Merge all data from the nested catalog into it's parent
914
1/2
✓ Branch 1 taken 55 times.
✗ Branch 2 not taken.
 55 nested_catalog->MergeIntoParent();
915 } else {
916 ✗ nested_catalog->RemoveFromParent();
917 }
918
919 // Delete the catalog database file from the working copy
920
2/6
✓ Branch 1 taken 55 times.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 55 times.
 55 if (unlink(nested_catalog->database_path().c_str()) != 0) {
921 ✗ PANIC(kLogStderr,
922 "unable to delete the removed nested catalog database file '%s'",
923 nested_catalog->database_path().c_str());
924 }
925
926 // Remove the catalog from internal data structures.
927 // On merge the children have been re-parented into this catalog's parent by
928 // MergeIntoParent (CopyCatalogsToParent), so only this catalog is detached.
929 // On removal (fast delete) the whole subtree is discarded together with the
930 // catalog, so any attached children must be detached as well - otherwise they
931 // would be left in the manager with a dangling parent pointer.
932
1/2
✓ Branch 0 taken 55 times.
✗ Branch 1 not taken.
 55 if (merge) {
933
1/2
✓ Branch 1 taken 55 times.
✗ Branch 2 not taken.
 55 DetachCatalog(nested_catalog);
934 } else {
935 ✗ DetachSubtree(nested_catalog);
936 }
937 55 SyncUnlock();
938 55 }
939
940
941 /**
942 * Swap in a new nested catalog
943 *
944 * The old nested catalog must not have been already attached to the
945 * catalog tree. This method will not attach the new nested catalog
946 * to the catalog tree.
947 *
948 * @param mountpoint - the path of the nested catalog to be removed
949 * @param new_hash - the hash of the new nested catalog
950 * @param new_size - the size of the new nested catalog
951 */
952 185 void WritableCatalogManager::SwapNestedCatalog(const string &mountpoint,
953 const shash::Any &new_hash,
954 const uint64_t new_size) {
955
1/2
✓ Branch 1 taken 185 times.
✗ Branch 2 not taken.
 185 const string nested_root_path = MakeRelativePath(mountpoint);
956
1/2
✓ Branch 1 taken 185 times.
✗ Branch 2 not taken.
 185 const string parent_path = GetParentPath(nested_root_path);
957
1/2
✓ Branch 1 taken 185 times.
✗ Branch 2 not taken.
 185 const PathString nested_root_ps = PathString(nested_root_path);
958
959 185 SyncLock();
960
961 // Find the immediate parent catalog
962 185 WritableCatalog *parent = NULL;
963
3/4
✓ Branch 1 taken 185 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 26 times.
✓ Branch 4 taken 159 times.
 185 if (!FindCatalog(parent_path, &parent)) {
964 26 SyncUnlock(); // this is needed for the unittest. otherwise they get stuck
965 26 PANIC(kLogStderr,
966 "failed to swap nested catalog '%s': could not find parent '%s'",
967 nested_root_path.c_str(), parent_path.c_str());
968 }
969
970 // Get old nested catalog counters
971
1/2
✓ Branch 1 taken 159 times.
✗ Branch 2 not taken.
 159 Catalog *old_attached_catalog = parent->FindChild(nested_root_ps);
972
1/2
✓ Branch 1 taken 159 times.
✗ Branch 2 not taken.
 159 Counters old_counters;
973
2/2
✓ Branch 0 taken 26 times.
✓ Branch 1 taken 133 times.
 159 if (old_attached_catalog) {
974 // Old catalog was already attached (e.g. as a child catalog
975 // attached by a prior call to CreateNestedCatalog()). Ensure
976 // that it has not been modified, get counters, and detach it.
977 26 WritableCatalogList list;
978
2/4
✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 26 times.
✗ Branch 4 not taken.
 26 if (GetModifiedCatalogLeafsRecursively(old_attached_catalog, &list)) {
979 26 SyncUnlock();
980 26 PANIC(kLogStderr,
981 "failed to swap nested catalog '%s': already modified",
982 nested_root_path.c_str());
983 }
984 ✗ old_counters = old_attached_catalog->GetCounters();
985 ✗ DetachSubtree(old_attached_catalog);
986
987 26 } else {
988 // Old catalog was not attached. Download a freely attached
989 // version and get counters.
990
1/2
✓ Branch 1 taken 133 times.
✗ Branch 2 not taken.
 133 shash::Any old_hash;
991 uint64_t old_size;
992
1/2
✓ Branch 1 taken 133 times.
✗ Branch 2 not taken.
 133 const bool old_found = parent->FindNested(nested_root_ps, &old_hash,
993 &old_size);
994
2/2
✓ Branch 0 taken 26 times.
✓ Branch 1 taken 107 times.
 133 if (!old_found) {
995 26 SyncUnlock();
996 26 PANIC(kLogStderr,
997 "failed to swap nested catalog '%s': not found in parent",
998 nested_root_path.c_str());
999 }
1000 const UniquePtr<Catalog> old_free_catalog(
1001
2/4
✓ Branch 1 taken 107 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 107 times.
✗ Branch 5 not taken.
 107 LoadFreeCatalog(nested_root_ps, old_hash));
1002
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 107 times.
 107 if (!old_free_catalog.IsValid()) {
1003 ✗ SyncUnlock();
1004 ✗ PANIC(kLogStderr,
1005 "failed to swap nested catalog '%s': failed to load old catalog",
1006 nested_root_path.c_str());
1007 }
1008 107 old_counters = old_free_catalog->GetCounters();
1009 107 }
1010
1011 // Load freely attached new catalog
1012 const UniquePtr<Catalog> new_catalog(
1013
3/4
✓ Branch 1 taken 81 times.
✓ Branch 2 taken 26 times.
✓ Branch 4 taken 81 times.
✗ Branch 5 not taken.
 107 LoadFreeCatalog(nested_root_ps, new_hash));
1014
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 81 times.
 81 if (!new_catalog.IsValid()) {
1015 ✗ SyncUnlock();
1016 ✗ PANIC(kLogStderr,
1017 "failed to swap nested catalog '%s': failed to load new catalog",
1018 nested_root_path.c_str());
1019 }
1020
1021 // Get new catalog root directory entry
1022
1/2
✓ Branch 1 taken 81 times.
✗ Branch 2 not taken.
 81 DirectoryEntry dirent;
1023 81 XattrList xattrs;
1024
1/2
✓ Branch 2 taken 81 times.
✗ Branch 3 not taken.
 81 const bool dirent_found = new_catalog->LookupPath(nested_root_ps, &dirent);
1025
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 81 times.
 81 if (!dirent_found) {
1026 ✗ SyncUnlock();
1027 ✗ PANIC(kLogStderr,
1028 "failed to swap nested catalog '%s': missing dirent in new catalog",
1029 nested_root_path.c_str());
1030 }
1031
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 81 times.
 81 if (dirent.HasXattrs()) {
1032 ✗ const bool xattrs_found = new_catalog->LookupXattrsPath(nested_root_ps,
1033 &xattrs);
1034 ✗ if (!xattrs_found) {
1035 ✗ SyncUnlock();
1036 ✗ PANIC(kLogStderr,
1037 "failed to swap nested catalog '%s': missing xattrs in new catalog",
1038 nested_root_path.c_str());
1039 }
1040 }
1041
1042 // Swap catalogs
1043
1/2
✓ Branch 1 taken 81 times.
✗ Branch 2 not taken.
 81 parent->RemoveNestedCatalog(nested_root_path, NULL);
1044
1/2
✓ Branch 1 taken 81 times.
✗ Branch 2 not taken.
 81 parent->InsertNestedCatalog(nested_root_path, NULL, new_hash, new_size);
1045
1046 // Update parent directory entry
1047 81 dirent.set_is_nested_catalog_mountpoint(true);
1048 81 dirent.set_is_nested_catalog_root(false);
1049
1/2
✓ Branch 1 taken 81 times.
✗ Branch 2 not taken.
 81 parent->UpdateEntry(dirent, nested_root_path);
1050
1/2
✓ Branch 1 taken 81 times.
✗ Branch 2 not taken.
 81 parent->TouchEntry(dirent, xattrs, nested_root_path);
1051
1052 // Update counters
1053
1/2
✓ Branch 3 taken 81 times.
✗ Branch 4 not taken.
 81 const DeltaCounters delta = Counters::Diff(old_counters,
1054 new_catalog->GetCounters());
1055
1/2
✓ Branch 1 taken 81 times.
✗ Branch 2 not taken.
 81 delta.PopulateToParent(&parent->delta_counters_);
1056
1057 81 SyncUnlock();
1058 393 }
1059
1060 /**
1061 * Install a nested catalog (catalog hierarchy) at a new, empty directory
1062 *
1063 * The mountpoint directory must not yet exist. Its parent directory, however
1064 * must exist. This method combines functionality from AddDirectory(),
1065 * CreateNestedCatalog() and SwapNestedCatalog().
1066 * The new nested catalog won't get attached.
1067 *
1068 * @param mountpoint - the path where the nested catalog should be installed
1069 * @param new_hash - the hash of the new nested catalog
1070 * @param new_size - the size of the new nested catalog
1071 */
1072 80 void WritableCatalogManager::GraftNestedCatalog(const string &mountpoint,
1073 const shash::Any &new_hash,
1074 const uint64_t new_size) {
1075
2/2
✓ Branch 1 taken 54 times.
✓ Branch 2 taken 26 times.
 80 if (!TryGraftNestedCatalog(mountpoint, new_hash, new_size)) {
1076 54 PANIC(kLogStderr, "failed to graft nested catalog '%s'",
1077 mountpoint.c_str());
1078 }
1079 26 }
1080
1081 80 bool WritableCatalogManager::TryGraftNestedCatalog(const string &mountpoint,
1082 const shash::Any &new_hash,
1083 const uint64_t new_size) {
1084
1/2
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
 80 const string nested_root_path = MakeRelativePath(mountpoint);
1085
1/2
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
 80 const string parent_path = GetParentPath(nested_root_path);
1086
1/2
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
 80 const PathString nested_root_ps = PathString(nested_root_path);
1087
1088
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 80 times.
 80 if (nested_root_path.empty()) {
1089 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1090 "failed to graft nested catalog: empty mountpoint");
1091 ✗ return false;
1092 }
1093
1094 // Load freely attached new catalog
1095 const UniquePtr<Catalog> new_catalog(
1096
2/4
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 80 times.
✗ Branch 5 not taken.
 80 LoadFreeCatalog(nested_root_ps, new_hash));
1097
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 80 times.
 80 if (!new_catalog.IsValid()) {
1098 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1099 "failed to graft nested catalog '%s': failed to load new catalog",
1100 nested_root_path.c_str());
1101 ✗ return false;
1102 }
1103
4/7
✓ Branch 2 taken 80 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 80 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 27 times.
✓ Branch 9 taken 53 times.
 80 if (new_catalog->root_prefix() != nested_root_ps) {
1104
1/7
✗ Branch 2 not taken.
✓ Branch 3 taken 27 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
 27 LogCvmfs(kLogCatalog, kLogStderr,
1105 "invalid nested catalog for grafting at '%s': catalog rooted at "
1106 "'%s'",
1107 nested_root_path.c_str(),
1108
2/4
✓ Branch 2 taken 27 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 27 times.
✗ Branch 6 not taken.
 54 new_catalog->root_prefix().ToString().c_str());
1109 27 return false;
1110 }
1111
1112 // Get new catalog root directory entry
1113
1/2
✓ Branch 1 taken 53 times.
✗ Branch 2 not taken.
 53 DirectoryEntry dirent;
1114 53 XattrList xattrs;
1115
1/2
✓ Branch 2 taken 53 times.
✗ Branch 3 not taken.
 53 const bool dirent_found = new_catalog->LookupPath(nested_root_ps, &dirent);
1116
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 53 times.
 53 if (!dirent_found) {
1117 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1118 "failed to graft nested catalog '%s': missing dirent in new "
1119 "catalog",
1120 nested_root_path.c_str());
1121 ✗ return false;
1122 }
1123
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 53 times.
 53 if (!dirent.IsDirectory()) {
1124 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1125 "failed to graft nested catalog '%s': root entry is not a "
1126 "directory",
1127 nested_root_path.c_str());
1128 ✗ return false;
1129 }
1130
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 53 times.
 53 if (dirent.HasXattrs()) {
1131 ✗ const bool xattrs_found = new_catalog->LookupXattrsPath(nested_root_ps,
1132 &xattrs);
1133 ✗ if (!xattrs_found) {
1134 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1135 "failed to graft nested catalog '%s': missing xattrs in new "
1136 "catalog",
1137 nested_root_path.c_str());
1138 ✗ return false;
1139 }
1140 }
1141 // Transform the nested catalog root into a transition point to be inserted
1142 // in the parent catalog
1143 53 dirent.set_is_nested_catalog_root(false);
1144 53 dirent.set_is_nested_catalog_mountpoint(true);
1145
1146 // Add directory and nested catalog
1147
1148 53 SyncLock();
1149 WritableCatalog *parent_catalog;
1150
1/2
✓ Branch 1 taken 53 times.
✗ Branch 2 not taken.
 53 DirectoryEntry parent_entry;
1151
2/4
✓ Branch 1 taken 53 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 53 times.
 53 if (!FindCatalog(parent_path, &parent_catalog, &parent_entry)) {
1152 ✗ SyncUnlock();
1153 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1154 "catalog for directory '%s' cannot be found", parent_path.c_str());
1155 ✗ return false;
1156 }
1157
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 53 times.
 53 if (!parent_entry.IsDirectory()) {
1158 ✗ SyncUnlock();
1159 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1160 "parent path '%s' is not a directory", parent_path.c_str());
1161 ✗ return false;
1162 }
1163
3/4
✓ Branch 1 taken 53 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 27 times.
✓ Branch 4 taken 26 times.
 53 if (parent_catalog->LookupPath(nested_root_ps, NULL)) {
1164 27 SyncUnlock();
1165
1/2
✓ Branch 2 taken 27 times.
✗ Branch 3 not taken.
 27 LogCvmfs(kLogCatalog, kLogStderr,
1166 "invalid attempt to graft nested catalog into existing directory "
1167 "'%s'",
1168 nested_root_path.c_str());
1169 27 return false;
1170 }
1171
1/2
✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
 26 parent_catalog->AddEntry(dirent, xattrs, nested_root_path, parent_path);
1172 26 parent_entry.set_linkcount(parent_entry.linkcount() + 1);
1173
1/2
✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
 26 parent_catalog->UpdateEntry(parent_entry, parent_path);
1174
1/2
✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
 26 if (parent_entry.IsNestedCatalogRoot()) {
1175 WritableCatalog *grand_parent_catalog = reinterpret_cast<WritableCatalog *>(
1176 26 parent_catalog->parent());
1177 26 parent_entry.set_is_nested_catalog_root(false);
1178 26 parent_entry.set_is_nested_catalog_mountpoint(true);
1179
1/2
✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
 26 grand_parent_catalog->UpdateEntry(parent_entry, parent_path);
1180 }
1181
1182
1/2
✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
 26 parent_catalog->InsertNestedCatalog(nested_root_path, NULL, new_hash,
1183 new_size);
1184
1185 // Fix-up counters
1186
1/2
✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
 26 const Counters counters;
1187
1/2
✓ Branch 3 taken 26 times.
✗ Branch 4 not taken.
 26 const DeltaCounters delta = Counters::Diff(counters,
1188 new_catalog->GetCounters());
1189
1/2
✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
 26 delta.PopulateToParent(&parent_catalog->delta_counters_);
1190
1191 26 SyncUnlock();
1192 26 return true;
1193 80 }
1194
1195 /**
1196 * Checks if a nested catalog starts at this path. The path must be valid.
1197 */
1198 ✗ bool WritableCatalogManager::IsTransitionPoint(const string &mountpoint) {
1199 ✗ const string path = MakeRelativePath(mountpoint);
1200
1201 ✗ SyncLock();
1202 WritableCatalog *catalog;
1203 ✗ DirectoryEntry entry;
1204 ✗ if (!FindCatalog(path, &catalog, &entry)) {
1205 ✗ PANIC(kLogStderr, "catalog for directory '%s' cannot be found",
1206 path.c_str());
1207 }
1208 ✗ const bool result = entry.IsNestedCatalogRoot();
1209 ✗ SyncUnlock();
1210 ✗ return result;
1211 }
1212
1213
1214 ✗ void WritableCatalogManager::PrecalculateListings() {
1215 // TODO(jblomer): meant for micro catalogs
1216 }
1217
1218
1219 ✗ void WritableCatalogManager::SetTTL(const uint64_t new_ttl) {
1220 ✗ SyncLock();
1221 ✗ reinterpret_cast<WritableCatalog *>(GetRootCatalog())->SetTTL(new_ttl);
1222 ✗ SyncUnlock();
1223 }
1224
1225
1226 ✗ bool WritableCatalogManager::SetVOMSAuthz(const std::string &voms_authz) {
1227 bool result;
1228 ✗ SyncLock();
1229 ✗ result = reinterpret_cast<WritableCatalog *>(GetRootCatalog())
1230 ✗ ->SetVOMSAuthz(voms_authz);
1231 ✗ SyncUnlock();
1232 ✗ return result;
1233 }
1234
1235
1236 656 bool WritableCatalogManager::Commit(const bool stop_for_tweaks,
1237 const uint64_t manual_revision,
1238 manifest::Manifest *manifest) {
1239 WritableCatalog *root_catalog = reinterpret_cast<WritableCatalog *>(
1240 656 GetRootCatalog());
1241
1/2
✓ Branch 1 taken 656 times.
✗ Branch 2 not taken.
 656 root_catalog->SetDirty();
1242
1243 // set root catalog revision to manually provided number if available
1244
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 656 times.
 656 if (manual_revision > 0) {
1245 ✗ const uint64_t revision = root_catalog->GetRevision();
1246 ✗ if (revision >= manual_revision) {
1247 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1248 "Manual revision (%" PRIu64 ") must not be "
1249 "smaller than the current root catalog's (%" PRIu64
1250 "). Skipped!",
1251 manual_revision, revision);
1252 } else {
1253 // Gets incremented by FinalizeCatalog() afterwards!
1254 ✗ root_catalog->SetRevision(manual_revision - 1);
1255 }
1256 }
1257
1258 // do the actual catalog snapshotting and upload
1259
1/2
✓ Branch 1 taken 656 times.
✗ Branch 2 not taken.
 656 CatalogInfo root_catalog_info;
1260
1/2
✓ Branch 1 taken 656 times.
✗ Branch 2 not taken.
 656 if (getenv("_CVMFS_SERIALIZED_CATALOG_PROCESSING_") == NULL)
1261
1/2
✓ Branch 1 taken 656 times.
✗ Branch 2 not taken.
 656 root_catalog_info = SnapshotCatalogs(stop_for_tweaks);
1262 else
1263 ✗ root_catalog_info = SnapshotCatalogsSerialized(stop_for_tweaks);
1264
2/4
✓ Branch 1 taken 656 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 656 times.
 656 if (spooler_->GetNumberOfErrors() > 0) {
1265 ✗ LogCvmfs(kLogCatalog, kLogStderr, "failed to commit catalogs");
1266 ✗ return false;
1267 }
1268
1269 // .cvmfspublished export
1270
1/2
✓ Branch 1 taken 656 times.
✗ Branch 2 not taken.
 656 LogCvmfs(kLogCatalog, kLogVerboseMsg, "Committing repository manifest");
1271 656 set_base_hash(root_catalog_info.content_hash);
1272
1273 656 manifest->set_catalog_hash(root_catalog_info.content_hash);
1274 656 manifest->set_catalog_size(root_catalog_info.size);
1275
2/4
✓ Branch 2 taken 656 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 656 times.
✗ Branch 6 not taken.
 656 manifest->set_root_path("");
1276 656 manifest->set_ttl(root_catalog_info.ttl);
1277 656 manifest->set_revision(root_catalog_info.revision);
1278
1279 656 return true;
1280 }
1281
1282
1283 /**
1284 * Handles the snapshotting of dirty (i.e. modified) catalogs while trying to
1285 * parallelize the compression and upload as much as possible. We use a parallel
1286 * depth first post order tree traversal based on 'continuations'.
1287 *
1288 * The idea is as follows:
1289 * 1. find all leaf-catalogs (i.e. dirty catalogs with no dirty children)
1290 * --> these can be processed and uploaded immediately and independently
1291 * see WritableCatalogManager::GetModifiedCatalogLeafs()
1292 * 2. annotate non-leaf catalogs with their number of dirty children
1293 * --> a finished child will notify it's parent and decrement this number
1294 * see WritableCatalogManager::CatalogUploadCallback()
1295 * 3. if a non-leaf catalog's dirty children number reaches 0, it is scheduled
1296 * for processing as well (continuation)
1297 * --> the parallel processing walks bottom-up through the catalog tree
1298 * see WritableCatalogManager::CatalogUploadCallback()
1299 * 4. when the root catalog is reached, we notify the main thread and return
1300 * --> done through a Future<> in WritableCatalogManager::SnapshotCatalogs
1301 *
1302 * Note: The catalog finalisation (see WritableCatalogManager::FinalizeCatalog)
1303 * happens in a worker thread (i.e. the callback method) for non-leaf
1304 * catalogs.
1305 *
1306 * TODO(rmeusel): since all leaf catalogs are finalized in the main thread, we
1307 * sacrifice some potential concurrency for simplicity.
1308 */
1309 656 WritableCatalogManager::CatalogInfo WritableCatalogManager::SnapshotCatalogs(
1310 const bool stop_for_tweaks) {
1311 // prepare environment for parallel processing
1312
1/2
✓ Branch 1 taken 656 times.
✗ Branch 2 not taken.
 656 Future<CatalogInfo> root_catalog_info_future;
1313 CatalogUploadContext upload_context;
1314 656 upload_context.root_catalog_info = &root_catalog_info_future;
1315 656 upload_context.stop_for_tweaks = stop_for_tweaks;
1316
1317
1/2
✓ Branch 1 taken 656 times.
✗ Branch 2 not taken.
 656 spooler_->RegisterListener(&WritableCatalogManager::CatalogUploadCallback,
1318 this, upload_context);
1319
1320 // find dirty leaf catalogs and annotate non-leaf catalogs (dirty child count)
1321 // post-condition: the entire catalog tree is ready for concurrent processing
1322 656 WritableCatalogList leafs_to_snapshot;
1323
1/2
✓ Branch 1 taken 656 times.
✗ Branch 2 not taken.
 656 GetModifiedCatalogLeafs(&leafs_to_snapshot);
1324
1325 // finalize and schedule the catalog processing
1326 656 WritableCatalogList::const_iterator i = leafs_to_snapshot.begin();
1327 656 const WritableCatalogList::const_iterator iend = leafs_to_snapshot.end();
1328
2/2
✓ Branch 2 taken 1059 times.
✓ Branch 3 taken 656 times.
 1715 for (; i != iend; ++i) {
1329
1/2
✓ Branch 2 taken 1059 times.
✗ Branch 3 not taken.
 1059 FinalizeCatalog(*i, stop_for_tweaks);
1330
1/2
✓ Branch 2 taken 1059 times.
✗ Branch 3 not taken.
 1059 ScheduleCatalogProcessing(*i);
1331 }
1332
1333
1/2
✓ Branch 1 taken 656 times.
✗ Branch 2 not taken.
 656 LogCvmfs(kLogCatalog, kLogVerboseMsg, "waiting for upload of catalogs");
1334
1/2
✓ Branch 1 taken 656 times.
✗ Branch 2 not taken.
 656 const CatalogInfo &root_catalog_info = root_catalog_info_future.Get();
1335
1/2
✓ Branch 1 taken 656 times.
✗ Branch 2 not taken.
 656 spooler_->WaitForUpload();
1336
1337
1/2
✓ Branch 1 taken 656 times.
✗ Branch 2 not taken.
 656 spooler_->UnregisterListeners();
1338 656 return root_catalog_info;
1339 656 }
1340
1341
1342 1767 void WritableCatalogManager::FinalizeCatalog(WritableCatalog *catalog,
1343 const bool stop_for_tweaks) {
1344 // update meta information of this catalog
1345
1/3
✓ Branch 2 taken 1767 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 1767 LogCvmfs(kLogCatalog, kLogVerboseMsg, "creating snapshot of catalog '%s'",
1346 3534 catalog->mountpoint().c_str());
1347
1348 1767 catalog->UpdateCounters();
1349 1767 catalog->UpdateLastModified();
1350 1767 catalog->IncrementRevision();
1351
1352 // update the previous catalog revision pointer
1353
2/2
✓ Branch 1 taken 656 times.
✓ Branch 2 taken 1111 times.
 1767 if (catalog->IsRoot()) {
1354
1/4
✓ Branch 2 taken 656 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 656 LogCvmfs(kLogCatalog, kLogVerboseMsg,
1355 "setting '%s' as previous revision "
1356 "for root catalog",
1357 1312 base_hash().ToStringWithSuffix().c_str());
1358 656 catalog->SetPreviousRevision(base_hash());
1359 } else {
1360 // Multiple catalogs might query the parent concurrently
1361 1111 SyncLock();
1362
1/2
✓ Branch 1 taken 1111 times.
✗ Branch 2 not taken.
 1111 shash::Any hash_previous;
1363 uint64_t size_previous;
1364
1/2
✓ Branch 2 taken 1111 times.
✗ Branch 3 not taken.
 2222 const bool retval = catalog->parent()->FindNested(
1365
1/2
✓ Branch 1 taken 1111 times.
✗ Branch 2 not taken.
 2222 catalog->mountpoint(), &hash_previous, &size_previous);
1366
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1111 times.
 1111 assert(retval);
1367 1111 SyncUnlock();
1368
1369
1/8
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 1111 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
 2222 LogCvmfs(kLogCatalog, kLogVerboseMsg,
1370 "found '%s' as previous revision "
1371 "for nested catalog '%s'",
1372
1/2
✓ Branch 1 taken 1111 times.
✗ Branch 2 not taken.
 2222 hash_previous.ToStringWithSuffix().c_str(),
1373
1/2
✓ Branch 1 taken 1111 times.
✗ Branch 2 not taken.
 2222 catalog->mountpoint().c_str());
1374
1/2
✓ Branch 1 taken 1111 times.
✗ Branch 2 not taken.
 1111 catalog->SetPreviousRevision(hash_previous);
1375 }
1376 1767 catalog->Commit();
1377
1378 // check if catalog has too many entries
1379 const uint64_t catalog_limit = uint64_t(1000)
1380
2/2
✓ Branch 1 taken 656 times.
✓ Branch 2 taken 1111 times.
 1767 * uint64_t((catalog->IsRoot()
1381 656 ? root_kcatalog_limit_
1382 1111 : nested_kcatalog_limit_));
1383 1767 if ((catalog_limit > 0)
1384
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 1767 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 1767 times.
 1767 && (catalog->GetCounters().GetSelfEntries() > catalog_limit)) {
1385 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1386 "%s: catalog at %s has more than %lu entries (%lu). "
1387 "Large catalogs stress the CernVM-FS transport infrastructure. "
1388 "Please split it into nested catalogs or increase the limit.",
1389 ✗ enforce_limits_ ? "FATAL" : "WARNING",
1390 ✗ (catalog->IsRoot() ? "/" : catalog->mountpoint().c_str()),
1391 ✗ catalog_limit, catalog->GetCounters().GetSelfEntries());
1392 ✗ if (enforce_limits_)
1393 ✗ PANIC(kLogStderr, "catalog at %s has more than %u entries (%u). ",
1394 (catalog->IsRoot() ? "/" : catalog->mountpoint().c_str()),
1395 catalog_limit, catalog->GetCounters().GetSelfEntries());
1396 }
1397
1398 // allow for manual adjustments in the catalog
1399
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1767 times.
 1767 if (stop_for_tweaks) {
1400 ✗ LogCvmfs(kLogCatalog, kLogStdout,
1401 "Allowing for tweaks in %s at %s "
1402 "(hit return to continue)",
1403 ✗ catalog->database_path().c_str(), catalog->mountpoint().c_str());
1404 ✗ const int read_char = getchar();
1405 ✗ assert(read_char != EOF);
1406 }
1407
1408 // compaction of bloated catalogs (usually after high database churn)
1409 1767 catalog->VacuumDatabaseIfNecessary();
1410 1767 }
1411
1412
1413 1767 void WritableCatalogManager::ScheduleCatalogProcessing(
1414 WritableCatalog *catalog) {
1415 {
1416 1767 const MutexLockGuard guard(catalog_processing_lock_);
1417 // register catalog object for WritableCatalogManager::CatalogUploadCallback
1418
2/4
✓ Branch 1 taken 1767 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1767 times.
✗ Branch 5 not taken.
 1767 catalog_processing_map_[catalog->database_path()] = catalog;
1419 1767 }
1420
1/2
✓ Branch 2 taken 1767 times.
✗ Branch 3 not taken.
 1767 spooler_->ProcessCatalog(catalog->database_path());
1421 1767 }
1422
1423 /**
1424 * Copy catalog to local cache.server
1425 * Must be an atomic write into the cache_dir
1426 * As such: create a temporary copy in cache_dir/txn and then do a
1427 * `rename` (which is atomic) to the actual cache path
1428 *
1429 * @returns true on success, otherwise false
1430 */
1431 ✗ bool WritableCatalogManager::CopyCatalogToLocalCache(
1432 const upload::SpoolerResult &result) {
1433 ✗ std::string tmp_catalog_path;
1434 ✗ const std::string cache_catalog_path = dir_cache_ + "/"
1435 + result.content_hash
1436 ✗ .MakePathWithoutSuffix();
1437 ✗ FILE *fcatalog = CreateTempFile(dir_cache_ + "/txn/catalog", 0666, "w",
1438 &tmp_catalog_path);
1439 ✗ if (!fcatalog) {
1440 ✗ PANIC(kLogDebug | kLogStderr,
1441 "Creating file for temporary catalog failed: %s",
1442 tmp_catalog_path.c_str());
1443 }
1444 ✗ CopyPath2File(result.local_path.c_str(), fcatalog);
1445 ✗ (void)fclose(fcatalog);
1446
1447 ✗ if (rename(tmp_catalog_path.c_str(), cache_catalog_path.c_str()) != 0) {
1448 ✗ PANIC(kLogDebug | kLogStderr, "Failed to copy catalog from %s to cache %s",
1449 result.local_path.c_str(), cache_catalog_path.c_str());
1450 }
1451 ✗ return true;
1452 }
1453
1454 1767 void WritableCatalogManager::CatalogUploadCallback(
1455 const upload::SpoolerResult &result,
1456 const CatalogUploadContext catalog_upload_context) {
1457
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1767 times.
 1767 if (result.return_code != 0) {
1458 ✗ PANIC(kLogStderr, "failed to upload '%s' (retval: %d)",
1459 result.local_path.c_str(), result.return_code);
1460 }
1461
1462 // retrieve the catalog object based on the callback information
1463 // see WritableCatalogManager::ScheduleCatalogProcessing()
1464 1767 WritableCatalog *catalog = NULL;
1465 {
1466 1767 const MutexLockGuard guard(catalog_processing_lock_);
1467 const std::map<std::string, WritableCatalog *>::iterator
1468
1/2
✓ Branch 1 taken 1767 times.
✗ Branch 2 not taken.
 1767 c = catalog_processing_map_.find(result.local_path);
1469
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 1767 times.
 1767 assert(c != catalog_processing_map_.end());
1470 1767 catalog = c->second;
1471 1767 }
1472
1473 1767 const uint64_t catalog_size = GetFileSize(result.local_path);
1474
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1767 times.
 1767 assert(catalog_size > 0);
1475
1476
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1767 times.
 1767 if (UseLocalCache()) {
1477 ✗ CopyCatalogToLocalCache(result);
1478 }
1479
1480 1767 SyncLock();
1481
2/2
✓ Branch 1 taken 1111 times.
✓ Branch 2 taken 656 times.
 1767 if (catalog->HasParent()) {
1482 // finalized nested catalogs will update their parent's pointer and schedule
1483 // them for processing (continuation) if the 'dirty children count' == 0
1484 1111 LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating nested catalog link");
1485 1111 WritableCatalog *parent = catalog->GetWritableParent();
1486
1487
2/4
✓ Branch 1 taken 1111 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1111 times.
✗ Branch 5 not taken.
 1111 parent->UpdateNestedCatalog(catalog->mountpoint().ToString(),
1488 1111 result.content_hash,
1489 catalog_size,
1490 1111 catalog->delta_counters_);
1491 1111 catalog->delta_counters_.SetZero();
1492
1493 const int remaining_dirty_children = catalog->GetWritableParent()
1494 1111 ->DecrementDirtyChildren();
1495
1496 1111 SyncUnlock();
1497
1498 // continuation of the dirty catalog tree traversal
1499 // see WritableCatalogManager::SnapshotCatalogs()
1500
2/2
✓ Branch 0 taken 708 times.
✓ Branch 1 taken 403 times.
 1111 if (remaining_dirty_children == 0) {
1501 708 FinalizeCatalog(parent, catalog_upload_context.stop_for_tweaks);
1502 708 ScheduleCatalogProcessing(parent);
1503 }
1504
1505
1/2
✓ Branch 1 taken 656 times.
✗ Branch 2 not taken.
 656 } else if (catalog->IsRoot()) {
1506 // once the root catalog is reached, we are done with processing and report
1507 // back to the main via a Future<> and provide the necessary information
1508
1/2
✓ Branch 1 taken 656 times.
✗ Branch 2 not taken.
 656 CatalogInfo root_catalog_info;
1509 656 root_catalog_info.size = catalog_size;
1510
1/2
✓ Branch 1 taken 656 times.
✗ Branch 2 not taken.
 656 root_catalog_info.ttl = catalog->GetTTL();
1511 656 root_catalog_info.content_hash = result.content_hash;
1512
1/2
✓ Branch 1 taken 656 times.
✗ Branch 2 not taken.
 656 root_catalog_info.revision = catalog->GetRevision();
1513
1/2
✓ Branch 1 taken 656 times.
✗ Branch 2 not taken.
 656 catalog_upload_context.root_catalog_info->Set(root_catalog_info);
1514 656 SyncUnlock();
1515 } else {
1516 ✗ PANIC(kLogStderr, "inconsistent state detected");
1517 }
1518 1767 }
1519
1520
1521 /**
1522 * Finds dirty catalogs that can be snapshot right away and annotates all the
1523 * other catalogs with their number of dirty descendants.
1524 * Note that there is a convenience wrapper to start the recursion:
1525 * WritableCatalogManager::GetModifiedCatalogLeafs()
1526 *
1527 * @param catalog the catalog for this recursion step
1528 * @param result the result list to be appended to
1529 * @return true if 'catalog' is dirty
1530 */
1531 1793 bool WritableCatalogManager::GetModifiedCatalogLeafsRecursively(
1532 Catalog *catalog, WritableCatalogList *result) const {
1533 1793 WritableCatalog *wr_catalog = static_cast<WritableCatalog *>(catalog);
1534
1535 // Look for dirty catalogs in the descendants of *catalog
1536 1793 int dirty_children = 0;
1537
1/2
✓ Branch 1 taken 1793 times.
✗ Branch 2 not taken.
 1793 CatalogList children = wr_catalog->GetChildren();
1538 1793 CatalogList::const_iterator i = children.begin();
1539 1793 const CatalogList::const_iterator iend = children.end();
1540
2/2
✓ Branch 2 taken 1111 times.
✓ Branch 3 taken 1793 times.
 2904 for (; i != iend; ++i) {
1541
2/4
✓ Branch 2 taken 1111 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1111 times.
✗ Branch 5 not taken.
 1111 if (GetModifiedCatalogLeafsRecursively(*i, result)) {
1542 1111 ++dirty_children;
1543 }
1544 }
1545
1546 // a catalog is dirty if itself or one of its children has changed
1547 // a leaf catalog doesn't have any dirty children
1548 1793 wr_catalog->set_dirty_children(dirty_children);
1549
1/4
✗ Branch 1 not taken.
✓ Branch 2 taken 1793 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 1793 const bool is_dirty = wr_catalog->IsDirty() || dirty_children > 0;
1550 1793 const bool is_leaf = dirty_children == 0;
1551
3/4
✓ Branch 0 taken 1793 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1085 times.
✓ Branch 3 taken 708 times.
 1793 if (is_dirty && is_leaf) {
1552
1/2
✓ Branch 1 taken 1085 times.
✗ Branch 2 not taken.
 1085 result->push_back(const_cast<WritableCatalog *>(wr_catalog));
1553 }
1554
1555 1793 return is_dirty;
1556 1793 }
1557
1558
1559 ✗ void WritableCatalogManager::DoBalance() {
1560 ✗ CatalogList catalog_list = GetCatalogs();
1561 ✗ reverse(catalog_list.begin(), catalog_list.end());
1562 ✗ for (unsigned i = 0; i < catalog_list.size(); ++i) {
1563 ✗ FixWeight(static_cast<WritableCatalog *>(catalog_list[i]));
1564 }
1565 }
1566
1567 ✗ void WritableCatalogManager::FixWeight(WritableCatalog *catalog) {
1568 // firstly check underflow because they can provoke overflows
1569 ✗ if (catalog->GetNumEntries() < min_weight_ && !catalog->IsRoot()
1570 ✗ && catalog->IsAutogenerated()) {
1571 ✗ LogCvmfs(kLogCatalog, kLogStdout,
1572 "Deleting an autogenerated catalog in '%s'",
1573 ✗ catalog->mountpoint().c_str());
1574 // Remove the .cvmfscatalog and .cvmfsautocatalog files first
1575 ✗ const string path = catalog->mountpoint().ToString();
1576 ✗ catalog->RemoveEntry(path + "/.cvmfscatalog");
1577 ✗ catalog->RemoveEntry(path + "/.cvmfsautocatalog");
1578 // Remove the actual catalog
1579 ✗ const string catalog_path = catalog->mountpoint().ToString().substr(1);
1580 ✗ RemoveNestedCatalog(catalog_path);
1581 ✗ } else if (catalog->GetNumEntries() > max_weight_) {
1582 ✗ CatalogBalancer<WritableCatalogManager> catalog_balancer(this);
1583 ✗ catalog_balancer.Balance(catalog);
1584 }
1585 }
1586
1587
1588 //****************************************************************************
1589 // Workaround -- Serialized Catalog Committing
1590
1591 ✗ int WritableCatalogManager::GetModifiedCatalogsRecursively(
1592 const Catalog *catalog, WritableCatalogList *result) const {
1593 // A catalog must be snapshot, if itself or one of it's descendants is dirty.
1594 // So we traverse the catalog tree recursively and look for dirty catalogs
1595 // on the way.
1596 ✗ const WritableCatalog *wr_catalog = static_cast<const WritableCatalog *>(
1597 catalog);
1598 // This variable will contain the number of dirty catalogs in the sub tree
1599 // with *catalog as it's root.
1600 ✗ int dirty_catalogs = (wr_catalog->IsDirty()) ? 1 : 0;
1601
1602 // Look for dirty catalogs in the descendants of *catalog
1603 ✗ CatalogList children = wr_catalog->GetChildren();
1604 ✗ for (CatalogList::const_iterator i = children.begin(), iEnd = children.end();
1605 ✗ i != iEnd; ++i) {
1606 ✗ dirty_catalogs += GetModifiedCatalogsRecursively(*i, result);
1607 }
1608
1609 // If we found a dirty catalog in the checked sub tree, the root (*catalog)
1610 // must be snapshot and ends up in the result list
1611 ✗ if (dirty_catalogs > 0)
1612 ✗ result->push_back(const_cast<WritableCatalog *>(wr_catalog));
1613
1614 // tell the upper layer about number of catalogs
1615 ✗ return dirty_catalogs;
1616 }
1617
1618
1619 ✗ void WritableCatalogManager::CatalogUploadSerializedCallback(
1620 const upload::SpoolerResult &result, const CatalogUploadContext unused) {
1621 ✗ if (result.return_code != 0) {
1622 ✗ PANIC(kLogStderr, "failed to upload '%s' (retval: %d)",
1623 result.local_path.c_str(), result.return_code);
1624 }
1625
1626 ✗ if (UseLocalCache()) {
1627 ✗ CopyCatalogToLocalCache(result);
1628 }
1629
1630 ✗ unlink(result.local_path.c_str());
1631 }
1632
1633
1634 WritableCatalogManager::CatalogInfo
1635 ✗ WritableCatalogManager::SnapshotCatalogsSerialized(const bool stop_for_tweaks) {
1636 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Serialized committing of file catalogs...");
1637 ✗ reinterpret_cast<WritableCatalog *>(GetRootCatalog())->SetDirty();
1638 ✗ WritableCatalogList catalogs_to_snapshot;
1639 ✗ GetModifiedCatalogs(&catalogs_to_snapshot);
1640 CatalogUploadContext unused;
1641 ✗ unused.root_catalog_info = NULL;
1642 ✗ unused.stop_for_tweaks = false;
1643 ✗ spooler_->RegisterListener(
1644 &WritableCatalogManager::CatalogUploadSerializedCallback, this, unused);
1645
1646 ✗ CatalogInfo root_catalog_info;
1647 ✗ WritableCatalogList::const_iterator i = catalogs_to_snapshot.begin();
1648 ✗ const WritableCatalogList::const_iterator iend = catalogs_to_snapshot.end();
1649 ✗ for (; i != iend; ++i) {
1650 ✗ FinalizeCatalog(*i, stop_for_tweaks);
1651
1652 // Compress and upload catalog
1653 ✗ shash::Any hash_catalog(spooler_->GetHashAlgorithm(),
1654 ✗ shash::kSuffixCatalog);
1655 ✗ if (!zlib::CompressPath2Null((*i)->database_path(), &hash_catalog)) {
1656 ✗ PANIC(kLogStderr, "could not compress catalog %s",
1657 (*i)->mountpoint().ToString().c_str());
1658 }
1659
1660 ✗ const int64_t catalog_size = GetFileSize((*i)->database_path());
1661 ✗ assert(catalog_size > 0);
1662
1663 ✗ if ((*i)->HasParent()) {
1664 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating nested catalog link");
1665 ✗ WritableCatalog *parent = (*i)->GetWritableParent();
1666 ✗ parent->UpdateNestedCatalog((*i)->mountpoint().ToString(), hash_catalog,
1667 ✗ catalog_size, (*i)->delta_counters_);
1668 ✗ (*i)->delta_counters_.SetZero();
1669 ✗ } else if ((*i)->IsRoot()) {
1670 ✗ root_catalog_info.size = catalog_size;
1671 ✗ root_catalog_info.ttl = (*i)->GetTTL();
1672 ✗ root_catalog_info.content_hash = hash_catalog;
1673 ✗ root_catalog_info.revision = (*i)->GetRevision();
1674 } else {
1675 ✗ PANIC(kLogStderr, "inconsistent state detected");
1676 }
1677
1678 ✗ spooler_->ProcessCatalog((*i)->database_path());
1679 }
1680 ✗ spooler_->WaitForUpload();
1681
1682 ✗ spooler_->UnregisterListeners();
1683 ✗ return root_catalog_info;
1684 }
1685
1686 ✗ void WritableCatalogManager::SetupSingleCatalogUploadCallback() {
1687 ✗ spooler_->RegisterListener(
1688 &WritableCatalogManager::SingleCatalogUploadCallback, this);
1689 }
1690
1691 ✗ void WritableCatalogManager::RemoveSingleCatalogUploadCallback() {
1692 ✗ spooler_->WaitForUpload(); // wait for all outstanding jobs to finish before
1693 // tearing it down
1694 ✗ spooler_->UnregisterListeners();
1695 ✗ pending_catalogs_ =
1696 ✗ {}; // whatever we couldn't process, leave it to the Commit
1697 }
1698
1699 ✗ void WritableCatalogManager::AddCatalogToQueue(const std::string &path) {
1700 ✗ SyncLock();
1701 ✗ WritableCatalog *catalog = NULL;
1702 ✗ bool const retval = FindCatalog(MakeRelativePath(path), &catalog, NULL);
1703 ✗ assert(retval);
1704 ✗ assert(catalog);
1705 ✗ catalog->SetDirty(); // ensure it's dirty so its parent will wait for it
1706 ✗ SyncUnlock();
1707 ✗ pending_catalogs_.push_back(catalog);
1708 }
1709
1710 ✗ void WritableCatalogManager::ScheduleReadyCatalogs() {
1711 // best effort to schedule as many catalogs for upload as possible
1712 ✗ for (auto it = pending_catalogs_.begin(); it != pending_catalogs_.end();) {
1713 ✗ if ((*it)->dirty_children() == 0) {
1714 ✗ FinalizeCatalog(*it, false /* stop_for_tweaks */);
1715 ✗ ScheduleCatalogProcessing(*it);
1716 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "scheduled %s for processing",
1717 ✗ (*it)->mountpoint().c_str());
1718 ✗ it = pending_catalogs_.erase(it);
1719 } else {
1720 ✗ ++it;
1721 }
1722 }
1723 }
1724
1725 // Callback for uploading a single catalog, similar to CatalogUploadCallback.
1726 // The main difference is that this callback would not trigger processing of the
1727 // parent
1728 ✗ void WritableCatalogManager::SingleCatalogUploadCallback(
1729 const upload::SpoolerResult &result) {
1730 ✗ if (result.return_code != 0) {
1731 ✗ PANIC(kLogStderr, "failed to upload '%s' (retval: %d)",
1732 result.local_path.c_str(), result.return_code);
1733 }
1734
1735 // retrieve the catalog object based on the callback information
1736 // see WritableCatalogManager::ScheduleCatalogProcessing()
1737 ✗ WritableCatalog *catalog = NULL;
1738 {
1739 ✗ MutexLockGuard const guard(catalog_processing_lock_);
1740 std::map<std::string, WritableCatalog *>::iterator const
1741 ✗ c = catalog_processing_map_.find(result.local_path);
1742 ✗ assert(c != catalog_processing_map_.end());
1743 ✗ catalog = c->second;
1744 }
1745
1746 ✗ uint64_t const catalog_size = GetFileSize(result.local_path);
1747 ✗ assert(catalog_size > 0);
1748
1749 ✗ SyncLock();
1750 ✗ if (catalog->HasParent()) {
1751 // finalized nested catalogs will update their parent's pointer
1752 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating nested catalog link");
1753 ✗ WritableCatalog *parent = catalog->GetWritableParent();
1754
1755 ✗ parent->UpdateNestedCatalog(catalog->mountpoint().ToString(),
1756 ✗ result.content_hash,
1757 catalog_size,
1758 ✗ catalog->delta_counters_);
1759 ✗ parent->DecrementDirtyChildren();
1760 ✗ catalog->delta_counters_.SetZero();
1761 }
1762 // JUMP: detach the catalog after uploading to free sqlite related resources
1763 ✗ DetachCatalog(catalog);
1764 ✗ SyncUnlock();
1765 }
1766 // using the given list of dirs, fetch all relevant catalogs
1767 ✗ void WritableCatalogManager::LoadCatalogs(
1768 const std::string &base_path, const std::unordered_set<std::string> &dirs) {
1769 // mount everything up to "base_path" first (this would be our lease_path
1770 // typically)
1771 Catalog *base_catalog;
1772 ✗ MountSubtree(PathString(base_path), NULL /* entry_point */,
1773 true /* is_listable */, &base_catalog);
1774
1775 // start up the downloader
1776 CatalogDownloadContext context;
1777 ✗ context.dirs = &dirs;
1778 ✗ catalog_download_pipeline_ = new CatalogDownloadPipeline(
1779 ✗ static_cast<SimpleCatalogManager *>(this));
1780 ✗ catalog_download_pipeline_->RegisterListener(
1781 &WritableCatalogManager::CatalogDownloadCallback, this, context);
1782 ✗ catalog_download_pipeline_->Spawn();
1783
1784 Catalog::NestedCatalogList nested_catalogs = base_catalog
1785 ✗ ->ListNestedCatalogs();
1786 ✗ for (auto it = nested_catalogs.begin(); it != nested_catalogs.end(); ++it) {
1787 // schedule relevant child nested catalogs for download
1788 ✗ std::string const mountpoint = it->mountpoint.ToString();
1789 ✗ if (dirs.find(mountpoint) != dirs.end()) {
1790 ✗ Catalog *catalog = CreateCatalog(it->mountpoint, it->hash,
1791 ✗ NULL /* parent */);
1792 {
1793 ✗ MutexLockGuard const guard(catalog_download_lock_);
1794 ✗ catalog_download_map_.insert(
1795 ✗ std::make_pair(it->hash.ToString(), catalog));
1796 }
1797 ✗ catalog_download_pipeline_->Process(it->hash);
1798 }
1799 }
1800
1801 ✗ catalog_download_pipeline_->WaitFor();
1802 ✗ delete catalog_download_pipeline_; // terminate all the threads
1803 }
1804
1805 ✗ bool WritableCatalogManager::LookupDirEntry(const string &path,
1806 const LookupOptions options,
1807 DirectoryEntry *dirent) {
1808 ✗ SyncLock();
1809 ✗ bool const exists = LookupPath(path, options, dirent);
1810 ✗ SyncUnlock();
1811 ✗ return exists;
1812 }
1813
1814 ✗ void WritableCatalogManager::CatalogHashSerializedCallback(
1815 const CompressHashResult &result) {
1816 ✗ MutexLockGuard const guard(catalog_hash_lock_);
1817 ✗ catalog_hash_map_[result.path] = result.hash;
1818 }
1819
1820 ✗ void WritableCatalogManager::CatalogDownloadCallback(
1821 const CatalogDownloadResult &result, CatalogDownloadContext context) {
1822 Catalog *downloaded_catalog;
1823 {
1824 ✗ MutexLockGuard const guard(catalog_download_lock_);
1825 ✗ auto it = catalog_download_map_.find(result.hash);
1826 ✗ assert(it != catalog_download_map_.end());
1827 ✗ downloaded_catalog = it->second;
1828 }
1829
1830 ✗ if (!downloaded_catalog->OpenDatabase(result.db_path)) {
1831 ✗ LogCvmfs(kLogCvmfs, kLogDebug, "failed to initialize catalog");
1832 ✗ delete downloaded_catalog;
1833 ✗ return;
1834 }
1835
1836 Catalog::NestedCatalogList nested_catalogs = downloaded_catalog
1837 ✗ ->ListNestedCatalogs();
1838 ✗ for (auto it = nested_catalogs.begin(); it != nested_catalogs.end(); ++it) {
1839 // schedule relevant child nested catalogs for download
1840 ✗ if (context.dirs->find(it->mountpoint.ToString()) != context.dirs->end()) {
1841 ✗ Catalog *child_catalog = CreateCatalog(it->mountpoint, it->hash,
1842 ✗ NULL /* parent */);
1843 {
1844 ✗ MutexLockGuard const guard(catalog_download_lock_);
1845 ✗ catalog_download_map_.insert(
1846 ✗ std::make_pair(it->hash.ToString(), child_catalog));
1847 }
1848 ✗ catalog_download_pipeline_->Process(it->hash);
1849 }
1850 }
1851 ✗ delete downloaded_catalog;
1852 }
1853
1854
1855 } // namespace catalog
1856