GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/catalog_mgr_rw.cc
Date: 2025-04-20 02:34:28
Exec Total Coverage
Lines: 461 734 62.8%
Branches: 356 1107 32.2%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM file system.
3 */
4
5 #define __STDC_FORMAT_MACROS
6
7 #include "catalog_mgr_rw.h"
8
9 #include <inttypes.h>
10 #include <unistd.h>
11
12 #include <cassert>
13 #include <cstdio>
14 #include <cstdlib>
15 #include <string>
16
17 #include "catalog_balancer.h"
18 #include "catalog_rw.h"
19 #include "manifest.h"
20 #include "statistics.h"
21 #include "upload.h"
22 #include "util/exception.h"
23 #include "util/logging.h"
24 #include "util/posix.h"
25 #include "util/smalloc.h"
26
27 using namespace std; // NOLINT
28
29 namespace catalog {
30
31 30 WritableCatalogManager::WritableCatalogManager(
32 const shash::Any &base_hash,
33 const std::string &stratum0,
34 const string &dir_temp,
35 upload::Spooler *spooler,
36 download::DownloadManager *download_manager,
37 bool enforce_limits,
38 const unsigned nested_kcatalog_limit,
39 const unsigned root_kcatalog_limit,
40 const unsigned file_mbyte_limit,
41 perf::Statistics *statistics,
42 bool is_balanceable,
43 unsigned max_weight,
44 unsigned min_weight,
45 30 const std::string &dir_cache)
46 : SimpleCatalogManager(base_hash, stratum0, dir_temp, download_manager,
47 statistics, false, dir_cache, true /* copy to tmpdir */)
48 30 , spooler_(spooler)
49 30 , enforce_limits_(enforce_limits)
50 30 , nested_kcatalog_limit_(nested_kcatalog_limit)
51 30 , root_kcatalog_limit_(root_kcatalog_limit)
52 30 , file_mbyte_limit_(file_mbyte_limit)
53 30 , is_balanceable_(is_balanceable)
54 30 , max_weight_(max_weight)
55 30 , min_weight_(min_weight)
56 30 , balance_weight_(max_weight / 2)
57 {
58 30 sync_lock_ =
59 30 reinterpret_cast<pthread_mutex_t *>(smalloc(sizeof(pthread_mutex_t)));
60 30 int retval = pthread_mutex_init(sync_lock_, NULL);
61
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 30 times.
 30 assert(retval == 0);
62 30 catalog_processing_lock_ =
63 30 reinterpret_cast<pthread_mutex_t *>(smalloc(sizeof(pthread_mutex_t)));
64 30 retval = pthread_mutex_init(catalog_processing_lock_, NULL);
65
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 30 times.
 30 assert(retval == 0);
66 30 }
67
68
69 120 WritableCatalogManager::~WritableCatalogManager() {
70 60 pthread_mutex_destroy(sync_lock_);
71 60 free(sync_lock_);
72 60 pthread_mutex_destroy(catalog_processing_lock_);
73 60 free(catalog_processing_lock_);
74 120 }
75
76
77 /**
78 * This method is virtual in AbstractCatalogManager. It returns a new catalog
79 * structure in the form the different CatalogManagers need it.
80 * In this case it returns a stub for a WritableCatalog.
81 * @param mountpoint the mount point of the catalog stub to create
82 * @param catalog_hash the content hash of the catalog to create
83 * @param parent_catalog the parent of the catalog stub to create
84 * @return a pointer to the catalog stub structure created
85 */
86 76 Catalog* WritableCatalogManager::CreateCatalog(
87 const PathString &mountpoint,
88 const shash::Any &catalog_hash,
89 Catalog *parent_catalog)
90 {
91 152 return new WritableCatalog(mountpoint.ToString(),
92 catalog_hash,
93
2/4
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 76 times.
✗ Branch 5 not taken.
 152 parent_catalog);
94 }
95
96
97 76 void WritableCatalogManager::ActivateCatalog(Catalog *catalog) {
98 76 catalog->TakeDatabaseFileOwnership();
99 76 }
100
101
102 /**
103 * This method is invoked if we create a completely new repository.
104 * The new root catalog will already contain a root entry.
105 * It is uploaded by a Forklift to the upstream storage.
106 * @return true on success, false otherwise
107 */
108 24 manifest::Manifest *WritableCatalogManager::CreateRepository(
109 const string &dir_temp,
110 const bool volatile_content,
111 const std::string &voms_authz,
112 upload::Spooler *spooler)
113 {
114 // Create a new root catalog at file_path
115
1/2
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
 24 string file_path = dir_temp + "/new_root_catalog";
116
117 24 shash::Algorithms hash_algorithm = spooler->GetHashAlgorithm();
118
119 // A newly created catalog always needs a root entry
120 // we create and configure this here
121
1/2
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
 24 DirectoryEntry root_entry;
122 24 root_entry.inode_ = DirectoryEntry::kInvalidInode;
123 24 root_entry.mode_ = 16877;
124 24 root_entry.size_ = 4096;
125 24 root_entry.mtime_ = time(NULL);
126 24 root_entry.uid_ = getuid();
127 24 root_entry.gid_ = getgid();
128
1/2
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
 24 root_entry.checksum_ = shash::Any(hash_algorithm);
129 24 root_entry.linkcount_ = 2;
130
1/2
✓ Branch 2 taken 24 times.
✗ Branch 3 not taken.
 24 string root_path = "";
131
132 // Create the database schema and the initial root entry
133 {
134
2/4
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 24 times.
✗ Branch 5 not taken.
 24 UniquePtr<CatalogDatabase> new_clg_db(CatalogDatabase::Create(file_path));
135
2/4
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 24 times.
 48 if (!new_clg_db.IsValid() ||
136
2/4
✓ Branch 2 taken 24 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 24 times.
 24 !new_clg_db->InsertInitialValues(root_path,
137 volatile_content,
138 voms_authz,
139 root_entry))
140 {
141 ✗ LogCvmfs(kLogCatalog, kLogStderr, "creation of catalog '%s' failed",
142 file_path.c_str());
143 ✗ return NULL;
144 }
145
1/2
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
 24 }
146
147 // Compress root catalog;
148
1/2
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
 24 int64_t catalog_size = GetFileSize(file_path);
149
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 24 times.
 24 if (catalog_size < 0) {
150 ✗ unlink(file_path.c_str());
151 ✗ return NULL;
152 }
153
1/2
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
 24 string file_path_compressed = file_path + ".compressed";
154
1/2
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
 24 shash::Any hash_catalog(hash_algorithm, shash::kSuffixCatalog);
155
1/2
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
 24 bool retval = zlib::CompressPath2Path(file_path, file_path_compressed,
156 &hash_catalog);
157
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 24 times.
 24 if (!retval) {
158 ✗ LogCvmfs(kLogCatalog, kLogStderr, "compression of catalog '%s' failed",
159 file_path.c_str());
160 ✗ unlink(file_path.c_str());
161 ✗ return NULL;
162 }
163 24 unlink(file_path.c_str());
164
165 // Create manifest
166
1/2
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
 24 const string manifest_path = dir_temp + "/manifest";
167 manifest::Manifest *manifest =
168
3/6
✓ Branch 2 taken 24 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 24 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 24 times.
✗ Branch 9 not taken.
 24 new manifest::Manifest(hash_catalog, catalog_size, "");
169
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 24 times.
 24 if (!voms_authz.empty()) {
170 ✗ manifest->set_has_alt_catalog_path(true);
171 }
172
173 // Upload catalog
174
3/6
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 24 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 24 times.
✗ Branch 8 not taken.
 24 spooler->Upload(file_path_compressed, "data/" + hash_catalog.MakePath());
175
1/2
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
 24 spooler->WaitForUpload();
176 24 unlink(file_path_compressed.c_str());
177
2/4
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 24 times.
 24 if (spooler->GetNumberOfErrors() > 0) {
178 ✗ LogCvmfs(kLogCatalog, kLogStderr, "failed to commit catalog %s",
179 file_path_compressed.c_str());
180 ✗ delete manifest;
181 ✗ return NULL;
182 }
183
184 24 return manifest;
185 24 }
186
187
188 /**
189 * Retrieve the catalog containing the given path.
190 * Other than AbstractCatalogManager::FindCatalog() this mounts nested
191 * catalogs if necessary and returns WritableCatalog objects.
192 * Furthermore it optionally returns the looked-up DirectoryEntry.
193 *
194 * @param path the path to look for
195 * @param result the retrieved catalog (as a pointer)
196 * @param dirent is set to looked up DirectoryEntry for 'path' if non-NULL
197 * @return true if catalog was found
198 */
199 350 bool WritableCatalogManager::FindCatalog(const string &path,
200 WritableCatalog **result,
201 DirectoryEntry *dirent) {
202
1/2
✓ Branch 1 taken 350 times.
✗ Branch 2 not taken.
 350 const PathString ps_path(path);
203
204 Catalog *best_fit =
205
1/2
✓ Branch 1 taken 350 times.
✗ Branch 2 not taken.
 350 AbstractCatalogManager<Catalog>::FindCatalog(ps_path);
206
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 350 times.
 350 assert(best_fit != NULL);
207 350 Catalog *catalog = NULL;
208 bool retval =
209
1/2
✓ Branch 1 taken 350 times.
✗ Branch 2 not taken.
 350 MountSubtree(ps_path, best_fit, true /* is_listable */, &catalog);
210
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 350 times.
 350 if (!retval)
211 ✗ return false;
212
213
1/2
✓ Branch 1 taken 350 times.
✗ Branch 2 not taken.
 350 catalog::DirectoryEntry dummy;
214
2/2
✓ Branch 0 taken 182 times.
✓ Branch 1 taken 168 times.
 350 if (NULL == dirent) {
215 182 dirent = &dummy;
216 }
217
1/2
✓ Branch 1 taken 350 times.
✗ Branch 2 not taken.
 350 bool found = catalog->LookupPath(ps_path, dirent);
218
6/8
✓ Branch 0 taken 349 times.
✓ Branch 1 taken 1 times.
✓ Branch 3 taken 349 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 349 times.
✓ Branch 7 taken 1 times.
✓ Branch 8 taken 349 times.
 350 if (!found || !catalog->IsWritable())
219 1 return false;
220
221 349 *result = static_cast<WritableCatalog *>(catalog);
222 349 return true;
223 350 }
224
225
226 ✗ WritableCatalog *WritableCatalogManager::GetHostingCatalog(
227 const std::string &path)
228 {
229 ✗ WritableCatalog *result = NULL;
230 ✗ bool retval = FindCatalog(MakeRelativePath(path), &result, NULL);
231 ✗ if (!retval) return NULL;
232 ✗ return result;
233 }
234
235
236 /**
237 * Remove the given file from the catalogs.
238 * @param file_path the full path to the file to be removed
239 * @return true on success, false otherwise
240 */
241 4 void WritableCatalogManager::RemoveFile(const std::string &path) {
242
1/2
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
 4 const string file_path = MakeRelativePath(path);
243
1/2
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
 4 const string parent_path = GetParentPath(file_path);
244
245 4 SyncLock();
246 WritableCatalog *catalog;
247
2/4
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 4 times.
 4 if (!FindCatalog(parent_path, &catalog)) {
248 ✗ PANIC(kLogStderr, "catalog for file '%s' cannot be found",
249 file_path.c_str());
250 }
251
252
1/2
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
 4 catalog->RemoveEntry(file_path);
253 4 SyncUnlock();
254 4 }
255
256
257 /**
258 * Remove the given directory from the catalogs.
259 * @param directory_path the full path to the directory to be removed
260 * @return true on success, false otherwise
261 */
262 5 void WritableCatalogManager::RemoveDirectory(const std::string &path) {
263
1/2
✓ Branch 1 taken 5 times.
✗ Branch 2 not taken.
 5 const string directory_path = MakeRelativePath(path);
264
1/2
✓ Branch 1 taken 5 times.
✗ Branch 2 not taken.
 5 const string parent_path = GetParentPath(directory_path);
265
266 5 SyncLock();
267 WritableCatalog *catalog;
268
1/2
✓ Branch 1 taken 5 times.
✗ Branch 2 not taken.
 5 DirectoryEntry parent_entry;
269
2/4
✓ Branch 1 taken 5 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 5 times.
 5 if (!FindCatalog(parent_path, &catalog, &parent_entry)) {
270 ✗ PANIC(kLogStderr, "catalog for directory '%s' cannot be found",
271 directory_path.c_str());
272 }
273
274 5 parent_entry.set_linkcount(parent_entry.linkcount() - 1);
275
276
1/2
✓ Branch 1 taken 5 times.
✗ Branch 2 not taken.
 5 catalog->RemoveEntry(directory_path);
277
1/2
✓ Branch 1 taken 5 times.
✗ Branch 2 not taken.
 5 catalog->UpdateEntry(parent_entry, parent_path);
278
2/2
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 3 times.
 5 if (parent_entry.IsNestedCatalogRoot()) {
279
1/2
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating transition point %s",
280 parent_path.c_str());
281 WritableCatalog *parent_catalog =
282 2 reinterpret_cast<WritableCatalog *>(catalog->parent());
283 2 parent_entry.set_is_nested_catalog_mountpoint(true);
284 2 parent_entry.set_is_nested_catalog_root(false);
285
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 parent_catalog->UpdateEntry(parent_entry, parent_path);
286 }
287 5 SyncUnlock();
288 5 }
289
290 /**
291 * Clone the file called `source` changing its name into `destination`, the
292 * source file is keep intact.
293 * @params destination, the name of the new file, complete path
294 * @params source, the name of the file to clone, which must be already in the
295 * repository
296 * @return void
297 */
298 ✗ void WritableCatalogManager::Clone(const std::string destination,
299 const std::string source) {
300 ✗ const std::string relative_source = MakeRelativePath(source);
301
302 ✗ DirectoryEntry source_dirent;
303 ✗ if (!LookupPath(relative_source, kLookupDefault, &source_dirent)) {
304 ✗ PANIC(kLogStderr, "catalog for file '%s' cannot be found, aborting",
305 source.c_str());
306 }
307 ✗ if (source_dirent.IsDirectory()) {
308 ✗ PANIC(kLogStderr, "Trying to clone a directory: '%s', aborting",
309 source.c_str());
310 }
311
312 // if the file is already there we remove it and we add it back
313 ✗ DirectoryEntry check_dirent;
314 bool destination_already_present =
315 ✗ LookupPath(MakeRelativePath(destination), kLookupDefault, &check_dirent);
316 ✗ if (destination_already_present) {
317 ✗ this->RemoveFile(destination);
318 }
319
320 ✗ DirectoryEntry destination_dirent(source_dirent);
321 ✗ std::string destination_dirname;
322 ✗ std::string destination_filename;
323 ✗ SplitPath(destination, &destination_dirname, &destination_filename);
324
325 ✗ destination_dirent.name_.Assign(
326 ✗ NameString(destination_filename.c_str(), destination_filename.length()));
327
328 // TODO(jblomer): clone is used by tarball engine and should eventually
329 // support extended attributes
330 ✗ this->AddFile(destination_dirent, empty_xattrs, destination_dirname);
331 }
332
333
334 /**
335 * Copies an entire directory tree from the existing from_dir to the
336 * non-existing to_dir. The destination's parent directory must exist. On the
337 * catalog level, the new entries will be identical to the old ones except
338 * for their path hash fields.
339 */
340 10 void WritableCatalogManager::CloneTree(const std::string &from_dir,
341 const std::string &to_dir)
342 {
343 // Sanitize input paths
344
6/6
✓ Branch 1 taken 8 times.
✓ Branch 2 taken 2 times.
✓ Branch 4 taken 1 times.
✓ Branch 5 taken 7 times.
✓ Branch 6 taken 3 times.
✓ Branch 7 taken 7 times.
 10 if (from_dir.empty() || to_dir.empty())
345 3 PANIC(kLogStderr, "clone tree from or to root impossible");
346
347
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 const std::string relative_source = MakeRelativePath(from_dir);
348
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 const std::string relative_dest = MakeRelativePath(to_dir);
349
350
2/2
✓ Branch 1 taken 1 times.
✓ Branch 2 taken 6 times.
 7 if (relative_source == relative_dest) {
351 1 PANIC(kLogStderr, "cannot clone tree into itself ('%s')", to_dir.c_str());
352 }
353
4/7
✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 6 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 1 times.
✓ Branch 8 taken 5 times.
 6 if (HasPrefix(relative_dest, relative_source + "/", false /*ignore_case*/)) {
354 1 PANIC(kLogStderr,
355 "cannot clone tree into sub directory of source '%s' --> '%s'",
356 from_dir.c_str(), to_dir.c_str());
357 }
358
359
1/2
✓ Branch 1 taken 5 times.
✗ Branch 2 not taken.
 5 DirectoryEntry source_dirent;
360
3/4
✓ Branch 1 taken 5 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 times.
✓ Branch 4 taken 4 times.
 5 if (!LookupPath(relative_source, kLookupDefault, &source_dirent)) {
361 1 PANIC(kLogStderr, "path '%s' cannot be found, aborting", from_dir.c_str());
362 }
363
2/2
✓ Branch 1 taken 1 times.
✓ Branch 2 taken 3 times.
 4 if (!source_dirent.IsDirectory()) {
364 1 PANIC(kLogStderr, "CloneTree: source '%s' not a directory, aborting",
365 from_dir.c_str());
366 }
367
368
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 DirectoryEntry dest_dirent;
369
3/4
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 times.
✓ Branch 4 taken 2 times.
 3 if (LookupPath(relative_dest, kLookupDefault, &dest_dirent)) {
370 1 PANIC(kLogStderr, "destination '%s' exists, aborting", to_dir.c_str());
371 }
372
373
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 const std::string dest_parent = GetParentPath(relative_dest);
374
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 DirectoryEntry dest_parent_dirent;
375
3/4
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 times.
✓ Branch 4 taken 1 times.
 2 if (!LookupPath(dest_parent, kLookupDefault, &dest_parent_dirent)) {
376 1 PANIC(kLogStderr, "destination '%s' not on a known path, aborting",
377 to_dir.c_str());
378 }
379
380
2/4
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1 times.
✗ Branch 5 not taken.
 1 CloneTreeImpl(PathString(from_dir),
381
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 2 GetParentPath(to_dir),
382
2/4
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1 times.
✗ Branch 5 not taken.
 2 NameString(GetFileName(to_dir)));
383 21 }
384
385
386 /**
387 * Called from CloneTree(), assumes that from_dir and to_dir are sufficiently
388 * sanitized
389 */
390 8 void WritableCatalogManager::CloneTreeImpl(
391 const PathString &source_dir,
392 const std::string &dest_parent_dir,
393 const NameString &dest_name)
394 {
395
1/2
✓ Branch 4 taken 8 times.
✗ Branch 5 not taken.
 8 LogCvmfs(kLogCatalog, kLogDebug, "cloning %s --> %s/%s", source_dir.c_str(),
396
1/2
✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
 16 dest_parent_dir.c_str(), dest_name.ToString().c_str());
397
3/6
✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 8 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 8 times.
✗ Branch 8 not taken.
 16 PathString relative_source(MakeRelativePath(source_dir.ToString()));
398
399
1/2
✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
 8 DirectoryEntry source_dirent;
400
1/2
✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
 8 bool retval = LookupPath(relative_source, kLookupDefault, &source_dirent);
401
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 8 times.
 8 assert(retval);
402
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 8 times.
 8 assert(!source_dirent.IsBindMountpoint());
403
404
1/2
✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
 8 DirectoryEntry dest_dirent(source_dirent);
405
1/2
✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
 8 dest_dirent.name_.Assign(dest_name);
406 // Just in case, reset the nested catalog markers
407 8 dest_dirent.set_is_nested_catalog_mountpoint(false);
408 8 dest_dirent.set_is_nested_catalog_root(false);
409
410 8 XattrList xattrs;
411
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 8 times.
 8 if (source_dirent.HasXattrs()) {
412 ✗ retval = LookupXattrs(relative_source, &xattrs);
413 ✗ assert(retval);
414 }
415
1/2
✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
 8 AddDirectory(dest_dirent, xattrs, dest_parent_dir);
416
417
1/2
✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
 8 std::string dest_dir = dest_parent_dir;
418
2/2
✓ Branch 1 taken 7 times.
✓ Branch 2 taken 1 times.
 8 if (!dest_dir.empty())
419
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 dest_dir.push_back('/');
420
2/4
✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 8 times.
✗ Branch 5 not taken.
 8 dest_dir += dest_name.ToString();
421
5/6
✓ Branch 1 taken 5 times.
✓ Branch 2 taken 3 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 5 times.
✓ Branch 5 taken 3 times.
✓ Branch 6 taken 5 times.
 13 if (source_dirent.IsNestedCatalogRoot() ||
422 5 source_dirent.IsNestedCatalogMountpoint())
423 {
424
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 CreateNestedCatalog(dest_dir);
425 }
426
427 8 DirectoryEntryList ls;
428
1/2
✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
 8 retval = Listing(relative_source, &ls, false /* expand_symlink */);
429
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 8 times.
 8 assert(retval);
430
2/2
✓ Branch 1 taken 17 times.
✓ Branch 2 taken 8 times.
 25 for (unsigned i = 0; i < ls.size(); ++i) {
431
1/2
✓ Branch 1 taken 17 times.
✗ Branch 2 not taken.
 17 PathString sub_path(source_dir);
432
2/4
✓ Branch 1 taken 17 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 17 times.
 17 assert(!sub_path.IsEmpty());
433
1/2
✓ Branch 1 taken 17 times.
✗ Branch 2 not taken.
 17 sub_path.Append("/", 1);
434
3/6
✓ Branch 2 taken 17 times.
✗ Branch 3 not taken.
✓ Branch 7 taken 17 times.
✗ Branch 8 not taken.
✓ Branch 11 taken 17 times.
✗ Branch 12 not taken.
 17 sub_path.Append(ls[i].name().GetChars(), ls[i].name().GetLength());
435
436
2/2
✓ Branch 2 taken 7 times.
✓ Branch 3 taken 10 times.
 17 if (ls[i].IsDirectory()) {
437
2/4
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 7 times.
✗ Branch 6 not taken.
 7 CloneTreeImpl(sub_path, dest_dir, ls[i].name());
438 7 continue;
439 }
440
441 // We break hard-links during cloning
442 10 ls[i].set_hardlink_group(0);
443 10 ls[i].set_linkcount(1);
444
445 10 xattrs.Clear();
446
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 10 times.
 10 if (ls[i].HasXattrs()) {
447 ✗ retval = LookupXattrs(sub_path, &xattrs);
448 ✗ assert(retval);
449 }
450
451
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 10 times.
 10 if (ls[i].IsChunkedFile()) {
452 ✗ FileChunkList chunks;
453 ✗ std::string relative_sub_path = MakeRelativePath(sub_path.ToString());
454 ✗ retval = ListFileChunks(
455 ✗ PathString(relative_sub_path), ls[i].hash_algorithm(), &chunks);
456 ✗ assert(retval);
457 ✗ AddChunkedFile(ls[i], xattrs, dest_dir, chunks);
458 ✗ } else {
459
1/2
✓ Branch 2 taken 10 times.
✗ Branch 3 not taken.
 10 AddFile(ls[i], xattrs, dest_dir);
460 }
461
2/2
✓ Branch 1 taken 10 times.
✓ Branch 2 taken 7 times.
 17 }
462 8 }
463
464
465 /**
466 * Add a new directory to the catalogs.
467 * @param entry a DirectoryEntry structure describing the new directory
468 * @param parent_directory the absolute path of the directory containing the
469 * directory to be created
470 * @return true on success, false otherwise
471 */
472 126 void WritableCatalogManager::AddDirectory(const DirectoryEntryBase &entry,
473 const XattrList &xattrs,
474 const std::string &parent_directory)
475 {
476
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 const string parent_path = MakeRelativePath(parent_directory);
477
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 string directory_path = parent_path + "/";
478
3/6
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 126 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 126 times.
✗ Branch 10 not taken.
 126 directory_path.append(entry.name().GetChars(), entry.name().GetLength());
479
480 126 SyncLock();
481 WritableCatalog *catalog;
482
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 DirectoryEntry parent_entry;
483
2/4
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 126 times.
 126 if (!FindCatalog(parent_path, &catalog, &parent_entry)) {
484 ✗ PANIC(kLogStderr, "catalog for directory '%s' cannot be found",
485 directory_path.c_str());
486 }
487
488
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 DirectoryEntry fixed_hardlink_count(entry);
489 126 fixed_hardlink_count.set_linkcount(2);
490
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 catalog->AddEntry(fixed_hardlink_count, xattrs,
491 directory_path, parent_path);
492
493 126 parent_entry.set_linkcount(parent_entry.linkcount() + 1);
494
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 catalog->UpdateEntry(parent_entry, parent_path);
495
2/2
✓ Branch 1 taken 3 times.
✓ Branch 2 taken 123 times.
 126 if (parent_entry.IsNestedCatalogRoot()) {
496
1/2
✓ Branch 2 taken 3 times.
✗ Branch 3 not taken.
 3 LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating transition point %s",
497 parent_path.c_str());
498 WritableCatalog *parent_catalog =
499 3 reinterpret_cast<WritableCatalog *>(catalog->parent());
500 3 parent_entry.set_is_nested_catalog_mountpoint(true);
501 3 parent_entry.set_is_nested_catalog_root(false);
502
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 parent_catalog->UpdateEntry(parent_entry, parent_path);
503 }
504 126 SyncUnlock();
505 126 }
506
507 /**
508 * Add a new file to the catalogs.
509 * @param entry a DirectoryEntry structure describing the new file
510 * @param parent_directory the absolute path of the directory containing the
511 * file to be created
512 * @return true on success, false otherwise
513 */
514 167 void WritableCatalogManager::AddFile(
515 const DirectoryEntry &entry,
516 const XattrList &xattrs,
517 const std::string &parent_directory)
518 {
519
1/2
✓ Branch 1 taken 167 times.
✗ Branch 2 not taken.
 167 const string parent_path = MakeRelativePath(parent_directory);
520
1/2
✓ Branch 1 taken 167 times.
✗ Branch 2 not taken.
 167 const string file_path = entry.GetFullPath(parent_path);
521
522 167 SyncLock();
523 WritableCatalog *catalog;
524
2/4
✓ Branch 1 taken 167 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 167 times.
 167 if (!FindCatalog(parent_path, &catalog)) {
525 ✗ PANIC(kLogStderr, "catalog for file '%s' cannot be found",
526 file_path.c_str());
527 }
528
529
4/6
✓ Branch 1 taken 164 times.
✓ Branch 2 taken 3 times.
✓ Branch 4 taken 164 times.
✗ Branch 5 not taken.
✗ Branch 8 not taken.
✓ Branch 9 taken 164 times.
 167 assert(!entry.IsRegular() || entry.IsChunkedFile() ||
530 !entry.checksum().IsNull());
531
3/4
✓ Branch 1 taken 3 times.
✓ Branch 2 taken 164 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 3 times.
 167 assert(entry.IsRegular() || !entry.IsExternalFile());
532
533 // check if file is too big
534
1/2
✓ Branch 1 taken 167 times.
✗ Branch 2 not taken.
 167 unsigned mbytes = entry.size() / (1024 * 1024);
535
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 167 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 167 if ((file_mbyte_limit_ > 0) && (mbytes > file_mbyte_limit_)) {
536 ✗ LogCvmfs(kLogCatalog, kLogStderr,
537 "%s: file at %s is larger than %u megabytes (%u). "
538 "CernVM-FS works best with small files. "
539 "Please remove the file or increase the limit.",
540 ✗ enforce_limits_ ? "FATAL" : "WARNING", file_path.c_str(),
541 file_mbyte_limit_, mbytes);
542 ✗ if (enforce_limits_)
543 ✗ PANIC(kLogStderr, "file at %s is larger than %u megabytes (%u).",
544 file_path.c_str(), file_mbyte_limit_, mbytes);
545 }
546
547
1/2
✓ Branch 1 taken 167 times.
✗ Branch 2 not taken.
 167 catalog->AddEntry(entry, xattrs, file_path, parent_path);
548 167 SyncUnlock();
549 167 }
550
551
552 ✗ void WritableCatalogManager::AddChunkedFile(
553 const DirectoryEntryBase &entry,
554 const XattrList &xattrs,
555 const std::string &parent_directory,
556 const FileChunkList &file_chunks)
557 {
558 ✗ assert(file_chunks.size() > 0);
559
560 ✗ DirectoryEntry full_entry(entry);
561 ✗ full_entry.set_is_chunked_file(true);
562
563 ✗ AddFile(full_entry, xattrs, parent_directory);
564
565 ✗ const string parent_path = MakeRelativePath(parent_directory);
566 ✗ const string file_path = entry.GetFullPath(parent_path);
567
568 ✗ SyncLock();
569 WritableCatalog *catalog;
570 ✗ if (!FindCatalog(parent_path, &catalog)) {
571 ✗ PANIC(kLogStderr, "catalog for file '%s' cannot be found",
572 file_path.c_str());
573 }
574
575 ✗ for (unsigned i = 0; i < file_chunks.size(); ++i) {
576 ✗ catalog->AddFileChunk(file_path, *file_chunks.AtPtr(i));
577 }
578 ✗ SyncUnlock();
579 }
580
581
582 /**
583 * Add a hardlink group to the catalogs.
584 * @param entries a list of DirectoryEntries describing the new files
585 * @param parent_directory the absolute path of the directory containing the
586 * files to be created
587 * @return true on success, false otherwise
588 */
589 ✗ void WritableCatalogManager::AddHardlinkGroup(
590 const DirectoryEntryBaseList &entries,
591 const XattrList &xattrs,
592 const std::string &parent_directory,
593 const FileChunkList &file_chunks)
594 {
595 ✗ assert(entries.size() >= 1);
596 ✗ assert(file_chunks.IsEmpty() || entries[0].IsRegular());
597 ✗ if (entries.size() == 1) {
598 ✗ DirectoryEntry fix_linkcount(entries[0]);
599 ✗ fix_linkcount.set_linkcount(1);
600 ✗ if (file_chunks.IsEmpty())
601 ✗ return AddFile(fix_linkcount, xattrs, parent_directory);
602 ✗ return AddChunkedFile(fix_linkcount, xattrs, parent_directory, file_chunks);
603 }
604
605 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "adding hardlink group %s/%s",
606 ✗ parent_directory.c_str(), entries[0].name().c_str());
607
608 // Hardlink groups have to reside in the same directory.
609 // Therefore we only have one parent directory here
610 ✗ const string parent_path = MakeRelativePath(parent_directory);
611
612 // check if hard link is too big
613 ✗ unsigned mbytes = entries[0].size() / (1024 * 1024);
614 ✗ if ((file_mbyte_limit_ > 0) && (mbytes > file_mbyte_limit_)) {
615 ✗ LogCvmfs(kLogCatalog, kLogStderr,
616 "%s: hard link at %s is larger than %u megabytes (%u). "
617 "CernVM-FS works best with small files. "
618 "Please remove the file or increase the limit.",
619 ✗ enforce_limits_ ? "FATAL" : "WARNING",
620 ✗ (parent_path + entries[0].name().ToString()).c_str(),
621 file_mbyte_limit_,
622 mbytes);
623 ✗ if (enforce_limits_)
624 ✗ PANIC(kLogStderr, "hard link at %s is larger than %u megabytes (%u)",
625 (parent_path + entries[0].name().ToString()).c_str(),
626 file_mbyte_limit_, mbytes);
627 }
628
629 ✗ SyncLock();
630 WritableCatalog *catalog;
631 ✗ if (!FindCatalog(parent_path, &catalog)) {
632 ✗ PANIC(kLogStderr,
633 "catalog for hardlink group containing '%s' cannot be found",
634 parent_path.c_str());
635 }
636
637 // Get a valid hardlink group id for the catalog the group will end up in
638 // TODO(unknown): Compaction
639 ✗ uint32_t new_group_id = catalog->GetMaxLinkId() + 1;
640 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "hardlink group id %u issued",
641 new_group_id);
642 ✗ assert(new_group_id > 0);
643
644 // Add the file entries to the catalog
645 ✗ for (DirectoryEntryBaseList::const_iterator i = entries.begin(),
646 ✗ iEnd = entries.end(); i != iEnd; ++i)
647 {
648 ✗ string file_path = parent_path + "/";
649 ✗ file_path.append(i->name().GetChars(), i->name().GetLength());
650
651 // create a fully fledged DirectoryEntry to add the hardlink group to it
652 // which is CVMFS specific meta data.
653 ✗ DirectoryEntry hardlink(*i);
654 ✗ hardlink.set_hardlink_group(new_group_id);
655 ✗ hardlink.set_linkcount(entries.size());
656 ✗ hardlink.set_is_chunked_file(!file_chunks.IsEmpty());
657
658 ✗ catalog->AddEntry(hardlink, xattrs, file_path, parent_path);
659 ✗ if (hardlink.IsChunkedFile()) {
660 ✗ for (unsigned i = 0; i < file_chunks.size(); ++i) {
661 ✗ catalog->AddFileChunk(file_path, *file_chunks.AtPtr(i));
662 }
663 }
664 }
665 ✗ SyncUnlock();
666 }
667
668
669 ✗ void WritableCatalogManager::ShrinkHardlinkGroup(const string &remove_path) {
670 ✗ const string relative_path = MakeRelativePath(remove_path);
671
672 ✗ SyncLock();
673 WritableCatalog *catalog;
674 ✗ if (!FindCatalog(relative_path, &catalog)) {
675 ✗ PANIC(kLogStderr,
676 "catalog for hardlink group containing '%s' cannot be found",
677 remove_path.c_str());
678 }
679
680 ✗ catalog->IncLinkcount(relative_path, -1);
681 ✗ SyncUnlock();
682 }
683
684
685 /**
686 * Update entry meta data (mode, owner, ...).
687 * CVMFS specific meta data (i.e. nested catalog transition points) are NOT
688 * changed by this method, although transition points intrinsics are taken into
689 * account, to keep nested catalogs consistent.
690 * @param entry the directory entry to be touched
691 * @param path the path of the directory entry to be touched
692 */
693 1 void WritableCatalogManager::TouchDirectory(const DirectoryEntryBase &entry,
694 const XattrList &xattrs,
695 const std::string &directory_path)
696 {
697
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1 times.
 1 assert(entry.IsDirectory());
698
699
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 const string entry_path = MakeRelativePath(directory_path);
700
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 const string parent_path = GetParentPath(entry_path);
701
702 1 SyncLock();
703 // find the catalog to be updated
704 WritableCatalog *catalog;
705
2/4
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 times.
 1 if (!FindCatalog(parent_path, &catalog)) {
706 ✗ PANIC(kLogStderr, "catalog for entry '%s' cannot be found",
707 entry_path.c_str());
708 }
709
710
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 catalog->TouchEntry(entry, xattrs, entry_path);
711
712 // since we deal with a directory here, we might just touch a
713 // nested catalog transition point. If this is the case we would need to
714 // update two catalog entries:
715 // * the nested catalog MOUNTPOINT in the parent catalog
716 // * the nested catalog ROOT in the nested catalog
717
718 // first check if we really have a nested catalog transition point
719
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 catalog::DirectoryEntry potential_transition_point;
720
1/2
✓ Branch 3 taken 1 times.
✗ Branch 4 not taken.
 1 PathString transition_path(entry_path.data(), entry_path.length());
721
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 bool retval = catalog->LookupPath(transition_path,
722 &potential_transition_point);
723
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 times.
 1 assert(retval);
724
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1 times.
 1 if (potential_transition_point.IsNestedCatalogMountpoint()) {
725 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg,
726 "updating transition point at %s", entry_path.c_str());
727
728 // find and mount nested catalog associated to this transition point
729 ✗ shash::Any nested_hash;
730 uint64_t nested_size;
731 ✗ retval = catalog->FindNested(transition_path, &nested_hash, &nested_size);
732 ✗ assert(retval);
733 Catalog *nested_catalog;
734 ✗ nested_catalog = MountCatalog(transition_path, nested_hash, catalog);
735 ✗ assert(nested_catalog != NULL);
736
737 // update nested catalog root in the child catalog
738 reinterpret_cast<WritableCatalog *>(nested_catalog)->
739 ✗ TouchEntry(entry, xattrs, entry_path);
740 }
741
742 1 SyncUnlock();
743 1 }
744
745
746 /**
747 * Create a new nested catalog. Includes moving all entries belonging there
748 * from it's parent catalog.
749 * @param mountpoint the path of the directory to become a nested root
750 * @return true on success, false otherwise
751 */
752 35 void WritableCatalogManager::CreateNestedCatalog(const std::string &mountpoint)
753 {
754
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 const string nested_root_path = MakeRelativePath(mountpoint);
755
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 const PathString ps_nested_root_path(nested_root_path);
756
757 35 SyncLock();
758 // Find the catalog currently containing the directory structure, which
759 // will be represented as a new nested catalog and its root-entry/mountpoint
760 // along the way
761 35 WritableCatalog *old_catalog = NULL;
762
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 DirectoryEntry new_root_entry;
763
2/4
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 35 times.
 35 if (!FindCatalog(nested_root_path, &old_catalog, &new_root_entry)) {
764 ✗ PANIC(kLogStderr,
765 "failed to create nested catalog '%s': "
766 "mountpoint was not found in current catalog structure",
767 nested_root_path.c_str());
768 }
769
770 // Create the database schema and the initial root entry
771 // for the new nested catalog
772
1/2
✓ Branch 2 taken 35 times.
✗ Branch 3 not taken.
 35 const string database_file_path = CreateTempPath(dir_temp() + "/catalog",
773
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 0666);
774 35 const bool volatile_content = false;
775
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 CatalogDatabase *new_catalog_db = CatalogDatabase::Create(database_file_path);
776
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 35 times.
 35 assert(NULL != new_catalog_db);
777 // Note we do not set the external_data bit for nested catalogs
778 bool retval =
779
2/4
✓ Branch 2 taken 35 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 35 times.
✗ Branch 6 not taken.
 35 new_catalog_db->InsertInitialValues(nested_root_path,
780 volatile_content,
781 "", // At this point, only root
782 // catalog gets VOMS authz
783 new_root_entry);
784
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 35 times.
 35 assert(retval);
785 // TODO(rmeusel): we need a way to attach a catalog directly from an open
786 // database to remove this indirection
787
1/2
✓ Branch 0 taken 35 times.
✗ Branch 1 not taken.
 35 delete new_catalog_db;
788 35 new_catalog_db = NULL;
789
790 // Attach the just created nested catalog
791 Catalog *new_catalog =
792
2/4
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 35 times.
✗ Branch 5 not taken.
 35 CreateCatalog(ps_nested_root_path, shash::Any(), old_catalog);
793
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 retval = AttachCatalog(database_file_path, new_catalog);
794
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 35 times.
 35 assert(retval);
795
796
2/4
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 35 times.
 35 assert(new_catalog->IsWritable());
797 35 WritableCatalog *wr_new_catalog = static_cast<WritableCatalog *>(new_catalog);
798
799
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 35 times.
 35 if (new_root_entry.HasXattrs()) {
800 ✗ XattrList xattrs;
801 ✗ retval = old_catalog->LookupXattrsPath(ps_nested_root_path, &xattrs);
802 ✗ assert(retval);
803 ✗ wr_new_catalog->TouchEntry(new_root_entry, xattrs, nested_root_path);
804 }
805
806 // From now on, there are two catalogs, spanning the same directory structure
807 // we have to split the overlapping directory entries from the old catalog
808 // to the new catalog to re-gain a valid catalog structure
809
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 old_catalog->Partition(wr_new_catalog);
810
811 // Add the newly created nested catalog to the references of the containing
812 // catalog
813
4/8
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 35 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 35 times.
✗ Branch 8 not taken.
✓ Branch 10 taken 35 times.
✗ Branch 11 not taken.
 35 old_catalog->InsertNestedCatalog(new_catalog->mountpoint().ToString(), NULL,
814 35 shash::Any(spooler_->GetHashAlgorithm()), 0);
815
816 // Fix subtree counters in new nested catalogs: subtree is the sum of all
817 // entries of all "grand-nested" catalogs
818 // Note: taking a copy of the nested catalog list here
819 const Catalog::NestedCatalogList &grand_nested =
820
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 wr_new_catalog->ListOwnNestedCatalogs();
821
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 DeltaCounters fix_subtree_counters;
822 70 for (Catalog::NestedCatalogList::const_iterator i = grand_nested.begin(),
823
1/2
✗ Branch 3 not taken.
✓ Branch 4 taken 35 times.
 70 iEnd = grand_nested.end(); i != iEnd; ++i)
824 {
825 WritableCatalog *grand_catalog;
826 ✗ retval = FindCatalog(i->mountpoint.ToString(), &grand_catalog);
827 ✗ assert(retval);
828 ✗ const Counters &grand_counters = grand_catalog->GetCounters();
829 ✗ grand_counters.AddAsSubtree(&fix_subtree_counters);
830 }
831 35 DeltaCounters save_counters = wr_new_catalog->delta_counters_;
832 35 wr_new_catalog->delta_counters_ = fix_subtree_counters;
833
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 wr_new_catalog->UpdateCounters();
834 35 wr_new_catalog->delta_counters_ = save_counters;
835
836 35 SyncUnlock();
837 35 }
838
839
840 /**
841 * Remove a nested catalog
842 *
843 * If the merged parameter is true, when you remove a nested catalog
844 * all entries currently held by it will be merged into its parent
845 * catalog.
846 * @param mountpoint - the path of the nested catalog to be removed
847 * @param merge - merge the subtree associated with the nested catalog
848 * into its parent catalog
849 * @return - true on success, false otherwise
850 */
851 3 void WritableCatalogManager::RemoveNestedCatalog(const string &mountpoint,
852 const bool merge) {
853
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 const string nested_root_path = MakeRelativePath(mountpoint);
854
855 3 SyncLock();
856 // Find the catalog which should be removed
857 3 WritableCatalog *nested_catalog = NULL;
858
2/4
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 3 times.
 3 if (!FindCatalog(nested_root_path, &nested_catalog)) {
859 ✗ PANIC(kLogStderr,
860 "failed to remove nested catalog '%s': "
861 "mountpoint was not found in current catalog structure",
862 nested_root_path.c_str());
863 }
864
865 // Check if the found catalog is really the nested catalog to be deleted
866
6/19
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 3 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 3 times.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 3 times.
✗ Branch 14 not taken.
✓ Branch 16 taken 3 times.
✗ Branch 17 not taken.
✗ Branch 19 not taken.
✗ Branch 20 not taken.
✗ Branch 22 not taken.
✗ Branch 23 not taken.
 6 assert(!nested_catalog->IsRoot() &&
867 (nested_catalog->mountpoint().ToString() == nested_root_path));
868
869
1/2
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
 3 if (merge) {
870 // Merge all data from the nested catalog into it's parent
871
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 nested_catalog->MergeIntoParent();
872 } else {
873 ✗ nested_catalog->RemoveFromParent();
874 }
875
876 // Delete the catalog database file from the working copy
877
2/6
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 3 times.
 3 if (unlink(nested_catalog->database_path().c_str()) != 0) {
878 ✗ PANIC(kLogStderr,
879 "unable to delete the removed nested catalog database file '%s'",
880 nested_catalog->database_path().c_str());
881 }
882
883 // Remove the catalog from internal data structures
884
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 DetachCatalog(nested_catalog);
885 3 SyncUnlock();
886 3 }
887
888
889 /**
890 * Swap in a new nested catalog
891 *
892 * The old nested catalog must not have been already attached to the
893 * catalog tree. This method will not attach the new nested catalog
894 * to the catalog tree.
895 *
896 * @param mountpoint - the path of the nested catalog to be removed
897 * @param new_hash - the hash of the new nested catalog
898 * @param new_size - the size of the new nested catalog
899 */
900 7 void WritableCatalogManager::SwapNestedCatalog(const string &mountpoint,
901 const shash::Any &new_hash,
902 const uint64_t new_size) {
903
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 const string nested_root_path = MakeRelativePath(mountpoint);
904
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 const string parent_path = GetParentPath(nested_root_path);
905
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 const PathString nested_root_ps = PathString(nested_root_path);
906
907 7 SyncLock();
908
909 // Find the immediate parent catalog
910 7 WritableCatalog *parent = NULL;
911
3/4
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 times.
✓ Branch 4 taken 6 times.
 7 if (!FindCatalog(parent_path, &parent)) {
912 1 SyncUnlock(); // this is needed for the unittest. otherwise they get stuck
913 1 PANIC(kLogStderr,
914 "failed to swap nested catalog '%s': could not find parent '%s'",
915 nested_root_path.c_str(), parent_path.c_str());
916 }
917
918 // Get old nested catalog counters
919
1/2
✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
 6 Catalog *old_attached_catalog = parent->FindChild(nested_root_ps);
920
1/2
✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
 6 Counters old_counters;
921
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 5 times.
 6 if (old_attached_catalog) {
922 // Old catalog was already attached (e.g. as a child catalog
923 // attached by a prior call to CreateNestedCatalog()). Ensure
924 // that it has not been modified, get counters, and detach it.
925 1 WritableCatalogList list;
926
2/4
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 times.
✗ Branch 4 not taken.
 1 if (GetModifiedCatalogLeafsRecursively(old_attached_catalog, &list)) {
927 1 SyncUnlock();
928 1 PANIC(kLogStderr,
929 "failed to swap nested catalog '%s': already modified",
930 nested_root_path.c_str());
931 }
932 ✗ old_counters = old_attached_catalog->GetCounters();
933 ✗ DetachSubtree(old_attached_catalog);
934
935 1 } else {
936 // Old catalog was not attached. Download a freely attached
937 // version and get counters.
938
1/2
✓ Branch 1 taken 5 times.
✗ Branch 2 not taken.
 5 shash::Any old_hash;
939 uint64_t old_size;
940
1/2
✓ Branch 1 taken 5 times.
✗ Branch 2 not taken.
 5 const bool old_found = parent->FindNested(nested_root_ps, &old_hash,
941 &old_size);
942
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 4 times.
 5 if (!old_found) {
943 1 SyncUnlock();
944 1 PANIC(kLogStderr,
945 "failed to swap nested catalog '%s': not found in parent",
946 nested_root_path.c_str());
947 }
948 UniquePtr<Catalog> old_free_catalog(
949
2/4
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 4 times.
✗ Branch 5 not taken.
 4 LoadFreeCatalog(nested_root_ps, old_hash));
950
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4 times.
 4 if (!old_free_catalog.IsValid()) {
951 ✗ SyncUnlock();
952 ✗ PANIC(kLogStderr,
953 "failed to swap nested catalog '%s': failed to load old catalog",
954 nested_root_path.c_str());
955 }
956 4 old_counters = old_free_catalog->GetCounters();
957 4 }
958
959 // Load freely attached new catalog
960
3/4
✓ Branch 1 taken 3 times.
✓ Branch 2 taken 1 times.
✓ Branch 4 taken 3 times.
✗ Branch 5 not taken.
 4 UniquePtr<Catalog> new_catalog(LoadFreeCatalog(nested_root_ps, new_hash));
961
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 3 times.
 3 if (!new_catalog.IsValid()) {
962 ✗ SyncUnlock();
963 ✗ PANIC(kLogStderr,
964 "failed to swap nested catalog '%s': failed to load new catalog",
965 nested_root_path.c_str());
966 }
967
968 // Get new catalog root directory entry
969
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 DirectoryEntry dirent;
970 3 XattrList xattrs;
971
1/2
✓ Branch 2 taken 3 times.
✗ Branch 3 not taken.
 3 const bool dirent_found = new_catalog->LookupPath(nested_root_ps, &dirent);
972
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3 times.
 3 if (!dirent_found) {
973 ✗ SyncUnlock();
974 ✗ PANIC(kLogStderr,
975 "failed to swap nested catalog '%s': missing dirent in new catalog",
976 nested_root_path.c_str());
977 }
978
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 3 times.
 3 if (dirent.HasXattrs()) {
979 ✗ const bool xattrs_found = new_catalog->LookupXattrsPath(nested_root_ps,
980 &xattrs);
981 ✗ if (!xattrs_found) {
982 ✗ SyncUnlock();
983 ✗ PANIC(kLogStderr,
984 "failed to swap nested catalog '%s': missing xattrs in new catalog",
985 nested_root_path.c_str());
986 }
987 }
988
989 // Swap catalogs
990
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 parent->RemoveNestedCatalog(nested_root_path, NULL);
991
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 parent->InsertNestedCatalog(nested_root_path, NULL, new_hash, new_size);
992
993 // Update parent directory entry
994 3 dirent.set_is_nested_catalog_mountpoint(true);
995 3 dirent.set_is_nested_catalog_root(false);
996
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 parent->UpdateEntry(dirent, nested_root_path);
997
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 parent->TouchEntry(dirent, xattrs, nested_root_path);
998
999 // Update counters
1000
1/2
✓ Branch 3 taken 3 times.
✗ Branch 4 not taken.
 3 DeltaCounters delta = Counters::Diff(old_counters,
1001 new_catalog->GetCounters());
1002
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 delta.PopulateToParent(&parent->delta_counters_);
1003
1004 3 SyncUnlock();
1005 15 }
1006
1007 /**
1008 * Install a nested catalog (catalog hierarchy) at a new, empty directory
1009 *
1010 * The mountpoint directory must not yet exist. Its parent directory, however
1011 * must exist. This method combines functionality from AddDirectory(),
1012 * CreateNestedCatalog() and SwapNestedCatalog().
1013 * The new nested catalog won't get attached.
1014 *
1015 * @param mountpoint - the path where the nested catalog should be installed
1016 * @param new_hash - the hash of the new nested catalog
1017 * @param new_size - the size of the new nested catalog
1018 */
1019 3 void WritableCatalogManager::GraftNestedCatalog(const string &mountpoint,
1020 const shash::Any &new_hash,
1021 const uint64_t new_size)
1022 {
1023
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 const string nested_root_path = MakeRelativePath(mountpoint);
1024
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 const string parent_path = GetParentPath(nested_root_path);
1025
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 const PathString nested_root_ps = PathString(nested_root_path);
1026
1027
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 3 times.
 3 assert(!nested_root_path.empty());
1028
1029 // Load freely attached new catalog
1030
2/4
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3 times.
✗ Branch 5 not taken.
 3 UniquePtr<Catalog> new_catalog(LoadFreeCatalog(nested_root_ps, new_hash));
1031
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 3 times.
 3 if (!new_catalog.IsValid()) {
1032 ✗ PANIC(kLogStderr,
1033 "failed to graft nested catalog '%s': failed to load new catalog",
1034 nested_root_path.c_str());
1035 }
1036
4/7
✓ Branch 2 taken 3 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 3 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1 times.
✓ Branch 9 taken 2 times.
 3 if (new_catalog->root_prefix() != nested_root_ps) {
1037
2/4
✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1 times.
✗ Branch 6 not taken.
 3 PANIC(kLogStderr,
1038 "invalid nested catalog for grafting at '%s': catalog rooted at '%s'",
1039 nested_root_path.c_str(),
1040 new_catalog->root_prefix().ToString().c_str());
1041 }
1042
1043 // Get new catalog root directory entry
1044
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 DirectoryEntry dirent;
1045 2 XattrList xattrs;
1046
1/2
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 const bool dirent_found = new_catalog->LookupPath(nested_root_ps, &dirent);
1047
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
 2 if (!dirent_found) {
1048 ✗ PANIC(kLogStderr,
1049 "failed to swap nested catalog '%s': missing dirent in new catalog",
1050 nested_root_path.c_str());
1051 }
1052
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 2 times.
 2 if (dirent.HasXattrs()) {
1053 ✗ const bool xattrs_found = new_catalog->LookupXattrsPath(nested_root_ps,
1054 &xattrs);
1055 ✗ if (!xattrs_found) {
1056 ✗ PANIC(kLogStderr,
1057 "failed to swap nested catalog '%s': missing xattrs in new catalog",
1058 nested_root_path.c_str());
1059 }
1060 }
1061 // Transform the nested catalog root into a transition point to be inserted
1062 // in the parent catalog
1063 2 dirent.set_is_nested_catalog_root(false);
1064 2 dirent.set_is_nested_catalog_mountpoint(true);
1065
1066 // Add directory and nested catalog
1067
1068 2 SyncLock();
1069 WritableCatalog *parent_catalog;
1070
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 DirectoryEntry parent_entry;
1071
2/4
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 2 times.
 2 if (!FindCatalog(parent_path, &parent_catalog, &parent_entry)) {
1072 ✗ SyncUnlock();
1073 ✗ PANIC(kLogStderr, "catalog for directory '%s' cannot be found",
1074 parent_path.c_str());
1075 }
1076
3/4
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 times.
✓ Branch 4 taken 1 times.
 2 if (parent_catalog->LookupPath(nested_root_ps, NULL)) {
1077 1 SyncUnlock();
1078 1 PANIC(kLogStderr, "invalid attempt to graft nested catalog into existing "
1079 "directory '%s'", nested_root_path.c_str());
1080 }
1081
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 parent_catalog->AddEntry(dirent, xattrs, nested_root_path, parent_path);
1082 1 parent_entry.set_linkcount(parent_entry.linkcount() + 1);
1083
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 parent_catalog->UpdateEntry(parent_entry, parent_path);
1084
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 if (parent_entry.IsNestedCatalogRoot()) {
1085 WritableCatalog *grand_parent_catalog =
1086 1 reinterpret_cast<WritableCatalog *>(parent_catalog->parent());
1087 1 parent_entry.set_is_nested_catalog_root(false);
1088 1 parent_entry.set_is_nested_catalog_mountpoint(true);
1089
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 grand_parent_catalog->UpdateEntry(parent_entry, parent_path);
1090 }
1091
1092
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 parent_catalog->InsertNestedCatalog(
1093 nested_root_path, NULL, new_hash, new_size);
1094
1095 // Fix-up counters
1096
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 Counters counters;
1097
1/2
✓ Branch 3 taken 1 times.
✗ Branch 4 not taken.
 1 DeltaCounters delta = Counters::Diff(counters, new_catalog->GetCounters());
1098
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 delta.PopulateToParent(&parent_catalog->delta_counters_);
1099
1100 1 SyncUnlock();
1101 12 }
1102
1103 /**
1104 * Checks if a nested catalog starts at this path. The path must be valid.
1105 */
1106 ✗ bool WritableCatalogManager::IsTransitionPoint(const string &mountpoint) {
1107 ✗ const string path = MakeRelativePath(mountpoint);
1108
1109 ✗ SyncLock();
1110 WritableCatalog *catalog;
1111 ✗ DirectoryEntry entry;
1112 ✗ if (!FindCatalog(path, &catalog, &entry)) {
1113 ✗ PANIC(kLogStderr, "catalog for directory '%s' cannot be found",
1114 path.c_str());
1115 }
1116 ✗ const bool result = entry.IsNestedCatalogRoot();
1117 ✗ SyncUnlock();
1118 ✗ return result;
1119 }
1120
1121
1122 ✗ void WritableCatalogManager::PrecalculateListings() {
1123 // TODO(jblomer): meant for micro catalogs
1124 }
1125
1126
1127 ✗ void WritableCatalogManager::SetTTL(const uint64_t new_ttl) {
1128 ✗ SyncLock();
1129 ✗ reinterpret_cast<WritableCatalog *>(GetRootCatalog())->SetTTL(new_ttl);
1130 ✗ SyncUnlock();
1131 }
1132
1133
1134 ✗ bool WritableCatalogManager::SetVOMSAuthz(const std::string &voms_authz) {
1135 bool result;
1136 ✗ SyncLock();
1137 result = reinterpret_cast<WritableCatalog *>(
1138 ✗ GetRootCatalog())->SetVOMSAuthz(voms_authz);
1139 ✗ SyncUnlock();
1140 ✗ return result;
1141 }
1142
1143
1144 22 bool WritableCatalogManager::Commit(const bool stop_for_tweaks,
1145 const uint64_t manual_revision,
1146 manifest::Manifest *manifest) {
1147 WritableCatalog *root_catalog =
1148 22 reinterpret_cast<WritableCatalog *>(GetRootCatalog());
1149
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 root_catalog->SetDirty();
1150
1151 // set root catalog revision to manually provided number if available
1152
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 22 times.
 22 if (manual_revision > 0) {
1153 ✗ const uint64_t revision = root_catalog->GetRevision();
1154 ✗ if (revision >= manual_revision) {
1155 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1156 "Manual revision (%" PRIu64 ") must not be "
1157 "smaller than the current root catalog's (%" PRIu64
1158 "). Skipped!", manual_revision, revision);
1159 } else {
1160 // Gets incremented by FinalizeCatalog() afterwards!
1161 ✗ root_catalog->SetRevision(manual_revision - 1);
1162 }
1163 }
1164
1165 // do the actual catalog snapshotting and upload
1166
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 CatalogInfo root_catalog_info;
1167
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 if (getenv("_CVMFS_SERIALIZED_CATALOG_PROCESSING_") == NULL)
1168
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 root_catalog_info = SnapshotCatalogs(stop_for_tweaks);
1169 else
1170 ✗ root_catalog_info = SnapshotCatalogsSerialized(stop_for_tweaks);
1171
2/4
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 22 times.
 22 if (spooler_->GetNumberOfErrors() > 0) {
1172 ✗ LogCvmfs(kLogCatalog, kLogStderr, "failed to commit catalogs");
1173 ✗ return false;
1174 }
1175
1176 // .cvmfspublished export
1177
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 LogCvmfs(kLogCatalog, kLogVerboseMsg, "Committing repository manifest");
1178 22 set_base_hash(root_catalog_info.content_hash);
1179
1180 22 manifest->set_catalog_hash(root_catalog_info.content_hash);
1181 22 manifest->set_catalog_size(root_catalog_info.size);
1182
2/4
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 22 times.
✗ Branch 6 not taken.
 22 manifest->set_root_path("");
1183 22 manifest->set_ttl(root_catalog_info.ttl);
1184 22 manifest->set_revision(root_catalog_info.revision);
1185
1186 22 return true;
1187 }
1188
1189
1190 /**
1191 * Handles the snapshotting of dirty (i.e. modified) catalogs while trying to
1192 * parallelize the compression and upload as much as possible. We use a parallel
1193 * depth first post order tree traversal based on 'continuations'.
1194 *
1195 * The idea is as follows:
1196 * 1. find all leaf-catalogs (i.e. dirty catalogs with no dirty children)
1197 * --> these can be processed and uploaded immediately and independently
1198 * see WritableCatalogManager::GetModifiedCatalogLeafs()
1199 * 2. annotate non-leaf catalogs with their number of dirty children
1200 * --> a finished child will notify it's parent and decrement this number
1201 * see WritableCatalogManager::CatalogUploadCallback()
1202 * 3. if a non-leaf catalog's dirty children number reaches 0, it is scheduled
1203 * for processing as well (continuation)
1204 * --> the parallel processing walks bottom-up through the catalog tree
1205 * see WritableCatalogManager::CatalogUploadCallback()
1206 * 4. when the root catalog is reached, we notify the main thread and return
1207 * --> done through a Future<> in WritableCatalogManager::SnapshotCatalogs
1208 *
1209 * Note: The catalog finalisation (see WritableCatalogManager::FinalizeCatalog)
1210 * happens in a worker thread (i.e. the callback method) for non-leaf
1211 * catalogs.
1212 *
1213 * TODO(rmeusel): since all leaf catalogs are finalized in the main thread, we
1214 * sacrifice some potential concurrency for simplicity.
1215 */
1216 22 WritableCatalogManager::CatalogInfo WritableCatalogManager::SnapshotCatalogs(
1217 const bool stop_for_tweaks) {
1218 // prepare environment for parallel processing
1219
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 Future<CatalogInfo> root_catalog_info_future;
1220 CatalogUploadContext upload_context;
1221 22 upload_context.root_catalog_info = &root_catalog_info_future;
1222 22 upload_context.stop_for_tweaks = stop_for_tweaks;
1223
1224
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 spooler_->RegisterListener(
1225 &WritableCatalogManager::CatalogUploadCallback, this, upload_context);
1226
1227 // find dirty leaf catalogs and annotate non-leaf catalogs (dirty child count)
1228 // post-condition: the entire catalog tree is ready for concurrent processing
1229 22 WritableCatalogList leafs_to_snapshot;
1230
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 GetModifiedCatalogLeafs(&leafs_to_snapshot);
1231
1232 // finalize and schedule the catalog processing
1233 22 WritableCatalogList::const_iterator i = leafs_to_snapshot.begin();
1234 22 const WritableCatalogList::const_iterator iend = leafs_to_snapshot.end();
1235
2/2
✓ Branch 2 taken 33 times.
✓ Branch 3 taken 22 times.
 55 for (; i != iend; ++i) {
1236
1/2
✓ Branch 2 taken 33 times.
✗ Branch 3 not taken.
 33 FinalizeCatalog(*i, stop_for_tweaks);
1237
1/2
✓ Branch 2 taken 33 times.
✗ Branch 3 not taken.
 33 ScheduleCatalogProcessing(*i);
1238 }
1239
1240
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 LogCvmfs(kLogCatalog, kLogVerboseMsg, "waiting for upload of catalogs");
1241
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 CatalogInfo& root_catalog_info = root_catalog_info_future.Get();
1242
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 spooler_->WaitForUpload();
1243
1244
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 spooler_->UnregisterListeners();
1245 22 return root_catalog_info;
1246 22 }
1247
1248
1249 54 void WritableCatalogManager::FinalizeCatalog(WritableCatalog *catalog,
1250 const bool stop_for_tweaks) {
1251 // update meta information of this catalog
1252
1/3
✓ Branch 2 taken 54 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 54 LogCvmfs(kLogCatalog, kLogVerboseMsg, "creating snapshot of catalog '%s'",
1253 108 catalog->mountpoint().c_str());
1254
1255 54 catalog->UpdateCounters();
1256 54 catalog->UpdateLastModified();
1257 54 catalog->IncrementRevision();
1258
1259 // update the previous catalog revision pointer
1260
2/2
✓ Branch 1 taken 22 times.
✓ Branch 2 taken 32 times.
 54 if (catalog->IsRoot()) {
1261
1/4
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 22 LogCvmfs(kLogCatalog, kLogVerboseMsg, "setting '%s' as previous revision "
1262 "for root catalog",
1263 44 base_hash().ToStringWithSuffix().c_str());
1264 22 catalog->SetPreviousRevision(base_hash());
1265 } else {
1266 // Multiple catalogs might query the parent concurrently
1267 32 SyncLock();
1268
1/2
✓ Branch 1 taken 32 times.
✗ Branch 2 not taken.
 32 shash::Any hash_previous;
1269 uint64_t size_previous;
1270 const bool retval =
1271
2/4
✓ Branch 2 taken 32 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 32 times.
✗ Branch 6 not taken.
 32 catalog->parent()->FindNested(catalog->mountpoint(),
1272 &hash_previous, &size_previous);
1273
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 32 times.
 32 assert(retval);
1274 32 SyncUnlock();
1275
1276
1/8
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 32 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
 64 LogCvmfs(kLogCatalog, kLogVerboseMsg, "found '%s' as previous revision "
1277 "for nested catalog '%s'",
1278
1/2
✓ Branch 1 taken 32 times.
✗ Branch 2 not taken.
 64 hash_previous.ToStringWithSuffix().c_str(),
1279
1/2
✓ Branch 1 taken 32 times.
✗ Branch 2 not taken.
 64 catalog->mountpoint().c_str());
1280
1/2
✓ Branch 1 taken 32 times.
✗ Branch 2 not taken.
 32 catalog->SetPreviousRevision(hash_previous);
1281 }
1282 54 catalog->Commit();
1283
1284 // check if catalog has too many entries
1285
2/2
✓ Branch 0 taken 22 times.
✓ Branch 1 taken 32 times.
 54 uint64_t catalog_limit = uint64_t(1000) *
1286 54 uint64_t((catalog->IsRoot()
1287 22 ? root_kcatalog_limit_
1288 32 : nested_kcatalog_limit_));
1289
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 54 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 54 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 54 if ((catalog_limit > 0) &&
1290 ✗ (catalog->GetCounters().GetSelfEntries() > catalog_limit)) {
1291 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1292 "%s: catalog at %s has more than %lu entries (%lu). "
1293 "Large catalogs stress the CernVM-FS transport infrastructure. "
1294 "Please split it into nested catalogs or increase the limit.",
1295 ✗ enforce_limits_ ? "FATAL" : "WARNING",
1296 ✗ (catalog->IsRoot() ? "/" : catalog->mountpoint().c_str()),
1297 ✗ catalog_limit, catalog->GetCounters().GetSelfEntries());
1298 ✗ if (enforce_limits_)
1299 ✗ PANIC(kLogStderr, "catalog at %s has more than %u entries (%u). ",
1300 (catalog->IsRoot() ? "/" : catalog->mountpoint().c_str()),
1301 catalog_limit, catalog->GetCounters().GetSelfEntries());
1302 }
1303
1304 // allow for manual adjustments in the catalog
1305
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 54 times.
 54 if (stop_for_tweaks) {
1306 ✗ LogCvmfs(kLogCatalog, kLogStdout, "Allowing for tweaks in %s at %s "
1307 "(hit return to continue)",
1308 ✗ catalog->database_path().c_str(), catalog->mountpoint().c_str());
1309 ✗ int read_char = getchar();
1310 ✗ assert(read_char != EOF);
1311 }
1312
1313 // compaction of bloated catalogs (usually after high database churn)
1314 54 catalog->VacuumDatabaseIfNecessary();
1315 54 }
1316
1317
1318 54 void WritableCatalogManager::ScheduleCatalogProcessing(
1319 WritableCatalog *catalog) {
1320 {
1321 54 MutexLockGuard guard(catalog_processing_lock_);
1322 // register catalog object for WritableCatalogManager::CatalogUploadCallback
1323
2/4
✓ Branch 1 taken 54 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 54 times.
✗ Branch 5 not taken.
 54 catalog_processing_map_[catalog->database_path()] = catalog;
1324 54 }
1325
1/2
✓ Branch 2 taken 54 times.
✗ Branch 3 not taken.
 54 spooler_->ProcessCatalog(catalog->database_path());
1326 54 }
1327
1328 /**
1329 * Copy catalog to local cache.server
1330 * Must be an atomic write into the cache_dir
1331 * As such: create a temporary copy in cache_dir/txn and then do a
1332 * `rename` (which is atomic) to the actual cache path
1333 *
1334 * @returns true on success, otherwise false
1335 */
1336 ✗ bool WritableCatalogManager::CopyCatalogToLocalCache(
1337 const upload::SpoolerResult &result) {
1338 ✗ std::string tmp_catalog_path;
1339 ✗ const std::string cache_catalog_path = dir_cache_ + "/"
1340 ✗ + result.content_hash.MakePathWithoutSuffix();
1341 ✗ FILE *fcatalog = CreateTempFile(dir_cache_ + "/txn/catalog", 0666,
1342 "w", &tmp_catalog_path);
1343 ✗ if (!fcatalog) {
1344 ✗ PANIC(kLogDebug | kLogStderr,
1345 "Creating file for temporary catalog failed: %s",
1346 tmp_catalog_path.c_str());
1347 }
1348 ✗ CopyPath2File(result.local_path.c_str(), fcatalog);
1349 ✗ (void) fclose(fcatalog);
1350
1351 ✗ if (rename(tmp_catalog_path.c_str(), cache_catalog_path.c_str()) != 0) {
1352 ✗ PANIC(kLogDebug | kLogStderr,
1353 "Failed to copy catalog from %s to cache %s",
1354 result.local_path.c_str(), cache_catalog_path.c_str());
1355 }
1356 ✗ return true;
1357 }
1358
1359 54 void WritableCatalogManager::CatalogUploadCallback(
1360 const upload::SpoolerResult &result,
1361 const CatalogUploadContext catalog_upload_context) {
1362
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 54 times.
 54 if (result.return_code != 0) {
1363 ✗ PANIC(kLogStderr, "failed to upload '%s' (retval: %d)",
1364 result.local_path.c_str(), result.return_code);
1365 }
1366
1367 // retrieve the catalog object based on the callback information
1368 // see WritableCatalogManager::ScheduleCatalogProcessing()
1369 54 WritableCatalog *catalog = NULL;
1370 {
1371 54 MutexLockGuard guard(catalog_processing_lock_);
1372 std::map<std::string, WritableCatalog*>::iterator c =
1373
1/2
✓ Branch 1 taken 54 times.
✗ Branch 2 not taken.
 54 catalog_processing_map_.find(result.local_path);
1374
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 54 times.
 54 assert(c != catalog_processing_map_.end());
1375 54 catalog = c->second;
1376 54 }
1377
1378 54 uint64_t catalog_size = GetFileSize(result.local_path);
1379
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 54 times.
 54 assert(catalog_size > 0);
1380
1381
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 54 times.
 54 if (UseLocalCache()) {
1382 ✗ CopyCatalogToLocalCache(result);
1383 }
1384
1385 54 SyncLock();
1386
2/2
✓ Branch 1 taken 32 times.
✓ Branch 2 taken 22 times.
 54 if (catalog->HasParent()) {
1387 // finalized nested catalogs will update their parent's pointer and schedule
1388 // them for processing (continuation) if the 'dirty children count' == 0
1389 32 LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating nested catalog link");
1390 32 WritableCatalog *parent = catalog->GetWritableParent();
1391
1392
2/4
✓ Branch 1 taken 32 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 32 times.
✗ Branch 5 not taken.
 32 parent->UpdateNestedCatalog(catalog->mountpoint().ToString(),
1393 32 result.content_hash,
1394 catalog_size,
1395 32 catalog->delta_counters_);
1396 32 catalog->delta_counters_.SetZero();
1397
1398 const int remaining_dirty_children =
1399 32 catalog->GetWritableParent()->DecrementDirtyChildren();
1400
1401 32 SyncUnlock();
1402
1403 // continuation of the dirty catalog tree traversal
1404 // see WritableCatalogManager::SnapshotCatalogs()
1405
2/2
✓ Branch 0 taken 21 times.
✓ Branch 1 taken 11 times.
 32 if (remaining_dirty_children == 0) {
1406 21 FinalizeCatalog(parent, catalog_upload_context.stop_for_tweaks);
1407 21 ScheduleCatalogProcessing(parent);
1408 }
1409
1410
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 } else if (catalog->IsRoot()) {
1411 // once the root catalog is reached, we are done with processing and report
1412 // back to the main via a Future<> and provide the necessary information
1413
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 CatalogInfo root_catalog_info;
1414 22 root_catalog_info.size = catalog_size;
1415
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 root_catalog_info.ttl = catalog->GetTTL();
1416 22 root_catalog_info.content_hash = result.content_hash;
1417
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 root_catalog_info.revision = catalog->GetRevision();
1418
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 catalog_upload_context.root_catalog_info->Set(root_catalog_info);
1419 22 SyncUnlock();
1420 } else {
1421 ✗ PANIC(kLogStderr, "inconsistent state detected");
1422 }
1423 54 }
1424
1425
1426 /**
1427 * Finds dirty catalogs that can be snapshot right away and annotates all the
1428 * other catalogs with their number of dirty descendants.
1429 * Note that there is a convenience wrapper to start the recursion:
1430 * WritableCatalogManager::GetModifiedCatalogLeafs()
1431 *
1432 * @param catalog the catalog for this recursion step
1433 * @param result the result list to be appended to
1434 * @return true if 'catalog' is dirty
1435 */
1436 55 bool WritableCatalogManager::GetModifiedCatalogLeafsRecursively(
1437 Catalog *catalog,
1438 WritableCatalogList *result) const {
1439 55 WritableCatalog *wr_catalog = static_cast<WritableCatalog *>(catalog);
1440
1441 // Look for dirty catalogs in the descendants of *catalog
1442 55 int dirty_children = 0;
1443
1/2
✓ Branch 1 taken 55 times.
✗ Branch 2 not taken.
 55 CatalogList children = wr_catalog->GetChildren();
1444 55 CatalogList::const_iterator i = children.begin();
1445 55 const CatalogList::const_iterator iend = children.end();
1446
2/2
✓ Branch 2 taken 32 times.
✓ Branch 3 taken 55 times.
 87 for (; i != iend; ++i) {
1447
2/4
✓ Branch 2 taken 32 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 32 times.
✗ Branch 5 not taken.
 32 if (GetModifiedCatalogLeafsRecursively(*i, result)) {
1448 32 ++dirty_children;
1449 }
1450 }
1451
1452 // a catalog is dirty if itself or one of its children has changed
1453 // a leaf catalog doesn't have any dirty children
1454 55 wr_catalog->set_dirty_children(dirty_children);
1455
1/4
✗ Branch 1 not taken.
✓ Branch 2 taken 55 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 55 const bool is_dirty = wr_catalog->IsDirty() || dirty_children > 0;
1456 55 const bool is_leaf = dirty_children == 0;
1457
3/4
✓ Branch 0 taken 55 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 34 times.
✓ Branch 3 taken 21 times.
 55 if (is_dirty && is_leaf) {
1458
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 result->push_back(const_cast<WritableCatalog *>(wr_catalog));
1459 }
1460
1461 55 return is_dirty;
1462 55 }
1463
1464
1465 ✗ void WritableCatalogManager::DoBalance() {
1466 ✗ CatalogList catalog_list = GetCatalogs();
1467 ✗ reverse(catalog_list.begin(), catalog_list.end());
1468 ✗ for (unsigned i = 0; i < catalog_list.size(); ++i) {
1469 ✗ FixWeight(static_cast<WritableCatalog*>(catalog_list[i]));
1470 }
1471 }
1472
1473 ✗ void WritableCatalogManager::FixWeight(WritableCatalog* catalog) {
1474 // firstly check underflow because they can provoke overflows
1475 ✗ if (catalog->GetNumEntries() < min_weight_ &&
1476 ✗ !catalog->IsRoot() &&
1477 ✗ catalog->IsAutogenerated()) {
1478 ✗ LogCvmfs(kLogCatalog, kLogStdout,
1479 "Deleting an autogenerated catalog in '%s'",
1480 ✗ catalog->mountpoint().c_str());
1481 // Remove the .cvmfscatalog and .cvmfsautocatalog files first
1482 ✗ string path = catalog->mountpoint().ToString();
1483 ✗ catalog->RemoveEntry(path + "/.cvmfscatalog");
1484 ✗ catalog->RemoveEntry(path + "/.cvmfsautocatalog");
1485 // Remove the actual catalog
1486 ✗ string catalog_path = catalog->mountpoint().ToString().substr(1);
1487 ✗ RemoveNestedCatalog(catalog_path);
1488 ✗ } else if (catalog->GetNumEntries() > max_weight_) {
1489 ✗ CatalogBalancer<WritableCatalogManager> catalog_balancer(this);
1490 ✗ catalog_balancer.Balance(catalog);
1491 }
1492 }
1493
1494
1495 //****************************************************************************
1496 // Workaround -- Serialized Catalog Committing
1497
1498 ✗ int WritableCatalogManager::GetModifiedCatalogsRecursively(
1499 const Catalog *catalog,
1500 WritableCatalogList *result) const
1501 {
1502 // A catalog must be snapshot, if itself or one of it's descendants is dirty.
1503 // So we traverse the catalog tree recursively and look for dirty catalogs
1504 // on the way.
1505 ✗ const WritableCatalog *wr_catalog =
1506 static_cast<const WritableCatalog *>(catalog);
1507 // This variable will contain the number of dirty catalogs in the sub tree
1508 // with *catalog as it's root.
1509 ✗ int dirty_catalogs = (wr_catalog->IsDirty()) ? 1 : 0;
1510
1511 // Look for dirty catalogs in the descendants of *catalog
1512 ✗ CatalogList children = wr_catalog->GetChildren();
1513 ✗ for (CatalogList::const_iterator i = children.begin(), iEnd = children.end();
1514 ✗ i != iEnd; ++i)
1515 {
1516 ✗ dirty_catalogs += GetModifiedCatalogsRecursively(*i, result);
1517 }
1518
1519 // If we found a dirty catalog in the checked sub tree, the root (*catalog)
1520 // must be snapshot and ends up in the result list
1521 ✗ if (dirty_catalogs > 0)
1522 ✗ result->push_back(const_cast<WritableCatalog *>(wr_catalog));
1523
1524 // tell the upper layer about number of catalogs
1525 ✗ return dirty_catalogs;
1526 }
1527
1528
1529 ✗ void WritableCatalogManager::CatalogUploadSerializedCallback(
1530 const upload::SpoolerResult &result,
1531 const CatalogUploadContext unused)
1532 {
1533 ✗ if (result.return_code != 0) {
1534 ✗ PANIC(kLogStderr, "failed to upload '%s' (retval: %d)",
1535 result.local_path.c_str(), result.return_code);
1536 }
1537
1538 ✗ if (UseLocalCache()) {
1539 ✗ CopyCatalogToLocalCache(result);
1540 }
1541
1542 ✗ unlink(result.local_path.c_str());
1543 }
1544
1545
1546 WritableCatalogManager::CatalogInfo
1547 ✗ WritableCatalogManager::SnapshotCatalogsSerialized(
1548 const bool stop_for_tweaks)
1549 {
1550 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Serialized committing of file catalogs...");
1551 ✗ reinterpret_cast<WritableCatalog *>(GetRootCatalog())->SetDirty();
1552 ✗ WritableCatalogList catalogs_to_snapshot;
1553 ✗ GetModifiedCatalogs(&catalogs_to_snapshot);
1554 CatalogUploadContext unused;
1555 ✗ unused.root_catalog_info = NULL;
1556 ✗ unused.stop_for_tweaks = false;
1557 ✗ spooler_->RegisterListener(
1558 &WritableCatalogManager::CatalogUploadSerializedCallback, this, unused);
1559
1560 ✗ CatalogInfo root_catalog_info;
1561 ✗ WritableCatalogList::const_iterator i = catalogs_to_snapshot.begin();
1562 ✗ const WritableCatalogList::const_iterator iend = catalogs_to_snapshot.end();
1563 ✗ for (; i != iend; ++i) {
1564 ✗ FinalizeCatalog(*i, stop_for_tweaks);
1565
1566 // Compress and upload catalog
1567 ✗ shash::Any hash_catalog(spooler_->GetHashAlgorithm(),
1568 ✗ shash::kSuffixCatalog);
1569 ✗ if (!zlib::CompressPath2Null((*i)->database_path(),
1570 &hash_catalog))
1571 {
1572 ✗ PANIC(kLogStderr, "could not compress catalog %s",
1573 (*i)->mountpoint().ToString().c_str());
1574 }
1575
1576 ✗ int64_t catalog_size = GetFileSize((*i)->database_path());
1577 ✗ assert(catalog_size > 0);
1578
1579 ✗ if ((*i)->HasParent()) {
1580 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating nested catalog link");
1581 ✗ WritableCatalog *parent = (*i)->GetWritableParent();
1582 ✗ parent->UpdateNestedCatalog((*i)->mountpoint().ToString(), hash_catalog,
1583 ✗ catalog_size, (*i)->delta_counters_);
1584 ✗ (*i)->delta_counters_.SetZero();
1585 ✗ } else if ((*i)->IsRoot()) {
1586 ✗ root_catalog_info.size = catalog_size;
1587 ✗ root_catalog_info.ttl = (*i)->GetTTL();
1588 ✗ root_catalog_info.content_hash = hash_catalog;
1589 ✗ root_catalog_info.revision = (*i)->GetRevision();
1590 } else {
1591 ✗ PANIC(kLogStderr, "inconsistent state detected");
1592 }
1593
1594 ✗ spooler_->ProcessCatalog((*i)->database_path());
1595 }
1596 ✗ spooler_->WaitForUpload();
1597
1598 ✗ spooler_->UnregisterListeners();
1599 ✗ return root_catalog_info;
1600 }
1601
1602 } // namespace catalog
1603