GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/catalog_mgr_rw.cc
Date: 2024-04-28 02:33:07
Exec Total Coverage
Lines: 419 675 62.1%
Branches: 321 1020 31.5%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM file system.
3 */
4
5 #define __STDC_FORMAT_MACROS
6
7 #include "catalog_mgr_rw.h"
8
9 #include <inttypes.h>
10 #include <unistd.h>
11
12 #include <cassert>
13 #include <cstdio>
14 #include <cstdlib>
15 #include <string>
16
17 #include "catalog_balancer.h"
18 #include "catalog_rw.h"
19 #include "manifest.h"
20 #include "statistics.h"
21 #include "upload.h"
22 #include "util/exception.h"
23 #include "util/logging.h"
24 #include "util/posix.h"
25 #include "util/smalloc.h"
26
27 using namespace std; // NOLINT
28
29 namespace catalog {
30
31 27 WritableCatalogManager::WritableCatalogManager(
32 const shash::Any &base_hash,
33 const std::string &stratum0,
34 const string &dir_temp,
35 upload::Spooler *spooler,
36 download::DownloadManager *download_manager,
37 bool enforce_limits,
38 const unsigned nested_kcatalog_limit,
39 const unsigned root_kcatalog_limit,
40 const unsigned file_mbyte_limit,
41 perf::Statistics *statistics,
42 bool is_balanceable,
43 unsigned max_weight,
44 27 unsigned min_weight)
45 : SimpleCatalogManager(base_hash, stratum0, dir_temp, download_manager,
46 statistics)
47 27 , spooler_(spooler)
48 27 , enforce_limits_(enforce_limits)
49 27 , nested_kcatalog_limit_(nested_kcatalog_limit)
50 27 , root_kcatalog_limit_(root_kcatalog_limit)
51 27 , file_mbyte_limit_(file_mbyte_limit)
52 27 , is_balanceable_(is_balanceable)
53 27 , max_weight_(max_weight)
54 27 , min_weight_(min_weight)
55 27 , balance_weight_(max_weight / 2)
56 {
57 27 sync_lock_ =
58 27 reinterpret_cast<pthread_mutex_t *>(smalloc(sizeof(pthread_mutex_t)));
59 27 int retval = pthread_mutex_init(sync_lock_, NULL);
60
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 27 times.
 27 assert(retval == 0);
61 27 catalog_processing_lock_ =
62 27 reinterpret_cast<pthread_mutex_t *>(smalloc(sizeof(pthread_mutex_t)));
63 27 retval = pthread_mutex_init(catalog_processing_lock_, NULL);
64
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 27 times.
 27 assert(retval == 0);
65 27 }
66
67
68 108 WritableCatalogManager::~WritableCatalogManager() {
69 54 pthread_mutex_destroy(sync_lock_);
70 54 free(sync_lock_);
71 54 pthread_mutex_destroy(catalog_processing_lock_);
72 54 free(catalog_processing_lock_);
73 108 }
74
75
76 /**
77 * This method is virtual in AbstractCatalogManager. It returns a new catalog
78 * structure in the form the different CatalogManagers need it.
79 * In this case it returns a stub for a WritableCatalog.
80 * @param mountpoint the mount point of the catalog stub to create
81 * @param catalog_hash the content hash of the catalog to create
82 * @param parent_catalog the parent of the catalog stub to create
83 * @return a pointer to the catalog stub structure created
84 */
85 66 Catalog* WritableCatalogManager::CreateCatalog(
86 const PathString &mountpoint,
87 const shash::Any &catalog_hash,
88 Catalog *parent_catalog)
89 {
90 132 return new WritableCatalog(mountpoint.ToString(),
91 catalog_hash,
92
2/4
✓ Branch 1 taken 66 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 66 times.
✗ Branch 5 not taken.
 132 parent_catalog);
93 }
94
95
96 66 void WritableCatalogManager::ActivateCatalog(Catalog *catalog) {
97 66 catalog->TakeDatabaseFileOwnership();
98 66 }
99
100
101 /**
102 * This method is invoked if we create a completely new repository.
103 * The new root catalog will already contain a root entry.
104 * It is uploaded by a Forklift to the upstream storage.
105 * @return true on success, false otherwise
106 */
107 22 manifest::Manifest *WritableCatalogManager::CreateRepository(
108 const string &dir_temp,
109 const bool volatile_content,
110 const std::string &voms_authz,
111 upload::Spooler *spooler)
112 {
113 // Create a new root catalog at file_path
114
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 string file_path = dir_temp + "/new_root_catalog";
115
116 22 shash::Algorithms hash_algorithm = spooler->GetHashAlgorithm();
117
118 // A newly created catalog always needs a root entry
119 // we create and configure this here
120
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 DirectoryEntry root_entry;
121 22 root_entry.inode_ = DirectoryEntry::kInvalidInode;
122 22 root_entry.mode_ = 16877;
123 22 root_entry.size_ = 4096;
124 22 root_entry.mtime_ = time(NULL);
125 22 root_entry.uid_ = getuid();
126 22 root_entry.gid_ = getgid();
127
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 root_entry.checksum_ = shash::Any(hash_algorithm);
128 22 root_entry.linkcount_ = 2;
129
1/2
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
 22 string root_path = "";
130
131 // Create the database schema and the initial root entry
132 {
133
2/4
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 22 times.
✗ Branch 5 not taken.
 22 UniquePtr<CatalogDatabase> new_clg_db(CatalogDatabase::Create(file_path));
134
2/4
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 22 times.
 44 if (!new_clg_db.IsValid() ||
135
2/4
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 22 times.
 22 !new_clg_db->InsertInitialValues(root_path,
136 volatile_content,
137 voms_authz,
138 root_entry))
139 {
140 ✗ LogCvmfs(kLogCatalog, kLogStderr, "creation of catalog '%s' failed",
141 file_path.c_str());
142 ✗ return NULL;
143 }
144
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 }
145
146 // Compress root catalog;
147
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 int64_t catalog_size = GetFileSize(file_path);
148
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 22 times.
 22 if (catalog_size < 0) {
149 ✗ unlink(file_path.c_str());
150 ✗ return NULL;
151 }
152
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 string file_path_compressed = file_path + ".compressed";
153
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 shash::Any hash_catalog(hash_algorithm, shash::kSuffixCatalog);
154
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 bool retval = zlib::CompressPath2Path(file_path, file_path_compressed,
155 &hash_catalog);
156
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 22 times.
 22 if (!retval) {
157 ✗ LogCvmfs(kLogCatalog, kLogStderr, "compression of catalog '%s' failed",
158 file_path.c_str());
159 ✗ unlink(file_path.c_str());
160 ✗ return NULL;
161 }
162 22 unlink(file_path.c_str());
163
164 // Create manifest
165
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 const string manifest_path = dir_temp + "/manifest";
166 manifest::Manifest *manifest =
167
3/6
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 22 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 22 times.
✗ Branch 9 not taken.
 22 new manifest::Manifest(hash_catalog, catalog_size, "");
168
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 22 times.
 22 if (!voms_authz.empty()) {
169 ✗ manifest->set_has_alt_catalog_path(true);
170 }
171
172 // Upload catalog
173
3/6
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 22 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 22 times.
✗ Branch 8 not taken.
 22 spooler->Upload(file_path_compressed, "data/" + hash_catalog.MakePath());
174
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 spooler->WaitForUpload();
175 22 unlink(file_path_compressed.c_str());
176
2/4
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 22 times.
 22 if (spooler->GetNumberOfErrors() > 0) {
177 ✗ LogCvmfs(kLogCatalog, kLogStderr, "failed to commit catalog %s",
178 file_path_compressed.c_str());
179 ✗ delete manifest;
180 ✗ return NULL;
181 }
182
183 22 return manifest;
184 22 }
185
186
187 /**
188 * Retrieve the catalog containing the given path.
189 * Other than AbstractCatalogManager::FindCatalog() this mounts nested
190 * catalogs if necessary and returns WritableCatalog objects.
191 * Furthermore it optionally returns the looked-up DirectoryEntry.
192 *
193 * @param path the path to look for
194 * @param result the retrieved catalog (as a pointer)
195 * @param dirent is set to looked up DirectoryEntry for 'path' if non-NULL
196 * @return true if catalog was found
197 */
198 299 bool WritableCatalogManager::FindCatalog(const string &path,
199 WritableCatalog **result,
200 DirectoryEntry *dirent) {
201
1/2
✓ Branch 1 taken 299 times.
✗ Branch 2 not taken.
 299 const PathString ps_path(path);
202
203 Catalog *best_fit =
204
1/2
✓ Branch 1 taken 299 times.
✗ Branch 2 not taken.
 299 AbstractCatalogManager<Catalog>::FindCatalog(ps_path);
205
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 299 times.
 299 assert(best_fit != NULL);
206 299 Catalog *catalog = NULL;
207 bool retval =
208
1/2
✓ Branch 1 taken 299 times.
✗ Branch 2 not taken.
 299 MountSubtree(ps_path, best_fit, true /* is_listable */, &catalog);
209
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 299 times.
 299 if (!retval)
210 ✗ return false;
211
212
1/2
✓ Branch 1 taken 299 times.
✗ Branch 2 not taken.
 299 catalog::DirectoryEntry dummy;
213
2/2
✓ Branch 0 taken 158 times.
✓ Branch 1 taken 141 times.
 299 if (NULL == dirent) {
214 158 dirent = &dummy;
215 }
216
1/2
✓ Branch 1 taken 299 times.
✗ Branch 2 not taken.
 299 bool found = catalog->LookupPath(ps_path, dirent);
217
6/8
✓ Branch 0 taken 298 times.
✓ Branch 1 taken 1 times.
✓ Branch 3 taken 298 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 298 times.
✓ Branch 7 taken 1 times.
✓ Branch 8 taken 298 times.
 299 if (!found || !catalog->IsWritable())
218 1 return false;
219
220 298 *result = static_cast<WritableCatalog *>(catalog);
221 298 return true;
222 299 }
223
224
225 ✗ WritableCatalog *WritableCatalogManager::GetHostingCatalog(
226 const std::string &path)
227 {
228 ✗ WritableCatalog *result = NULL;
229 ✗ bool retval = FindCatalog(MakeRelativePath(path), &result, NULL);
230 ✗ if (!retval) return NULL;
231 ✗ return result;
232 }
233
234
235 /**
236 * Remove the given file from the catalogs.
237 * @param file_path the full path to the file to be removed
238 * @return true on success, false otherwise
239 */
240 4 void WritableCatalogManager::RemoveFile(const std::string &path) {
241
1/2
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
 4 const string file_path = MakeRelativePath(path);
242
1/2
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
 4 const string parent_path = GetParentPath(file_path);
243
244 4 SyncLock();
245 WritableCatalog *catalog;
246
2/4
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 4 times.
 4 if (!FindCatalog(parent_path, &catalog)) {
247 ✗ PANIC(kLogStderr, "catalog for file '%s' cannot be found",
248 file_path.c_str());
249 }
250
251
1/2
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
 4 catalog->RemoveEntry(file_path);
252 4 SyncUnlock();
253 4 }
254
255
256 /**
257 * Remove the given directory from the catalogs.
258 * @param directory_path the full path to the directory to be removed
259 * @return true on success, false otherwise
260 */
261 2 void WritableCatalogManager::RemoveDirectory(const std::string &path) {
262
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 const string directory_path = MakeRelativePath(path);
263
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 const string parent_path = GetParentPath(directory_path);
264
265 2 SyncLock();
266 WritableCatalog *catalog;
267
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 DirectoryEntry parent_entry;
268
2/4
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 2 times.
 2 if (!FindCatalog(parent_path, &catalog, &parent_entry)) {
269 ✗ PANIC(kLogStderr, "catalog for directory '%s' cannot be found",
270 directory_path.c_str());
271 }
272
273 2 parent_entry.set_linkcount(parent_entry.linkcount() - 1);
274
275
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 catalog->RemoveEntry(directory_path);
276
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 catalog->UpdateEntry(parent_entry, parent_path);
277
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 2 times.
 2 if (parent_entry.IsNestedCatalogRoot()) {
278 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating transition point %s",
279 parent_path.c_str());
280 WritableCatalog *parent_catalog =
281 ✗ reinterpret_cast<WritableCatalog *>(catalog->parent());
282 ✗ parent_entry.set_is_nested_catalog_mountpoint(true);
283 ✗ parent_entry.set_is_nested_catalog_root(false);
284 ✗ parent_catalog->UpdateEntry(parent_entry, parent_path);
285 }
286 2 SyncUnlock();
287 2 }
288
289 /**
290 * Clone the file called `source` changing its name into `destination`, the
291 * source file is keep intact.
292 * @params destination, the name of the new file, complete path
293 * @params source, the name of the file to clone, which must be already in the
294 * repository
295 * @return void
296 */
297 ✗ void WritableCatalogManager::Clone(const std::string destination,
298 const std::string source) {
299 ✗ const std::string relative_source = MakeRelativePath(source);
300
301 ✗ DirectoryEntry source_dirent;
302 ✗ if (!LookupPath(relative_source, kLookupDefault, &source_dirent)) {
303 ✗ PANIC(kLogStderr, "catalog for file '%s' cannot be found, aborting",
304 source.c_str());
305 }
306 ✗ if (source_dirent.IsDirectory()) {
307 ✗ PANIC(kLogStderr, "Trying to clone a directory: '%s', aborting",
308 source.c_str());
309 }
310
311 // if the file is already there we remove it and we add it back
312 ✗ DirectoryEntry check_dirent;
313 bool destination_already_present =
314 ✗ LookupPath(MakeRelativePath(destination), kLookupDefault, &check_dirent);
315 ✗ if (destination_already_present) {
316 ✗ this->RemoveFile(destination);
317 }
318
319 ✗ DirectoryEntry destination_dirent(source_dirent);
320 ✗ std::string destination_dirname;
321 ✗ std::string destination_filename;
322 ✗ SplitPath(destination, &destination_dirname, &destination_filename);
323
324 ✗ destination_dirent.name_.Assign(
325 ✗ NameString(destination_filename.c_str(), destination_filename.length()));
326
327 // TODO(jblomer): clone is used by tarball engine and should eventually
328 // support extended attributes
329 ✗ this->AddFile(destination_dirent, empty_xattrs, destination_dirname);
330 }
331
332
333 /**
334 * Copies an entire directory tree from the existing from_dir to the
335 * non-existing to_dir. The destination's parent directory must exist. On the
336 * catalog level, the new entries will be identical to the old ones except
337 * for their path hash fields.
338 */
339 10 void WritableCatalogManager::CloneTree(const std::string &from_dir,
340 const std::string &to_dir)
341 {
342 // Sanitize input paths
343
6/6
✓ Branch 1 taken 8 times.
✓ Branch 2 taken 2 times.
✓ Branch 4 taken 1 times.
✓ Branch 5 taken 7 times.
✓ Branch 6 taken 3 times.
✓ Branch 7 taken 7 times.
 10 if (from_dir.empty() || to_dir.empty())
344 3 PANIC(kLogStderr, "clone tree from or to root impossible");
345
346
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 const std::string relative_source = MakeRelativePath(from_dir);
347
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 const std::string relative_dest = MakeRelativePath(to_dir);
348
349
2/2
✓ Branch 1 taken 1 times.
✓ Branch 2 taken 6 times.
 7 if (relative_source == relative_dest) {
350 1 PANIC(kLogStderr, "cannot clone tree into itself ('%s')", to_dir.c_str());
351 }
352
4/7
✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 6 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 1 times.
✓ Branch 8 taken 5 times.
 6 if (HasPrefix(relative_dest, relative_source + "/", false /*ignore_case*/)) {
353 1 PANIC(kLogStderr,
354 "cannot clone tree into sub directory of source '%s' --> '%s'",
355 from_dir.c_str(), to_dir.c_str());
356 }
357
358
1/2
✓ Branch 1 taken 5 times.
✗ Branch 2 not taken.
 5 DirectoryEntry source_dirent;
359
3/4
✓ Branch 1 taken 5 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 times.
✓ Branch 4 taken 4 times.
 5 if (!LookupPath(relative_source, kLookupDefault, &source_dirent)) {
360 1 PANIC(kLogStderr, "path '%s' cannot be found, aborting", from_dir.c_str());
361 }
362
2/2
✓ Branch 1 taken 1 times.
✓ Branch 2 taken 3 times.
 4 if (!source_dirent.IsDirectory()) {
363 1 PANIC(kLogStderr, "CloneTree: source '%s' not a directory, aborting",
364 from_dir.c_str());
365 }
366
367
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 DirectoryEntry dest_dirent;
368
3/4
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 times.
✓ Branch 4 taken 2 times.
 3 if (LookupPath(relative_dest, kLookupDefault, &dest_dirent)) {
369 1 PANIC(kLogStderr, "destination '%s' exists, aborting", to_dir.c_str());
370 }
371
372
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 const std::string dest_parent = GetParentPath(relative_dest);
373
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 DirectoryEntry dest_parent_dirent;
374
3/4
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 times.
✓ Branch 4 taken 1 times.
 2 if (!LookupPath(dest_parent, kLookupDefault, &dest_parent_dirent)) {
375 1 PANIC(kLogStderr, "destination '%s' not on a known path, aborting",
376 to_dir.c_str());
377 }
378
379
2/4
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1 times.
✗ Branch 5 not taken.
 1 CloneTreeImpl(PathString(from_dir),
380
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 2 GetParentPath(to_dir),
381
2/4
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1 times.
✗ Branch 5 not taken.
 2 NameString(GetFileName(to_dir)));
382 21 }
383
384
385 /**
386 * Called from CloneTree(), assumes that from_dir and to_dir are sufficiently
387 * sanitized
388 */
389 8 void WritableCatalogManager::CloneTreeImpl(
390 const PathString &source_dir,
391 const std::string &dest_parent_dir,
392 const NameString &dest_name)
393 {
394
1/2
✓ Branch 4 taken 8 times.
✗ Branch 5 not taken.
 8 LogCvmfs(kLogCatalog, kLogDebug, "cloning %s --> %s/%s", source_dir.c_str(),
395
1/2
✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
 16 dest_parent_dir.c_str(), dest_name.ToString().c_str());
396
3/6
✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 8 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 8 times.
✗ Branch 8 not taken.
 16 PathString relative_source(MakeRelativePath(source_dir.ToString()));
397
398
1/2
✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
 8 DirectoryEntry source_dirent;
399
1/2
✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
 8 bool retval = LookupPath(relative_source, kLookupDefault, &source_dirent);
400
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 8 times.
 8 assert(retval);
401
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 8 times.
 8 assert(!source_dirent.IsBindMountpoint());
402
403
1/2
✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
 8 DirectoryEntry dest_dirent(source_dirent);
404
1/2
✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
 8 dest_dirent.name_.Assign(dest_name);
405 // Just in case, reset the nested catalog markers
406 8 dest_dirent.set_is_nested_catalog_mountpoint(false);
407 8 dest_dirent.set_is_nested_catalog_root(false);
408
409 8 XattrList xattrs;
410
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 8 times.
 8 if (source_dirent.HasXattrs()) {
411 ✗ retval = LookupXattrs(relative_source, &xattrs);
412 ✗ assert(retval);
413 }
414
1/2
✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
 8 AddDirectory(dest_dirent, xattrs, dest_parent_dir);
415
416
1/2
✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
 8 std::string dest_dir = dest_parent_dir;
417
2/2
✓ Branch 1 taken 7 times.
✓ Branch 2 taken 1 times.
 8 if (!dest_dir.empty())
418
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 dest_dir.push_back('/');
419
2/4
✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 8 times.
✗ Branch 5 not taken.
 8 dest_dir += dest_name.ToString();
420
5/6
✓ Branch 1 taken 5 times.
✓ Branch 2 taken 3 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 5 times.
✓ Branch 5 taken 3 times.
✓ Branch 6 taken 5 times.
 13 if (source_dirent.IsNestedCatalogRoot() ||
421 5 source_dirent.IsNestedCatalogMountpoint())
422 {
423
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 CreateNestedCatalog(dest_dir);
424 }
425
426 8 DirectoryEntryList ls;
427
1/2
✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
 8 retval = Listing(relative_source, &ls, false /* expand_symlink */);
428
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 8 times.
 8 assert(retval);
429
2/2
✓ Branch 1 taken 17 times.
✓ Branch 2 taken 8 times.
 25 for (unsigned i = 0; i < ls.size(); ++i) {
430
1/2
✓ Branch 1 taken 17 times.
✗ Branch 2 not taken.
 17 PathString sub_path(source_dir);
431
2/4
✓ Branch 1 taken 17 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 17 times.
 17 assert(!sub_path.IsEmpty());
432
1/2
✓ Branch 1 taken 17 times.
✗ Branch 2 not taken.
 17 sub_path.Append("/", 1);
433
3/6
✓ Branch 2 taken 17 times.
✗ Branch 3 not taken.
✓ Branch 7 taken 17 times.
✗ Branch 8 not taken.
✓ Branch 11 taken 17 times.
✗ Branch 12 not taken.
 17 sub_path.Append(ls[i].name().GetChars(), ls[i].name().GetLength());
434
435
2/2
✓ Branch 2 taken 7 times.
✓ Branch 3 taken 10 times.
 17 if (ls[i].IsDirectory()) {
436
2/4
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 7 times.
✗ Branch 6 not taken.
 7 CloneTreeImpl(sub_path, dest_dir, ls[i].name());
437 7 continue;
438 }
439
440 // We break hard-links during cloning
441 10 ls[i].set_hardlink_group(0);
442 10 ls[i].set_linkcount(1);
443
444 10 xattrs.Clear();
445
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 10 times.
 10 if (ls[i].HasXattrs()) {
446 ✗ retval = LookupXattrs(sub_path, &xattrs);
447 ✗ assert(retval);
448 }
449
450
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 10 times.
 10 if (ls[i].IsChunkedFile()) {
451 ✗ FileChunkList chunks;
452 ✗ std::string relative_sub_path = MakeRelativePath(sub_path.ToString());
453 ✗ retval = ListFileChunks(
454 ✗ PathString(relative_sub_path), ls[i].hash_algorithm(), &chunks);
455 ✗ assert(retval);
456 ✗ AddChunkedFile(ls[i], xattrs, dest_dir, chunks);
457 ✗ } else {
458
1/2
✓ Branch 2 taken 10 times.
✗ Branch 3 not taken.
 10 AddFile(ls[i], xattrs, dest_dir);
459 }
460
2/2
✓ Branch 1 taken 10 times.
✓ Branch 2 taken 7 times.
 17 }
461 8 }
462
463
464 /**
465 * Add a new directory to the catalogs.
466 * @param entry a DirectoryEntry structure describing the new directory
467 * @param parent_directory the absolute path of the directory containing the
468 * directory to be created
469 * @return true on success, false otherwise
470 */
471 110 void WritableCatalogManager::AddDirectory(const DirectoryEntryBase &entry,
472 const XattrList &xattrs,
473 const std::string &parent_directory)
474 {
475
1/2
✓ Branch 1 taken 110 times.
✗ Branch 2 not taken.
 110 const string parent_path = MakeRelativePath(parent_directory);
476
1/2
✓ Branch 1 taken 110 times.
✗ Branch 2 not taken.
 110 string directory_path = parent_path + "/";
477
3/6
✓ Branch 1 taken 110 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 110 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 110 times.
✗ Branch 10 not taken.
 110 directory_path.append(entry.name().GetChars(), entry.name().GetLength());
478
479 110 SyncLock();
480 WritableCatalog *catalog;
481
1/2
✓ Branch 1 taken 110 times.
✗ Branch 2 not taken.
 110 DirectoryEntry parent_entry;
482
2/4
✓ Branch 1 taken 110 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 110 times.
 110 if (!FindCatalog(parent_path, &catalog, &parent_entry)) {
483 ✗ PANIC(kLogStderr, "catalog for directory '%s' cannot be found",
484 directory_path.c_str());
485 }
486
487
1/2
✓ Branch 1 taken 110 times.
✗ Branch 2 not taken.
 110 DirectoryEntry fixed_hardlink_count(entry);
488 110 fixed_hardlink_count.set_linkcount(2);
489
1/2
✓ Branch 1 taken 110 times.
✗ Branch 2 not taken.
 110 catalog->AddEntry(fixed_hardlink_count, xattrs,
490 directory_path, parent_path);
491
492 110 parent_entry.set_linkcount(parent_entry.linkcount() + 1);
493
1/2
✓ Branch 1 taken 110 times.
✗ Branch 2 not taken.
 110 catalog->UpdateEntry(parent_entry, parent_path);
494
2/2
✓ Branch 1 taken 3 times.
✓ Branch 2 taken 107 times.
 110 if (parent_entry.IsNestedCatalogRoot()) {
495
1/2
✓ Branch 2 taken 3 times.
✗ Branch 3 not taken.
 3 LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating transition point %s",
496 parent_path.c_str());
497 WritableCatalog *parent_catalog =
498 3 reinterpret_cast<WritableCatalog *>(catalog->parent());
499 3 parent_entry.set_is_nested_catalog_mountpoint(true);
500 3 parent_entry.set_is_nested_catalog_root(false);
501
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 parent_catalog->UpdateEntry(parent_entry, parent_path);
502 }
503 110 SyncUnlock();
504 110 }
505
506 /**
507 * Add a new file to the catalogs.
508 * @param entry a DirectoryEntry structure describing the new file
509 * @param parent_directory the absolute path of the directory containing the
510 * file to be created
511 * @return true on success, false otherwise
512 */
513 145 void WritableCatalogManager::AddFile(
514 const DirectoryEntry &entry,
515 const XattrList &xattrs,
516 const std::string &parent_directory)
517 {
518
1/2
✓ Branch 1 taken 145 times.
✗ Branch 2 not taken.
 145 const string parent_path = MakeRelativePath(parent_directory);
519
1/2
✓ Branch 1 taken 145 times.
✗ Branch 2 not taken.
 145 const string file_path = entry.GetFullPath(parent_path);
520
521 145 SyncLock();
522 WritableCatalog *catalog;
523
2/4
✓ Branch 1 taken 145 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 145 times.
 145 if (!FindCatalog(parent_path, &catalog)) {
524 ✗ PANIC(kLogStderr, "catalog for file '%s' cannot be found",
525 file_path.c_str());
526 }
527
528
4/6
✓ Branch 1 taken 142 times.
✓ Branch 2 taken 3 times.
✓ Branch 4 taken 142 times.
✗ Branch 5 not taken.
✗ Branch 8 not taken.
✓ Branch 9 taken 142 times.
 145 assert(!entry.IsRegular() || entry.IsChunkedFile() ||
529 !entry.checksum().IsNull());
530
3/4
✓ Branch 1 taken 3 times.
✓ Branch 2 taken 142 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 3 times.
 145 assert(entry.IsRegular() || !entry.IsExternalFile());
531
532 // check if file is too big
533
1/2
✓ Branch 1 taken 145 times.
✗ Branch 2 not taken.
 145 unsigned mbytes = entry.size() / (1024 * 1024);
534
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 145 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 145 if ((file_mbyte_limit_ > 0) && (mbytes > file_mbyte_limit_)) {
535 ✗ LogCvmfs(kLogCatalog, kLogStderr,
536 "%s: file at %s is larger than %u megabytes (%u). "
537 "CernVM-FS works best with small files. "
538 "Please remove the file or increase the limit.",
539 ✗ enforce_limits_ ? "FATAL" : "WARNING", file_path.c_str(),
540 file_mbyte_limit_, mbytes);
541 ✗ if (enforce_limits_)
542 ✗ PANIC(kLogStderr, "file at %s is larger than %u megabytes (%u).",
543 file_path.c_str(), file_mbyte_limit_, mbytes);
544 }
545
546
1/2
✓ Branch 1 taken 145 times.
✗ Branch 2 not taken.
 145 catalog->AddEntry(entry, xattrs, file_path, parent_path);
547 145 SyncUnlock();
548 145 }
549
550
551 ✗ void WritableCatalogManager::AddChunkedFile(
552 const DirectoryEntryBase &entry,
553 const XattrList &xattrs,
554 const std::string &parent_directory,
555 const FileChunkList &file_chunks)
556 {
557 ✗ assert(file_chunks.size() > 0);
558
559 ✗ DirectoryEntry full_entry(entry);
560 ✗ full_entry.set_is_chunked_file(true);
561
562 ✗ AddFile(full_entry, xattrs, parent_directory);
563
564 ✗ const string parent_path = MakeRelativePath(parent_directory);
565 ✗ const string file_path = entry.GetFullPath(parent_path);
566
567 ✗ SyncLock();
568 WritableCatalog *catalog;
569 ✗ if (!FindCatalog(parent_path, &catalog)) {
570 ✗ PANIC(kLogStderr, "catalog for file '%s' cannot be found",
571 file_path.c_str());
572 }
573
574 ✗ for (unsigned i = 0; i < file_chunks.size(); ++i) {
575 ✗ catalog->AddFileChunk(file_path, *file_chunks.AtPtr(i));
576 }
577 ✗ SyncUnlock();
578 }
579
580
581 /**
582 * Add a hardlink group to the catalogs.
583 * @param entries a list of DirectoryEntries describing the new files
584 * @param parent_directory the absolute path of the directory containing the
585 * files to be created
586 * @return true on success, false otherwise
587 */
588 ✗ void WritableCatalogManager::AddHardlinkGroup(
589 const DirectoryEntryBaseList &entries,
590 const XattrList &xattrs,
591 const std::string &parent_directory,
592 const FileChunkList &file_chunks)
593 {
594 ✗ assert(entries.size() >= 1);
595 ✗ assert(file_chunks.IsEmpty() || entries[0].IsRegular());
596 ✗ if (entries.size() == 1) {
597 ✗ DirectoryEntry fix_linkcount(entries[0]);
598 ✗ fix_linkcount.set_linkcount(1);
599 ✗ if (file_chunks.IsEmpty())
600 ✗ return AddFile(fix_linkcount, xattrs, parent_directory);
601 ✗ return AddChunkedFile(fix_linkcount, xattrs, parent_directory, file_chunks);
602 }
603
604 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "adding hardlink group %s/%s",
605 ✗ parent_directory.c_str(), entries[0].name().c_str());
606
607 // Hardlink groups have to reside in the same directory.
608 // Therefore we only have one parent directory here
609 ✗ const string parent_path = MakeRelativePath(parent_directory);
610
611 // check if hard link is too big
612 ✗ unsigned mbytes = entries[0].size() / (1024 * 1024);
613 ✗ if ((file_mbyte_limit_ > 0) && (mbytes > file_mbyte_limit_)) {
614 ✗ LogCvmfs(kLogCatalog, kLogStderr,
615 "%s: hard link at %s is larger than %u megabytes (%u). "
616 "CernVM-FS works best with small files. "
617 "Please remove the file or increase the limit.",
618 ✗ enforce_limits_ ? "FATAL" : "WARNING",
619 ✗ (parent_path + entries[0].name().ToString()).c_str(),
620 file_mbyte_limit_,
621 mbytes);
622 ✗ if (enforce_limits_)
623 ✗ PANIC(kLogStderr, "hard link at %s is larger than %u megabytes (%u)",
624 (parent_path + entries[0].name().ToString()).c_str(),
625 file_mbyte_limit_, mbytes);
626 }
627
628 ✗ SyncLock();
629 WritableCatalog *catalog;
630 ✗ if (!FindCatalog(parent_path, &catalog)) {
631 ✗ PANIC(kLogStderr,
632 "catalog for hardlink group containing '%s' cannot be found",
633 parent_path.c_str());
634 }
635
636 // Get a valid hardlink group id for the catalog the group will end up in
637 // TODO(unknown): Compaction
638 ✗ uint32_t new_group_id = catalog->GetMaxLinkId() + 1;
639 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "hardlink group id %u issued",
640 new_group_id);
641 ✗ assert(new_group_id > 0);
642
643 // Add the file entries to the catalog
644 ✗ for (DirectoryEntryBaseList::const_iterator i = entries.begin(),
645 ✗ iEnd = entries.end(); i != iEnd; ++i)
646 {
647 ✗ string file_path = parent_path + "/";
648 ✗ file_path.append(i->name().GetChars(), i->name().GetLength());
649
650 // create a fully fledged DirectoryEntry to add the hardlink group to it
651 // which is CVMFS specific meta data.
652 ✗ DirectoryEntry hardlink(*i);
653 ✗ hardlink.set_hardlink_group(new_group_id);
654 ✗ hardlink.set_linkcount(entries.size());
655 ✗ hardlink.set_is_chunked_file(!file_chunks.IsEmpty());
656
657 ✗ catalog->AddEntry(hardlink, xattrs, file_path, parent_path);
658 ✗ if (hardlink.IsChunkedFile()) {
659 ✗ for (unsigned i = 0; i < file_chunks.size(); ++i) {
660 ✗ catalog->AddFileChunk(file_path, *file_chunks.AtPtr(i));
661 }
662 }
663 }
664 ✗ SyncUnlock();
665 }
666
667
668 ✗ void WritableCatalogManager::ShrinkHardlinkGroup(const string &remove_path) {
669 ✗ const string relative_path = MakeRelativePath(remove_path);
670
671 ✗ SyncLock();
672 WritableCatalog *catalog;
673 ✗ if (!FindCatalog(relative_path, &catalog)) {
674 ✗ PANIC(kLogStderr,
675 "catalog for hardlink group containing '%s' cannot be found",
676 remove_path.c_str());
677 }
678
679 ✗ catalog->IncLinkcount(relative_path, -1);
680 ✗ SyncUnlock();
681 }
682
683
684 /**
685 * Update entry meta data (mode, owner, ...).
686 * CVMFS specific meta data (i.e. nested catalog transition points) are NOT
687 * changed by this method, although transition points intrinsics are taken into
688 * account, to keep nested catalogs consistent.
689 * @param entry the directory entry to be touched
690 * @param path the path of the directory entry to be touched
691 */
692 1 void WritableCatalogManager::TouchDirectory(const DirectoryEntryBase &entry,
693 const XattrList &xattrs,
694 const std::string &directory_path)
695 {
696
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1 times.
 1 assert(entry.IsDirectory());
697
698
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 const string entry_path = MakeRelativePath(directory_path);
699
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 const string parent_path = GetParentPath(entry_path);
700
701 1 SyncLock();
702 // find the catalog to be updated
703 WritableCatalog *catalog;
704
2/4
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 times.
 1 if (!FindCatalog(parent_path, &catalog)) {
705 ✗ PANIC(kLogStderr, "catalog for entry '%s' cannot be found",
706 entry_path.c_str());
707 }
708
709
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 catalog->TouchEntry(entry, xattrs, entry_path);
710
711 // since we deal with a directory here, we might just touch a
712 // nested catalog transition point. If this is the case we would need to
713 // update two catalog entries:
714 // * the nested catalog MOUNTPOINT in the parent catalog
715 // * the nested catalog ROOT in the nested catalog
716
717 // first check if we really have a nested catalog transition point
718
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 catalog::DirectoryEntry potential_transition_point;
719
1/2
✓ Branch 3 taken 1 times.
✗ Branch 4 not taken.
 1 PathString transition_path(entry_path.data(), entry_path.length());
720
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 bool retval = catalog->LookupPath(transition_path,
721 &potential_transition_point);
722
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 times.
 1 assert(retval);
723
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1 times.
 1 if (potential_transition_point.IsNestedCatalogMountpoint()) {
724 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg,
725 "updating transition point at %s", entry_path.c_str());
726
727 // find and mount nested catalog associated to this transition point
728 ✗ shash::Any nested_hash;
729 uint64_t nested_size;
730 ✗ retval = catalog->FindNested(transition_path, &nested_hash, &nested_size);
731 ✗ assert(retval);
732 Catalog *nested_catalog;
733 ✗ nested_catalog = MountCatalog(transition_path, nested_hash, catalog);
734 ✗ assert(nested_catalog != NULL);
735
736 // update nested catalog root in the child catalog
737 reinterpret_cast<WritableCatalog *>(nested_catalog)->
738 ✗ TouchEntry(entry, xattrs, entry_path);
739 }
740
741 1 SyncUnlock();
742 1 }
743
744
745 /**
746 * Create a new nested catalog. Includes moving all entries belonging there
747 * from it's parent catalog.
748 * @param mountpoint the path of the directory to become a nested root
749 * @return true on success, false otherwise
750 */
751 29 void WritableCatalogManager::CreateNestedCatalog(const std::string &mountpoint)
752 {
753
1/2
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
 29 const string nested_root_path = MakeRelativePath(mountpoint);
754
1/2
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
 29 const PathString ps_nested_root_path(nested_root_path);
755
756 29 SyncLock();
757 // Find the catalog currently containing the directory structure, which
758 // will be represented as a new nested catalog and its root-entry/mountpoint
759 // along the way
760 29 WritableCatalog *old_catalog = NULL;
761
1/2
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
 29 DirectoryEntry new_root_entry;
762
2/4
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 29 times.
 29 if (!FindCatalog(nested_root_path, &old_catalog, &new_root_entry)) {
763 ✗ PANIC(kLogStderr,
764 "failed to create nested catalog '%s': "
765 "mountpoint was not found in current catalog structure",
766 nested_root_path.c_str());
767 }
768
769 // Create the database schema and the initial root entry
770 // for the new nested catalog
771
1/2
✓ Branch 2 taken 29 times.
✗ Branch 3 not taken.
 29 const string database_file_path = CreateTempPath(dir_temp() + "/catalog",
772
1/2
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
 29 0666);
773 29 const bool volatile_content = false;
774
1/2
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
 29 CatalogDatabase *new_catalog_db = CatalogDatabase::Create(database_file_path);
775
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 29 times.
 29 assert(NULL != new_catalog_db);
776 // Note we do not set the external_data bit for nested catalogs
777 bool retval =
778
2/4
✓ Branch 2 taken 29 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 29 times.
✗ Branch 6 not taken.
 29 new_catalog_db->InsertInitialValues(nested_root_path,
779 volatile_content,
780 "", // At this point, only root
781 // catalog gets VOMS authz
782 new_root_entry);
783
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 29 times.
 29 assert(retval);
784 // TODO(rmeusel): we need a way to attach a catalog directly from an open
785 // database to remove this indirection
786
1/2
✓ Branch 0 taken 29 times.
✗ Branch 1 not taken.
 29 delete new_catalog_db;
787 29 new_catalog_db = NULL;
788
789 // Attach the just created nested catalog
790 Catalog *new_catalog =
791
2/4
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 29 times.
✗ Branch 5 not taken.
 29 CreateCatalog(ps_nested_root_path, shash::Any(), old_catalog);
792
1/2
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
 29 retval = AttachCatalog(database_file_path, new_catalog);
793
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 29 times.
 29 assert(retval);
794
795
2/4
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 29 times.
 29 assert(new_catalog->IsWritable());
796 29 WritableCatalog *wr_new_catalog = static_cast<WritableCatalog *>(new_catalog);
797
798
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 29 times.
 29 if (new_root_entry.HasXattrs()) {
799 ✗ XattrList xattrs;
800 ✗ retval = old_catalog->LookupXattrsPath(ps_nested_root_path, &xattrs);
801 ✗ assert(retval);
802 ✗ wr_new_catalog->TouchEntry(new_root_entry, xattrs, nested_root_path);
803 }
804
805 // From now on, there are two catalogs, spanning the same directory structure
806 // we have to split the overlapping directory entries from the old catalog
807 // to the new catalog to re-gain a valid catalog structure
808
1/2
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
 29 old_catalog->Partition(wr_new_catalog);
809
810 // Add the newly created nested catalog to the references of the containing
811 // catalog
812
4/8
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 29 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 29 times.
✗ Branch 8 not taken.
✓ Branch 10 taken 29 times.
✗ Branch 11 not taken.
 29 old_catalog->InsertNestedCatalog(new_catalog->mountpoint().ToString(), NULL,
813 29 shash::Any(spooler_->GetHashAlgorithm()), 0);
814
815 // Fix subtree counters in new nested catalogs: subtree is the sum of all
816 // entries of all "grand-nested" catalogs
817 // Note: taking a copy of the nested catalog list here
818 const Catalog::NestedCatalogList &grand_nested =
819
1/2
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
 29 wr_new_catalog->ListOwnNestedCatalogs();
820
1/2
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
 29 DeltaCounters fix_subtree_counters;
821 58 for (Catalog::NestedCatalogList::const_iterator i = grand_nested.begin(),
822
1/2
✗ Branch 3 not taken.
✓ Branch 4 taken 29 times.
 58 iEnd = grand_nested.end(); i != iEnd; ++i)
823 {
824 WritableCatalog *grand_catalog;
825 ✗ retval = FindCatalog(i->mountpoint.ToString(), &grand_catalog);
826 ✗ assert(retval);
827 ✗ const Counters &grand_counters = grand_catalog->GetCounters();
828 ✗ grand_counters.AddAsSubtree(&fix_subtree_counters);
829 }
830 29 DeltaCounters save_counters = wr_new_catalog->delta_counters_;
831 29 wr_new_catalog->delta_counters_ = fix_subtree_counters;
832
1/2
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
 29 wr_new_catalog->UpdateCounters();
833 29 wr_new_catalog->delta_counters_ = save_counters;
834
835 29 SyncUnlock();
836 29 }
837
838
839 /**
840 * Remove a nested catalog
841 *
842 * If the merged parameter is true, when you remove a nested catalog
843 * all entries currently held by it will be merged into its parent
844 * catalog.
845 * @param mountpoint - the path of the nested catalog to be removed
846 * @param merge - merge the subtree associated with the nested catalog
847 * into its parent catalog
848 * @return - true on success, false otherwise
849 */
850 1 void WritableCatalogManager::RemoveNestedCatalog(const string &mountpoint,
851 const bool merge) {
852
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 const string nested_root_path = MakeRelativePath(mountpoint);
853
854 1 SyncLock();
855 // Find the catalog which should be removed
856 1 WritableCatalog *nested_catalog = NULL;
857
2/4
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 times.
 1 if (!FindCatalog(nested_root_path, &nested_catalog)) {
858 ✗ PANIC(kLogStderr,
859 "failed to remove nested catalog '%s': "
860 "mountpoint was not found in current catalog structure",
861 nested_root_path.c_str());
862 }
863
864 // Check if the found catalog is really the nested catalog to be deleted
865
6/19
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 1 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 1 times.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 1 times.
✗ Branch 14 not taken.
✓ Branch 16 taken 1 times.
✗ Branch 17 not taken.
✗ Branch 19 not taken.
✗ Branch 20 not taken.
✗ Branch 22 not taken.
✗ Branch 23 not taken.
 2 assert(!nested_catalog->IsRoot() &&
866 (nested_catalog->mountpoint().ToString() == nested_root_path));
867
868
1/2
✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
 1 if (merge) {
869 // Merge all data from the nested catalog into it's parent
870
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 nested_catalog->MergeIntoParent();
871 } else {
872 ✗ nested_catalog->RemoveFromParent();
873 }
874
875 // Delete the catalog database file from the working copy
876
2/6
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 1 times.
 1 if (unlink(nested_catalog->database_path().c_str()) != 0) {
877 ✗ PANIC(kLogStderr,
878 "unable to delete the removed nested catalog database file '%s'",
879 nested_catalog->database_path().c_str());
880 }
881
882 // Remove the catalog from internal data structures
883
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 DetachCatalog(nested_catalog);
884 1 SyncUnlock();
885 1 }
886
887
888 /**
889 * Swap in a new nested catalog
890 *
891 * The old nested catalog must not have been already attached to the
892 * catalog tree. This method will not attach the new nested catalog
893 * to the catalog tree.
894 *
895 * @param mountpoint - the path of the nested catalog to be removed
896 * @param new_hash - the hash of the new nested catalog
897 * @param new_size - the size of the new nested catalog
898 */
899 7 void WritableCatalogManager::SwapNestedCatalog(const string &mountpoint,
900 const shash::Any &new_hash,
901 const uint64_t new_size) {
902
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 const string nested_root_path = MakeRelativePath(mountpoint);
903
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 const string parent_path = GetParentPath(nested_root_path);
904
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 const PathString nested_root_ps = PathString(nested_root_path);
905
906 7 SyncLock();
907
908 // Find the immediate parent catalog
909 7 WritableCatalog *parent = NULL;
910
3/4
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 times.
✓ Branch 4 taken 6 times.
 7 if (!FindCatalog(parent_path, &parent)) {
911 1 SyncUnlock(); // this is needed for the unittest. otherwise they get stuck
912 1 PANIC(kLogStderr,
913 "failed to swap nested catalog '%s': could not find parent '%s'",
914 nested_root_path.c_str(), parent_path.c_str());
915 }
916
917 // Get old nested catalog counters
918
1/2
✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
 6 Catalog *old_attached_catalog = parent->FindChild(nested_root_ps);
919
1/2
✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
 6 Counters old_counters;
920
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 5 times.
 6 if (old_attached_catalog) {
921 // Old catalog was already attached (e.g. as a child catalog
922 // attached by a prior call to CreateNestedCatalog()). Ensure
923 // that it has not been modified, get counters, and detach it.
924 1 WritableCatalogList list;
925
2/4
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 times.
✗ Branch 4 not taken.
 1 if (GetModifiedCatalogLeafsRecursively(old_attached_catalog, &list)) {
926 1 SyncUnlock();
927 1 PANIC(kLogStderr,
928 "failed to swap nested catalog '%s': already modified",
929 nested_root_path.c_str());
930 }
931 ✗ old_counters = old_attached_catalog->GetCounters();
932 ✗ DetachSubtree(old_attached_catalog);
933
934 1 } else {
935 // Old catalog was not attached. Download a freely attached
936 // version and get counters.
937
1/2
✓ Branch 1 taken 5 times.
✗ Branch 2 not taken.
 5 shash::Any old_hash;
938 uint64_t old_size;
939
1/2
✓ Branch 1 taken 5 times.
✗ Branch 2 not taken.
 5 const bool old_found = parent->FindNested(nested_root_ps, &old_hash,
940 &old_size);
941
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 4 times.
 5 if (!old_found) {
942 1 SyncUnlock();
943 1 PANIC(kLogStderr,
944 "failed to swap nested catalog '%s': not found in parent",
945 nested_root_path.c_str());
946 }
947 UniquePtr<Catalog> old_free_catalog(
948
2/4
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 4 times.
✗ Branch 5 not taken.
 4 LoadFreeCatalog(nested_root_ps, old_hash));
949
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4 times.
 4 if (!old_free_catalog.IsValid()) {
950 ✗ SyncUnlock();
951 ✗ PANIC(kLogStderr,
952 "failed to swap nested catalog '%s': failed to load old catalog",
953 nested_root_path.c_str());
954 }
955 4 old_counters = old_free_catalog->GetCounters();
956 4 }
957
958 // Load freely attached new catalog
959
3/4
✓ Branch 1 taken 3 times.
✓ Branch 2 taken 1 times.
✓ Branch 4 taken 3 times.
✗ Branch 5 not taken.
 4 UniquePtr<Catalog> new_catalog(LoadFreeCatalog(nested_root_ps, new_hash));
960
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 3 times.
 3 if (!new_catalog.IsValid()) {
961 ✗ SyncUnlock();
962 ✗ PANIC(kLogStderr,
963 "failed to swap nested catalog '%s': failed to load new catalog",
964 nested_root_path.c_str());
965 }
966
967 // Get new catalog root directory entry
968
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 DirectoryEntry dirent;
969 3 XattrList xattrs;
970
1/2
✓ Branch 2 taken 3 times.
✗ Branch 3 not taken.
 3 const bool dirent_found = new_catalog->LookupPath(nested_root_ps, &dirent);
971
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3 times.
 3 if (!dirent_found) {
972 ✗ SyncUnlock();
973 ✗ PANIC(kLogStderr,
974 "failed to swap nested catalog '%s': missing dirent in new catalog",
975 nested_root_path.c_str());
976 }
977
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 3 times.
 3 if (dirent.HasXattrs()) {
978 ✗ const bool xattrs_found = new_catalog->LookupXattrsPath(nested_root_ps,
979 &xattrs);
980 ✗ if (!xattrs_found) {
981 ✗ SyncUnlock();
982 ✗ PANIC(kLogStderr,
983 "failed to swap nested catalog '%s': missing xattrs in new catalog",
984 nested_root_path.c_str());
985 }
986 }
987
988 // Swap catalogs
989
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 parent->RemoveNestedCatalog(nested_root_path, NULL);
990
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 parent->InsertNestedCatalog(nested_root_path, NULL, new_hash, new_size);
991
992 // Update parent directory entry
993 3 dirent.set_is_nested_catalog_mountpoint(true);
994 3 dirent.set_is_nested_catalog_root(false);
995
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 parent->UpdateEntry(dirent, nested_root_path);
996
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 parent->TouchEntry(dirent, xattrs, nested_root_path);
997
998 // Update counters
999
1/2
✓ Branch 3 taken 3 times.
✗ Branch 4 not taken.
 3 DeltaCounters delta = Counters::Diff(old_counters,
1000 new_catalog->GetCounters());
1001
1/2
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
 3 delta.PopulateToParent(&parent->delta_counters_);
1002
1003 3 SyncUnlock();
1004 15 }
1005
1006
1007 /**
1008 * Checks if a nested catalog starts at this path. The path must be valid.
1009 */
1010 ✗ bool WritableCatalogManager::IsTransitionPoint(const string &mountpoint) {
1011 ✗ const string path = MakeRelativePath(mountpoint);
1012
1013 ✗ SyncLock();
1014 WritableCatalog *catalog;
1015 ✗ DirectoryEntry entry;
1016 ✗ if (!FindCatalog(path, &catalog, &entry)) {
1017 ✗ PANIC(kLogStderr, "catalog for directory '%s' cannot be found",
1018 path.c_str());
1019 }
1020 ✗ const bool result = entry.IsNestedCatalogRoot();
1021 ✗ SyncUnlock();
1022 ✗ return result;
1023 }
1024
1025
1026 ✗ void WritableCatalogManager::PrecalculateListings() {
1027 // TODO(jblomer): meant for micro catalogs
1028 }
1029
1030
1031 ✗ void WritableCatalogManager::SetTTL(const uint64_t new_ttl) {
1032 ✗ SyncLock();
1033 ✗ reinterpret_cast<WritableCatalog *>(GetRootCatalog())->SetTTL(new_ttl);
1034 ✗ SyncUnlock();
1035 }
1036
1037
1038 ✗ bool WritableCatalogManager::SetVOMSAuthz(const std::string &voms_authz) {
1039 bool result;
1040 ✗ SyncLock();
1041 result = reinterpret_cast<WritableCatalog *>(
1042 ✗ GetRootCatalog())->SetVOMSAuthz(voms_authz);
1043 ✗ SyncUnlock();
1044 ✗ return result;
1045 }
1046
1047
1048 20 bool WritableCatalogManager::Commit(const bool stop_for_tweaks,
1049 const uint64_t manual_revision,
1050 manifest::Manifest *manifest) {
1051 WritableCatalog *root_catalog =
1052 20 reinterpret_cast<WritableCatalog *>(GetRootCatalog());
1053
1/2
✓ Branch 1 taken 20 times.
✗ Branch 2 not taken.
 20 root_catalog->SetDirty();
1054
1055 // set root catalog revision to manually provided number if available
1056
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 20 times.
 20 if (manual_revision > 0) {
1057 ✗ const uint64_t revision = root_catalog->GetRevision();
1058 ✗ if (revision >= manual_revision) {
1059 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1060 "Manual revision (%" PRIu64 ") must not be "
1061 "smaller than the current root catalog's (%" PRIu64
1062 "). Skipped!", manual_revision, revision);
1063 } else {
1064 // Gets incremented by FinalizeCatalog() afterwards!
1065 ✗ root_catalog->SetRevision(manual_revision - 1);
1066 }
1067 }
1068
1069 // do the actual catalog snapshotting and upload
1070
1/2
✓ Branch 1 taken 20 times.
✗ Branch 2 not taken.
 20 CatalogInfo root_catalog_info;
1071
1/2
✓ Branch 1 taken 20 times.
✗ Branch 2 not taken.
 20 if (getenv("_CVMFS_SERIALIZED_CATALOG_PROCESSING_") == NULL)
1072
1/2
✓ Branch 1 taken 20 times.
✗ Branch 2 not taken.
 20 root_catalog_info = SnapshotCatalogs(stop_for_tweaks);
1073 else
1074 ✗ root_catalog_info = SnapshotCatalogsSerialized(stop_for_tweaks);
1075
2/4
✓ Branch 1 taken 20 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 20 times.
 20 if (spooler_->GetNumberOfErrors() > 0) {
1076 ✗ LogCvmfs(kLogCatalog, kLogStderr, "failed to commit catalogs");
1077 ✗ return false;
1078 }
1079
1080 // .cvmfspublished export
1081
1/2
✓ Branch 1 taken 20 times.
✗ Branch 2 not taken.
 20 LogCvmfs(kLogCatalog, kLogVerboseMsg, "Committing repository manifest");
1082 20 set_base_hash(root_catalog_info.content_hash);
1083
1084 20 manifest->set_catalog_hash(root_catalog_info.content_hash);
1085 20 manifest->set_catalog_size(root_catalog_info.size);
1086
2/4
✓ Branch 2 taken 20 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 20 times.
✗ Branch 6 not taken.
 20 manifest->set_root_path("");
1087 20 manifest->set_ttl(root_catalog_info.ttl);
1088 20 manifest->set_revision(root_catalog_info.revision);
1089
1090 20 return true;
1091 }
1092
1093
1094 /**
1095 * Handles the snapshotting of dirty (i.e. modified) catalogs while trying to
1096 * parallelize the compression and upload as much as possible. We use a parallel
1097 * depth first post order tree traversal based on 'continuations'.
1098 *
1099 * The idea is as follows:
1100 * 1. find all leaf-catalogs (i.e. dirty catalogs with no dirty children)
1101 * --> these can be processed and uploaded immediately and independently
1102 * see WritableCatalogManager::GetModifiedCatalogLeafs()
1103 * 2. annotate non-leaf catalogs with their number of dirty children
1104 * --> a finished child will notify it's parent and decrement this number
1105 * see WritableCatalogManager::CatalogUploadCallback()
1106 * 3. if a non-leaf catalog's dirty children number reaches 0, it is scheduled
1107 * for processing as well (continuation)
1108 * --> the parallel processing walks bottom-up through the catalog tree
1109 * see WritableCatalogManager::CatalogUploadCallback()
1110 * 4. when the root catalog is reached, we notify the main thread and return
1111 * --> done through a Future<> in WritableCatalogManager::SnapshotCatalogs
1112 *
1113 * Note: The catalog finalisation (see WritableCatalogManager::FinalizeCatalog)
1114 * happens in a worker thread (i.e. the callback method) for non-leaf
1115 * catalogs.
1116 *
1117 * TODO(rmeusel): since all leaf catalogs are finalized in the main thread, we
1118 * sacrifice some potential concurrency for simplicity.
1119 */
1120 20 WritableCatalogManager::CatalogInfo WritableCatalogManager::SnapshotCatalogs(
1121 const bool stop_for_tweaks) {
1122 // prepare environment for parallel processing
1123
1/2
✓ Branch 1 taken 20 times.
✗ Branch 2 not taken.
 20 Future<CatalogInfo> root_catalog_info_future;
1124 CatalogUploadContext upload_context;
1125 20 upload_context.root_catalog_info = &root_catalog_info_future;
1126 20 upload_context.stop_for_tweaks = stop_for_tweaks;
1127
1128
1/2
✓ Branch 1 taken 20 times.
✗ Branch 2 not taken.
 20 spooler_->RegisterListener(
1129 &WritableCatalogManager::CatalogUploadCallback, this, upload_context);
1130
1131 // find dirty leaf catalogs and annotate non-leaf catalogs (dirty child count)
1132 // post-condition: the entire catalog tree is ready for concurrent processing
1133 20 WritableCatalogList leafs_to_snapshot;
1134
1/2
✓ Branch 1 taken 20 times.
✗ Branch 2 not taken.
 20 GetModifiedCatalogLeafs(&leafs_to_snapshot);
1135
1136 // finalize and schedule the catalog processing
1137 20 WritableCatalogList::const_iterator i = leafs_to_snapshot.begin();
1138 20 const WritableCatalogList::const_iterator iend = leafs_to_snapshot.end();
1139
2/2
✓ Branch 2 taken 29 times.
✓ Branch 3 taken 20 times.
 49 for (; i != iend; ++i) {
1140
1/2
✓ Branch 2 taken 29 times.
✗ Branch 3 not taken.
 29 FinalizeCatalog(*i, stop_for_tweaks);
1141
1/2
✓ Branch 2 taken 29 times.
✗ Branch 3 not taken.
 29 ScheduleCatalogProcessing(*i);
1142 }
1143
1144
1/2
✓ Branch 1 taken 20 times.
✗ Branch 2 not taken.
 20 LogCvmfs(kLogCatalog, kLogVerboseMsg, "waiting for upload of catalogs");
1145
1/2
✓ Branch 1 taken 20 times.
✗ Branch 2 not taken.
 20 CatalogInfo& root_catalog_info = root_catalog_info_future.Get();
1146
1/2
✓ Branch 1 taken 20 times.
✗ Branch 2 not taken.
 20 spooler_->WaitForUpload();
1147
1148
1/2
✓ Branch 1 taken 20 times.
✗ Branch 2 not taken.
 20 spooler_->UnregisterListeners();
1149 20 return root_catalog_info;
1150 20 }
1151
1152
1153 46 void WritableCatalogManager::FinalizeCatalog(WritableCatalog *catalog,
1154 const bool stop_for_tweaks) {
1155 // update meta information of this catalog
1156
1/3
✓ Branch 2 taken 46 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 46 LogCvmfs(kLogCatalog, kLogVerboseMsg, "creating snapshot of catalog '%s'",
1157 92 catalog->mountpoint().c_str());
1158
1159 46 catalog->UpdateCounters();
1160 46 catalog->UpdateLastModified();
1161 46 catalog->IncrementRevision();
1162
1163 // update the previous catalog revision pointer
1164
2/2
✓ Branch 1 taken 20 times.
✓ Branch 2 taken 26 times.
 46 if (catalog->IsRoot()) {
1165
1/4
✓ Branch 2 taken 20 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 20 LogCvmfs(kLogCatalog, kLogVerboseMsg, "setting '%s' as previous revision "
1166 "for root catalog",
1167 40 base_hash().ToStringWithSuffix().c_str());
1168 20 catalog->SetPreviousRevision(base_hash());
1169 } else {
1170 // Multiple catalogs might query the parent concurrently
1171 26 SyncLock();
1172
1/2
✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
 26 shash::Any hash_previous;
1173 uint64_t size_previous;
1174 const bool retval =
1175
2/4
✓ Branch 2 taken 26 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 26 times.
✗ Branch 6 not taken.
 26 catalog->parent()->FindNested(catalog->mountpoint(),
1176 &hash_previous, &size_previous);
1177
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 26 times.
 26 assert(retval);
1178 26 SyncUnlock();
1179
1180
1/8
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 26 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
 52 LogCvmfs(kLogCatalog, kLogVerboseMsg, "found '%s' as previous revision "
1181 "for nested catalog '%s'",
1182
1/2
✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
 52 hash_previous.ToStringWithSuffix().c_str(),
1183
1/2
✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
 52 catalog->mountpoint().c_str());
1184
1/2
✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
 26 catalog->SetPreviousRevision(hash_previous);
1185 }
1186 46 catalog->Commit();
1187
1188 // check if catalog has too many entries
1189
2/2
✓ Branch 0 taken 20 times.
✓ Branch 1 taken 26 times.
 46 uint64_t catalog_limit = uint64_t(1000) *
1190 46 uint64_t((catalog->IsRoot()
1191 20 ? root_kcatalog_limit_
1192 26 : nested_kcatalog_limit_));
1193
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 46 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 46 if ((catalog_limit > 0) &&
1194 ✗ (catalog->GetCounters().GetSelfEntries() > catalog_limit)) {
1195 ✗ LogCvmfs(kLogCatalog, kLogStderr,
1196 "%s: catalog at %s has more than %lu entries (%lu). "
1197 "Large catalogs stress the CernVM-FS transport infrastructure. "
1198 "Please split it into nested catalogs or increase the limit.",
1199 ✗ enforce_limits_ ? "FATAL" : "WARNING",
1200 ✗ (catalog->IsRoot() ? "/" : catalog->mountpoint().c_str()),
1201 ✗ catalog_limit, catalog->GetCounters().GetSelfEntries());
1202 ✗ if (enforce_limits_)
1203 ✗ PANIC(kLogStderr, "catalog at %s has more than %u entries (%u). ",
1204 (catalog->IsRoot() ? "/" : catalog->mountpoint().c_str()),
1205 catalog_limit, catalog->GetCounters().GetSelfEntries());
1206 }
1207
1208 // allow for manual adjustments in the catalog
1209
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 46 times.
 46 if (stop_for_tweaks) {
1210 ✗ LogCvmfs(kLogCatalog, kLogStdout, "Allowing for tweaks in %s at %s "
1211 "(hit return to continue)",
1212 ✗ catalog->database_path().c_str(), catalog->mountpoint().c_str());
1213 ✗ int read_char = getchar();
1214 ✗ assert(read_char != EOF);
1215 }
1216
1217 // compaction of bloated catalogs (usually after high database churn)
1218 46 catalog->VacuumDatabaseIfNecessary();
1219 46 }
1220
1221
1222 46 void WritableCatalogManager::ScheduleCatalogProcessing(
1223 WritableCatalog *catalog) {
1224 {
1225 46 MutexLockGuard guard(catalog_processing_lock_);
1226 // register catalog object for WritableCatalogManager::CatalogUploadCallback
1227
2/4
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 46 times.
✗ Branch 5 not taken.
 46 catalog_processing_map_[catalog->database_path()] = catalog;
1228 46 }
1229
1/2
✓ Branch 2 taken 46 times.
✗ Branch 3 not taken.
 46 spooler_->ProcessCatalog(catalog->database_path());
1230 46 }
1231
1232
1233 46 void WritableCatalogManager::CatalogUploadCallback(
1234 const upload::SpoolerResult &result,
1235 const CatalogUploadContext catalog_upload_context) {
1236
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 46 times.
 46 if (result.return_code != 0) {
1237 ✗ PANIC(kLogStderr, "failed to upload '%s' (retval: %d)",
1238 result.local_path.c_str(), result.return_code);
1239 }
1240
1241 // retrieve the catalog object based on the callback information
1242 // see WritableCatalogManager::ScheduleCatalogProcessing()
1243 46 WritableCatalog *catalog = NULL;
1244 {
1245 46 MutexLockGuard guard(catalog_processing_lock_);
1246 std::map<std::string, WritableCatalog*>::iterator c =
1247
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 catalog_processing_map_.find(result.local_path);
1248
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 46 times.
 46 assert(c != catalog_processing_map_.end());
1249 46 catalog = c->second;
1250 46 }
1251
1252 46 uint64_t catalog_size = GetFileSize(result.local_path);
1253
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 46 times.
 46 assert(catalog_size > 0);
1254
1255 46 SyncLock();
1256
2/2
✓ Branch 1 taken 26 times.
✓ Branch 2 taken 20 times.
 46 if (catalog->HasParent()) {
1257 // finalized nested catalogs will update their parent's pointer and schedule
1258 // them for processing (continuation) if the 'dirty children count' == 0
1259 26 LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating nested catalog link");
1260 26 WritableCatalog *parent = catalog->GetWritableParent();
1261
1262
2/4
✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 26 times.
✗ Branch 5 not taken.
 26 parent->UpdateNestedCatalog(catalog->mountpoint().ToString(),
1263 26 result.content_hash,
1264 catalog_size,
1265 26 catalog->delta_counters_);
1266 26 catalog->delta_counters_.SetZero();
1267
1268 const int remaining_dirty_children =
1269 26 catalog->GetWritableParent()->DecrementDirtyChildren();
1270
1271 26 SyncUnlock();
1272
1273 // continuation of the dirty catalog tree traversal
1274 // see WritableCatalogManager::SnapshotCatalogs()
1275
2/2
✓ Branch 0 taken 17 times.
✓ Branch 1 taken 9 times.
 26 if (remaining_dirty_children == 0) {
1276 17 FinalizeCatalog(parent, catalog_upload_context.stop_for_tweaks);
1277 17 ScheduleCatalogProcessing(parent);
1278 }
1279
1280
1/2
✓ Branch 1 taken 20 times.
✗ Branch 2 not taken.
 20 } else if (catalog->IsRoot()) {
1281 // once the root catalog is reached, we are done with processing and report
1282 // back to the main via a Future<> and provide the necessary information
1283
1/2
✓ Branch 1 taken 20 times.
✗ Branch 2 not taken.
 20 CatalogInfo root_catalog_info;
1284 20 root_catalog_info.size = catalog_size;
1285
1/2
✓ Branch 1 taken 20 times.
✗ Branch 2 not taken.
 20 root_catalog_info.ttl = catalog->GetTTL();
1286 20 root_catalog_info.content_hash = result.content_hash;
1287
1/2
✓ Branch 1 taken 20 times.
✗ Branch 2 not taken.
 20 root_catalog_info.revision = catalog->GetRevision();
1288
1/2
✓ Branch 1 taken 20 times.
✗ Branch 2 not taken.
 20 catalog_upload_context.root_catalog_info->Set(root_catalog_info);
1289 20 SyncUnlock();
1290 } else {
1291 ✗ PANIC(kLogStderr, "inconsistent state detected");
1292 }
1293 46 }
1294
1295
1296 /**
1297 * Finds dirty catalogs that can be snapshot right away and annotates all the
1298 * other catalogs with their number of dirty descendants.
1299 * Note that there is a convenience wrapper to start the recursion:
1300 * WritableCatalogManager::GetModifiedCatalogLeafs()
1301 *
1302 * @param catalog the catalog for this recursion step
1303 * @param result the result list to be appended to
1304 * @return true if 'catalog' is dirty
1305 */
1306 47 bool WritableCatalogManager::GetModifiedCatalogLeafsRecursively(
1307 Catalog *catalog,
1308 WritableCatalogList *result) const {
1309 47 WritableCatalog *wr_catalog = static_cast<WritableCatalog *>(catalog);
1310
1311 // Look for dirty catalogs in the descendants of *catalog
1312 47 int dirty_children = 0;
1313
1/2
✓ Branch 1 taken 47 times.
✗ Branch 2 not taken.
 47 CatalogList children = wr_catalog->GetChildren();
1314 47 CatalogList::const_iterator i = children.begin();
1315 47 const CatalogList::const_iterator iend = children.end();
1316
2/2
✓ Branch 2 taken 26 times.
✓ Branch 3 taken 47 times.
 73 for (; i != iend; ++i) {
1317
2/4
✓ Branch 2 taken 26 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 26 times.
✗ Branch 5 not taken.
 26 if (GetModifiedCatalogLeafsRecursively(*i, result)) {
1318 26 ++dirty_children;
1319 }
1320 }
1321
1322 // a catalog is dirty if itself or one of its children has changed
1323 // a leaf catalog doesn't have any dirty children
1324 47 wr_catalog->set_dirty_children(dirty_children);
1325
1/4
✗ Branch 1 not taken.
✓ Branch 2 taken 47 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 47 const bool is_dirty = wr_catalog->IsDirty() || dirty_children > 0;
1326 47 const bool is_leaf = dirty_children == 0;
1327
3/4
✓ Branch 0 taken 47 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 30 times.
✓ Branch 3 taken 17 times.
 47 if (is_dirty && is_leaf) {
1328
1/2
✓ Branch 1 taken 30 times.
✗ Branch 2 not taken.
 30 result->push_back(const_cast<WritableCatalog *>(wr_catalog));
1329 }
1330
1331 47 return is_dirty;
1332 47 }
1333
1334
1335 ✗ void WritableCatalogManager::DoBalance() {
1336 ✗ CatalogList catalog_list = GetCatalogs();
1337 ✗ reverse(catalog_list.begin(), catalog_list.end());
1338 ✗ for (unsigned i = 0; i < catalog_list.size(); ++i) {
1339 ✗ FixWeight(static_cast<WritableCatalog*>(catalog_list[i]));
1340 }
1341 }
1342
1343 ✗ void WritableCatalogManager::FixWeight(WritableCatalog* catalog) {
1344 // firstly check underflow because they can provoke overflows
1345 ✗ if (catalog->GetNumEntries() < min_weight_ &&
1346 ✗ !catalog->IsRoot() &&
1347 ✗ catalog->IsAutogenerated()) {
1348 ✗ LogCvmfs(kLogCatalog, kLogStdout,
1349 "Deleting an autogenerated catalog in '%s'",
1350 ✗ catalog->mountpoint().c_str());
1351 // Remove the .cvmfscatalog and .cvmfsautocatalog files first
1352 ✗ string path = catalog->mountpoint().ToString();
1353 ✗ catalog->RemoveEntry(path + "/.cvmfscatalog");
1354 ✗ catalog->RemoveEntry(path + "/.cvmfsautocatalog");
1355 // Remove the actual catalog
1356 ✗ string catalog_path = catalog->mountpoint().ToString().substr(1);
1357 ✗ RemoveNestedCatalog(catalog_path);
1358 ✗ } else if (catalog->GetNumEntries() > max_weight_) {
1359 ✗ CatalogBalancer<WritableCatalogManager> catalog_balancer(this);
1360 ✗ catalog_balancer.Balance(catalog);
1361 }
1362 }
1363
1364
1365 //****************************************************************************
1366 // Workaround -- Serialized Catalog Committing
1367
1368 ✗ int WritableCatalogManager::GetModifiedCatalogsRecursively(
1369 const Catalog *catalog,
1370 WritableCatalogList *result) const
1371 {
1372 // A catalog must be snapshot, if itself or one of it's descendants is dirty.
1373 // So we traverse the catalog tree recursively and look for dirty catalogs
1374 // on the way.
1375 ✗ const WritableCatalog *wr_catalog =
1376 static_cast<const WritableCatalog *>(catalog);
1377 // This variable will contain the number of dirty catalogs in the sub tree
1378 // with *catalog as it's root.
1379 ✗ int dirty_catalogs = (wr_catalog->IsDirty()) ? 1 : 0;
1380
1381 // Look for dirty catalogs in the descendants of *catalog
1382 ✗ CatalogList children = wr_catalog->GetChildren();
1383 ✗ for (CatalogList::const_iterator i = children.begin(), iEnd = children.end();
1384 ✗ i != iEnd; ++i)
1385 {
1386 ✗ dirty_catalogs += GetModifiedCatalogsRecursively(*i, result);
1387 }
1388
1389 // If we found a dirty catalog in the checked sub tree, the root (*catalog)
1390 // must be snapshot and ends up in the result list
1391 ✗ if (dirty_catalogs > 0)
1392 ✗ result->push_back(const_cast<WritableCatalog *>(wr_catalog));
1393
1394 // tell the upper layer about number of catalogs
1395 ✗ return dirty_catalogs;
1396 }
1397
1398
1399 ✗ void WritableCatalogManager::CatalogUploadSerializedCallback(
1400 const upload::SpoolerResult &result,
1401 const CatalogUploadContext unused)
1402 {
1403 ✗ if (result.return_code != 0) {
1404 ✗ PANIC(kLogStderr, "failed to upload '%s' (retval: %d)",
1405 result.local_path.c_str(), result.return_code);
1406 }
1407 ✗ unlink(result.local_path.c_str());
1408 }
1409
1410
1411 WritableCatalogManager::CatalogInfo
1412 ✗ WritableCatalogManager::SnapshotCatalogsSerialized(
1413 const bool stop_for_tweaks)
1414 {
1415 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Serialized committing of file catalogs...");
1416 ✗ reinterpret_cast<WritableCatalog *>(GetRootCatalog())->SetDirty();
1417 ✗ WritableCatalogList catalogs_to_snapshot;
1418 ✗ GetModifiedCatalogs(&catalogs_to_snapshot);
1419 CatalogUploadContext unused;
1420 ✗ unused.root_catalog_info = NULL;
1421 ✗ unused.stop_for_tweaks = false;
1422 ✗ spooler_->RegisterListener(
1423 &WritableCatalogManager::CatalogUploadSerializedCallback, this, unused);
1424
1425 ✗ CatalogInfo root_catalog_info;
1426 ✗ WritableCatalogList::const_iterator i = catalogs_to_snapshot.begin();
1427 ✗ const WritableCatalogList::const_iterator iend = catalogs_to_snapshot.end();
1428 ✗ for (; i != iend; ++i) {
1429 ✗ FinalizeCatalog(*i, stop_for_tweaks);
1430
1431 // Compress and upload catalog
1432 ✗ shash::Any hash_catalog(spooler_->GetHashAlgorithm(),
1433 ✗ shash::kSuffixCatalog);
1434 ✗ if (!zlib::CompressPath2Null((*i)->database_path(),
1435 &hash_catalog))
1436 {
1437 ✗ PANIC(kLogStderr, "could not compress catalog %s",
1438 (*i)->mountpoint().ToString().c_str());
1439 }
1440
1441 ✗ int64_t catalog_size = GetFileSize((*i)->database_path());
1442 ✗ assert(catalog_size > 0);
1443
1444 ✗ if ((*i)->HasParent()) {
1445 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating nested catalog link");
1446 ✗ WritableCatalog *parent = (*i)->GetWritableParent();
1447 ✗ parent->UpdateNestedCatalog((*i)->mountpoint().ToString(), hash_catalog,
1448 ✗ catalog_size, (*i)->delta_counters_);
1449 ✗ (*i)->delta_counters_.SetZero();
1450 ✗ } else if ((*i)->IsRoot()) {
1451 ✗ root_catalog_info.size = catalog_size;
1452 ✗ root_catalog_info.ttl = (*i)->GetTTL();
1453 ✗ root_catalog_info.content_hash = hash_catalog;
1454 ✗ root_catalog_info.revision = (*i)->GetRevision();
1455 } else {
1456 ✗ PANIC(kLogStderr, "inconsistent state detected");
1457 }
1458
1459 ✗ spooler_->ProcessCatalog((*i)->database_path());
1460 }
1461 ✗ spooler_->WaitForUpload();
1462
1463 ✗ spooler_->UnregisterListeners();
1464 ✗ return root_catalog_info;
1465 }
1466
1467 } // namespace catalog
1468