GCC Code Coverage Report


Directory: cvmfs/
File: cvmfs/swissknife_overlay.cc
Date: 2026-03-01 02:36:14
Exec Total Coverage
Lines: 0 366 0.0%
Branches: 0 317 0.0%

Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * Implementation of the overlay swissknife command that merges multiple
5 * CVMFS subdirectory catalogs using overlay semantics (similar to OverlayFS)
6 * and publishes the result as a repository subdirectory.
7 */
8
9 #define __STDC_FORMAT_MACROS
10
11 #include "swissknife_overlay.h"
12
13 #include <inttypes.h>
14 #include <sys/stat.h>
15 #include <unistd.h>
16
17 #include <algorithm>
18 #include <cassert>
19 #include <ctime>
20 #include <map>
21 #include <string>
22 #include <vector>
23
24 #include "catalog.h"
25 #include "catalog_mgr_rw.h"
26 #include "catalog_rw.h"
27 #include "catalog_sql.h"
28 #include "compression/compression.h"
29 #include "crypto/hash.h"
30 #include "directory_entry.h"
31 #include "manifest.h"
32 #include "network/download.h"
33 #include "network/sink_path.h"
34 #include "repository_tag.h"
35 #include "shortstring.h"
36 #include "statistics.h"
37 #include "upload.h"
38 #include "upload_spooler_definition.h"
39 #include "util/logging.h"
40 #include "util/pointer.h"
41 #include "util/posix.h"
42 #include "util/string.h"
43 #include "xattr.h"
44
45 using namespace std; // NOLINT
46
47 namespace swissknife {
48
49 ParameterList CommandOverlay::GetParams() const {
50 ParameterList r;
51 // Publish workflow parameters (same convention as ingest/sync)
52 r.push_back(Parameter::Mandatory('r', "upstream storage definition"));
53 r.push_back(Parameter::Mandatory('w', "stratum 0 URL"));
54 r.push_back(Parameter::Mandatory('t', "temporary directory"));
55 r.push_back(Parameter::Mandatory('o', "manifest output path"));
56 r.push_back(Parameter::Mandatory('b', "base hash of current root catalog"));
57 r.push_back(Parameter::Mandatory('K', "public key path"));
58 r.push_back(Parameter::Mandatory('N', "repository name"));
59
60 // Overlay-specific parameters
61 r.push_back(Parameter::Mandatory('l', "comma-separated layer paths "
62 "(bottom-to-top order)"));
63 r.push_back(Parameter::Mandatory('d', "destination subdirectory path in "
64 "repository for the merged overlay"));
65 r.push_back(Parameter::Optional('e', "hash algorithm (default: sha1)"));
66 r.push_back(Parameter::Optional('Z', "compression algorithm "
67 "(default: zlib)"));
68 r.push_back(Parameter::Optional('@', "proxy URL"));
69 r.push_back(Parameter::Switch('L', "follow HTTP redirects"));
70 return r;
71 }
72
73
74 bool CommandOverlay::IsWhiteoutFile(const string &name) {
75 return HasPrefix(name, ".wh.", false) && !IsOpaqueMarker(name);
76 }
77
78
79 string CommandOverlay::GetWhiteoutTarget(const string &name) {
80 // ".wh." is 4 characters
81 if (name.length() <= 4) return "";
82 return name.substr(4);
83 }
84
85
86 bool CommandOverlay::IsOpaqueMarker(const string &name) {
87 return name == ".wh..wh..opq";
88 }
89
90
91 bool CommandOverlay::ReadCatalogEntries(
92 catalog::Catalog *catalog,
93 const string &catalog_root_path,
94 const string &relative_prefix,
95 const string &repo_base,
96 const string &temp_dir,
97 map<string, OverlayEntry> *entries) {
98 // List entries at this path in the catalog
99 catalog::DirectoryEntryList listing;
100 const PathString ps_path(catalog_root_path.data(),
101 catalog_root_path.length());
102 const bool has_entries = catalog->ListingPath(ps_path, &listing);
103
104 if (!has_entries && catalog_root_path != catalog->mountpoint().ToString()) {
105 // No entries at this path - it might be a file, not a directory
106 return true;
107 }
108
109 for (size_t i = 0; i < listing.size(); ++i) {
110 const catalog::DirectoryEntry &dirent = listing[i];
111 const string name = dirent.name().ToString();
112
113 // Skip CVMFS bookkeeping files — they are internal metadata and must not
114 // be carried over into the merged overlay. PublishMergedEntries() will
115 // create its own .cvmfscatalog marker for the destination catalog.
116 if (name == ".cvmfscatalog" || name == ".cvmfsdirtab"
117 || name == ".cvmfsautocatalog") {
118 continue;
119 }
120
121 const string child_catalog_path =
122 catalog_root_path.empty() ? "/" + name : catalog_root_path + "/" + name;
123 const string child_relative =
124 relative_prefix.empty() ? name : relative_prefix + "/" + name;
125
126 OverlayEntry oe;
127 oe.entry = dirent;
128 oe.path = child_relative;
129 oe.parent = relative_prefix;
130 oe.is_whiteout = IsWhiteoutFile(name);
131 oe.is_opaque_dir = false;
132
133 // Look up xattrs for this entry
134 XattrList xattrs;
135 const PathString ps_child(child_catalog_path.data(),
136 child_catalog_path.length());
137 catalog->LookupXattrsPath(ps_child, &xattrs);
138 oe.xattrs = xattrs;
139
140 (*entries)[child_relative] = oe;
141
142 if (dirent.IsDirectory()) {
143 if (dirent.IsNestedCatalogMountpoint() && !repo_base.empty()) {
144 // Load the nested catalog and recurse into it
145 shash::Any nested_hash;
146 uint64_t nested_size;
147 if (!catalog->FindNested(ps_child, &nested_hash, &nested_size)) {
148 LogCvmfs(kLogCvmfs, kLogStderr,
149 "Failed to find nested catalog hash for %s",
150 child_catalog_path.c_str());
151 return false;
152 }
153
154 catalog::Catalog *nested = LoadCatalogForPath(
155 repo_base, child_catalog_path, temp_dir, nested_hash);
156 if (nested == NULL) {
157 LogCvmfs(kLogCvmfs, kLogStderr,
158 "Failed to load nested catalog for %s",
159 child_catalog_path.c_str());
160 return false;
161 }
162
163 // Check for opaque marker in the nested catalog root
164 catalog::DirectoryEntryList sub_listing;
165 nested->ListingPath(ps_child, &sub_listing);
166 for (size_t j = 0; j < sub_listing.size(); ++j) {
167 if (IsOpaqueMarker(sub_listing[j].name().ToString())) {
168 (*entries)[child_relative].is_opaque_dir = true;
169 break;
170 }
171 }
172
173 if (!ReadCatalogEntries(nested, child_catalog_path,
174 child_relative, repo_base, temp_dir, entries)) {
175 delete nested;
176 return false;
177 }
178 delete nested;
179 } else if (!dirent.IsNestedCatalogMountpoint()) {
180 // Regular directory — check for opaque marker among children
181 catalog::DirectoryEntryList sub_listing;
182 catalog->ListingPath(ps_child, &sub_listing);
183 for (size_t j = 0; j < sub_listing.size(); ++j) {
184 if (IsOpaqueMarker(sub_listing[j].name().ToString())) {
185 (*entries)[child_relative].is_opaque_dir = true;
186 break;
187 }
188 }
189
190 if (!ReadCatalogEntries(catalog, child_catalog_path,
191 child_relative, repo_base, temp_dir, entries)) {
192 return false;
193 }
194 }
195 // else: nested catalog mountpoint but no repo_base — skip (e.g. cache)
196 }
197 }
198
199 return true;
200 }
201
202
203 void CommandOverlay::MergeLayer(
204 const map<string, OverlayEntry> &layer_entries,
205 map<string, OverlayEntry> *merged) const {
206 // First pass: collect whiteouts and opaque directories
207 vector<string> whiteout_targets;
208 vector<string> opaque_dirs;
209
210 for (map<string, OverlayEntry>::const_iterator it = layer_entries.begin();
211 it != layer_entries.end(); ++it) {
212 const OverlayEntry &oe = it->second;
213 const string &path = it->first;
214
215 if (oe.is_whiteout) {
216 // Whiteout: mark the target for deletion from lower layers
217 const string target_name = GetWhiteoutTarget(
218 GetFileName(path));
219 const string target_path =
220 oe.parent.empty() ? target_name : oe.parent + "/" + target_name;
221 whiteout_targets.push_back(target_path);
222 continue;
223 }
224
225 if (IsOpaqueMarker(GetFileName(path))) {
226 // Don't add the opaque marker itself to the merged output
227 continue;
228 }
229
230 if (oe.is_opaque_dir) {
231 opaque_dirs.push_back(path);
232 }
233 }
234
235 // Apply opaque directory semantics: remove all entries from lower layers
236 // that are under opaque directories
237 for (size_t i = 0; i < opaque_dirs.size(); ++i) {
238 const string &opaque_path = opaque_dirs[i];
239 const string prefix = opaque_path + "/";
240
241 // Remove children of this directory from merged (lower layer entries)
242 vector<string> to_remove;
243 for (map<string, OverlayEntry>::iterator it = merged->begin();
244 it != merged->end(); ++it) {
245 if (HasPrefix(it->first, prefix, false)) {
246 to_remove.push_back(it->first);
247 }
248 }
249 for (size_t j = 0; j < to_remove.size(); ++j) {
250 merged->erase(to_remove[j]);
251 }
252 }
253
254 // Apply whiteout semantics: remove targeted entries and their children
255 for (size_t i = 0; i < whiteout_targets.size(); ++i) {
256 const string &target = whiteout_targets[i];
257 const string prefix = target + "/";
258
259 // Remove the target entry itself
260 merged->erase(target);
261
262 // Remove all children of the target
263 vector<string> to_remove;
264 for (map<string, OverlayEntry>::iterator it = merged->begin();
265 it != merged->end(); ++it) {
266 if (HasPrefix(it->first, prefix, false)) {
267 to_remove.push_back(it->first);
268 }
269 }
270 for (size_t j = 0; j < to_remove.size(); ++j) {
271 merged->erase(to_remove[j]);
272 }
273 }
274
275 // Second pass: add/override entries from this layer
276 for (map<string, OverlayEntry>::const_iterator it = layer_entries.begin();
277 it != layer_entries.end(); ++it) {
278 const OverlayEntry &oe = it->second;
279 const string &path = it->first;
280
281 // Skip whiteout files and opaque markers - they are control files
282 if (oe.is_whiteout || IsOpaqueMarker(GetFileName(path))) {
283 continue;
284 }
285
286 // Upper layer overrides lower layer for the same path
287 (*merged)[path] = oe;
288 }
289 }
290
291
292
293 bool CommandOverlay::PublishMergedEntries(
294 catalog::WritableCatalogManager *catalog_mgr,
295 const map<string, OverlayEntry> &merged,
296 const string &dest_path) const {
297 // dest_path starts with '/' for LookupPath, but AddDirectory/AddFile
298 // expect parent_directory without leading '/' because MakeRelativePath
299 // (called internally) prepends it. Create a stripped copy for add calls.
300 const string dest_path_rel = (!dest_path.empty() && dest_path[0] == '/')
301 ? dest_path.substr(1) : dest_path;
302
303 // Ensure the destination directory itself exists in the catalog.
304 // Check if dest_path already exists; if not, create it.
305 catalog::DirectoryEntry dest_dirent;
306 if (!catalog_mgr->LookupPath(dest_path, catalog::kLookupDefault,
307 &dest_dirent)) {
308 // Create the destination directory (and any missing parents)
309 // Walk up to find the deepest existing ancestor
310 vector<string> dirs_to_create;
311 string check_path = dest_path;
312 while (!check_path.empty() && check_path != "/") {
313 catalog::DirectoryEntry check_dirent;
314 if (catalog_mgr->LookupPath(check_path, catalog::kLookupDefault,
315 &check_dirent)) {
316 break;
317 }
318 dirs_to_create.push_back(check_path);
319 check_path = GetParentPath(check_path);
320 }
321
322 // Create directories from outermost to innermost
323 for (int i = static_cast<int>(dirs_to_create.size()) - 1; i >= 0; --i) {
324 const string &dir = dirs_to_create[i];
325 string parent = GetParentPath(dir);
326 const string name = GetFileName(dir);
327
328 // Strip leading '/' — AddDirectory calls MakeRelativePath which adds it
329 if (!parent.empty() && parent[0] == '/') {
330 parent = parent.substr(1);
331 }
332
333 catalog::DirectoryEntryBase new_dir;
334 new_dir.name_.Assign(name.data(), name.length());
335 new_dir.mode_ = S_IFDIR | 0755;
336 new_dir.uid_ = 0;
337 new_dir.gid_ = 0;
338 new_dir.size_ = 4096;
339 new_dir.mtime_ = time(NULL);
340 new_dir.linkcount_ = 2;
341
342 catalog_mgr->AddDirectory(new_dir, XattrList(), parent);
343 LogCvmfs(kLogCvmfs, kLogDebug,
344 "Created destination directory: %s", dir.c_str());
345 }
346 }
347
348 // Add entries in sorted order. The map is sorted lexicographically,
349 // so parent directories appear before their children.
350 for (map<string, OverlayEntry>::const_iterator it = merged.begin();
351 it != merged.end(); ++it) {
352 const OverlayEntry &oe = it->second;
353
354 // Build parent path without leading '/' for AddDirectory/AddFile
355 // (MakeRelativePath inside those functions adds it back)
356 const string parent_path = oe.parent.empty()
357 ? dest_path_rel
358 : dest_path_rel + "/" + oe.parent;
359
360 if (oe.entry.IsDirectory()) {
361 catalog_mgr->AddDirectory(oe.entry, oe.xattrs, parent_path);
362 } else {
363 catalog_mgr->AddFile(
364 static_cast<const catalog::DirectoryEntryBase &>(oe.entry),
365 oe.xattrs, parent_path);
366 }
367 }
368
369 // Turn the destination directory into a nested catalog so that the overlay
370 // content lives in its own catalog database file.
371
372 // Add a .cvmfscatalog marker file
373 catalog::DirectoryEntryBase catalog_marker;
374 catalog_marker.name_ = NameString(".cvmfscatalog");
375 catalog_marker.mode_ = (S_IFREG | 0666);
376 catalog_marker.size_ = 0;
377 catalog_marker.mtime_ = time(NULL);
378 catalog_marker.uid_ = 0;
379 catalog_marker.gid_ = 0;
380 catalog_marker.linkcount_ = 1;
381 // Hash of the compressed empty file
382 catalog_marker.checksum_ = shash::MkFromHexPtr(
383 shash::HexPtr("e8ec3d88b62ebf526e4e5a4ff6162a3aa48a6b78"),
384 shash::kSuffixNone); // hash of ""
385 catalog_mgr->AddFile(catalog_marker, XattrList(), dest_path_rel);
386 // CreateNestedCatalog calls MakeRelativePath internally which prepends '/'.
387 // Pass the stripped version to avoid a double leading slash.
388 catalog_mgr->CreateNestedCatalog(dest_path_rel);
389
390 LogCvmfs(kLogCvmfs, kLogStdout,
391 "Published %zu entries under %s (nested catalog)",
392 merged.size(), dest_path.c_str());
393 return true;
394 }
395
396
397 catalog::Catalog *CommandOverlay::LoadCatalogForPath(
398 const string &repo_base,
399 const string &subdirectory,
400 const string &temp_dir,
401 const shash::Any &root_hash) {
402 // Fetch the root catalog from the repository
403 const string hash_path = "data/" + root_hash.MakePath();
404 string catalog_path;
405
406 if (IsHttpUrl(repo_base)) {
407 // Download and decompress from remote
408 const string url = repo_base + "/" + hash_path;
409 catalog_path = temp_dir + "/" + root_hash.ToString();
410
411 cvmfs::PathSink pathsink(catalog_path);
412 download::JobInfo download_job(&url, true, false, &root_hash, &pathsink);
413 const download::Failures retval = download_manager()->Fetch(&download_job);
414 if (retval != download::kFailOk) {
415 LogCvmfs(kLogCvmfs, kLogStderr, "Failed to download catalog %s (%d)",
416 root_hash.ToString().c_str(), retval);
417 return NULL;
418 }
419 } else {
420 // Local repository: decompress the catalog
421 const string source_path = repo_base + "/" + hash_path;
422 catalog_path = temp_dir + "/" + root_hash.ToString();
423
424 if (!zlib::DecompressPath2Path(source_path, catalog_path)) {
425 LogCvmfs(kLogCvmfs, kLogStderr,
426 "Failed to decompress catalog %s from %s",
427 root_hash.ToString().c_str(), source_path.c_str());
428 return NULL;
429 }
430 }
431
432 catalog::Catalog *catalog = catalog::Catalog::AttachFreely(
433 subdirectory, catalog_path, root_hash);
434 if (catalog == NULL) {
435 LogCvmfs(kLogCvmfs, kLogStderr,
436 "Failed to attach catalog for path %s",
437 subdirectory.c_str());
438 unlink(catalog_path.c_str());
439 return NULL;
440 }
441
442 catalog->TakeDatabaseFileOwnership();
443 return catalog;
444 }
445
446
447 catalog::Catalog *CommandOverlay::FindCatalogForLayer(
448 const string &repo_base,
449 const string &temp_dir,
450 catalog::Catalog *catalog,
451 const string &layer_path,
452 vector<catalog::Catalog *> *loaded_catalogs) {
453 // First try a direct lookup in the given catalog
454 catalog::DirectoryEntry test_entry;
455 const PathString ps_layer(layer_path.data(), layer_path.length());
456 if (catalog->LookupPath(ps_layer, &test_entry)) {
457 return catalog;
458 }
459
460 // The path was not found directly. Walk the path components *below*
461 // this catalog's mountpoint to find a nested catalog mountpoint that
462 // is an ancestor of layer_path.
463 const string mountpoint = catalog->mountpoint().ToString();
464
465 // Verify layer_path starts with the mountpoint (or mountpoint is empty
466 // for the root catalog)
467 if (!mountpoint.empty() && layer_path.substr(0, mountpoint.length())
468 != mountpoint) {
469 return NULL;
470 }
471
472 // Get the suffix of layer_path below the mountpoint
473 const string suffix = mountpoint.empty() ? layer_path
474 : layer_path.substr(
475 mountpoint.length());
476 const vector<string> components = SplitString(suffix, '/');
477 string prefix = mountpoint;
478 for (size_t i = 0; i < components.size(); ++i) {
479 if (components[i].empty()) continue;
480 prefix += "/" + components[i];
481
482 catalog::DirectoryEntry dir_entry;
483 const PathString ps_prefix(prefix.data(), prefix.length());
484 if (!catalog->LookupPath(ps_prefix, &dir_entry)) {
485 break;
486 }
487
488 if (dir_entry.IsNestedCatalogMountpoint()) {
489 shash::Any nested_hash;
490 uint64_t nested_size;
491 if (!catalog->FindNested(ps_prefix, &nested_hash, &nested_size)) {
492 LogCvmfs(kLogCvmfs, kLogStderr,
493 "Failed to find nested catalog hash for %s", prefix.c_str());
494 return NULL;
495 }
496
497 catalog::Catalog *nested = LoadCatalogForPath(
498 repo_base, prefix, temp_dir, nested_hash);
499 if (nested == NULL) {
500 LogCvmfs(kLogCvmfs, kLogStderr,
501 "Failed to load nested catalog at %s", prefix.c_str());
502 return NULL;
503 }
504 loaded_catalogs->push_back(nested);
505
506 // Recurse: the layer path may be directly in this nested catalog
507 // or in an even deeper nested catalog
508 return FindCatalogForLayer(
509 repo_base, temp_dir, nested, layer_path, loaded_catalogs);
510 }
511 }
512
513 LogCvmfs(kLogCvmfs, kLogStderr, "Layer path not found: %s",
514 layer_path.c_str());
515 return NULL;
516 }
517
518
519 int CommandOverlay::Main(const ArgumentList &args) {
520 // Parse publish workflow parameters
521 const string spooler_definition_str = *args.find('r')->second;
522 const string stratum0 = *args.find('w')->second;
523 const string temp_dir = MakeCanonicalPath(*args.find('t')->second);
524 const string manifest_path = *args.find('o')->second;
525 const shash::Any base_hash =
526 shash::MkFromHexPtr(shash::HexPtr(*args.find('b')->second),
527 shash::kSuffixCatalog);
528 const string public_keys = *args.find('K')->second;
529 const string repo_name = *args.find('N')->second;
530
531 // Parse overlay-specific parameters
532 const string layers_str = *args.find('l')->second;
533 string dest_path = MakeCanonicalPath(*args.find('d')->second);
534 // Ensure dest_path starts with exactly one '/'
535 while (dest_path.length() > 1 && dest_path[0] == '/' && dest_path[1] == '/') {
536 dest_path = dest_path.substr(1);
537 }
538 if (dest_path.empty() || dest_path[0] != '/') {
539 dest_path = "/" + dest_path;
540 }
541 shash::Algorithms hash_algorithm = shash::kSha1;
542 if (args.find('e') != args.end()) {
543 hash_algorithm = shash::ParseHashAlgorithm(*args.find('e')->second);
544 if (hash_algorithm == shash::kAny) {
545 PrintError("unknown hash algorithm");
546 return 1;
547 }
548 }
549 zlib::Algorithms compression_alg = zlib::kZlibDefault;
550 if (args.find('Z') != args.end()) {
551 compression_alg = zlib::ParseCompressionAlgorithm(
552 *args.find('Z')->second);
553 }
554
555 // Parse comma-separated layer paths
556 const vector<string> layers = SplitString(layers_str, ',');
557 if (layers.empty()) {
558 LogCvmfs(kLogCvmfs, kLogStderr, "No layers specified");
559 return 1;
560 }
561
562 LogCvmfs(kLogCvmfs, kLogStdout, "Overlay merge of %zu layers into %s",
563 layers.size(), dest_path.c_str());
564 for (size_t i = 0; i < layers.size(); ++i) {
565 LogCvmfs(kLogCvmfs, kLogStdout, " Layer %zu: %s", i, layers[i].c_str());
566 }
567
568 // Set up spoolers (following the ingest pattern)
569 perf::StatisticsTemplate publish_statistics("publish", this->statistics());
570
571 const upload::SpoolerDefinition spooler_definition(
572 spooler_definition_str, hash_algorithm, compression_alg,
573 false /* generate_legacy_bulk_chunks */,
574 false /* use_file_chunking */,
575 0, 0, 0 /* chunk sizes: unused */,
576 "" /* session_token_file */, "" /* key_file */);
577
578 const upload::SpoolerDefinition spooler_definition_catalogs(
579 spooler_definition.Dup2DefaultCompression());
580
581 const UniquePtr<upload::Spooler> spooler_files(
582 upload::Spooler::Construct(spooler_definition, &publish_statistics));
583 if (!spooler_files.IsValid()) {
584 PrintError("Failed to create file spooler");
585 return 3;
586 }
587 const UniquePtr<upload::Spooler> spooler_catalogs(
588 upload::Spooler::Construct(spooler_definition_catalogs,
589 &publish_statistics));
590 if (!spooler_catalogs.IsValid()) {
591 PrintError("Failed to create catalog spooler");
592 return 3;
593 }
594
595 // Initialize download manager and signature manager
596 const bool follow_redirects = (args.count('L') > 0);
597 const string proxy = (args.count('@') > 0) ? *args.find('@')->second : "";
598 if (!InitDownloadManager(follow_redirects, proxy)) {
599 PrintError("Failed to initialize download manager");
600 return 3;
601 }
602 if (!InitSignatureManager(public_keys)) {
603 PrintError("Failed to initialize signature manager");
604 return 3;
605 }
606
607 // Fetch repository manifest
608 const UniquePtr<manifest::Manifest> manifest(
609 FetchRemoteManifest(stratum0, repo_name, base_hash));
610 if (!manifest.IsValid()) {
611 PrintError("Failed to load repository manifest");
612 return 3;
613 }
614
615 const string old_root_hash = manifest->catalog_hash().ToString(true);
616 LogCvmfs(kLogCvmfs, kLogStdout, "Root catalog hash: %s",
617 old_root_hash.c_str());
618
619 // Load root catalog for reading layer entries
620 map<string, OverlayEntry> merged;
621 catalog::Catalog *root_catalog = LoadCatalogForPath(
622 stratum0, "", temp_dir, manifest->catalog_hash());
623 if (root_catalog == NULL) {
624 PrintError("Failed to load root catalog");
625 return 1;
626 }
627
628 // Process layers bottom-to-top
629 for (size_t i = 0; i < layers.size(); ++i) {
630 string layer_path = MakeCanonicalPath(layers[i]);
631 // Ensure layer path starts with exactly one '/'
632 while (layer_path.length() > 1
633 && layer_path[0] == '/' && layer_path[1] == '/') {
634 layer_path = layer_path.substr(1);
635 }
636 if (layer_path.empty() || layer_path[0] != '/') {
637 layer_path = "/" + layer_path;
638 }
639
640 LogCvmfs(kLogCvmfs, kLogStdout, "Processing layer %zu: %s",
641 i, layer_path.c_str());
642
643 map<string, OverlayEntry> layer_entries;
644
645 // Find the catalog that contains this layer path (may be nested)
646 vector<catalog::Catalog *> loaded_catalogs;
647 catalog::Catalog *layer_catalog = FindCatalogForLayer(
648 stratum0, temp_dir, root_catalog, layer_path, &loaded_catalogs);
649 if (layer_catalog == NULL) {
650 for (size_t j = 0; j < loaded_catalogs.size(); ++j)
651 delete loaded_catalogs[j];
652 delete root_catalog;
653 return 1;
654 }
655
656 catalog::DirectoryEntry subdir_entry;
657 const PathString ps_layer_path(layer_path.data(), layer_path.length());
658 if (!layer_catalog->LookupPath(ps_layer_path, &subdir_entry)) {
659 LogCvmfs(kLogCvmfs, kLogStderr,
660 "Unexpected: layer path not found after catalog resolution: %s",
661 layer_path.c_str());
662 for (size_t j = 0; j < loaded_catalogs.size(); ++j)
663 delete loaded_catalogs[j];
664 delete root_catalog;
665 return 1;
666 }
667
668 // Check if the layer path itself is a nested catalog mountpoint;
669 // if so, load that catalog and read its entries.
670 if (subdir_entry.IsNestedCatalogMountpoint()) {
671 shash::Any nested_hash;
672 uint64_t nested_size;
673 if (!layer_catalog->FindNested(ps_layer_path, &nested_hash,
674 &nested_size)) {
675 LogCvmfs(kLogCvmfs, kLogStderr,
676 "Failed to find nested catalog for %s",
677 layer_path.c_str());
678 for (size_t j = 0; j < loaded_catalogs.size(); ++j)
679 delete loaded_catalogs[j];
680 delete root_catalog;
681 return 1;
682 }
683
684 catalog::Catalog *nested_catalog = LoadCatalogForPath(
685 stratum0, layer_path, temp_dir, nested_hash);
686 if (nested_catalog == NULL) {
687 LogCvmfs(kLogCvmfs, kLogStderr,
688 "Failed to load nested catalog for %s",
689 layer_path.c_str());
690 for (size_t j = 0; j < loaded_catalogs.size(); ++j)
691 delete loaded_catalogs[j];
692 delete root_catalog;
693 return 1;
694 }
695
696 ReadCatalogEntries(nested_catalog, layer_path, "",
697 stratum0, temp_dir, &layer_entries);
698 delete nested_catalog;
699 } else {
700 ReadCatalogEntries(layer_catalog, layer_path, "",
701 stratum0, temp_dir, &layer_entries);
702 }
703
704 // Clean up any intermediate catalogs loaded during hierarchy walk
705 for (size_t j = 0; j < loaded_catalogs.size(); ++j)
706 delete loaded_catalogs[j];
707
708 LogCvmfs(kLogCvmfs, kLogStdout, " Read %zu entries from layer %s",
709 layer_entries.size(), layer_path.c_str());
710
711 MergeLayer(layer_entries, &merged);
712
713 LogCvmfs(kLogCvmfs, kLogStdout, " Merged total: %zu entries",
714 merged.size());
715 }
716
717 delete root_catalog;
718
719 // Set up WritableCatalogManager and publish merged entries
720 LogCvmfs(kLogCvmfs, kLogStdout,
721 "Publishing %zu merged entries under %s",
722 merged.size(), dest_path.c_str());
723
724 catalog::WritableCatalogManager catalog_manager(
725 base_hash, stratum0, temp_dir,
726 spooler_catalogs.weak_ref(), download_manager(),
727 false /* enforce_limits */,
728 0 /* nested_kcatalog_limit */,
729 0 /* root_kcatalog_limit */,
730 0 /* file_mbyte_limit */,
731 statistics(),
732 false /* is_balanceable */,
733 0 /* max_weight */, 0 /* min_weight */);
734 catalog_manager.Init();
735
736 if (!PublishMergedEntries(&catalog_manager, merged, dest_path)) {
737 PrintError("Failed to publish merged entries");
738 return 5;
739 }
740
741 // Commit catalog changes and produce updated manifest
742 catalog_manager.PrecalculateListings();
743 if (!catalog_manager.Commit(false, 0, manifest.weak_ref())) {
744 PrintError("Failed to commit catalog changes");
745 return 5;
746 }
747
748 // Finalize spoolers
749 LogCvmfs(kLogCvmfs, kLogStdout, "Waiting for uploads to finish...");
750 spooler_files->WaitForUpload();
751 spooler_catalogs->WaitForUpload();
752 spooler_files->FinalizeSession(false);
753
754 const string new_root_hash = manifest->catalog_hash().ToString(true);
755 if (!spooler_catalogs->FinalizeSession(true, old_root_hash, new_root_hash,
756 RepositoryTag())) {
757 PrintError("Failed to finalize session");
758 return 5;
759 }
760
761 // Export manifest
762 if (!manifest->Export(manifest_path)) {
763 PrintError("Failed to export manifest");
764 return 6;
765 }
766
767 LogCvmfs(kLogCvmfs, kLogStdout,
768 "Overlay published successfully to %s", dest_path.c_str());
769 return 0;
770 }
771
772 } // namespace swissknife
773