| Line |
Branch |
Exec |
Source |
| 1 |
|
|
/** |
| 2 |
|
|
* This file is part of the CernVM File System. |
| 3 |
|
|
* |
| 4 |
|
|
* Implementation of the overlay swissknife command that merges multiple |
| 5 |
|
|
* CVMFS subdirectory catalogs using overlay semantics (similar to OverlayFS) |
| 6 |
|
|
* and publishes the result as a repository subdirectory. |
| 7 |
|
|
*/ |
| 8 |
|
|
|
| 9 |
|
|
#include "swissknife_overlay.h" |
| 10 |
|
|
|
| 11 |
|
|
#include <fcntl.h> |
| 12 |
|
|
#include <inttypes.h> |
| 13 |
|
|
#include <sys/stat.h> |
| 14 |
|
|
#include <unistd.h> |
| 15 |
|
|
|
| 16 |
|
|
#include <cassert> |
| 17 |
|
|
#include <ctime> |
| 18 |
|
|
#include <map> |
| 19 |
|
|
#include <memory> |
| 20 |
|
|
#include <string> |
| 21 |
|
|
#include <vector> |
| 22 |
|
|
|
| 23 |
|
|
#include "catalog.h" |
| 24 |
|
|
#include "catalog_mgr_rw.h" |
| 25 |
|
|
#include "catalog_rw.h" |
| 26 |
|
|
#include "catalog_sql.h" |
| 27 |
|
|
#include "compression/compression.h" |
| 28 |
|
|
#include "crypto/hash.h" |
| 29 |
|
|
#include "directory_entry.h" |
| 30 |
|
|
#include "ingestion/ingestion_source.h" |
| 31 |
|
|
#include "json_document.h" |
| 32 |
|
|
#include "manifest.h" |
| 33 |
|
|
#include "network/download.h" |
| 34 |
|
|
#include "network/sink_path.h" |
| 35 |
|
|
#include "repository_tag.h" |
| 36 |
|
|
#include "shortstring.h" |
| 37 |
|
|
#include "statistics.h" |
| 38 |
|
|
#include "upload.h" |
| 39 |
|
|
#include "upload_spooler_definition.h" |
| 40 |
|
|
#include "upload_spooler_result.h" |
| 41 |
|
|
#include "util/logging.h" |
| 42 |
|
|
#include "util/posix.h" |
| 43 |
|
|
#include "util/string.h" |
| 44 |
|
|
#include "xattr.h" |
| 45 |
|
|
|
| 46 |
|
|
using namespace std; // NOLINT |
| 47 |
|
|
|
| 48 |
|
|
namespace swissknife { |
| 49 |
|
|
|
| 50 |
|
✗ |
ParameterList CommandOverlay::GetParams() const { |
| 51 |
|
✗ |
ParameterList r; |
| 52 |
|
|
// Publish workflow parameters (same convention as ingest/sync) |
| 53 |
|
✗ |
r.push_back(Parameter::Mandatory('r', "upstream storage definition")); |
| 54 |
|
✗ |
r.push_back(Parameter::Mandatory('w', "stratum 0 URL")); |
| 55 |
|
✗ |
r.push_back(Parameter::Mandatory('t', "temporary directory")); |
| 56 |
|
✗ |
r.push_back(Parameter::Mandatory('o', "manifest output path")); |
| 57 |
|
✗ |
r.push_back(Parameter::Mandatory('b', "base hash of current root catalog")); |
| 58 |
|
✗ |
r.push_back(Parameter::Mandatory('K', "public key path")); |
| 59 |
|
✗ |
r.push_back(Parameter::Mandatory('N', "repository name")); |
| 60 |
|
|
// Gateway publishing (same convention as ingest): when the upstream is a |
| 61 |
|
|
// repository gateway these carry the lease so the merge can be committed |
| 62 |
|
|
// without a local FUSE mount (mountless publishing). |
| 63 |
|
✗ |
r.push_back(Parameter::Optional('P', "session token file (gateway)")); |
| 64 |
|
✗ |
r.push_back(Parameter::Optional('H', "gateway key file")); |
| 65 |
|
|
|
| 66 |
|
|
// Overlay-specific parameters |
| 67 |
|
✗ |
r.push_back(Parameter::Mandatory('l', "comma-separated layer paths " |
| 68 |
|
|
"(bottom-to-top order)")); |
| 69 |
|
✗ |
r.push_back(Parameter::Mandatory('d', "destination subdirectory path in " |
| 70 |
|
|
"repository for the merged overlay")); |
| 71 |
|
✗ |
r.push_back(Parameter::Optional('e', "hash algorithm (default: sha1)")); |
| 72 |
|
✗ |
r.push_back(Parameter::Optional('Z', "compression algorithm " |
| 73 |
|
|
"(default: zlib)")); |
| 74 |
|
✗ |
r.push_back(Parameter::Optional('@', "proxy URL")); |
| 75 |
|
✗ |
r.push_back(Parameter::Switch('L', "follow HTTP redirects")); |
| 76 |
|
✗ |
r.push_back(Parameter::Optional( |
| 77 |
|
|
'c', "OCI image config JSON file path " |
| 78 |
|
|
"(when provided, Singularity .singularity.d dotfiles are " |
| 79 |
|
|
"injected into the merged overlay)")); |
| 80 |
|
✗ |
r.push_back(Parameter::Switch('S', "skip Singularity dotfile injection " |
| 81 |
|
|
"even when an OCI config is provided")); |
| 82 |
|
✗ |
return r; |
| 83 |
|
|
} |
| 84 |
|
|
|
| 85 |
|
|
|
| 86 |
|
✗ |
bool CommandOverlay::IsWhiteoutFile(const string &name) { |
| 87 |
|
✗ |
return HasPrefix(name, ".wh.", false) && !IsOpaqueMarker(name); |
| 88 |
|
|
} |
| 89 |
|
|
|
| 90 |
|
|
|
| 91 |
|
✗ |
string CommandOverlay::GetWhiteoutTarget(const string &name) { |
| 92 |
|
|
// ".wh." is 4 characters |
| 93 |
|
✗ |
if (name.length() <= 4) |
| 94 |
|
✗ |
return ""; |
| 95 |
|
✗ |
return name.substr(4); |
| 96 |
|
|
} |
| 97 |
|
|
|
| 98 |
|
|
|
| 99 |
|
✗ |
bool CommandOverlay::IsOpaqueMarker(const string &name) { |
| 100 |
|
✗ |
return name == ".wh..wh..opq"; |
| 101 |
|
|
} |
| 102 |
|
|
|
| 103 |
|
|
|
| 104 |
|
✗ |
bool CommandOverlay::ReadCatalogEntries(catalog::Catalog *catalog, |
| 105 |
|
|
const string &catalog_root_path, |
| 106 |
|
|
const string &relative_prefix, |
| 107 |
|
|
const string &repo_base, |
| 108 |
|
|
const string &temp_dir, |
| 109 |
|
|
map<string, OverlayEntry> *entries) { |
| 110 |
|
|
// List entries at this path in the catalog |
| 111 |
|
✗ |
catalog::DirectoryEntryList listing; |
| 112 |
|
|
const PathString ps_path(catalog_root_path.data(), |
| 113 |
|
✗ |
catalog_root_path.length()); |
| 114 |
|
✗ |
const bool has_entries = catalog->ListingPath(ps_path, &listing); |
| 115 |
|
|
|
| 116 |
|
✗ |
if (!has_entries && catalog_root_path != catalog->mountpoint().ToString()) { |
| 117 |
|
|
// No entries at this path - it might be a file, not a directory |
| 118 |
|
✗ |
return true; |
| 119 |
|
|
} |
| 120 |
|
|
|
| 121 |
|
✗ |
for (size_t i = 0; i < listing.size(); ++i) { |
| 122 |
|
✗ |
const catalog::DirectoryEntry &dirent = listing[i]; |
| 123 |
|
✗ |
const string name = dirent.name().ToString(); |
| 124 |
|
|
|
| 125 |
|
|
// Skip CVMFS bookkeeping files — they are internal metadata and must not |
| 126 |
|
|
// be carried over into the merged overlay. PublishMergedEntries() will |
| 127 |
|
|
// create its own .cvmfscatalog marker for the destination catalog. |
| 128 |
|
✗ |
if (name == ".cvmfscatalog" || name == ".cvmfsdirtab" |
| 129 |
|
✗ |
|| name == ".cvmfsautocatalog") { |
| 130 |
|
✗ |
continue; |
| 131 |
|
|
} |
| 132 |
|
|
|
| 133 |
|
✗ |
const string child_catalog_path = catalog_root_path.empty() |
| 134 |
|
|
? "/" + name |
| 135 |
|
✗ |
: catalog_root_path + "/" + name; |
| 136 |
|
✗ |
const string child_relative = relative_prefix.empty() |
| 137 |
|
|
? name |
| 138 |
|
✗ |
: relative_prefix + "/" + name; |
| 139 |
|
|
|
| 140 |
|
✗ |
OverlayEntry oe; |
| 141 |
|
✗ |
oe.entry = dirent; |
| 142 |
|
✗ |
oe.path = child_relative; |
| 143 |
|
✗ |
oe.parent = relative_prefix; |
| 144 |
|
✗ |
oe.is_whiteout = IsWhiteoutFile(name); |
| 145 |
|
✗ |
oe.is_opaque_dir = false; |
| 146 |
|
|
|
| 147 |
|
|
// Look up xattrs for this entry |
| 148 |
|
✗ |
XattrList xattrs; |
| 149 |
|
|
const PathString ps_child(child_catalog_path.data(), |
| 150 |
|
✗ |
child_catalog_path.length()); |
| 151 |
|
✗ |
catalog->LookupXattrsPath(ps_child, &xattrs); |
| 152 |
|
✗ |
oe.xattrs = xattrs; |
| 153 |
|
|
|
| 154 |
|
✗ |
(*entries)[child_relative] = oe; |
| 155 |
|
|
|
| 156 |
|
✗ |
if (dirent.IsDirectory()) { |
| 157 |
|
✗ |
if (dirent.IsNestedCatalogMountpoint() && !repo_base.empty()) { |
| 158 |
|
|
// Load the nested catalog and recurse into it |
| 159 |
|
✗ |
shash::Any nested_hash; |
| 160 |
|
|
uint64_t nested_size; |
| 161 |
|
✗ |
if (!catalog->FindNested(ps_child, &nested_hash, &nested_size)) { |
| 162 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStderr, |
| 163 |
|
|
"Failed to find nested catalog hash for %s", |
| 164 |
|
|
child_catalog_path.c_str()); |
| 165 |
|
✗ |
return false; |
| 166 |
|
|
} |
| 167 |
|
|
|
| 168 |
|
✗ |
catalog::Catalog *nested = LoadCatalogForPath( |
| 169 |
|
|
repo_base, child_catalog_path, temp_dir, nested_hash); |
| 170 |
|
✗ |
if (nested == NULL) { |
| 171 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStderr, |
| 172 |
|
|
"Failed to load nested catalog for %s", |
| 173 |
|
|
child_catalog_path.c_str()); |
| 174 |
|
✗ |
return false; |
| 175 |
|
|
} |
| 176 |
|
|
|
| 177 |
|
|
// Check for opaque marker in the nested catalog root |
| 178 |
|
✗ |
catalog::DirectoryEntryList sub_listing; |
| 179 |
|
✗ |
nested->ListingPath(ps_child, &sub_listing); |
| 180 |
|
✗ |
for (size_t j = 0; j < sub_listing.size(); ++j) { |
| 181 |
|
✗ |
if (IsOpaqueMarker(sub_listing[j].name().ToString())) { |
| 182 |
|
✗ |
(*entries)[child_relative].is_opaque_dir = true; |
| 183 |
|
✗ |
break; |
| 184 |
|
|
} |
| 185 |
|
|
} |
| 186 |
|
|
|
| 187 |
|
✗ |
if (!ReadCatalogEntries(nested, child_catalog_path, child_relative, |
| 188 |
|
|
repo_base, temp_dir, entries)) { |
| 189 |
|
✗ |
delete nested; |
| 190 |
|
✗ |
return false; |
| 191 |
|
|
} |
| 192 |
|
✗ |
delete nested; |
| 193 |
|
✗ |
} else if (!dirent.IsNestedCatalogMountpoint()) { |
| 194 |
|
|
// Regular directory — check for opaque marker among children |
| 195 |
|
✗ |
catalog::DirectoryEntryList sub_listing; |
| 196 |
|
✗ |
catalog->ListingPath(ps_child, &sub_listing); |
| 197 |
|
✗ |
for (size_t j = 0; j < sub_listing.size(); ++j) { |
| 198 |
|
✗ |
if (IsOpaqueMarker(sub_listing[j].name().ToString())) { |
| 199 |
|
✗ |
(*entries)[child_relative].is_opaque_dir = true; |
| 200 |
|
✗ |
break; |
| 201 |
|
|
} |
| 202 |
|
|
} |
| 203 |
|
|
|
| 204 |
|
✗ |
if (!ReadCatalogEntries(catalog, child_catalog_path, child_relative, |
| 205 |
|
|
repo_base, temp_dir, entries)) { |
| 206 |
|
✗ |
return false; |
| 207 |
|
|
} |
| 208 |
|
|
} |
| 209 |
|
|
// else: nested catalog mountpoint but no repo_base — skip (e.g. cache) |
| 210 |
|
|
} |
| 211 |
|
|
} |
| 212 |
|
|
|
| 213 |
|
✗ |
return true; |
| 214 |
|
|
} |
| 215 |
|
|
|
| 216 |
|
|
|
| 217 |
|
✗ |
void CommandOverlay::MergeLayer(const map<string, OverlayEntry> &layer_entries, |
| 218 |
|
|
map<string, OverlayEntry> *merged) const { |
| 219 |
|
|
// First pass: collect whiteouts and opaque directories |
| 220 |
|
✗ |
vector<string> whiteout_targets; |
| 221 |
|
✗ |
vector<string> opaque_dirs; |
| 222 |
|
|
|
| 223 |
|
✗ |
for (map<string, OverlayEntry>::const_iterator it = layer_entries.begin(); |
| 224 |
|
✗ |
it != layer_entries.end(); |
| 225 |
|
✗ |
++it) { |
| 226 |
|
✗ |
const OverlayEntry &oe = it->second; |
| 227 |
|
✗ |
const string &path = it->first; |
| 228 |
|
|
|
| 229 |
|
✗ |
if (oe.is_whiteout) { |
| 230 |
|
|
// Whiteout: mark the target for deletion from lower layers |
| 231 |
|
✗ |
const string target_name = GetWhiteoutTarget(GetFileName(path)); |
| 232 |
|
✗ |
const string target_path = oe.parent.empty() |
| 233 |
|
|
? target_name |
| 234 |
|
✗ |
: oe.parent + "/" + target_name; |
| 235 |
|
✗ |
whiteout_targets.push_back(target_path); |
| 236 |
|
✗ |
continue; |
| 237 |
|
|
} |
| 238 |
|
|
|
| 239 |
|
✗ |
if (IsOpaqueMarker(GetFileName(path))) { |
| 240 |
|
|
// Don't add the opaque marker itself to the merged output |
| 241 |
|
✗ |
continue; |
| 242 |
|
|
} |
| 243 |
|
|
|
| 244 |
|
✗ |
if (oe.is_opaque_dir) { |
| 245 |
|
✗ |
opaque_dirs.push_back(path); |
| 246 |
|
|
} |
| 247 |
|
|
} |
| 248 |
|
|
|
| 249 |
|
|
// Apply opaque directory semantics: remove all entries from lower layers |
| 250 |
|
|
// that are under opaque directories |
| 251 |
|
✗ |
for (size_t i = 0; i < opaque_dirs.size(); ++i) { |
| 252 |
|
✗ |
const string &opaque_path = opaque_dirs[i]; |
| 253 |
|
✗ |
const string prefix = opaque_path + "/"; |
| 254 |
|
|
|
| 255 |
|
|
// Remove children of this directory from merged (lower layer entries) |
| 256 |
|
✗ |
vector<string> to_remove; |
| 257 |
|
✗ |
for (map<string, OverlayEntry>::iterator it = merged->begin(); |
| 258 |
|
✗ |
it != merged->end(); |
| 259 |
|
✗ |
++it) { |
| 260 |
|
✗ |
if (HasPrefix(it->first, prefix, false)) { |
| 261 |
|
✗ |
to_remove.push_back(it->first); |
| 262 |
|
|
} |
| 263 |
|
|
} |
| 264 |
|
✗ |
for (size_t j = 0; j < to_remove.size(); ++j) { |
| 265 |
|
✗ |
merged->erase(to_remove[j]); |
| 266 |
|
|
} |
| 267 |
|
|
} |
| 268 |
|
|
|
| 269 |
|
|
// Apply whiteout semantics: remove targeted entries and their children |
| 270 |
|
✗ |
for (size_t i = 0; i < whiteout_targets.size(); ++i) { |
| 271 |
|
✗ |
const string &target = whiteout_targets[i]; |
| 272 |
|
✗ |
const string prefix = target + "/"; |
| 273 |
|
|
|
| 274 |
|
|
// Remove the target entry itself |
| 275 |
|
✗ |
merged->erase(target); |
| 276 |
|
|
|
| 277 |
|
|
// Remove all children of the target |
| 278 |
|
✗ |
vector<string> to_remove; |
| 279 |
|
✗ |
for (map<string, OverlayEntry>::iterator it = merged->begin(); |
| 280 |
|
✗ |
it != merged->end(); |
| 281 |
|
✗ |
++it) { |
| 282 |
|
✗ |
if (HasPrefix(it->first, prefix, false)) { |
| 283 |
|
✗ |
to_remove.push_back(it->first); |
| 284 |
|
|
} |
| 285 |
|
|
} |
| 286 |
|
✗ |
for (size_t j = 0; j < to_remove.size(); ++j) { |
| 287 |
|
✗ |
merged->erase(to_remove[j]); |
| 288 |
|
|
} |
| 289 |
|
|
} |
| 290 |
|
|
|
| 291 |
|
|
// Second pass: add/override entries from this layer |
| 292 |
|
✗ |
for (map<string, OverlayEntry>::const_iterator it = layer_entries.begin(); |
| 293 |
|
✗ |
it != layer_entries.end(); |
| 294 |
|
✗ |
++it) { |
| 295 |
|
✗ |
const OverlayEntry &oe = it->second; |
| 296 |
|
✗ |
const string &path = it->first; |
| 297 |
|
|
|
| 298 |
|
|
// Skip whiteout files and opaque markers - they are control files |
| 299 |
|
✗ |
if (oe.is_whiteout || IsOpaqueMarker(GetFileName(path))) { |
| 300 |
|
✗ |
continue; |
| 301 |
|
|
} |
| 302 |
|
|
|
| 303 |
|
|
// Upper layer overrides lower layer for the same path |
| 304 |
|
✗ |
(*merged)[path] = oe; |
| 305 |
|
|
} |
| 306 |
|
|
} |
| 307 |
|
|
|
| 308 |
|
|
|
| 309 |
|
|
// --------------------------------------------------------------------------- |
| 310 |
|
|
// Singularity dotfile generation |
| 311 |
|
|
// --------------------------------------------------------------------------- |
| 312 |
|
|
|
| 313 |
|
|
// Static file contents for /.singularity.d — these mirror the Go |
| 314 |
|
|
// constants in singularity/dotfiles.go (originally from Sylabs/Singularity). |
| 315 |
|
|
|
| 316 |
|
|
static const char * const |
| 317 |
|
|
kSingExec = "#!/bin/sh\n" |
| 318 |
|
|
"for script in /.singularity.d/env/*.sh; do\n" |
| 319 |
|
|
" if [ -f \"$script\" ]; then\n" |
| 320 |
|
|
" . \"$script\"\n" |
| 321 |
|
|
" fi\n" |
| 322 |
|
|
"done\n" |
| 323 |
|
|
"exec \"$@\"\n"; |
| 324 |
|
|
|
| 325 |
|
|
static const char * const kSingRun = |
| 326 |
|
|
"#!/bin/sh\n" |
| 327 |
|
|
"for script in /.singularity.d/env/*.sh; do\n" |
| 328 |
|
|
" if [ -f \"$script\" ]; then\n" |
| 329 |
|
|
" . \"$script\"\n" |
| 330 |
|
|
" fi\n" |
| 331 |
|
|
"done\n" |
| 332 |
|
|
"if test -n \"${SINGULARITY_APPNAME:-}\"; then\n" |
| 333 |
|
|
" if test -x \"/scif/apps/${SINGULARITY_APPNAME:-}/scif/runscript\"; " |
| 334 |
|
|
"then\n" |
| 335 |
|
|
" exec \"/scif/apps/${SINGULARITY_APPNAME:-}/scif/runscript\" " |
| 336 |
|
|
"\"$@\"\n" |
| 337 |
|
|
" else\n" |
| 338 |
|
|
" echo \"No Singularity runscript for contained app: " |
| 339 |
|
|
"${SINGULARITY_APPNAME:-}\"\n" |
| 340 |
|
|
" exit 1\n" |
| 341 |
|
|
" fi\n" |
| 342 |
|
|
"elif test -x \"/.singularity.d/runscript\"; then\n" |
| 343 |
|
|
" exec \"/.singularity.d/runscript\" \"$@\"\n" |
| 344 |
|
|
"else\n" |
| 345 |
|
|
" echo \"No Singularity runscript found, executing /bin/sh\"\n" |
| 346 |
|
|
" exec /bin/sh \"$@\"\n" |
| 347 |
|
|
"fi\n"; |
| 348 |
|
|
|
| 349 |
|
|
static const char * const kSingShell = |
| 350 |
|
|
"#!/bin/sh\n" |
| 351 |
|
|
"for script in /.singularity.d/env/*.sh; do\n" |
| 352 |
|
|
" if [ -f \"$script\" ]; then\n" |
| 353 |
|
|
" . \"$script\"\n" |
| 354 |
|
|
" fi\n" |
| 355 |
|
|
"done\n" |
| 356 |
|
|
"if test -n \"$SINGULARITY_SHELL\" -a -x \"$SINGULARITY_SHELL\"; then\n" |
| 357 |
|
|
" exec $SINGULARITY_SHELL \"$@\"\n" |
| 358 |
|
|
" echo \"ERROR: Failed running shell as defined by " |
| 359 |
|
|
"'\\$SINGULARITY_SHELL'\" 1>&2\n" |
| 360 |
|
|
" exit 1\n" |
| 361 |
|
|
"elif test -x /bin/bash; then\n" |
| 362 |
|
|
" SHELL=/bin/bash\n" |
| 363 |
|
|
" PS1=\"Singularity $SINGULARITY_NAME:\\w> \"\n" |
| 364 |
|
|
" export SHELL PS1\n" |
| 365 |
|
|
" exec /bin/bash --norc \"$@\"\n" |
| 366 |
|
|
"elif test -x /bin/sh; then\n" |
| 367 |
|
|
" SHELL=/bin/sh\n" |
| 368 |
|
|
" export SHELL\n" |
| 369 |
|
|
" exec /bin/sh \"$@\"\n" |
| 370 |
|
|
"else\n" |
| 371 |
|
|
" echo \"ERROR: /bin/sh does not exist in container\" 1>&2\n" |
| 372 |
|
|
"fi\n" |
| 373 |
|
|
"exit 1\n"; |
| 374 |
|
|
|
| 375 |
|
|
static const char * const |
| 376 |
|
|
kSingStart = "#!/bin/sh\n" |
| 377 |
|
|
"# if we are here start notify PID 1 to continue\n" |
| 378 |
|
|
"# DON'T REMOVE\n" |
| 379 |
|
|
"kill -CONT 1\n" |
| 380 |
|
|
"for script in /.singularity.d/env/*.sh; do\n" |
| 381 |
|
|
" if [ -f \"$script\" ]; then\n" |
| 382 |
|
|
" . \"$script\"\n" |
| 383 |
|
|
" fi\n" |
| 384 |
|
|
"done\n" |
| 385 |
|
|
"if test -x \"/.singularity.d/startscript\"; then\n" |
| 386 |
|
|
" exec \"/.singularity.d/startscript\"\n" |
| 387 |
|
|
"fi\n"; |
| 388 |
|
|
|
| 389 |
|
|
static const char * const kSingTest = |
| 390 |
|
|
"#!/bin/sh\n" |
| 391 |
|
|
"for script in /.singularity.d/env/*.sh; do\n" |
| 392 |
|
|
" if [ -f \"$script\" ]; then\n" |
| 393 |
|
|
" . \"$script\"\n" |
| 394 |
|
|
" fi\n" |
| 395 |
|
|
"done\n" |
| 396 |
|
|
"if test -n \"${SINGULARITY_APPNAME:-}\"; then\n" |
| 397 |
|
|
" if test -x \"/scif/apps/${SINGULARITY_APPNAME:-}/scif/test\"; then\n" |
| 398 |
|
|
" exec \"/scif/apps/${SINGULARITY_APPNAME:-}/scif/test\" \"$@\"\n" |
| 399 |
|
|
" else\n" |
| 400 |
|
|
" echo \"No tests for contained app: ${SINGULARITY_APPNAME:-}\"\n" |
| 401 |
|
|
" exit 1\n" |
| 402 |
|
|
" fi\n" |
| 403 |
|
|
"elif test -x \"/.singularity.d/test\"; then\n" |
| 404 |
|
|
" exec \"/.singularity.d/test\" \"$@\"\n" |
| 405 |
|
|
"else\n" |
| 406 |
|
|
" echo \"No test found in container, executing /bin/sh -c true\"\n" |
| 407 |
|
|
" exec /bin/sh -c true\n" |
| 408 |
|
|
"fi\n"; |
| 409 |
|
|
|
| 410 |
|
|
static const char * const kSingEnv01Base = |
| 411 |
|
|
"#!/bin/sh\n" |
| 412 |
|
|
"# \n" |
| 413 |
|
|
"# Copyright (c) 2017, SingularityWare, LLC. All rights reserved.\n" |
| 414 |
|
|
"# Copyright (c) 2015-2017, Gregory M. Kurtzer. All rights reserved.\n" |
| 415 |
|
|
"# \n" |
| 416 |
|
|
"# Copyright (c) 2016-2017, The Regents of the University of California,\n" |
| 417 |
|
|
"# through Lawrence Berkeley National Laboratory (subject to receipt of " |
| 418 |
|
|
"any\n" |
| 419 |
|
|
"# required approvals from the U.S. Dept. of Energy). All rights " |
| 420 |
|
|
"reserved.\n" |
| 421 |
|
|
"# \n"; |
| 422 |
|
|
|
| 423 |
|
|
static const char * const |
| 424 |
|
|
kSingEnv90 = "#!/bin/sh\n" |
| 425 |
|
|
"# Custom environment shell code should follow\n"; |
| 426 |
|
|
|
| 427 |
|
|
static const char * const kSingEnv95Apps = |
| 428 |
|
|
"#!/bin/sh\n" |
| 429 |
|
|
"#\n" |
| 430 |
|
|
"# Copyright (c) 2017, SingularityWare, LLC. All rights reserved.\n" |
| 431 |
|
|
"#\n" |
| 432 |
|
|
"if test -n \"${SINGULARITY_APPNAME:-}\"; then\n" |
| 433 |
|
|
" # The active app should be exported\n" |
| 434 |
|
|
" export SINGULARITY_APPNAME\n" |
| 435 |
|
|
" if test -d \"/scif/apps/${SINGULARITY_APPNAME:-}/\"; then\n" |
| 436 |
|
|
" SCIF_APPS=\"/scif/apps\"\n" |
| 437 |
|
|
" SCIF_APPROOT=\"/scif/apps/${SINGULARITY_APPNAME:-}\"\n" |
| 438 |
|
|
" export SCIF_APPROOT SCIF_APPS\n" |
| 439 |
|
|
" PATH=\"/scif/apps/${SINGULARITY_APPNAME:-}:$PATH\"\n" |
| 440 |
|
|
" if test -d \"/scif/apps/${SINGULARITY_APPNAME:-}/bin\"; then\n" |
| 441 |
|
|
" PATH=\"/scif/apps/${SINGULARITY_APPNAME:-}/bin:$PATH\"\n" |
| 442 |
|
|
" fi\n" |
| 443 |
|
|
" if test -d \"/scif/apps/${SINGULARITY_APPNAME:-}/lib\"; then\n" |
| 444 |
|
|
" " |
| 445 |
|
|
"LD_LIBRARY_PATH=\"/scif/apps/${SINGULARITY_APPNAME:-}/" |
| 446 |
|
|
"lib:$LD_LIBRARY_PATH\"\n" |
| 447 |
|
|
" export LD_LIBRARY_PATH\n" |
| 448 |
|
|
" fi\n" |
| 449 |
|
|
" if [ -f " |
| 450 |
|
|
"\"/scif/apps/${SINGULARITY_APPNAME:-}/scif/env/01-base.sh\" ]; then\n" |
| 451 |
|
|
" . " |
| 452 |
|
|
"\"/scif/apps/${SINGULARITY_APPNAME:-}/scif/env/01-base.sh\"\n" |
| 453 |
|
|
" fi\n" |
| 454 |
|
|
" if [ -f " |
| 455 |
|
|
"\"/scif/apps/${SINGULARITY_APPNAME:-}/scif/env/90-environment.sh\" ]; " |
| 456 |
|
|
"then\n" |
| 457 |
|
|
" . " |
| 458 |
|
|
"\"/scif/apps/${SINGULARITY_APPNAME:-}/scif/env/90-environment.sh\"\n" |
| 459 |
|
|
" fi\n" |
| 460 |
|
|
" export PATH\n" |
| 461 |
|
|
" else\n" |
| 462 |
|
|
" echo \"Could not locate the container application: " |
| 463 |
|
|
"${SINGULARITY_APPNAME}\"\n" |
| 464 |
|
|
" exit 1\n" |
| 465 |
|
|
" fi\n" |
| 466 |
|
|
"fi\n"; |
| 467 |
|
|
|
| 468 |
|
|
static const char * const kSingEnv99Base = |
| 469 |
|
|
"#!/bin/sh\n" |
| 470 |
|
|
"# \n" |
| 471 |
|
|
"# Copyright (c) 2017, SingularityWare, LLC. All rights reserved.\n" |
| 472 |
|
|
"# Copyright (c) 2015-2017, Gregory M. Kurtzer. All rights reserved.\n" |
| 473 |
|
|
"# \n" |
| 474 |
|
|
"if [ -z \"$LD_LIBRARY_PATH\" ]; then\n" |
| 475 |
|
|
" LD_LIBRARY_PATH=\"/.singularity.d/libs\"\n" |
| 476 |
|
|
"else\n" |
| 477 |
|
|
" LD_LIBRARY_PATH=\"$LD_LIBRARY_PATH:/.singularity.d/libs\"\n" |
| 478 |
|
|
"fi\n" |
| 479 |
|
|
"PS1=\"Singularity> \"\n" |
| 480 |
|
|
"export LD_LIBRARY_PATH PS1\n"; |
| 481 |
|
|
|
| 482 |
|
|
static const char * const kSingEnv99Runtimevars = |
| 483 |
|
|
"#!/bin/sh\n" |
| 484 |
|
|
"if [ -n \"${SING_USER_DEFINED_PREPEND_PATH:-}\" ]; then\n" |
| 485 |
|
|
"\tPATH=\"${SING_USER_DEFINED_PREPEND_PATH}:${PATH}\"\n" |
| 486 |
|
|
"fi\n" |
| 487 |
|
|
"if [ -n \"${SING_USER_DEFINED_APPEND_PATH:-}\" ]; then\n" |
| 488 |
|
|
"\tPATH=\"${PATH}:${SING_USER_DEFINED_APPEND_PATH}\"\n" |
| 489 |
|
|
"fi\n" |
| 490 |
|
|
"if [ -n \"${SING_USER_DEFINED_PATH:-}\" ]; then\n" |
| 491 |
|
|
"\tPATH=\"${SING_USER_DEFINED_PATH}\"\n" |
| 492 |
|
|
"fi\n" |
| 493 |
|
|
"unset SING_USER_DEFINED_PREPEND_PATH \\\n" |
| 494 |
|
|
"\t SING_USER_DEFINED_APPEND_PATH \\\n" |
| 495 |
|
|
"\t SING_USER_DEFINED_PATH\n" |
| 496 |
|
|
"export PATH\n"; |
| 497 |
|
|
|
| 498 |
|
|
static const char * const kSingStartscript = "#!/bin/sh\n"; |
| 499 |
|
|
|
| 500 |
|
|
|
| 501 |
|
✗ |
string CommandOverlay::ShellEscape(const string &s) { |
| 502 |
|
✗ |
string escaped = ReplaceAll(s, "\\", "\\\\"); |
| 503 |
|
✗ |
escaped = ReplaceAll(escaped, "\"", "\\\""); |
| 504 |
|
✗ |
escaped = ReplaceAll(escaped, "`", "\\`"); |
| 505 |
|
✗ |
escaped = ReplaceAll(escaped, "$", "\\$"); |
| 506 |
|
✗ |
return escaped; |
| 507 |
|
|
} |
| 508 |
|
|
|
| 509 |
|
|
|
| 510 |
|
✗ |
string CommandOverlay::ArgsQuoted(const vector<string> &args) { |
| 511 |
|
✗ |
string quoted; |
| 512 |
|
✗ |
for (size_t i = 0; i < args.size(); ++i) { |
| 513 |
|
✗ |
if (i > 0) |
| 514 |
|
✗ |
quoted += " "; |
| 515 |
|
✗ |
quoted += "\"" + ShellEscape(args[i]) + "\""; |
| 516 |
|
|
} |
| 517 |
|
✗ |
return quoted; |
| 518 |
|
|
} |
| 519 |
|
|
|
| 520 |
|
|
|
| 521 |
|
✗ |
string CommandOverlay::GenerateRunscript(const vector<string> &entrypoint, |
| 522 |
|
|
const vector<string> &cmd) { |
| 523 |
|
✗ |
string script = "#!/bin/sh\n"; |
| 524 |
|
✗ |
if (!entrypoint.empty()) { |
| 525 |
|
✗ |
script += "OCI_ENTRYPOINT='" + ArgsQuoted(entrypoint) + "'\n"; |
| 526 |
|
|
} else { |
| 527 |
|
✗ |
script += "OCI_ENTRYPOINT=''\n"; |
| 528 |
|
|
} |
| 529 |
|
✗ |
if (!cmd.empty()) { |
| 530 |
|
✗ |
script += "OCI_CMD='" + ArgsQuoted(cmd) + "'\n"; |
| 531 |
|
|
} else { |
| 532 |
|
✗ |
script += "OCI_CMD=''\n"; |
| 533 |
|
|
} |
| 534 |
|
|
script += |
| 535 |
|
|
"CMDLINE_ARGS=\"\"\n" |
| 536 |
|
|
"# prepare command line arguments for evaluation\n" |
| 537 |
|
|
"for arg in \"$@\"; do\n" |
| 538 |
|
|
" CMDLINE_ARGS=\"${CMDLINE_ARGS} \\\"$arg\\\"\"\n" |
| 539 |
|
|
"done\n" |
| 540 |
|
|
"# ENTRYPOINT only - run entrypoint plus args\n" |
| 541 |
|
|
"if [ -z \"$OCI_CMD\" ] && [ -n \"$OCI_ENTRYPOINT\" ]; then\n" |
| 542 |
|
|
" if [ $# -gt 0 ]; then\n" |
| 543 |
|
|
" SINGULARITY_OCI_RUN=\"${OCI_ENTRYPOINT} ${CMDLINE_ARGS}\"\n" |
| 544 |
|
|
" else\n" |
| 545 |
|
|
" SINGULARITY_OCI_RUN=\"${OCI_ENTRYPOINT}\"\n" |
| 546 |
|
|
" fi\n" |
| 547 |
|
|
"fi\n" |
| 548 |
|
|
"# CMD only - run CMD or override with args\n" |
| 549 |
|
|
"if [ -n \"$OCI_CMD\" ] && [ -z \"$OCI_ENTRYPOINT\" ]; then\n" |
| 550 |
|
|
" if [ $# -gt 0 ]; then\n" |
| 551 |
|
|
" SINGULARITY_OCI_RUN=\"${CMDLINE_ARGS}\"\n" |
| 552 |
|
|
" else\n" |
| 553 |
|
|
" SINGULARITY_OCI_RUN=\"${OCI_CMD}\"\n" |
| 554 |
|
|
" fi\n" |
| 555 |
|
|
"fi\n" |
| 556 |
|
|
"# ENTRYPOINT and CMD - run ENTRYPOINT with CMD as default args\n" |
| 557 |
|
|
"# override with user provided args\n" |
| 558 |
|
|
"if [ $# -gt 0 ]; then\n" |
| 559 |
|
|
" SINGULARITY_OCI_RUN=\"${OCI_ENTRYPOINT} ${CMDLINE_ARGS}\"\n" |
| 560 |
|
|
"else\n" |
| 561 |
|
|
" SINGULARITY_OCI_RUN=\"${OCI_ENTRYPOINT} ${OCI_CMD}\"\n" |
| 562 |
|
|
"fi\n" |
| 563 |
|
|
"# Evaluate shell expressions first and set arguments accordingly,\n" |
| 564 |
|
|
"# then execute final command as first container process\n" |
| 565 |
|
|
"eval \"set ${SINGULARITY_OCI_RUN}\"\n" |
| 566 |
|
✗ |
"exec \"$@\"\n"; |
| 567 |
|
✗ |
return script; |
| 568 |
|
|
} |
| 569 |
|
|
|
| 570 |
|
|
|
| 571 |
|
✗ |
string CommandOverlay::GenerateEnvScript(const vector<string> &env) { |
| 572 |
|
✗ |
string script = "#!/bin/sh\n"; |
| 573 |
|
✗ |
for (size_t i = 0; i < env.size(); ++i) { |
| 574 |
|
✗ |
const string &element = env[i]; |
| 575 |
|
✗ |
const size_t eq = element.find('='); |
| 576 |
|
✗ |
if (eq == string::npos) { |
| 577 |
|
|
// No '=' — just export empty default |
| 578 |
|
✗ |
script += "export " + element + "=\"${" + element + ":-}\"\n"; |
| 579 |
|
|
} else { |
| 580 |
|
✗ |
const string key = element.substr(0, eq); |
| 581 |
|
✗ |
const string val = element.substr(eq + 1); |
| 582 |
|
✗ |
if (key == "PATH") { |
| 583 |
|
✗ |
script += "export PATH=\"" + ShellEscape(val) + "\"\n"; |
| 584 |
|
|
} else { |
| 585 |
|
✗ |
script += "export " + key + "=\"${" + key + ":-\"" + ShellEscape(val) |
| 586 |
|
✗ |
+ "\"}\"\n"; |
| 587 |
|
|
} |
| 588 |
|
|
} |
| 589 |
|
|
} |
| 590 |
|
✗ |
return script; |
| 591 |
|
|
} |
| 592 |
|
|
|
| 593 |
|
|
|
| 594 |
|
✗ |
OverlayEntry CommandOverlay::MakeDirEntry(const string &path, |
| 595 |
|
|
const string &parent) { |
| 596 |
|
✗ |
OverlayEntry oe; |
| 597 |
|
✗ |
oe.path = path; |
| 598 |
|
✗ |
oe.parent = parent; |
| 599 |
|
✗ |
oe.is_whiteout = false; |
| 600 |
|
✗ |
oe.is_opaque_dir = false; |
| 601 |
|
✗ |
oe.entry.name_ = NameString(GetFileName(path)); |
| 602 |
|
✗ |
oe.entry.mode_ = S_IFDIR | 0755; |
| 603 |
|
✗ |
oe.entry.uid_ = 0; |
| 604 |
|
✗ |
oe.entry.gid_ = 0; |
| 605 |
|
✗ |
oe.entry.size_ = 4096; |
| 606 |
|
✗ |
oe.entry.mtime_ = time(NULL); |
| 607 |
|
✗ |
oe.entry.linkcount_ = 2; |
| 608 |
|
✗ |
return oe; |
| 609 |
|
|
} |
| 610 |
|
|
|
| 611 |
|
|
|
| 612 |
|
|
/** |
| 613 |
|
|
* Helper class to collect spooler results for singularity dotfiles. |
| 614 |
|
|
* Registered as a listener on the spooler, it stores the content hash |
| 615 |
|
|
* for each processed file keyed by path. |
| 616 |
|
|
*/ |
| 617 |
|
|
class SingularitySpoolerSink { |
| 618 |
|
|
public: |
| 619 |
|
✗ |
void OnFileProcessed(const upload::SpoolerResult &result) { |
| 620 |
|
✗ |
hashes_[result.local_path] = result.content_hash; |
| 621 |
|
|
} |
| 622 |
|
|
|
| 623 |
|
✗ |
bool GetHash(const string &path, shash::Any *hash) const { |
| 624 |
|
✗ |
const map<string, shash::Any>::const_iterator it = hashes_.find(path); |
| 625 |
|
✗ |
if (it == hashes_.end()) |
| 626 |
|
✗ |
return false; |
| 627 |
|
✗ |
*hash = it->second; |
| 628 |
|
✗ |
return true; |
| 629 |
|
|
} |
| 630 |
|
|
|
| 631 |
|
|
private: |
| 632 |
|
|
map<string, shash::Any> hashes_; |
| 633 |
|
|
}; |
| 634 |
|
|
|
| 635 |
|
|
|
| 636 |
|
✗ |
OverlayEntry CommandOverlay::MakeFileEntry(const string &path, |
| 637 |
|
|
const string &parent, |
| 638 |
|
|
const string &content, |
| 639 |
|
|
upload::Spooler *spooler) { |
| 640 |
|
|
// Process the content through the spooler to get a content hash. |
| 641 |
|
|
// We use a StringIngestionSource so no temp file is needed. |
| 642 |
|
|
// The spooler is used in a synchronous fashion: process one file, |
| 643 |
|
|
// wait, then read the result via a temporary listener. |
| 644 |
|
✗ |
SingularitySpoolerSink sink; |
| 645 |
|
✗ |
typename upload::Spooler::CallbackPtr cb = spooler->RegisterListener( |
| 646 |
|
|
&SingularitySpoolerSink::OnFileProcessed, &sink); |
| 647 |
|
|
|
| 648 |
|
✗ |
spooler->Process(new StringIngestionSource(content, path), |
| 649 |
|
|
false /* no chunking */); |
| 650 |
|
✗ |
spooler->WaitForUpload(); |
| 651 |
|
✗ |
spooler->UnregisterListener(cb); |
| 652 |
|
|
|
| 653 |
|
✗ |
shash::Any content_hash; |
| 654 |
|
✗ |
if (!sink.GetHash(path, &content_hash)) { |
| 655 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStderr, |
| 656 |
|
|
"Failed to get content hash for singularity file %s", |
| 657 |
|
|
path.c_str()); |
| 658 |
|
|
} |
| 659 |
|
|
|
| 660 |
|
✗ |
OverlayEntry oe; |
| 661 |
|
✗ |
oe.path = path; |
| 662 |
|
✗ |
oe.parent = parent; |
| 663 |
|
✗ |
oe.is_whiteout = false; |
| 664 |
|
✗ |
oe.is_opaque_dir = false; |
| 665 |
|
✗ |
oe.entry.name_ = NameString(GetFileName(path)); |
| 666 |
|
✗ |
oe.entry.mode_ = S_IFREG | 0755; |
| 667 |
|
✗ |
oe.entry.uid_ = 0; |
| 668 |
|
✗ |
oe.entry.gid_ = 0; |
| 669 |
|
✗ |
oe.entry.size_ = content.size(); |
| 670 |
|
✗ |
oe.entry.mtime_ = time(NULL); |
| 671 |
|
✗ |
oe.entry.linkcount_ = 1; |
| 672 |
|
✗ |
oe.entry.checksum_ = content_hash; |
| 673 |
|
✗ |
return oe; |
| 674 |
|
|
} |
| 675 |
|
|
|
| 676 |
|
|
|
| 677 |
|
✗ |
OverlayEntry CommandOverlay::MakeSymlinkEntry(const string &path, |
| 678 |
|
|
const string &parent, |
| 679 |
|
|
const string &target) { |
| 680 |
|
✗ |
OverlayEntry oe; |
| 681 |
|
✗ |
oe.path = path; |
| 682 |
|
✗ |
oe.parent = parent; |
| 683 |
|
✗ |
oe.is_whiteout = false; |
| 684 |
|
✗ |
oe.is_opaque_dir = false; |
| 685 |
|
✗ |
oe.entry.name_ = NameString(GetFileName(path)); |
| 686 |
|
✗ |
oe.entry.mode_ = S_IFLNK | 0777; |
| 687 |
|
✗ |
oe.entry.uid_ = 0; |
| 688 |
|
✗ |
oe.entry.gid_ = 0; |
| 689 |
|
✗ |
oe.entry.size_ = target.size(); |
| 690 |
|
✗ |
oe.entry.mtime_ = time(NULL); |
| 691 |
|
✗ |
oe.entry.linkcount_ = 1; |
| 692 |
|
✗ |
oe.entry.symlink_ = LinkString(target); |
| 693 |
|
✗ |
return oe; |
| 694 |
|
|
} |
| 695 |
|
|
|
| 696 |
|
|
|
| 697 |
|
✗ |
bool CommandOverlay::InjectSingularityDotfiles( |
| 698 |
|
|
const string &oci_config_path, |
| 699 |
|
|
upload::Spooler *spooler, |
| 700 |
|
|
map<string, OverlayEntry> *merged) { |
| 701 |
|
|
// --------------------------------------------------------------- |
| 702 |
|
|
// 1. Parse the OCI image config JSON |
| 703 |
|
|
// --------------------------------------------------------------- |
| 704 |
|
✗ |
const int fd = open(oci_config_path.c_str(), O_RDONLY); |
| 705 |
|
✗ |
if (fd < 0) { |
| 706 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStderr, "Failed to open OCI config file %s", |
| 707 |
|
|
oci_config_path.c_str()); |
| 708 |
|
✗ |
return false; |
| 709 |
|
|
} |
| 710 |
|
✗ |
string config_json; |
| 711 |
|
✗ |
if (!SafeReadToString(fd, &config_json)) { |
| 712 |
|
✗ |
close(fd); |
| 713 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStderr, "Failed to read OCI config from %s", |
| 714 |
|
|
oci_config_path.c_str()); |
| 715 |
|
✗ |
return false; |
| 716 |
|
|
} |
| 717 |
|
✗ |
close(fd); |
| 718 |
|
|
|
| 719 |
|
✗ |
const std::unique_ptr<JsonDocument> json(JsonDocument::Create(config_json)); |
| 720 |
|
✗ |
if (json.get() == nullptr) { |
| 721 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStderr, "Failed to parse OCI config JSON from %s", |
| 722 |
|
|
oci_config_path.c_str()); |
| 723 |
|
✗ |
return false; |
| 724 |
|
|
} |
| 725 |
|
|
|
| 726 |
|
|
// Extract config.Env, config.Entrypoint, config.Cmd |
| 727 |
|
✗ |
vector<string> entrypoint; |
| 728 |
|
✗ |
vector<string> cmd; |
| 729 |
|
✗ |
vector<string> env; |
| 730 |
|
|
|
| 731 |
|
✗ |
const JSON *config_obj = JsonDocument::SearchInObject(json->root(), "config", |
| 732 |
|
|
JSON_OBJECT); |
| 733 |
|
✗ |
if (config_obj != NULL) { |
| 734 |
|
✗ |
const JSON *ep_arr = JsonDocument::SearchInObject(config_obj, "Entrypoint", |
| 735 |
|
|
JSON_ARRAY); |
| 736 |
|
✗ |
if (ep_arr != NULL) { |
| 737 |
|
✗ |
for (JSON::const_iterator it = ep_arr->begin(); it != ep_arr->end(); |
| 738 |
|
✗ |
++it) { |
| 739 |
|
✗ |
if (it->is_string()) |
| 740 |
|
✗ |
entrypoint.push_back(it->get<string>()); |
| 741 |
|
|
} |
| 742 |
|
|
} |
| 743 |
|
✗ |
const JSON *cmd_arr = JsonDocument::SearchInObject(config_obj, "Cmd", |
| 744 |
|
|
JSON_ARRAY); |
| 745 |
|
✗ |
if (cmd_arr != NULL) { |
| 746 |
|
✗ |
for (JSON::const_iterator it = cmd_arr->begin(); it != cmd_arr->end(); |
| 747 |
|
✗ |
++it) { |
| 748 |
|
✗ |
if (it->is_string()) |
| 749 |
|
✗ |
cmd.push_back(it->get<string>()); |
| 750 |
|
|
} |
| 751 |
|
|
} |
| 752 |
|
✗ |
const JSON *env_arr = JsonDocument::SearchInObject(config_obj, "Env", |
| 753 |
|
|
JSON_ARRAY); |
| 754 |
|
✗ |
if (env_arr != NULL) { |
| 755 |
|
✗ |
for (JSON::const_iterator it = env_arr->begin(); it != env_arr->end(); |
| 756 |
|
✗ |
++it) { |
| 757 |
|
✗ |
if (it->is_string()) |
| 758 |
|
✗ |
env.push_back(it->get<string>()); |
| 759 |
|
|
} |
| 760 |
|
|
} |
| 761 |
|
|
} |
| 762 |
|
|
|
| 763 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStdout, |
| 764 |
|
|
"Injecting Singularity dotfiles (Entrypoint: %zu, Cmd: %zu, " |
| 765 |
|
|
"Env: %zu entries)", |
| 766 |
|
|
entrypoint.size(), cmd.size(), env.size()); |
| 767 |
|
|
|
| 768 |
|
|
// --------------------------------------------------------------- |
| 769 |
|
|
// 2. Create directory entries |
| 770 |
|
|
// --------------------------------------------------------------- |
| 771 |
|
✗ |
(*merged)[".singularity.d"] = MakeDirEntry(".singularity.d", ""); |
| 772 |
|
✗ |
(*merged)[".singularity.d/libs"] = MakeDirEntry(".singularity.d/libs", |
| 773 |
|
✗ |
".singularity.d"); |
| 774 |
|
✗ |
(*merged)[".singularity.d/actions"] = MakeDirEntry(".singularity.d/actions", |
| 775 |
|
✗ |
".singularity.d"); |
| 776 |
|
✗ |
(*merged)[".singularity.d/env"] = MakeDirEntry(".singularity.d/env", |
| 777 |
|
✗ |
".singularity.d"); |
| 778 |
|
|
|
| 779 |
|
|
// Also create common FHS directories if missing |
| 780 |
|
✗ |
const char *fhs_dirs[] = {"dev", "proc", "root", "var", "var/tmp", |
| 781 |
|
|
"tmp", "etc", "sys", "home", NULL}; |
| 782 |
|
✗ |
for (int i = 0; fhs_dirs[i] != NULL; ++i) { |
| 783 |
|
✗ |
const string d = fhs_dirs[i]; |
| 784 |
|
✗ |
if (merged->find(d) == merged->end()) { |
| 785 |
|
✗ |
const string par = (d.find('/') != string::npos) ? GetParentPath(d) : ""; |
| 786 |
|
✗ |
(*merged)[d] = MakeDirEntry(d, par); |
| 787 |
|
|
} |
| 788 |
|
|
} |
| 789 |
|
|
|
| 790 |
|
|
// --------------------------------------------------------------- |
| 791 |
|
|
// 3. Create file entries (content is uploaded via spooler) |
| 792 |
|
|
// --------------------------------------------------------------- |
| 793 |
|
|
// Action scripts |
| 794 |
|
✗ |
(*merged)[".singularity.d/actions/exec"] = MakeFileEntry( |
| 795 |
|
|
".singularity.d/actions/exec", ".singularity.d/actions", kSingExec, |
| 796 |
|
✗ |
spooler); |
| 797 |
|
✗ |
(*merged)[".singularity.d/actions/run"] = MakeFileEntry( |
| 798 |
|
|
".singularity.d/actions/run", ".singularity.d/actions", kSingRun, |
| 799 |
|
✗ |
spooler); |
| 800 |
|
✗ |
(*merged)[".singularity.d/actions/shell"] = MakeFileEntry( |
| 801 |
|
|
".singularity.d/actions/shell", ".singularity.d/actions", kSingShell, |
| 802 |
|
✗ |
spooler); |
| 803 |
|
✗ |
(*merged)[".singularity.d/actions/start"] = MakeFileEntry( |
| 804 |
|
|
".singularity.d/actions/start", ".singularity.d/actions", kSingStart, |
| 805 |
|
✗ |
spooler); |
| 806 |
|
✗ |
(*merged)[".singularity.d/actions/test"] = MakeFileEntry( |
| 807 |
|
|
".singularity.d/actions/test", ".singularity.d/actions", kSingTest, |
| 808 |
|
✗ |
spooler); |
| 809 |
|
|
|
| 810 |
|
|
// Environment scripts |
| 811 |
|
✗ |
(*merged)[".singularity.d/env/01-base.sh"] = MakeFileEntry( |
| 812 |
|
|
".singularity.d/env/01-base.sh", ".singularity.d/env", kSingEnv01Base, |
| 813 |
|
✗ |
spooler); |
| 814 |
|
✗ |
(*merged)[".singularity.d/env/90-environment.sh"] = MakeFileEntry( |
| 815 |
|
|
".singularity.d/env/90-environment.sh", ".singularity.d/env", kSingEnv90, |
| 816 |
|
✗ |
spooler); |
| 817 |
|
✗ |
(*merged)[".singularity.d/env/91-environment.sh"] = MakeFileEntry( |
| 818 |
|
|
".singularity.d/env/91-environment.sh", ".singularity.d/env", kSingEnv90, |
| 819 |
|
✗ |
spooler); |
| 820 |
|
✗ |
(*merged)[".singularity.d/env/95-apps.sh"] = MakeFileEntry( |
| 821 |
|
|
".singularity.d/env/95-apps.sh", ".singularity.d/env", kSingEnv95Apps, |
| 822 |
|
✗ |
spooler); |
| 823 |
|
✗ |
(*merged)[".singularity.d/env/99-base.sh"] = MakeFileEntry( |
| 824 |
|
|
".singularity.d/env/99-base.sh", ".singularity.d/env", kSingEnv99Base, |
| 825 |
|
✗ |
spooler); |
| 826 |
|
✗ |
(*merged)[".singularity.d/env/99-runtimevars.sh"] = MakeFileEntry( |
| 827 |
|
|
".singularity.d/env/99-runtimevars.sh", ".singularity.d/env", |
| 828 |
|
✗ |
kSingEnv99Runtimevars, spooler); |
| 829 |
|
|
|
| 830 |
|
|
// OCI-config-dependent files |
| 831 |
|
✗ |
const string runscript = GenerateRunscript(entrypoint, cmd); |
| 832 |
|
✗ |
(*merged)[".singularity.d/runscript"] = MakeFileEntry( |
| 833 |
|
✗ |
".singularity.d/runscript", ".singularity.d", runscript, spooler); |
| 834 |
|
✗ |
(*merged)[".singularity.d/startscript"] = MakeFileEntry( |
| 835 |
|
|
".singularity.d/startscript", ".singularity.d", kSingStartscript, |
| 836 |
|
✗ |
spooler); |
| 837 |
|
|
|
| 838 |
|
✗ |
const string env_script = GenerateEnvScript(env); |
| 839 |
|
✗ |
(*merged)[".singularity.d/env/10-docker2singularity.sh"] = MakeFileEntry( |
| 840 |
|
|
".singularity.d/env/10-docker2singularity.sh", ".singularity.d/env", |
| 841 |
|
✗ |
env_script, spooler); |
| 842 |
|
|
|
| 843 |
|
|
// --------------------------------------------------------------- |
| 844 |
|
|
// 4. Create symlinks |
| 845 |
|
|
// --------------------------------------------------------------- |
| 846 |
|
|
// Only create if not already present from a layer |
| 847 |
|
✗ |
if (merged->find("singularity") == merged->end()) { |
| 848 |
|
✗ |
(*merged)["singularity"] = MakeSymlinkEntry("singularity", "", |
| 849 |
|
✗ |
".singularity.d/runscript"); |
| 850 |
|
|
} |
| 851 |
|
✗ |
if (merged->find(".run") == merged->end()) { |
| 852 |
|
✗ |
(*merged)[".run"] = MakeSymlinkEntry(".run", "", |
| 853 |
|
✗ |
".singularity.d/actions/run"); |
| 854 |
|
|
} |
| 855 |
|
✗ |
if (merged->find(".shell") == merged->end()) { |
| 856 |
|
✗ |
(*merged)[".shell"] = MakeSymlinkEntry(".shell", "", |
| 857 |
|
✗ |
".singularity.d/actions/shell"); |
| 858 |
|
|
} |
| 859 |
|
✗ |
if (merged->find(".exec") == merged->end()) { |
| 860 |
|
✗ |
(*merged)[".exec"] = MakeSymlinkEntry(".exec", "", |
| 861 |
|
✗ |
".singularity.d/actions/exec"); |
| 862 |
|
|
} |
| 863 |
|
✗ |
if (merged->find(".test") == merged->end()) { |
| 864 |
|
✗ |
(*merged)[".test"] = MakeSymlinkEntry(".test", "", |
| 865 |
|
✗ |
".singularity.d/actions/test"); |
| 866 |
|
|
} |
| 867 |
|
✗ |
if (merged->find("environment") == merged->end()) { |
| 868 |
|
✗ |
(*merged)["environment"] = MakeSymlinkEntry( |
| 869 |
|
✗ |
"environment", "", ".singularity.d/env/90-environment.sh"); |
| 870 |
|
|
} |
| 871 |
|
|
|
| 872 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStdout, |
| 873 |
|
|
"Injected Singularity dotfiles into merged overlay"); |
| 874 |
|
✗ |
return true; |
| 875 |
|
|
} |
| 876 |
|
|
|
| 877 |
|
|
|
| 878 |
|
✗ |
bool CommandOverlay::PublishMergedEntries( |
| 879 |
|
|
catalog::WritableCatalogManager *catalog_mgr, |
| 880 |
|
|
const map<string, OverlayEntry> &merged, |
| 881 |
|
|
const string &dest_path) const { |
| 882 |
|
|
// dest_path starts with '/' for LookupPath, but AddDirectory/AddFile |
| 883 |
|
|
// expect parent_directory without leading '/' because MakeRelativePath |
| 884 |
|
|
// (called internally) prepends it. Create a stripped copy for add calls. |
| 885 |
|
✗ |
const string dest_path_rel = (!dest_path.empty() && dest_path[0] == '/') |
| 886 |
|
✗ |
? dest_path.substr(1) |
| 887 |
|
✗ |
: dest_path; |
| 888 |
|
|
|
| 889 |
|
|
// Ensure the destination directory itself exists in the catalog. |
| 890 |
|
|
// Check if dest_path already exists; if not, create it. |
| 891 |
|
✗ |
catalog::DirectoryEntry dest_dirent; |
| 892 |
|
✗ |
if (!catalog_mgr->LookupPath(dest_path, catalog::kLookupDefault, |
| 893 |
|
|
&dest_dirent)) { |
| 894 |
|
|
// Create the destination directory (and any missing parents) |
| 895 |
|
|
// Walk up to find the deepest existing ancestor |
| 896 |
|
✗ |
vector<string> dirs_to_create; |
| 897 |
|
✗ |
string check_path = dest_path; |
| 898 |
|
✗ |
while (!check_path.empty() && check_path != "/") { |
| 899 |
|
✗ |
catalog::DirectoryEntry check_dirent; |
| 900 |
|
✗ |
if (catalog_mgr->LookupPath(check_path, catalog::kLookupDefault, |
| 901 |
|
|
&check_dirent)) { |
| 902 |
|
✗ |
break; |
| 903 |
|
|
} |
| 904 |
|
✗ |
dirs_to_create.push_back(check_path); |
| 905 |
|
✗ |
check_path = GetParentPath(check_path); |
| 906 |
|
|
} |
| 907 |
|
|
|
| 908 |
|
|
// Create directories from outermost to innermost |
| 909 |
|
✗ |
for (int i = static_cast<int>(dirs_to_create.size()) - 1; i >= 0; --i) { |
| 910 |
|
✗ |
const string &dir = dirs_to_create[i]; |
| 911 |
|
✗ |
string parent = GetParentPath(dir); |
| 912 |
|
✗ |
const string name = GetFileName(dir); |
| 913 |
|
|
|
| 914 |
|
|
// Strip leading '/' — AddDirectory calls MakeRelativePath which adds it |
| 915 |
|
✗ |
if (!parent.empty() && parent[0] == '/') { |
| 916 |
|
✗ |
parent = parent.substr(1); |
| 917 |
|
|
} |
| 918 |
|
|
|
| 919 |
|
✗ |
catalog::DirectoryEntryBase new_dir; |
| 920 |
|
✗ |
new_dir.name_.Assign(name.data(), name.length()); |
| 921 |
|
✗ |
new_dir.mode_ = S_IFDIR | 0755; |
| 922 |
|
✗ |
new_dir.uid_ = 0; |
| 923 |
|
✗ |
new_dir.gid_ = 0; |
| 924 |
|
✗ |
new_dir.size_ = 4096; |
| 925 |
|
✗ |
new_dir.mtime_ = time(NULL); |
| 926 |
|
✗ |
new_dir.linkcount_ = 2; |
| 927 |
|
|
|
| 928 |
|
✗ |
catalog_mgr->AddDirectory(new_dir, XattrList(), parent); |
| 929 |
|
|
LogCvmfs(kLogCvmfs, kLogDebug, "Created destination directory: %s", |
| 930 |
|
|
dir.c_str()); |
| 931 |
|
|
} |
| 932 |
|
|
} |
| 933 |
|
|
|
| 934 |
|
|
// Add entries in sorted order. The map is sorted lexicographically, |
| 935 |
|
|
// so parent directories appear before their children. |
| 936 |
|
✗ |
for (map<string, OverlayEntry>::const_iterator it = merged.begin(); |
| 937 |
|
✗ |
it != merged.end(); |
| 938 |
|
✗ |
++it) { |
| 939 |
|
✗ |
const OverlayEntry &oe = it->second; |
| 940 |
|
|
|
| 941 |
|
|
// Build parent path without leading '/' for AddDirectory/AddFile |
| 942 |
|
|
// (MakeRelativePath inside those functions adds it back) |
| 943 |
|
✗ |
const string parent_path = oe.parent.empty() |
| 944 |
|
|
? dest_path_rel |
| 945 |
|
✗ |
: dest_path_rel + "/" + oe.parent; |
| 946 |
|
|
|
| 947 |
|
✗ |
if (oe.entry.IsDirectory()) { |
| 948 |
|
✗ |
catalog_mgr->AddDirectory(oe.entry, oe.xattrs, parent_path); |
| 949 |
|
|
} else { |
| 950 |
|
✗ |
catalog_mgr->AddFile( |
| 951 |
|
✗ |
static_cast<const catalog::DirectoryEntryBase &>(oe.entry), oe.xattrs, |
| 952 |
|
|
parent_path); |
| 953 |
|
|
} |
| 954 |
|
|
} |
| 955 |
|
|
|
| 956 |
|
|
// Turn the destination directory into a nested catalog so that the overlay |
| 957 |
|
|
// content lives in its own catalog database file. |
| 958 |
|
|
|
| 959 |
|
|
// Add a .cvmfscatalog marker file |
| 960 |
|
✗ |
catalog::DirectoryEntryBase catalog_marker; |
| 961 |
|
✗ |
catalog_marker.name_ = NameString(".cvmfscatalog"); |
| 962 |
|
✗ |
catalog_marker.mode_ = (S_IFREG | 0666); |
| 963 |
|
✗ |
catalog_marker.size_ = 0; |
| 964 |
|
✗ |
catalog_marker.mtime_ = time(NULL); |
| 965 |
|
✗ |
catalog_marker.uid_ = 0; |
| 966 |
|
✗ |
catalog_marker.gid_ = 0; |
| 967 |
|
✗ |
catalog_marker.linkcount_ = 1; |
| 968 |
|
|
// Hash of the compressed empty file |
| 969 |
|
✗ |
catalog_marker.checksum_ = shash::MkFromHexPtr( |
| 970 |
|
|
shash::HexPtr("e8ec3d88b62ebf526e4e5a4ff6162a3aa48a6b78"), |
| 971 |
|
|
shash::kSuffixNone); // hash of "" |
| 972 |
|
✗ |
catalog_mgr->AddFile(catalog_marker, XattrList(), dest_path_rel); |
| 973 |
|
|
// CreateNestedCatalog calls MakeRelativePath internally which prepends '/'. |
| 974 |
|
|
// Pass the stripped version to avoid a double leading slash. |
| 975 |
|
✗ |
catalog_mgr->CreateNestedCatalog(dest_path_rel); |
| 976 |
|
|
|
| 977 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStdout, |
| 978 |
|
|
"Published %zu entries under %s (nested catalog)", merged.size(), |
| 979 |
|
|
dest_path.c_str()); |
| 980 |
|
✗ |
return true; |
| 981 |
|
|
} |
| 982 |
|
|
|
| 983 |
|
|
|
| 984 |
|
✗ |
catalog::Catalog *CommandOverlay::LoadCatalogForPath( |
| 985 |
|
|
const string &repo_base, |
| 986 |
|
|
const string &subdirectory, |
| 987 |
|
|
const string &temp_dir, |
| 988 |
|
|
const shash::Any &root_hash) { |
| 989 |
|
|
// Fetch the root catalog from the repository |
| 990 |
|
✗ |
const string hash_path = "data/" + root_hash.MakePath(); |
| 991 |
|
✗ |
string catalog_path; |
| 992 |
|
|
|
| 993 |
|
✗ |
if (IsHttpUrl(repo_base)) { |
| 994 |
|
|
// Download and decompress from remote |
| 995 |
|
✗ |
const string url = repo_base + "/" + hash_path; |
| 996 |
|
✗ |
catalog_path = temp_dir + "/" + root_hash.ToString(); |
| 997 |
|
|
|
| 998 |
|
✗ |
cvmfs::PathSink pathsink(catalog_path); |
| 999 |
|
✗ |
download::JobInfo download_job(&url, true, false, &root_hash, &pathsink); |
| 1000 |
|
✗ |
const download::Failures retval = download_manager()->Fetch(&download_job); |
| 1001 |
|
✗ |
if (retval != download::kFailOk) { |
| 1002 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStderr, "Failed to download catalog %s (%d)", |
| 1003 |
|
|
root_hash.ToString().c_str(), retval); |
| 1004 |
|
✗ |
return NULL; |
| 1005 |
|
|
} |
| 1006 |
|
✗ |
} else { |
| 1007 |
|
|
// Local repository: decompress the catalog |
| 1008 |
|
✗ |
const string source_path = repo_base + "/" + hash_path; |
| 1009 |
|
✗ |
catalog_path = temp_dir + "/" + root_hash.ToString(); |
| 1010 |
|
|
|
| 1011 |
|
✗ |
if (!zlib::DecompressPath2Path(source_path, catalog_path)) { |
| 1012 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStderr, "Failed to decompress catalog %s from %s", |
| 1013 |
|
|
root_hash.ToString().c_str(), source_path.c_str()); |
| 1014 |
|
✗ |
return NULL; |
| 1015 |
|
|
} |
| 1016 |
|
|
} |
| 1017 |
|
|
|
| 1018 |
|
✗ |
catalog::Catalog *catalog = catalog::Catalog::AttachFreely( |
| 1019 |
|
|
subdirectory, catalog_path, root_hash); |
| 1020 |
|
✗ |
if (catalog == NULL) { |
| 1021 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStderr, "Failed to attach catalog for path %s", |
| 1022 |
|
|
subdirectory.c_str()); |
| 1023 |
|
✗ |
unlink(catalog_path.c_str()); |
| 1024 |
|
✗ |
return NULL; |
| 1025 |
|
|
} |
| 1026 |
|
|
|
| 1027 |
|
✗ |
catalog->TakeDatabaseFileOwnership(); |
| 1028 |
|
✗ |
return catalog; |
| 1029 |
|
|
} |
| 1030 |
|
|
|
| 1031 |
|
|
|
| 1032 |
|
✗ |
catalog::Catalog *CommandOverlay::FindCatalogForLayer( |
| 1033 |
|
|
const string &repo_base, |
| 1034 |
|
|
const string &temp_dir, |
| 1035 |
|
|
catalog::Catalog *catalog, |
| 1036 |
|
|
const string &layer_path, |
| 1037 |
|
|
vector<catalog::Catalog *> *loaded_catalogs) { |
| 1038 |
|
|
// First try a direct lookup in the given catalog |
| 1039 |
|
✗ |
catalog::DirectoryEntry test_entry; |
| 1040 |
|
✗ |
const PathString ps_layer(layer_path.data(), layer_path.length()); |
| 1041 |
|
✗ |
if (catalog->LookupPath(ps_layer, &test_entry)) { |
| 1042 |
|
✗ |
return catalog; |
| 1043 |
|
|
} |
| 1044 |
|
|
|
| 1045 |
|
|
// The path was not found directly. Walk the path components *below* |
| 1046 |
|
|
// this catalog's mountpoint to find a nested catalog mountpoint that |
| 1047 |
|
|
// is an ancestor of layer_path. |
| 1048 |
|
✗ |
const string mountpoint = catalog->mountpoint().ToString(); |
| 1049 |
|
|
|
| 1050 |
|
|
// Verify layer_path starts with the mountpoint (or mountpoint is empty |
| 1051 |
|
|
// for the root catalog) |
| 1052 |
|
✗ |
if (!mountpoint.empty() |
| 1053 |
|
✗ |
&& layer_path.substr(0, mountpoint.length()) != mountpoint) { |
| 1054 |
|
✗ |
return NULL; |
| 1055 |
|
|
} |
| 1056 |
|
|
|
| 1057 |
|
|
// Get the suffix of layer_path below the mountpoint |
| 1058 |
|
✗ |
const string suffix = mountpoint.empty() |
| 1059 |
|
|
? layer_path |
| 1060 |
|
✗ |
: layer_path.substr(mountpoint.length()); |
| 1061 |
|
✗ |
const vector<string> components = SplitString(suffix, '/'); |
| 1062 |
|
✗ |
string prefix = mountpoint; |
| 1063 |
|
✗ |
for (size_t i = 0; i < components.size(); ++i) { |
| 1064 |
|
✗ |
if (components[i].empty()) |
| 1065 |
|
✗ |
continue; |
| 1066 |
|
✗ |
prefix += "/" + components[i]; |
| 1067 |
|
|
|
| 1068 |
|
✗ |
catalog::DirectoryEntry dir_entry; |
| 1069 |
|
✗ |
const PathString ps_prefix(prefix.data(), prefix.length()); |
| 1070 |
|
✗ |
if (!catalog->LookupPath(ps_prefix, &dir_entry)) { |
| 1071 |
|
✗ |
break; |
| 1072 |
|
|
} |
| 1073 |
|
|
|
| 1074 |
|
✗ |
if (dir_entry.IsNestedCatalogMountpoint()) { |
| 1075 |
|
✗ |
shash::Any nested_hash; |
| 1076 |
|
|
uint64_t nested_size; |
| 1077 |
|
✗ |
if (!catalog->FindNested(ps_prefix, &nested_hash, &nested_size)) { |
| 1078 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStderr, |
| 1079 |
|
|
"Failed to find nested catalog hash for %s", prefix.c_str()); |
| 1080 |
|
✗ |
return NULL; |
| 1081 |
|
|
} |
| 1082 |
|
|
|
| 1083 |
|
✗ |
catalog::Catalog *nested = LoadCatalogForPath(repo_base, prefix, temp_dir, |
| 1084 |
|
✗ |
nested_hash); |
| 1085 |
|
✗ |
if (nested == NULL) { |
| 1086 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStderr, "Failed to load nested catalog at %s", |
| 1087 |
|
|
prefix.c_str()); |
| 1088 |
|
✗ |
return NULL; |
| 1089 |
|
|
} |
| 1090 |
|
✗ |
loaded_catalogs->push_back(nested); |
| 1091 |
|
|
|
| 1092 |
|
|
// Recurse: the layer path may be directly in this nested catalog |
| 1093 |
|
|
// or in an even deeper nested catalog |
| 1094 |
|
✗ |
return FindCatalogForLayer(repo_base, temp_dir, nested, layer_path, |
| 1095 |
|
✗ |
loaded_catalogs); |
| 1096 |
|
|
} |
| 1097 |
|
|
} |
| 1098 |
|
|
|
| 1099 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStderr, "Layer path not found: %s", |
| 1100 |
|
|
layer_path.c_str()); |
| 1101 |
|
✗ |
return NULL; |
| 1102 |
|
|
} |
| 1103 |
|
|
|
| 1104 |
|
|
|
| 1105 |
|
✗ |
int CommandOverlay::Main(const ArgumentList &args) { |
| 1106 |
|
|
// Parse publish workflow parameters |
| 1107 |
|
✗ |
const string spooler_definition_str = *args.find('r')->second; |
| 1108 |
|
✗ |
const string stratum0 = *args.find('w')->second; |
| 1109 |
|
✗ |
const string temp_dir = MakeCanonicalPath(*args.find('t')->second); |
| 1110 |
|
✗ |
const string manifest_path = *args.find('o')->second; |
| 1111 |
|
✗ |
const shash::Any base_hash = shash::MkFromHexPtr( |
| 1112 |
|
✗ |
shash::HexPtr(*args.find('b')->second), shash::kSuffixCatalog); |
| 1113 |
|
✗ |
const string public_keys = *args.find('K')->second; |
| 1114 |
|
✗ |
const string repo_name = *args.find('N')->second; |
| 1115 |
|
|
|
| 1116 |
|
|
// Parse overlay-specific parameters |
| 1117 |
|
✗ |
const string layers_str = *args.find('l')->second; |
| 1118 |
|
✗ |
string dest_path = MakeCanonicalPath(*args.find('d')->second); |
| 1119 |
|
|
// Ensure dest_path starts with exactly one '/' |
| 1120 |
|
✗ |
while (dest_path.length() > 1 && dest_path[0] == '/' && dest_path[1] == '/') { |
| 1121 |
|
✗ |
dest_path = dest_path.substr(1); |
| 1122 |
|
|
} |
| 1123 |
|
✗ |
if (dest_path.empty() || dest_path[0] != '/') { |
| 1124 |
|
✗ |
dest_path = "/" + dest_path; |
| 1125 |
|
|
} |
| 1126 |
|
✗ |
shash::Algorithms hash_algorithm = shash::kSha1; |
| 1127 |
|
✗ |
if (args.find('e') != args.end()) { |
| 1128 |
|
✗ |
hash_algorithm = shash::ParseHashAlgorithm(*args.find('e')->second); |
| 1129 |
|
✗ |
if (hash_algorithm == shash::kAny) { |
| 1130 |
|
✗ |
PrintError("unknown hash algorithm"); |
| 1131 |
|
✗ |
return 1; |
| 1132 |
|
|
} |
| 1133 |
|
|
} |
| 1134 |
|
✗ |
zlib::Algorithms compression_alg = zlib::kZlibDefault; |
| 1135 |
|
✗ |
if (args.find('Z') != args.end()) { |
| 1136 |
|
✗ |
compression_alg = zlib::ParseCompressionAlgorithm(*args.find('Z')->second); |
| 1137 |
|
|
} |
| 1138 |
|
|
|
| 1139 |
|
✗ |
const string oci_config_path = (args.count('c') > 0) ? *args.find('c')->second |
| 1140 |
|
✗ |
: ""; |
| 1141 |
|
✗ |
const bool skip_singularity = (args.count('S') > 0); |
| 1142 |
|
|
|
| 1143 |
|
|
// Gateway lease: empty for a directly-writable (S3/local) upstream, set when |
| 1144 |
|
|
// committing through a repository gateway (mountless publishing). |
| 1145 |
|
✗ |
const string session_token_file = (args.count('P') > 0) |
| 1146 |
|
✗ |
? *args.find('P')->second |
| 1147 |
|
✗ |
: ""; |
| 1148 |
|
✗ |
const string key_file = (args.count('H') > 0) ? *args.find('H')->second : ""; |
| 1149 |
|
|
|
| 1150 |
|
|
// Parse comma-separated layer paths |
| 1151 |
|
✗ |
const vector<string> layers = SplitString(layers_str, ','); |
| 1152 |
|
✗ |
if (layers.empty()) { |
| 1153 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStderr, "No layers specified"); |
| 1154 |
|
✗ |
return 1; |
| 1155 |
|
|
} |
| 1156 |
|
|
|
| 1157 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStdout, "Overlay merge of %zu layers into %s", |
| 1158 |
|
|
layers.size(), dest_path.c_str()); |
| 1159 |
|
✗ |
for (size_t i = 0; i < layers.size(); ++i) { |
| 1160 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStdout, " Layer %zu: %s", i, layers[i].c_str()); |
| 1161 |
|
|
} |
| 1162 |
|
|
|
| 1163 |
|
|
// Set up spoolers (following the ingest pattern) |
| 1164 |
|
✗ |
perf::StatisticsTemplate publish_statistics("publish", this->statistics()); |
| 1165 |
|
|
|
| 1166 |
|
|
const upload::SpoolerDefinition spooler_definition( |
| 1167 |
|
|
spooler_definition_str, hash_algorithm, compression_alg, |
| 1168 |
|
|
false /* generate_legacy_bulk_chunks */, false /* use_file_chunking */, 0, |
| 1169 |
|
✗ |
0, 0 /* chunk sizes: unused */, session_token_file, key_file); |
| 1170 |
|
|
|
| 1171 |
|
|
const upload::SpoolerDefinition spooler_definition_catalogs( |
| 1172 |
|
✗ |
spooler_definition.Dup2DefaultCompression()); |
| 1173 |
|
|
|
| 1174 |
|
|
const std::unique_ptr<upload::Spooler> spooler_files( |
| 1175 |
|
✗ |
upload::Spooler::Construct(spooler_definition, &publish_statistics)); |
| 1176 |
|
✗ |
if (spooler_files.get() == nullptr) { |
| 1177 |
|
✗ |
PrintError("Failed to create file spooler"); |
| 1178 |
|
✗ |
return 3; |
| 1179 |
|
|
} |
| 1180 |
|
|
const std::unique_ptr<upload::Spooler> spooler_catalogs( |
| 1181 |
|
|
upload::Spooler::Construct(spooler_definition_catalogs, |
| 1182 |
|
✗ |
&publish_statistics)); |
| 1183 |
|
✗ |
if (spooler_catalogs.get() == nullptr) { |
| 1184 |
|
✗ |
PrintError("Failed to create catalog spooler"); |
| 1185 |
|
✗ |
return 3; |
| 1186 |
|
|
} |
| 1187 |
|
|
|
| 1188 |
|
|
// Initialize download manager and signature manager |
| 1189 |
|
✗ |
const bool follow_redirects = (args.count('L') > 0); |
| 1190 |
|
✗ |
const string proxy = (args.count('@') > 0) ? *args.find('@')->second : ""; |
| 1191 |
|
✗ |
if (!InitDownloadManager(follow_redirects, proxy)) { |
| 1192 |
|
✗ |
PrintError("Failed to initialize download manager"); |
| 1193 |
|
✗ |
return 3; |
| 1194 |
|
|
} |
| 1195 |
|
✗ |
if (!InitSignatureManager(public_keys)) { |
| 1196 |
|
✗ |
PrintError("Failed to initialize signature manager"); |
| 1197 |
|
✗ |
return 3; |
| 1198 |
|
|
} |
| 1199 |
|
|
|
| 1200 |
|
|
// Fetch repository manifest |
| 1201 |
|
|
const std::unique_ptr<manifest::Manifest> manifest( |
| 1202 |
|
✗ |
FetchRemoteManifest(stratum0, repo_name, base_hash)); |
| 1203 |
|
✗ |
if (manifest.get() == nullptr) { |
| 1204 |
|
✗ |
PrintError("Failed to load repository manifest"); |
| 1205 |
|
✗ |
return 3; |
| 1206 |
|
|
} |
| 1207 |
|
|
|
| 1208 |
|
✗ |
const string old_root_hash = manifest->catalog_hash().ToString(true); |
| 1209 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStdout, "Root catalog hash: %s", |
| 1210 |
|
|
old_root_hash.c_str()); |
| 1211 |
|
|
|
| 1212 |
|
|
// Load root catalog for reading layer entries |
| 1213 |
|
✗ |
map<string, OverlayEntry> merged; |
| 1214 |
|
✗ |
catalog::Catalog *root_catalog = LoadCatalogForPath(stratum0, "", temp_dir, |
| 1215 |
|
✗ |
manifest->catalog_hash()); |
| 1216 |
|
✗ |
if (root_catalog == NULL) { |
| 1217 |
|
✗ |
PrintError("Failed to load root catalog"); |
| 1218 |
|
✗ |
return 1; |
| 1219 |
|
|
} |
| 1220 |
|
|
|
| 1221 |
|
|
// Process layers bottom-to-top |
| 1222 |
|
✗ |
for (size_t i = 0; i < layers.size(); ++i) { |
| 1223 |
|
✗ |
string layer_path = MakeCanonicalPath(layers[i]); |
| 1224 |
|
|
// Ensure layer path starts with exactly one '/' |
| 1225 |
|
✗ |
while (layer_path.length() > 1 && layer_path[0] == '/' |
| 1226 |
|
✗ |
&& layer_path[1] == '/') { |
| 1227 |
|
✗ |
layer_path = layer_path.substr(1); |
| 1228 |
|
|
} |
| 1229 |
|
✗ |
if (layer_path.empty() || layer_path[0] != '/') { |
| 1230 |
|
✗ |
layer_path = "/" + layer_path; |
| 1231 |
|
|
} |
| 1232 |
|
|
|
| 1233 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStdout, "Processing layer %zu: %s", i, |
| 1234 |
|
|
layer_path.c_str()); |
| 1235 |
|
|
|
| 1236 |
|
✗ |
map<string, OverlayEntry> layer_entries; |
| 1237 |
|
|
|
| 1238 |
|
|
// Find the catalog that contains this layer path (may be nested) |
| 1239 |
|
✗ |
vector<catalog::Catalog *> loaded_catalogs; |
| 1240 |
|
✗ |
catalog::Catalog *layer_catalog = FindCatalogForLayer( |
| 1241 |
|
|
stratum0, temp_dir, root_catalog, layer_path, &loaded_catalogs); |
| 1242 |
|
✗ |
if (layer_catalog == NULL) { |
| 1243 |
|
✗ |
for (size_t j = 0; j < loaded_catalogs.size(); ++j) |
| 1244 |
|
✗ |
delete loaded_catalogs[j]; |
| 1245 |
|
✗ |
delete root_catalog; |
| 1246 |
|
✗ |
return 1; |
| 1247 |
|
|
} |
| 1248 |
|
|
|
| 1249 |
|
✗ |
catalog::DirectoryEntry subdir_entry; |
| 1250 |
|
✗ |
const PathString ps_layer_path(layer_path.data(), layer_path.length()); |
| 1251 |
|
✗ |
if (!layer_catalog->LookupPath(ps_layer_path, &subdir_entry)) { |
| 1252 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStderr, |
| 1253 |
|
|
"Unexpected: layer path not found after catalog resolution: %s", |
| 1254 |
|
|
layer_path.c_str()); |
| 1255 |
|
✗ |
for (size_t j = 0; j < loaded_catalogs.size(); ++j) |
| 1256 |
|
✗ |
delete loaded_catalogs[j]; |
| 1257 |
|
✗ |
delete root_catalog; |
| 1258 |
|
✗ |
return 1; |
| 1259 |
|
|
} |
| 1260 |
|
|
|
| 1261 |
|
|
// Check if the layer path itself is a nested catalog mountpoint; |
| 1262 |
|
|
// if so, load that catalog and read its entries. |
| 1263 |
|
✗ |
if (subdir_entry.IsNestedCatalogMountpoint()) { |
| 1264 |
|
✗ |
shash::Any nested_hash; |
| 1265 |
|
|
uint64_t nested_size; |
| 1266 |
|
✗ |
if (!layer_catalog->FindNested(ps_layer_path, &nested_hash, |
| 1267 |
|
|
&nested_size)) { |
| 1268 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStderr, "Failed to find nested catalog for %s", |
| 1269 |
|
|
layer_path.c_str()); |
| 1270 |
|
✗ |
for (size_t j = 0; j < loaded_catalogs.size(); ++j) |
| 1271 |
|
✗ |
delete loaded_catalogs[j]; |
| 1272 |
|
✗ |
delete root_catalog; |
| 1273 |
|
✗ |
return 1; |
| 1274 |
|
|
} |
| 1275 |
|
|
|
| 1276 |
|
✗ |
catalog::Catalog *nested_catalog = LoadCatalogForPath( |
| 1277 |
|
|
stratum0, layer_path, temp_dir, nested_hash); |
| 1278 |
|
✗ |
if (nested_catalog == NULL) { |
| 1279 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStderr, "Failed to load nested catalog for %s", |
| 1280 |
|
|
layer_path.c_str()); |
| 1281 |
|
✗ |
for (size_t j = 0; j < loaded_catalogs.size(); ++j) |
| 1282 |
|
✗ |
delete loaded_catalogs[j]; |
| 1283 |
|
✗ |
delete root_catalog; |
| 1284 |
|
✗ |
return 1; |
| 1285 |
|
|
} |
| 1286 |
|
|
|
| 1287 |
|
✗ |
ReadCatalogEntries(nested_catalog, layer_path, "", stratum0, temp_dir, |
| 1288 |
|
|
&layer_entries); |
| 1289 |
|
✗ |
delete nested_catalog; |
| 1290 |
|
|
} else { |
| 1291 |
|
✗ |
ReadCatalogEntries(layer_catalog, layer_path, "", stratum0, temp_dir, |
| 1292 |
|
|
&layer_entries); |
| 1293 |
|
|
} |
| 1294 |
|
|
|
| 1295 |
|
|
// Clean up any intermediate catalogs loaded during hierarchy walk |
| 1296 |
|
✗ |
for (size_t j = 0; j < loaded_catalogs.size(); ++j) |
| 1297 |
|
✗ |
delete loaded_catalogs[j]; |
| 1298 |
|
|
|
| 1299 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStdout, " Read %zu entries from layer %s", |
| 1300 |
|
|
layer_entries.size(), layer_path.c_str()); |
| 1301 |
|
|
|
| 1302 |
|
✗ |
MergeLayer(layer_entries, &merged); |
| 1303 |
|
|
|
| 1304 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStdout, " Merged total: %zu entries", |
| 1305 |
|
|
merged.size()); |
| 1306 |
|
|
} |
| 1307 |
|
|
|
| 1308 |
|
✗ |
delete root_catalog; |
| 1309 |
|
|
|
| 1310 |
|
|
// Inject Singularity dotfiles if requested |
| 1311 |
|
✗ |
if (!oci_config_path.empty() && !skip_singularity) { |
| 1312 |
|
✗ |
if (!InjectSingularityDotfiles(oci_config_path, spooler_files.get(), |
| 1313 |
|
|
&merged)) { |
| 1314 |
|
✗ |
PrintError("Failed to inject Singularity dotfiles"); |
| 1315 |
|
✗ |
return 4; |
| 1316 |
|
|
} |
| 1317 |
|
|
} |
| 1318 |
|
|
|
| 1319 |
|
|
// Set up WritableCatalogManager and publish merged entries |
| 1320 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStdout, "Publishing %zu merged entries under %s", |
| 1321 |
|
|
merged.size(), dest_path.c_str()); |
| 1322 |
|
|
|
| 1323 |
|
|
catalog::WritableCatalogManager catalog_manager( |
| 1324 |
|
|
base_hash, stratum0, temp_dir, spooler_catalogs.get(), download_manager(), |
| 1325 |
|
|
false /* enforce_limits */, 0 /* nested_kcatalog_limit */, |
| 1326 |
|
|
0 /* root_kcatalog_limit */, 0 /* file_mbyte_limit */, statistics(), |
| 1327 |
|
✗ |
false /* is_balanceable */, 0 /* max_weight */, 0 /* min_weight */); |
| 1328 |
|
✗ |
catalog_manager.Init(); |
| 1329 |
|
|
|
| 1330 |
|
✗ |
if (!PublishMergedEntries(&catalog_manager, merged, dest_path)) { |
| 1331 |
|
✗ |
PrintError("Failed to publish merged entries"); |
| 1332 |
|
✗ |
return 5; |
| 1333 |
|
|
} |
| 1334 |
|
|
|
| 1335 |
|
|
// Commit catalog changes and produce updated manifest |
| 1336 |
|
✗ |
catalog_manager.PrecalculateListings(); |
| 1337 |
|
✗ |
if (!catalog_manager.Commit(false, 0, manifest.get())) { |
| 1338 |
|
✗ |
PrintError("Failed to commit catalog changes"); |
| 1339 |
|
✗ |
return 5; |
| 1340 |
|
|
} |
| 1341 |
|
|
|
| 1342 |
|
|
// Finalize spoolers |
| 1343 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStdout, "Waiting for uploads to finish..."); |
| 1344 |
|
✗ |
spooler_files->WaitForUpload(); |
| 1345 |
|
✗ |
spooler_catalogs->WaitForUpload(); |
| 1346 |
|
✗ |
spooler_files->FinalizeSession(false); |
| 1347 |
|
|
|
| 1348 |
|
✗ |
const string new_root_hash = manifest->catalog_hash().ToString(true); |
| 1349 |
|
✗ |
if (!spooler_catalogs->FinalizeSession(true, old_root_hash, new_root_hash, |
| 1350 |
|
✗ |
RepositoryTag())) { |
| 1351 |
|
✗ |
PrintError("Failed to finalize session"); |
| 1352 |
|
✗ |
return 5; |
| 1353 |
|
|
} |
| 1354 |
|
|
|
| 1355 |
|
|
// Export manifest |
| 1356 |
|
✗ |
if (!manifest->Export(manifest_path)) { |
| 1357 |
|
✗ |
PrintError("Failed to export manifest"); |
| 1358 |
|
✗ |
return 6; |
| 1359 |
|
|
} |
| 1360 |
|
|
|
| 1361 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStdout, "Overlay published successfully to %s", |
| 1362 |
|
|
dest_path.c_str()); |
| 1363 |
|
✗ |
return 0; |
| 1364 |
|
|
} |
| 1365 |
|
|
|
| 1366 |
|
|
} // namespace swissknife |
| 1367 |
|
|
|