| Directory: | cvmfs/ |
|---|---|
| File: | cvmfs/catalog_mgr_rw.cc |
| Date: | 2026-11-08 02:41:18 |
| Exec | Total | Coverage | |
|---|---|---|---|
| Lines: | 489 | 897 | 54.5% |
| Branches: | 363 | 1293 | 28.1% |
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /** | ||
| 2 | * This file is part of the CernVM file system. | ||
| 3 | */ | ||
| 4 | |||
| 5 | #include "catalog_mgr_rw.h" | ||
| 6 | |||
| 7 | #include <inttypes.h> | ||
| 8 | #include <unistd.h> | ||
| 9 | |||
| 10 | #include <cassert> | ||
| 11 | #include <cstdio> | ||
| 12 | #include <cstdlib> | ||
| 13 | #include <string> | ||
| 14 | |||
| 15 | #include "catalog_balancer.h" | ||
| 16 | #include "catalog_rw.h" | ||
| 17 | #include "manifest.h" | ||
| 18 | #include "statistics.h" | ||
| 19 | #include "upload.h" | ||
| 20 | #include "util/exception.h" | ||
| 21 | #include "util/logging.h" | ||
| 22 | #include "util/posix.h" | ||
| 23 | #include "util/smalloc.h" | ||
| 24 | |||
| 25 | using namespace std; // NOLINT | ||
| 26 | |||
| 27 | namespace catalog { | ||
| 28 | |||
| 29 | 442 | WritableCatalogManager::WritableCatalogManager( | |
| 30 | const shash::Any &base_hash, | ||
| 31 | const std::string &stratum0, | ||
| 32 | const string &dir_temp, | ||
| 33 | upload::Spooler *spooler, | ||
| 34 | download::DownloadManager *download_manager, | ||
| 35 | bool enforce_limits, | ||
| 36 | const unsigned nested_kcatalog_limit, | ||
| 37 | const unsigned root_kcatalog_limit, | ||
| 38 | const unsigned file_mbyte_limit, | ||
| 39 | perf::Statistics *statistics, | ||
| 40 | bool is_balanceable, | ||
| 41 | unsigned max_weight, | ||
| 42 | unsigned min_weight, | ||
| 43 | 442 | const std::string &dir_cache) | |
| 44 | : SimpleCatalogManager(base_hash, stratum0, dir_temp, download_manager, | ||
| 45 | statistics, false, dir_cache, | ||
| 46 | true /* copy to tmpdir */) | ||
| 47 | 442 | , spooler_(spooler) | |
| 48 | 442 | , enforce_limits_(enforce_limits) | |
| 49 | 442 | , nested_kcatalog_limit_(nested_kcatalog_limit) | |
| 50 | 442 | , root_kcatalog_limit_(root_kcatalog_limit) | |
| 51 | 442 | , file_mbyte_limit_(file_mbyte_limit) | |
| 52 | 442 | , is_balanceable_(is_balanceable) | |
| 53 | 442 | , max_weight_(max_weight) | |
| 54 | 442 | , min_weight_(min_weight) | |
| 55 | 442 | , balance_weight_(max_weight / 2) { | |
| 56 | 442 | sync_lock_ = reinterpret_cast<pthread_mutex_t *>( | |
| 57 | 442 | smalloc(sizeof(pthread_mutex_t))); | |
| 58 | 442 | int retval = pthread_mutex_init(sync_lock_, NULL); | |
| 59 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 442 times.
|
442 | assert(retval == 0); |
| 60 | 442 | catalog_processing_lock_ = reinterpret_cast<pthread_mutex_t *>( | |
| 61 | 442 | smalloc(sizeof(pthread_mutex_t))); | |
| 62 | 442 | retval = pthread_mutex_init(catalog_processing_lock_, NULL); | |
| 63 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 442 times.
|
442 | assert(retval == 0); |
| 64 | 442 | catalog_download_lock_ = reinterpret_cast<pthread_mutex_t *>( | |
| 65 | 442 | smalloc(sizeof(pthread_mutex_t))); | |
| 66 | 442 | retval = pthread_mutex_init(catalog_download_lock_, NULL); | |
| 67 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 442 times.
|
442 | assert(retval == 0); |
| 68 | 442 | catalog_hash_lock_ = reinterpret_cast<pthread_mutex_t *>( | |
| 69 | 442 | smalloc(sizeof(pthread_mutex_t))); | |
| 70 | 442 | retval = pthread_mutex_init(catalog_hash_lock_, NULL); | |
| 71 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 442 times.
|
442 | assert(retval == 0); |
| 72 | 442 | } | |
| 73 | |||
| 74 | |||
| 75 | 1764 | WritableCatalogManager::~WritableCatalogManager() { | |
| 76 | 882 | pthread_mutex_destroy(sync_lock_); | |
| 77 | 882 | free(sync_lock_); | |
| 78 | 882 | pthread_mutex_destroy(catalog_processing_lock_); | |
| 79 | 882 | free(catalog_processing_lock_); | |
| 80 | 882 | pthread_mutex_destroy(catalog_download_lock_); | |
| 81 | 882 | free(catalog_download_lock_); | |
| 82 | 882 | pthread_mutex_destroy(catalog_hash_lock_); | |
| 83 | 882 | free(catalog_hash_lock_); | |
| 84 | 1764 | } | |
| 85 | |||
| 86 | |||
| 87 | /** | ||
| 88 | * This method is virtual in AbstractCatalogManager. It returns a new catalog | ||
| 89 | * structure in the form the different CatalogManagers need it. | ||
| 90 | * In this case it returns a stub for a WritableCatalog. | ||
| 91 | * @param mountpoint the mount point of the catalog stub to create | ||
| 92 | * @param catalog_hash the content hash of the catalog to create | ||
| 93 | * @param parent_catalog the parent of the catalog stub to create | ||
| 94 | * @return a pointer to the catalog stub structure created | ||
| 95 | */ | ||
| 96 | 1071 | Catalog *WritableCatalogManager::CreateCatalog(const PathString &mountpoint, | |
| 97 | const shash::Any &catalog_hash, | ||
| 98 | Catalog *parent_catalog) { | ||
| 99 | 2142 | return new WritableCatalog(mountpoint.ToString(), catalog_hash, | |
| 100 |
2/4✓ Branch 1 taken 1071 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1071 times.
✗ Branch 5 not taken.
|
2142 | parent_catalog); |
| 101 | } | ||
| 102 | |||
| 103 | |||
| 104 | 1071 | void WritableCatalogManager::ActivateCatalog(Catalog *catalog) { | |
| 105 | 1071 | catalog->TakeDatabaseFileOwnership(); | |
| 106 | 1071 | } | |
| 107 | |||
| 108 | |||
| 109 | /** | ||
| 110 | * This method is invoked if we create a completely new repository. | ||
| 111 | * The new root catalog will already contain a root entry. | ||
| 112 | * It is uploaded by a Forklift to the upstream storage. | ||
| 113 | * @return true on success, false otherwise | ||
| 114 | */ | ||
| 115 | 374 | manifest::Manifest *WritableCatalogManager::CreateRepository( | |
| 116 | const string &dir_temp, | ||
| 117 | const bool volatile_content, | ||
| 118 | const std::string &voms_authz, | ||
| 119 | upload::Spooler *spooler) { | ||
| 120 | // Create a new root catalog at file_path | ||
| 121 |
1/2✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
|
374 | const string file_path = dir_temp + "/new_root_catalog"; |
| 122 | |||
| 123 | 374 | const shash::Algorithms hash_algorithm = spooler->GetHashAlgorithm(); | |
| 124 | |||
| 125 | // A newly created catalog always needs a root entry | ||
| 126 | // we create and configure this here | ||
| 127 |
1/2✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
|
374 | DirectoryEntry root_entry; |
| 128 | 374 | root_entry.inode_ = DirectoryEntry::kInvalidInode; | |
| 129 | 374 | root_entry.mode_ = 16877; | |
| 130 | 374 | root_entry.size_ = 4096; | |
| 131 | 374 | root_entry.mtime_ = time(NULL); | |
| 132 | 374 | root_entry.uid_ = getuid(); | |
| 133 | 374 | root_entry.gid_ = getgid(); | |
| 134 |
1/2✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
|
374 | root_entry.checksum_ = shash::Any(hash_algorithm); |
| 135 | 374 | root_entry.linkcount_ = 2; | |
| 136 |
1/2✓ Branch 2 taken 374 times.
✗ Branch 3 not taken.
|
374 | const string root_path = ""; |
| 137 | |||
| 138 | // Create the database schema and the initial root entry | ||
| 139 | { | ||
| 140 | const std::unique_ptr<CatalogDatabase> new_clg_db( | ||
| 141 |
1/2✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
|
374 | CatalogDatabase::Create(file_path)); |
| 142 | 374 | if (new_clg_db.get() == nullptr | |
| 143 |
4/8✓ Branch 0 taken 374 times.
✗ Branch 1 not taken.
✓ Branch 4 taken 374 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 374 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 374 times.
|
374 | || !new_clg_db->InsertInitialValues(root_path, volatile_content, |
| 144 | voms_authz, root_entry)) { | ||
| 145 | ✗ | LogCvmfs(kLogCatalog, kLogStderr, "creation of catalog '%s' failed", | |
| 146 | file_path.c_str()); | ||
| 147 | ✗ | return NULL; | |
| 148 | } | ||
| 149 |
1/2✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
|
374 | } |
| 150 | |||
| 151 | // Compress root catalog; | ||
| 152 |
1/2✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
|
374 | const int64_t catalog_size = GetFileSize(file_path); |
| 153 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 374 times.
|
374 | if (catalog_size < 0) { |
| 154 | ✗ | unlink(file_path.c_str()); | |
| 155 | ✗ | return NULL; | |
| 156 | } | ||
| 157 |
1/2✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
|
374 | const string file_path_compressed = file_path + ".compressed"; |
| 158 |
1/2✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
|
374 | shash::Any hash_catalog(hash_algorithm, shash::kSuffixCatalog); |
| 159 |
1/2✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
|
374 | const bool retval = zlib::CompressPath2Path(file_path, file_path_compressed, |
| 160 | &hash_catalog); | ||
| 161 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 374 times.
|
374 | if (!retval) { |
| 162 | ✗ | LogCvmfs(kLogCatalog, kLogStderr, "compression of catalog '%s' failed", | |
| 163 | file_path.c_str()); | ||
| 164 | ✗ | unlink(file_path.c_str()); | |
| 165 | ✗ | return NULL; | |
| 166 | } | ||
| 167 | 374 | unlink(file_path.c_str()); | |
| 168 | |||
| 169 | // Create manifest | ||
| 170 |
1/2✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
|
374 | const string manifest_path = dir_temp + "/manifest"; |
| 171 | manifest::Manifest *manifest = new manifest::Manifest(hash_catalog, | ||
| 172 |
3/6✓ Branch 2 taken 374 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 374 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 374 times.
✗ Branch 9 not taken.
|
374 | catalog_size, ""); |
| 173 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 374 times.
|
374 | if (!voms_authz.empty()) { |
| 174 | ✗ | manifest->set_has_alt_catalog_path(true); | |
| 175 | } | ||
| 176 | |||
| 177 | // Upload catalog | ||
| 178 |
3/6✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 374 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 374 times.
✗ Branch 8 not taken.
|
374 | spooler->Upload(file_path_compressed, "data/" + hash_catalog.MakePath()); |
| 179 |
1/2✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
|
374 | spooler->WaitForUpload(); |
| 180 | 374 | unlink(file_path_compressed.c_str()); | |
| 181 |
2/4✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 374 times.
|
374 | if (spooler->GetNumberOfErrors() > 0) { |
| 182 | ✗ | LogCvmfs(kLogCatalog, kLogStderr, "failed to commit catalog %s", | |
| 183 | file_path_compressed.c_str()); | ||
| 184 | ✗ | delete manifest; | |
| 185 | ✗ | return NULL; | |
| 186 | } | ||
| 187 | |||
| 188 | 374 | return manifest; | |
| 189 | 374 | } | |
| 190 | |||
| 191 | |||
| 192 | /** | ||
| 193 | * Retrieve the catalog containing the given path. | ||
| 194 | * Other than AbstractCatalogManager::FindCatalog() this mounts nested | ||
| 195 | * catalogs if necessary and returns WritableCatalog objects. | ||
| 196 | * Furthermore it optionally returns the looked-up DirectoryEntry. | ||
| 197 | * | ||
| 198 | * @param path the path to look for | ||
| 199 | * @param result the retrieved catalog (as a pointer) | ||
| 200 | * @param dirent is set to looked up DirectoryEntry for 'path' if non-NULL | ||
| 201 | * @return true if catalog was found | ||
| 202 | */ | ||
| 203 | 4967 | bool WritableCatalogManager::FindCatalog(const string &path, | |
| 204 | WritableCatalog **result, | ||
| 205 | DirectoryEntry *dirent) { | ||
| 206 |
1/2✓ Branch 1 taken 4967 times.
✗ Branch 2 not taken.
|
4967 | const PathString ps_path(path); |
| 207 | |||
| 208 |
1/2✓ Branch 1 taken 4967 times.
✗ Branch 2 not taken.
|
4967 | Catalog *best_fit = AbstractCatalogManager<Catalog>::FindCatalog(ps_path); |
| 209 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 4967 times.
|
4967 | assert(best_fit != NULL); |
| 210 | 4967 | Catalog *catalog = NULL; | |
| 211 |
1/2✓ Branch 1 taken 4967 times.
✗ Branch 2 not taken.
|
4967 | const bool retval = MountSubtree(ps_path, best_fit, true /* is_listable */, |
| 212 | &catalog); | ||
| 213 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 4967 times.
|
4967 | if (!retval) |
| 214 | ✗ | return false; | |
| 215 | |||
| 216 | 4967 | *result = static_cast<WritableCatalog *>(catalog); | |
| 217 | |||
| 218 |
1/2✓ Branch 1 taken 4967 times.
✗ Branch 2 not taken.
|
4967 | catalog::DirectoryEntry dummy; |
| 219 |
2/2✓ Branch 0 taken 2521 times.
✓ Branch 1 taken 2446 times.
|
4967 | if (NULL == dirent) { |
| 220 | 2521 | dirent = &dummy; | |
| 221 | } | ||
| 222 |
1/2✓ Branch 1 taken 4967 times.
✗ Branch 2 not taken.
|
4967 | const bool found = catalog->LookupPath(ps_path, dirent); |
| 223 |
6/8✓ Branch 0 taken 4955 times.
✓ Branch 1 taken 12 times.
✓ Branch 3 taken 4955 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 4955 times.
✓ Branch 7 taken 12 times.
✓ Branch 8 taken 4955 times.
|
4967 | if (!found || !catalog->IsWritable()) |
| 224 | 12 | return false; | |
| 225 | |||
| 226 | 4955 | return true; | |
| 227 | 4967 | } | |
| 228 | |||
| 229 | |||
| 230 | ✗ | WritableCatalog *WritableCatalogManager::GetHostingCatalog( | |
| 231 | const std::string &path) { | ||
| 232 | ✗ | WritableCatalog *result = NULL; | |
| 233 | ✗ | const bool retval = FindCatalog(MakeRelativePath(path), &result, NULL); | |
| 234 | ✗ | if (!retval) | |
| 235 | ✗ | return NULL; | |
| 236 | ✗ | return result; | |
| 237 | } | ||
| 238 | |||
| 239 | |||
| 240 | /** | ||
| 241 | * Remove the given file from the catalogs. | ||
| 242 | * @param file_path the full path to the file to be removed | ||
| 243 | * @return true on success, false otherwise | ||
| 244 | */ | ||
| 245 | 30 | void WritableCatalogManager::RemoveFile(const std::string &path) { | |
| 246 |
1/2✓ Branch 1 taken 30 times.
✗ Branch 2 not taken.
|
30 | const string file_path = MakeRelativePath(path); |
| 247 | |||
| 248 | 30 | SyncLock(); | |
| 249 | 30 | WritableCatalog *catalog = NULL; | |
| 250 |
1/2✓ Branch 1 taken 30 times.
✗ Branch 2 not taken.
|
30 | DirectoryEntry entry; |
| 251 |
2/4✓ Branch 1 taken 30 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 30 times.
✗ Branch 4 not taken.
|
30 | if (FindCatalog(file_path, &catalog, &entry)) { |
| 252 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 30 times.
|
30 | if (entry.IsBundleTrigger()) { |
| 253 | ✗ | catalog->RemoveEntry(GetParentPath(file_path) + "/.cvmfsbundle-" | |
| 254 | ✗ | + GetFileName(file_path)); | |
| 255 | } | ||
| 256 |
1/2✓ Branch 1 taken 30 times.
✗ Branch 2 not taken.
|
30 | catalog->RemoveEntry(file_path); |
| 257 | } | ||
| 258 | 30 | SyncUnlock(); | |
| 259 | |||
| 260 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 30 times.
|
30 | if (catalog == NULL) { |
| 261 | ✗ | PANIC(kLogStderr, "catalog for file '%s' cannot be found", | |
| 262 | file_path.c_str()); | ||
| 263 | } | ||
| 264 | 30 | } | |
| 265 | |||
| 266 | |||
| 267 | /** | ||
| 268 | * Remove the given directory from the catalogs. | ||
| 269 | * @param directory_path the full path to the directory to be removed | ||
| 270 | * @return true on success, false otherwise | ||
| 271 | */ | ||
| 272 | 51 | void WritableCatalogManager::RemoveDirectory(const std::string &path) { | |
| 273 |
1/2✓ Branch 1 taken 51 times.
✗ Branch 2 not taken.
|
51 | const string directory_path = MakeRelativePath(path); |
| 274 |
1/2✓ Branch 1 taken 51 times.
✗ Branch 2 not taken.
|
51 | const string parent_path = GetParentPath(directory_path); |
| 275 | |||
| 276 | 51 | SyncLock(); | |
| 277 | WritableCatalog *catalog; | ||
| 278 |
1/2✓ Branch 1 taken 51 times.
✗ Branch 2 not taken.
|
51 | DirectoryEntry parent_entry; |
| 279 |
2/4✓ Branch 1 taken 51 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 51 times.
|
51 | if (!FindCatalog(parent_path, &catalog, &parent_entry)) { |
| 280 | ✗ | PANIC(kLogStderr, "catalog for directory '%s' cannot be found", | |
| 281 | directory_path.c_str()); | ||
| 282 | } | ||
| 283 | |||
| 284 | 51 | parent_entry.set_linkcount(parent_entry.linkcount() - 1); | |
| 285 | |||
| 286 |
1/2✓ Branch 1 taken 51 times.
✗ Branch 2 not taken.
|
51 | catalog->RemoveEntry(directory_path); |
| 287 |
1/2✓ Branch 1 taken 51 times.
✗ Branch 2 not taken.
|
51 | catalog->UpdateEntry(parent_entry, parent_path); |
| 288 |
2/2✓ Branch 1 taken 26 times.
✓ Branch 2 taken 25 times.
|
51 | if (parent_entry.IsNestedCatalogRoot()) { |
| 289 |
1/2✓ Branch 2 taken 26 times.
✗ Branch 3 not taken.
|
26 | LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating transition point %s", |
| 290 | parent_path.c_str()); | ||
| 291 | WritableCatalog *parent_catalog = reinterpret_cast<WritableCatalog *>( | ||
| 292 | 26 | catalog->parent()); | |
| 293 | 26 | parent_entry.set_is_nested_catalog_mountpoint(true); | |
| 294 | 26 | parent_entry.set_is_nested_catalog_root(false); | |
| 295 |
1/2✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
|
26 | parent_catalog->UpdateEntry(parent_entry, parent_path); |
| 296 | } | ||
| 297 | 51 | SyncUnlock(); | |
| 298 | 51 | } | |
| 299 | |||
| 300 | /** | ||
| 301 | * Clone the file called `source` changing its name into `destination`, the | ||
| 302 | * source file is keep intact. | ||
| 303 | * @params destination, the name of the new file, complete path | ||
| 304 | * @params source, the name of the file to clone, which must be already in the | ||
| 305 | * repository | ||
| 306 | * @params fail_if_source_missing, when true (default) abort if the source is | ||
| 307 | * not in the catalog; when false, skip the clone and return false instead. | ||
| 308 | * This is used by the tarball ingestion engine to tolerate hardlinks whose | ||
| 309 | * target is not part of the same archive (e.g. cross-layer hardlinks in OCI | ||
| 310 | * image layers). | ||
| 311 | * @return true if the file was cloned, false if the source was missing and | ||
| 312 | * fail_if_source_missing was false | ||
| 313 | */ | ||
| 314 | ✗ | bool WritableCatalogManager::Clone(const std::string destination, | |
| 315 | const std::string source, | ||
| 316 | const bool fail_if_source_missing) { | ||
| 317 | ✗ | const std::string relative_source = MakeRelativePath(source); | |
| 318 | |||
| 319 | ✗ | DirectoryEntry source_dirent; | |
| 320 | ✗ | if (!LookupPath(relative_source, kLookupDefault, &source_dirent)) { | |
| 321 | ✗ | if (fail_if_source_missing) { | |
| 322 | ✗ | PANIC(kLogStderr, "catalog for file '%s' cannot be found, aborting", | |
| 323 | source.c_str()); | ||
| 324 | } | ||
| 325 | // The caller opted out of aborting and is expected to handle the missing | ||
| 326 | // source (e.g. by materializing a replacement file), so only log at debug | ||
| 327 | // level here to avoid duplicate user-facing warnings. | ||
| 328 | ✗ | LogCvmfs(kLogCatalog, kLogDebug, | |
| 329 | "catalog for clone source '%s' cannot be found, not cloning " | ||
| 330 | "to '%s'", | ||
| 331 | source.c_str(), destination.c_str()); | ||
| 332 | ✗ | return false; | |
| 333 | } | ||
| 334 | ✗ | if (source_dirent.IsDirectory()) { | |
| 335 | ✗ | PANIC(kLogStderr, "Trying to clone a directory: '%s', aborting", | |
| 336 | source.c_str()); | ||
| 337 | } | ||
| 338 | |||
| 339 | // if the file is already there we remove it and we add it back | ||
| 340 | ✗ | DirectoryEntry check_dirent; | |
| 341 | ✗ | const bool destination_already_present = LookupPath( | |
| 342 | ✗ | MakeRelativePath(destination), kLookupDefault, &check_dirent); | |
| 343 | ✗ | if (destination_already_present) { | |
| 344 | ✗ | this->RemoveFile(destination); | |
| 345 | } | ||
| 346 | |||
| 347 | ✗ | DirectoryEntry destination_dirent(source_dirent); | |
| 348 | ✗ | std::string destination_dirname; | |
| 349 | ✗ | std::string destination_filename; | |
| 350 | ✗ | SplitPath(destination, &destination_dirname, &destination_filename); | |
| 351 | |||
| 352 | ✗ | destination_dirent.name_.Assign( | |
| 353 | ✗ | NameString(destination_filename.c_str(), destination_filename.length())); | |
| 354 | |||
| 355 | // TODO(jblomer): clone is used by tarball engine and should eventually | ||
| 356 | // support extended attributes | ||
| 357 | ✗ | this->AddFile(destination_dirent, empty_xattrs, destination_dirname); | |
| 358 | ✗ | return true; | |
| 359 | } | ||
| 360 | |||
| 361 | |||
| 362 | /** | ||
| 363 | * Copies an entire directory tree from the existing from_dir to the | ||
| 364 | * non-existing to_dir. The destination's parent directory must exist. On the | ||
| 365 | * catalog level, the new entries will be identical to the old ones except | ||
| 366 | * for their path hash fields. | ||
| 367 | */ | ||
| 368 | 121 | void WritableCatalogManager::CloneTree(const std::string &from_dir, | |
| 369 | const std::string &to_dir) { | ||
| 370 | // Sanitize input paths | ||
| 371 |
6/6✓ Branch 1 taken 97 times.
✓ Branch 2 taken 24 times.
✓ Branch 4 taken 12 times.
✓ Branch 5 taken 85 times.
✓ Branch 6 taken 36 times.
✓ Branch 7 taken 85 times.
|
121 | if (from_dir.empty() || to_dir.empty()) |
| 372 | 36 | PANIC(kLogStderr, "clone tree from or to root impossible"); | |
| 373 | |||
| 374 |
1/2✓ Branch 1 taken 85 times.
✗ Branch 2 not taken.
|
85 | const std::string relative_source = MakeRelativePath(from_dir); |
| 375 |
1/2✓ Branch 1 taken 85 times.
✗ Branch 2 not taken.
|
85 | const std::string relative_dest = MakeRelativePath(to_dir); |
| 376 | |||
| 377 |
2/2✓ Branch 1 taken 12 times.
✓ Branch 2 taken 73 times.
|
85 | if (relative_source == relative_dest) { |
| 378 | 12 | PANIC(kLogStderr, "cannot clone tree into itself ('%s')", to_dir.c_str()); | |
| 379 | } | ||
| 380 |
4/7✓ Branch 1 taken 73 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 73 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 12 times.
✓ Branch 8 taken 61 times.
|
73 | if (HasPrefix(relative_dest, relative_source + "/", false /*ignore_case*/)) { |
| 381 | 12 | PANIC(kLogStderr, | |
| 382 | "cannot clone tree into sub directory of source '%s' --> '%s'", | ||
| 383 | from_dir.c_str(), to_dir.c_str()); | ||
| 384 | } | ||
| 385 | |||
| 386 |
1/2✓ Branch 1 taken 61 times.
✗ Branch 2 not taken.
|
61 | DirectoryEntry source_dirent; |
| 387 |
3/4✓ Branch 1 taken 61 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 12 times.
✓ Branch 4 taken 49 times.
|
61 | if (!LookupPath(relative_source, kLookupDefault, &source_dirent)) { |
| 388 | 12 | PANIC(kLogStderr, "path '%s' cannot be found, aborting", from_dir.c_str()); | |
| 389 | } | ||
| 390 |
2/2✓ Branch 1 taken 12 times.
✓ Branch 2 taken 37 times.
|
49 | if (!source_dirent.IsDirectory()) { |
| 391 | 12 | PANIC(kLogStderr, "CloneTree: source '%s' not a directory, aborting", | |
| 392 | from_dir.c_str()); | ||
| 393 | } | ||
| 394 | |||
| 395 |
1/2✓ Branch 1 taken 37 times.
✗ Branch 2 not taken.
|
37 | DirectoryEntry dest_dirent; |
| 396 |
3/4✓ Branch 1 taken 37 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 12 times.
✓ Branch 4 taken 25 times.
|
37 | if (LookupPath(relative_dest, kLookupDefault, &dest_dirent)) { |
| 397 | 12 | PANIC(kLogStderr, "destination '%s' exists, aborting", to_dir.c_str()); | |
| 398 | } | ||
| 399 | |||
| 400 |
1/2✓ Branch 1 taken 25 times.
✗ Branch 2 not taken.
|
25 | const std::string dest_parent = GetParentPath(relative_dest); |
| 401 |
1/2✓ Branch 1 taken 25 times.
✗ Branch 2 not taken.
|
25 | DirectoryEntry dest_parent_dirent; |
| 402 |
3/4✓ Branch 1 taken 25 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 12 times.
✓ Branch 4 taken 13 times.
|
25 | if (!LookupPath(dest_parent, kLookupDefault, &dest_parent_dirent)) { |
| 403 | 12 | PANIC(kLogStderr, "destination '%s' not on a known path, aborting", | |
| 404 | to_dir.c_str()); | ||
| 405 | } | ||
| 406 | |||
| 407 |
2/4✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 13 times.
✗ Branch 5 not taken.
|
13 | CloneTreeImpl(PathString(from_dir), |
| 408 |
1/2✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
|
26 | GetParentPath(to_dir), |
| 409 |
2/4✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 13 times.
✗ Branch 5 not taken.
|
26 | NameString(GetFileName(to_dir))); |
| 410 | 253 | } | |
| 411 | |||
| 412 | |||
| 413 | /** | ||
| 414 | * Called from CloneTree(), assumes that from_dir and to_dir are sufficiently | ||
| 415 | * sanitized | ||
| 416 | */ | ||
| 417 | 104 | void WritableCatalogManager::CloneTreeImpl(const PathString &source_dir, | |
| 418 | const std::string &dest_parent_dir, | ||
| 419 | const NameString &dest_name) { | ||
| 420 |
1/2✓ Branch 4 taken 104 times.
✗ Branch 5 not taken.
|
104 | LogCvmfs(kLogCatalog, kLogDebug, "cloning %s --> %s/%s", source_dir.c_str(), |
| 421 |
1/2✓ Branch 1 taken 104 times.
✗ Branch 2 not taken.
|
208 | dest_parent_dir.c_str(), dest_name.ToString().c_str()); |
| 422 |
3/6✓ Branch 1 taken 104 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 104 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 104 times.
✗ Branch 8 not taken.
|
208 | const PathString relative_source(MakeRelativePath(source_dir.ToString())); |
| 423 | |||
| 424 |
1/2✓ Branch 1 taken 104 times.
✗ Branch 2 not taken.
|
104 | DirectoryEntry source_dirent; |
| 425 |
1/2✓ Branch 1 taken 104 times.
✗ Branch 2 not taken.
|
104 | bool retval = LookupPath(relative_source, kLookupDefault, &source_dirent); |
| 426 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 104 times.
|
104 | assert(retval); |
| 427 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 104 times.
|
104 | assert(!source_dirent.IsBindMountpoint()); |
| 428 | |||
| 429 |
1/2✓ Branch 1 taken 104 times.
✗ Branch 2 not taken.
|
104 | DirectoryEntry dest_dirent(source_dirent); |
| 430 |
1/2✓ Branch 1 taken 104 times.
✗ Branch 2 not taken.
|
104 | dest_dirent.name_.Assign(dest_name); |
| 431 | // Just in case, reset the nested catalog markers | ||
| 432 | 104 | dest_dirent.set_is_nested_catalog_mountpoint(false); | |
| 433 | 104 | dest_dirent.set_is_nested_catalog_root(false); | |
| 434 | |||
| 435 | 104 | XattrList xattrs; | |
| 436 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 104 times.
|
104 | if (source_dirent.HasXattrs()) { |
| 437 | ✗ | retval = LookupXattrs(relative_source, &xattrs); | |
| 438 | ✗ | assert(retval); | |
| 439 | } | ||
| 440 |
1/2✓ Branch 1 taken 104 times.
✗ Branch 2 not taken.
|
104 | AddDirectory(dest_dirent, xattrs, dest_parent_dir); |
| 441 | |||
| 442 |
1/2✓ Branch 1 taken 104 times.
✗ Branch 2 not taken.
|
104 | std::string dest_dir = dest_parent_dir; |
| 443 |
2/2✓ Branch 1 taken 91 times.
✓ Branch 2 taken 13 times.
|
104 | if (!dest_dir.empty()) |
| 444 |
1/2✓ Branch 1 taken 91 times.
✗ Branch 2 not taken.
|
91 | dest_dir.push_back('/'); |
| 445 |
2/4✓ Branch 1 taken 104 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 104 times.
✗ Branch 5 not taken.
|
104 | dest_dir += dest_name.ToString(); |
| 446 | 104 | if (source_dirent.IsNestedCatalogRoot() | |
| 447 |
5/6✓ Branch 0 taken 65 times.
✓ Branch 1 taken 39 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 65 times.
✓ Branch 5 taken 39 times.
✓ Branch 6 taken 65 times.
|
104 | || source_dirent.IsNestedCatalogMountpoint()) { |
| 448 |
1/2✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
|
39 | CreateNestedCatalog(dest_dir); |
| 449 | } | ||
| 450 | |||
| 451 | 104 | DirectoryEntryList ls; | |
| 452 |
1/2✓ Branch 1 taken 104 times.
✗ Branch 2 not taken.
|
104 | retval = Listing(relative_source, &ls, false /* expand_symlink */); |
| 453 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 104 times.
|
104 | assert(retval); |
| 454 |
2/2✓ Branch 1 taken 221 times.
✓ Branch 2 taken 104 times.
|
325 | for (unsigned i = 0; i < ls.size(); ++i) { |
| 455 |
1/2✓ Branch 1 taken 221 times.
✗ Branch 2 not taken.
|
221 | PathString sub_path(source_dir); |
| 456 |
2/4✓ Branch 1 taken 221 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 221 times.
|
221 | assert(!sub_path.IsEmpty()); |
| 457 |
1/2✓ Branch 1 taken 221 times.
✗ Branch 2 not taken.
|
221 | sub_path.Append("/", 1); |
| 458 |
3/6✓ Branch 2 taken 221 times.
✗ Branch 3 not taken.
✓ Branch 7 taken 221 times.
✗ Branch 8 not taken.
✓ Branch 11 taken 221 times.
✗ Branch 12 not taken.
|
221 | sub_path.Append(ls[i].name().GetChars(), ls[i].name().GetLength()); |
| 459 | |||
| 460 |
2/2✓ Branch 2 taken 91 times.
✓ Branch 3 taken 130 times.
|
221 | if (ls[i].IsDirectory()) { |
| 461 |
2/4✓ Branch 2 taken 91 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 91 times.
✗ Branch 6 not taken.
|
91 | CloneTreeImpl(sub_path, dest_dir, ls[i].name()); |
| 462 | 91 | continue; | |
| 463 | } | ||
| 464 | |||
| 465 | // We break hard-links during cloning | ||
| 466 | 130 | ls[i].set_hardlink_group(0); | |
| 467 | 130 | ls[i].set_linkcount(1); | |
| 468 | |||
| 469 | 130 | xattrs.Clear(); | |
| 470 |
1/2✗ Branch 2 not taken.
✓ Branch 3 taken 130 times.
|
130 | if (ls[i].HasXattrs()) { |
| 471 | ✗ | retval = LookupXattrs(sub_path, &xattrs); | |
| 472 | ✗ | assert(retval); | |
| 473 | } | ||
| 474 | |||
| 475 |
1/2✗ Branch 2 not taken.
✓ Branch 3 taken 130 times.
|
130 | if (ls[i].IsChunkedFile()) { |
| 476 | ✗ | FileChunkList chunks; | |
| 477 | const std::string relative_sub_path = MakeRelativePath( | ||
| 478 | ✗ | sub_path.ToString()); | |
| 479 | ✗ | retval = ListFileChunks(PathString(relative_sub_path), | |
| 480 | ✗ | ls[i].hash_algorithm(), &chunks); | |
| 481 | ✗ | assert(retval); | |
| 482 | ✗ | AddChunkedFile(ls[i], xattrs, dest_dir, chunks); | |
| 483 | ✗ | } else { | |
| 484 |
1/2✓ Branch 2 taken 130 times.
✗ Branch 3 not taken.
|
130 | AddFile(ls[i], xattrs, dest_dir); |
| 485 | } | ||
| 486 |
2/2✓ Branch 1 taken 130 times.
✓ Branch 2 taken 91 times.
|
221 | } |
| 487 | 104 | } | |
| 488 | |||
| 489 | |||
| 490 | /** | ||
| 491 | * Add a new directory to the catalogs. | ||
| 492 | * @param entry a DirectoryEntry structure describing the new directory | ||
| 493 | * @param parent_directory the absolute path of the directory containing the | ||
| 494 | * directory to be created | ||
| 495 | * @return true on success, false otherwise | ||
| 496 | */ | ||
| 497 | 1846 | void WritableCatalogManager::AddDirectory(const DirectoryEntryBase &entry, | |
| 498 | const XattrList &xattrs, | ||
| 499 | const std::string &parent_directory) { | ||
| 500 |
1/2✓ Branch 1 taken 1846 times.
✗ Branch 2 not taken.
|
1846 | const string parent_path = MakeRelativePath(parent_directory); |
| 501 |
1/2✓ Branch 1 taken 1846 times.
✗ Branch 2 not taken.
|
1846 | string directory_path = parent_path + "/"; |
| 502 |
3/6✓ Branch 1 taken 1846 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 1846 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 1846 times.
✗ Branch 10 not taken.
|
1846 | directory_path.append(entry.name().GetChars(), entry.name().GetLength()); |
| 503 | |||
| 504 | 1846 | SyncLock(); | |
| 505 | WritableCatalog *catalog; | ||
| 506 |
1/2✓ Branch 1 taken 1846 times.
✗ Branch 2 not taken.
|
1846 | DirectoryEntry parent_entry; |
| 507 |
2/4✓ Branch 1 taken 1846 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 1846 times.
|
1846 | if (!FindCatalog(parent_path, &catalog, &parent_entry)) { |
| 508 | ✗ | PANIC(kLogStderr, "catalog for directory '%s' cannot be found", | |
| 509 | directory_path.c_str()); | ||
| 510 | } | ||
| 511 | |||
| 512 |
1/2✓ Branch 1 taken 1846 times.
✗ Branch 2 not taken.
|
1846 | DirectoryEntry fixed_hardlink_count(entry); |
| 513 | 1846 | fixed_hardlink_count.set_linkcount(2); | |
| 514 |
1/2✓ Branch 1 taken 1846 times.
✗ Branch 2 not taken.
|
1846 | catalog->AddEntry(fixed_hardlink_count, xattrs, directory_path, parent_path); |
| 515 | |||
| 516 | 1846 | parent_entry.set_linkcount(parent_entry.linkcount() + 1); | |
| 517 |
1/2✓ Branch 1 taken 1846 times.
✗ Branch 2 not taken.
|
1846 | catalog->UpdateEntry(parent_entry, parent_path); |
| 518 |
2/2✓ Branch 1 taken 39 times.
✓ Branch 2 taken 1807 times.
|
1846 | if (parent_entry.IsNestedCatalogRoot()) { |
| 519 |
1/2✓ Branch 2 taken 39 times.
✗ Branch 3 not taken.
|
39 | LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating transition point %s", |
| 520 | parent_path.c_str()); | ||
| 521 | WritableCatalog *parent_catalog = reinterpret_cast<WritableCatalog *>( | ||
| 522 | 39 | catalog->parent()); | |
| 523 | 39 | parent_entry.set_is_nested_catalog_mountpoint(true); | |
| 524 | 39 | parent_entry.set_is_nested_catalog_root(false); | |
| 525 |
1/2✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
|
39 | parent_catalog->UpdateEntry(parent_entry, parent_path); |
| 526 | } | ||
| 527 | 1846 | SyncUnlock(); | |
| 528 | 1846 | } | |
| 529 | |||
| 530 | /** | ||
| 531 | * Add a new file to the catalogs. | ||
| 532 | * @param entry a DirectoryEntry structure describing the new file | ||
| 533 | * @param parent_directory the absolute path of the directory containing the | ||
| 534 | * file to be created | ||
| 535 | * @return true on success, false otherwise | ||
| 536 | */ | ||
| 537 | 2396 | void WritableCatalogManager::AddFile(const DirectoryEntry &entry, | |
| 538 | const XattrList &xattrs, | ||
| 539 | const std::string &parent_directory) { | ||
| 540 |
1/2✓ Branch 1 taken 2396 times.
✗ Branch 2 not taken.
|
2396 | const string parent_path = MakeRelativePath(parent_directory); |
| 541 |
1/2✓ Branch 1 taken 2396 times.
✗ Branch 2 not taken.
|
2396 | const string file_path = entry.GetFullPath(parent_path); |
| 542 | |||
| 543 | 2396 | SyncLock(); | |
| 544 | WritableCatalog *catalog; | ||
| 545 |
2/4✓ Branch 1 taken 2396 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 2396 times.
|
2396 | if (!FindCatalog(parent_path, &catalog)) { |
| 546 | ✗ | PANIC(kLogStderr, "catalog for file '%s' cannot be found", | |
| 547 | file_path.c_str()); | ||
| 548 | } | ||
| 549 | |||
| 550 |
4/6✓ Branch 1 taken 2368 times.
✓ Branch 2 taken 28 times.
✓ Branch 4 taken 2368 times.
✗ Branch 5 not taken.
✗ Branch 8 not taken.
✓ Branch 9 taken 2368 times.
|
2396 | assert(!entry.IsRegular() || entry.IsChunkedFile() |
| 551 | || !entry.checksum().IsNull()); | ||
| 552 |
3/4✓ Branch 1 taken 28 times.
✓ Branch 2 taken 2368 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 28 times.
|
2396 | assert(entry.IsRegular() || !entry.IsExternalFile()); |
| 553 | |||
| 554 | // check if file is too big | ||
| 555 |
1/2✓ Branch 1 taken 2396 times.
✗ Branch 2 not taken.
|
2396 | const unsigned mbytes = entry.size() / (1024 * 1024); |
| 556 |
1/4✗ Branch 0 not taken.
✓ Branch 1 taken 2396 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
2396 | if ((file_mbyte_limit_ > 0) && (mbytes > file_mbyte_limit_)) { |
| 557 | ✗ | LogCvmfs(kLogCatalog, kLogStderr, | |
| 558 | "%s: file at %s is larger than %u megabytes (%u). " | ||
| 559 | "CernVM-FS works best with small files. " | ||
| 560 | "Please remove the file or increase the limit.", | ||
| 561 | ✗ | enforce_limits_ ? "FATAL" : "WARNING", file_path.c_str(), | |
| 562 | file_mbyte_limit_, mbytes); | ||
| 563 | ✗ | if (enforce_limits_) | |
| 564 | ✗ | PANIC(kLogStderr, "file at %s is larger than %u megabytes (%u).", | |
| 565 | file_path.c_str(), file_mbyte_limit_, mbytes); | ||
| 566 | } | ||
| 567 | |||
| 568 |
1/2✓ Branch 1 taken 2396 times.
✗ Branch 2 not taken.
|
2396 | catalog->AddEntry(entry, xattrs, file_path, parent_path); |
| 569 | 2396 | SyncUnlock(); | |
| 570 | 2396 | } | |
| 571 | |||
| 572 | |||
| 573 | ✗ | void WritableCatalogManager::AddChunkedFile(const DirectoryEntryBase &entry, | |
| 574 | const XattrList &xattrs, | ||
| 575 | const std::string &parent_directory, | ||
| 576 | const FileChunkList &file_chunks) { | ||
| 577 | ✗ | assert(file_chunks.size() > 0); | |
| 578 | |||
| 579 | ✗ | DirectoryEntry full_entry(entry); | |
| 580 | ✗ | full_entry.set_is_chunked_file(true); | |
| 581 | |||
| 582 | ✗ | AddFile(full_entry, xattrs, parent_directory); | |
| 583 | |||
| 584 | ✗ | const string parent_path = MakeRelativePath(parent_directory); | |
| 585 | ✗ | const string file_path = entry.GetFullPath(parent_path); | |
| 586 | |||
| 587 | ✗ | SyncLock(); | |
| 588 | WritableCatalog *catalog; | ||
| 589 | ✗ | if (!FindCatalog(parent_path, &catalog)) { | |
| 590 | ✗ | PANIC(kLogStderr, "catalog for file '%s' cannot be found", | |
| 591 | file_path.c_str()); | ||
| 592 | } | ||
| 593 | |||
| 594 | ✗ | for (unsigned i = 0; i < file_chunks.size(); ++i) { | |
| 595 | ✗ | catalog->AddFileChunk(file_path, *file_chunks.AtPtr(i)); | |
| 596 | } | ||
| 597 | ✗ | SyncUnlock(); | |
| 598 | } | ||
| 599 | |||
| 600 | /** | ||
| 601 | * Reads the entry given by file_path and sets the bundle trigger flag | ||
| 602 | * accordingly. If new_value is true, the regular file at file_path must exist. | ||
| 603 | * If new_value is false, a missing file_path is ignored (otherwise it would | ||
| 604 | * be hard to get rid of dangling bundle trigger markers). | ||
| 605 | */ | ||
| 606 | ✗ | void WritableCatalogManager::UpdateBundleTrigger(const std::string &file_path, | |
| 607 | bool new_value) { | ||
| 608 | ✗ | SyncLock(); | |
| 609 | |||
| 610 | ✗ | WritableCatalog *catalog = NULL; | |
| 611 | ✗ | DirectoryEntry entry; | |
| 612 | ✗ | if (!FindCatalog(file_path, &catalog, &entry)) { | |
| 613 | ✗ | SyncUnlock(); | |
| 614 | |||
| 615 | ✗ | if (new_value) { | |
| 616 | ✗ | PANIC(kLogStderr, "failed to find catalog of %s", file_path.c_str()); | |
| 617 | } | ||
| 618 | |||
| 619 | ✗ | LogCvmfs(kLogCatalog, kLogDebug, "dangling bundle trigger %s", | |
| 620 | file_path.c_str()); | ||
| 621 | ✗ | return; | |
| 622 | } | ||
| 623 | |||
| 624 | ✗ | if (new_value && !entry.IsRegular()) { | |
| 625 | ✗ | SyncUnlock(); | |
| 626 | ✗ | PANIC(kLogStderr, "failed to set bundle trigger on non-regular file %s", | |
| 627 | file_path.c_str()); | ||
| 628 | } | ||
| 629 | |||
| 630 | ✗ | entry.set_is_bundle_trigger(new_value); | |
| 631 | ✗ | catalog->UpdateEntry(entry, file_path); | |
| 632 | |||
| 633 | ✗ | SyncUnlock(); | |
| 634 | } | ||
| 635 | |||
| 636 | |||
| 637 | /** | ||
| 638 | * Add a hardlink group to the catalogs. | ||
| 639 | * @param entries a list of DirectoryEntries describing the new files | ||
| 640 | * @param parent_directory the absolute path of the directory containing the | ||
| 641 | * files to be created | ||
| 642 | * @return true on success, false otherwise | ||
| 643 | */ | ||
| 644 | ✗ | void WritableCatalogManager::AddHardlinkGroup( | |
| 645 | const DirectoryEntryBaseList &entries, | ||
| 646 | const XattrList &xattrs, | ||
| 647 | const std::string &parent_directory, | ||
| 648 | const FileChunkList &file_chunks) { | ||
| 649 | ✗ | assert(entries.size() >= 1); | |
| 650 | ✗ | assert(file_chunks.IsEmpty() || entries[0].IsRegular()); | |
| 651 | ✗ | if (entries.size() == 1) { | |
| 652 | ✗ | DirectoryEntry fix_linkcount(entries[0]); | |
| 653 | ✗ | fix_linkcount.set_linkcount(1); | |
| 654 | ✗ | if (file_chunks.IsEmpty()) | |
| 655 | ✗ | return AddFile(fix_linkcount, xattrs, parent_directory); | |
| 656 | ✗ | return AddChunkedFile(fix_linkcount, xattrs, parent_directory, file_chunks); | |
| 657 | } | ||
| 658 | |||
| 659 | ✗ | LogCvmfs(kLogCatalog, kLogVerboseMsg, "adding hardlink group %s/%s", | |
| 660 | ✗ | parent_directory.c_str(), entries[0].name().c_str()); | |
| 661 | |||
| 662 | // Hardlink groups have to reside in the same directory. | ||
| 663 | // Therefore we only have one parent directory here | ||
| 664 | ✗ | const string parent_path = MakeRelativePath(parent_directory); | |
| 665 | |||
| 666 | // check if hard link is too big | ||
| 667 | ✗ | const unsigned mbytes = entries[0].size() / (1024 * 1024); | |
| 668 | ✗ | if ((file_mbyte_limit_ > 0) && (mbytes > file_mbyte_limit_)) { | |
| 669 | ✗ | LogCvmfs(kLogCatalog, kLogStderr, | |
| 670 | "%s: hard link at %s is larger than %u megabytes (%u). " | ||
| 671 | "CernVM-FS works best with small files. " | ||
| 672 | "Please remove the file or increase the limit.", | ||
| 673 | ✗ | enforce_limits_ ? "FATAL" : "WARNING", | |
| 674 | ✗ | (parent_path + entries[0].name().ToString()).c_str(), | |
| 675 | file_mbyte_limit_, mbytes); | ||
| 676 | ✗ | if (enforce_limits_) | |
| 677 | ✗ | PANIC(kLogStderr, "hard link at %s is larger than %u megabytes (%u)", | |
| 678 | (parent_path + entries[0].name().ToString()).c_str(), | ||
| 679 | file_mbyte_limit_, mbytes); | ||
| 680 | } | ||
| 681 | |||
| 682 | ✗ | SyncLock(); | |
| 683 | WritableCatalog *catalog; | ||
| 684 | ✗ | if (!FindCatalog(parent_path, &catalog)) { | |
| 685 | ✗ | PANIC(kLogStderr, | |
| 686 | "catalog for hardlink group containing '%s' cannot be found", | ||
| 687 | parent_path.c_str()); | ||
| 688 | } | ||
| 689 | |||
| 690 | // Get a valid hardlink group id for the catalog the group will end up in | ||
| 691 | // TODO(unknown): Compaction | ||
| 692 | ✗ | const uint32_t new_group_id = catalog->GetMaxLinkId() + 1; | |
| 693 | ✗ | LogCvmfs(kLogCatalog, kLogVerboseMsg, "hardlink group id %u issued", | |
| 694 | new_group_id); | ||
| 695 | ✗ | assert(new_group_id > 0); | |
| 696 | |||
| 697 | // Add the file entries to the catalog | ||
| 698 | ✗ | for (DirectoryEntryBaseList::const_iterator i = entries.begin(), | |
| 699 | ✗ | iEnd = entries.end(); | |
| 700 | ✗ | i != iEnd; | |
| 701 | ✗ | ++i) { | |
| 702 | ✗ | string file_path = parent_path + "/"; | |
| 703 | ✗ | file_path.append(i->name().GetChars(), i->name().GetLength()); | |
| 704 | |||
| 705 | // create a fully fledged DirectoryEntry to add the hardlink group to it | ||
| 706 | // which is CVMFS specific meta data. | ||
| 707 | ✗ | DirectoryEntry hardlink(*i); | |
| 708 | ✗ | hardlink.set_hardlink_group(new_group_id); | |
| 709 | ✗ | hardlink.set_linkcount(entries.size()); | |
| 710 | ✗ | hardlink.set_is_chunked_file(!file_chunks.IsEmpty()); | |
| 711 | |||
| 712 | ✗ | catalog->AddEntry(hardlink, xattrs, file_path, parent_path); | |
| 713 | ✗ | if (hardlink.IsChunkedFile()) { | |
| 714 | ✗ | for (unsigned i = 0; i < file_chunks.size(); ++i) { | |
| 715 | ✗ | catalog->AddFileChunk(file_path, *file_chunks.AtPtr(i)); | |
| 716 | } | ||
| 717 | } | ||
| 718 | } | ||
| 719 | ✗ | SyncUnlock(); | |
| 720 | } | ||
| 721 | |||
| 722 | |||
| 723 | ✗ | void WritableCatalogManager::ShrinkHardlinkGroup(const string &remove_path) { | |
| 724 | ✗ | const string relative_path = MakeRelativePath(remove_path); | |
| 725 | |||
| 726 | ✗ | SyncLock(); | |
| 727 | WritableCatalog *catalog; | ||
| 728 | ✗ | if (!FindCatalog(relative_path, &catalog)) { | |
| 729 | ✗ | PANIC(kLogStderr, | |
| 730 | "catalog for hardlink group containing '%s' cannot be found", | ||
| 731 | remove_path.c_str()); | ||
| 732 | } | ||
| 733 | |||
| 734 | ✗ | catalog->IncLinkcount(relative_path, -1); | |
| 735 | ✗ | SyncUnlock(); | |
| 736 | } | ||
| 737 | |||
| 738 | |||
| 739 | /** | ||
| 740 | * Update entry meta data (mode, owner, ...). | ||
| 741 | * CVMFS specific meta data (i.e. nested catalog transition points) are NOT | ||
| 742 | * changed by this method, although transition points intrinsics are taken into | ||
| 743 | * account, to keep nested catalogs consistent. | ||
| 744 | * @param entry the directory entry to be touched | ||
| 745 | * @param path the path of the directory entry to be touched | ||
| 746 | */ | ||
| 747 | 6 | void WritableCatalogManager::TouchDirectory(const DirectoryEntryBase &entry, | |
| 748 | const XattrList &xattrs, | ||
| 749 | const std::string &directory_path) { | ||
| 750 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 6 times.
|
6 | assert(entry.IsDirectory()); |
| 751 | |||
| 752 |
1/2✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
|
6 | const string entry_path = MakeRelativePath(directory_path); |
| 753 |
1/2✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
|
6 | const string parent_path = GetParentPath(entry_path); |
| 754 | |||
| 755 | 6 | SyncLock(); | |
| 756 | // find the catalog to be updated | ||
| 757 | WritableCatalog *catalog; | ||
| 758 |
2/4✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 6 times.
|
6 | if (!FindCatalog(parent_path, &catalog)) { |
| 759 | ✗ | PANIC(kLogStderr, "catalog for entry '%s' cannot be found", | |
| 760 | entry_path.c_str()); | ||
| 761 | } | ||
| 762 | |||
| 763 |
1/2✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
|
6 | catalog->TouchEntry(entry, xattrs, entry_path); |
| 764 | |||
| 765 | // since we deal with a directory here, we might just touch a | ||
| 766 | // nested catalog transition point. If this is the case we would need to | ||
| 767 | // update two catalog entries: | ||
| 768 | // * the nested catalog MOUNTPOINT in the parent catalog | ||
| 769 | // * the nested catalog ROOT in the nested catalog | ||
| 770 | |||
| 771 | // first check if we really have a nested catalog transition point | ||
| 772 |
1/2✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
|
6 | catalog::DirectoryEntry potential_transition_point; |
| 773 |
1/2✓ Branch 3 taken 6 times.
✗ Branch 4 not taken.
|
6 | const PathString transition_path(entry_path.data(), entry_path.length()); |
| 774 |
1/2✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
|
6 | bool retval = catalog->LookupPath(transition_path, |
| 775 | &potential_transition_point); | ||
| 776 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 6 times.
|
6 | assert(retval); |
| 777 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 6 times.
|
6 | if (potential_transition_point.IsNestedCatalogMountpoint()) { |
| 778 | ✗ | LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating transition point at %s", | |
| 779 | entry_path.c_str()); | ||
| 780 | |||
| 781 | // find and mount nested catalog associated to this transition point | ||
| 782 | ✗ | shash::Any nested_hash; | |
| 783 | uint64_t nested_size; | ||
| 784 | ✗ | retval = catalog->FindNested(transition_path, &nested_hash, &nested_size); | |
| 785 | ✗ | assert(retval); | |
| 786 | Catalog *nested_catalog; | ||
| 787 | ✗ | nested_catalog = MountCatalog(transition_path, nested_hash, catalog); | |
| 788 | ✗ | assert(nested_catalog != NULL); | |
| 789 | |||
| 790 | // update nested catalog root in the child catalog | ||
| 791 | reinterpret_cast<WritableCatalog *>(nested_catalog) | ||
| 792 | ✗ | ->TouchEntry(entry, xattrs, entry_path); | |
| 793 | } | ||
| 794 | |||
| 795 | 6 | SyncUnlock(); | |
| 796 | 6 | } | |
| 797 | |||
| 798 | |||
| 799 | /** | ||
| 800 | * Create a new nested catalog. Includes moving all entries belonging there | ||
| 801 | * from it's parent catalog. | ||
| 802 | * @param mountpoint the path of the directory to become a nested root | ||
| 803 | * @return true on success, false otherwise | ||
| 804 | */ | ||
| 805 | 493 | void WritableCatalogManager::CreateNestedCatalog( | |
| 806 | const std::string &mountpoint) { | ||
| 807 |
1/2✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
|
493 | const string nested_root_path = MakeRelativePath(mountpoint); |
| 808 |
1/2✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
|
493 | const PathString ps_nested_root_path(nested_root_path); |
| 809 | |||
| 810 | 493 | SyncLock(); | |
| 811 | // Find the catalog currently containing the directory structure, which | ||
| 812 | // will be represented as a new nested catalog and its root-entry/mountpoint | ||
| 813 | // along the way | ||
| 814 | 493 | WritableCatalog *old_catalog = NULL; | |
| 815 |
1/2✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
|
493 | DirectoryEntry new_root_entry; |
| 816 |
2/4✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 493 times.
|
493 | if (!FindCatalog(nested_root_path, &old_catalog, &new_root_entry)) { |
| 817 | ✗ | PANIC(kLogStderr, | |
| 818 | "failed to create nested catalog '%s': " | ||
| 819 | "mountpoint was not found in current catalog structure", | ||
| 820 | nested_root_path.c_str()); | ||
| 821 | } | ||
| 822 | |||
| 823 | // Create the database schema and the initial root entry | ||
| 824 | // for the new nested catalog | ||
| 825 |
1/2✓ Branch 2 taken 493 times.
✗ Branch 3 not taken.
|
493 | const string database_file_path = CreateTempPath(dir_temp() + "/catalog", |
| 826 |
1/2✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
|
493 | 0666); |
| 827 | 493 | const bool volatile_content = false; | |
| 828 |
1/2✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
|
493 | CatalogDatabase *new_catalog_db = CatalogDatabase::Create(database_file_path); |
| 829 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 493 times.
|
493 | assert(NULL != new_catalog_db); |
| 830 | // Note we do not set the external_data bit for nested catalogs | ||
| 831 |
2/4✓ Branch 2 taken 493 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 493 times.
✗ Branch 6 not taken.
|
493 | bool retval = new_catalog_db->InsertInitialValues( |
| 832 | nested_root_path, | ||
| 833 | volatile_content, | ||
| 834 | "", // At this point, only root | ||
| 835 | // catalog gets VOMS authz | ||
| 836 | new_root_entry); | ||
| 837 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 493 times.
|
493 | assert(retval); |
| 838 | // TODO(rmeusel): we need a way to attach a catalog directly from an open | ||
| 839 | // database to remove this indirection | ||
| 840 |
1/2✓ Branch 0 taken 493 times.
✗ Branch 1 not taken.
|
493 | delete new_catalog_db; |
| 841 | 493 | new_catalog_db = NULL; | |
| 842 | |||
| 843 | // Attach the just created nested catalog | ||
| 844 |
2/4✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 493 times.
✗ Branch 5 not taken.
|
493 | Catalog *new_catalog = CreateCatalog(ps_nested_root_path, shash::Any(), |
| 845 | old_catalog); | ||
| 846 |
1/2✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
|
493 | retval = AttachCatalog(database_file_path, new_catalog); |
| 847 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 493 times.
|
493 | assert(retval); |
| 848 | |||
| 849 |
2/4✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 493 times.
|
493 | assert(new_catalog->IsWritable()); |
| 850 | 493 | WritableCatalog *wr_new_catalog = static_cast<WritableCatalog *>(new_catalog); | |
| 851 | |||
| 852 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 493 times.
|
493 | if (new_root_entry.HasXattrs()) { |
| 853 | ✗ | XattrList xattrs; | |
| 854 | ✗ | retval = old_catalog->LookupXattrsPath(ps_nested_root_path, &xattrs); | |
| 855 | ✗ | assert(retval); | |
| 856 | ✗ | wr_new_catalog->TouchEntry(new_root_entry, xattrs, nested_root_path); | |
| 857 | } | ||
| 858 | |||
| 859 | // From now on, there are two catalogs, spanning the same directory structure | ||
| 860 | // we have to split the overlapping directory entries from the old catalog | ||
| 861 | // to the new catalog to re-gain a valid catalog structure | ||
| 862 |
1/2✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
|
493 | old_catalog->Partition(wr_new_catalog); |
| 863 | |||
| 864 | // Add the newly created nested catalog to the references of the containing | ||
| 865 | // catalog | ||
| 866 |
4/8✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 493 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 493 times.
✗ Branch 8 not taken.
✓ Branch 10 taken 493 times.
✗ Branch 11 not taken.
|
493 | old_catalog->InsertNestedCatalog(new_catalog->mountpoint().ToString(), NULL, |
| 867 | 493 | shash::Any(spooler_->GetHashAlgorithm()), 0); | |
| 868 | |||
| 869 | // Fix subtree counters in new nested catalogs: subtree is the sum of all | ||
| 870 | // entries of all "grand-nested" catalogs | ||
| 871 | // Note: taking a copy of the nested catalog list here | ||
| 872 | const Catalog::NestedCatalogList | ||
| 873 |
1/2✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
|
493 | &grand_nested = wr_new_catalog->ListOwnNestedCatalogs(); |
| 874 |
1/2✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
|
493 | DeltaCounters fix_subtree_counters; |
| 875 | 986 | for (Catalog::NestedCatalogList::const_iterator i = grand_nested.begin(), | |
| 876 | 493 | iEnd = grand_nested.end(); | |
| 877 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 493 times.
|
493 | i != iEnd; |
| 878 | ✗ | ++i) { | |
| 879 | WritableCatalog *grand_catalog; | ||
| 880 | ✗ | retval = FindCatalog(i->mountpoint.ToString(), &grand_catalog); | |
| 881 | ✗ | assert(retval); | |
| 882 | ✗ | const Counters &grand_counters = grand_catalog->GetCounters(); | |
| 883 | ✗ | grand_counters.AddAsSubtree(&fix_subtree_counters); | |
| 884 | } | ||
| 885 | 493 | const DeltaCounters save_counters = wr_new_catalog->delta_counters_; | |
| 886 | 493 | wr_new_catalog->delta_counters_ = fix_subtree_counters; | |
| 887 |
1/2✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
|
493 | wr_new_catalog->UpdateCounters(); |
| 888 | 493 | wr_new_catalog->delta_counters_ = save_counters; | |
| 889 | |||
| 890 | 493 | SyncUnlock(); | |
| 891 | 493 | } | |
| 892 | |||
| 893 | |||
| 894 | /** | ||
| 895 | * Remove a nested catalog | ||
| 896 | * | ||
| 897 | * If the merged parameter is true, when you remove a nested catalog | ||
| 898 | * all entries currently held by it will be merged into its parent | ||
| 899 | * catalog. | ||
| 900 | * @param mountpoint - the path of the nested catalog to be removed | ||
| 901 | * @param merge - merge the subtree associated with the nested catalog | ||
| 902 | * into its parent catalog | ||
| 903 | * @return - true on success, false otherwise | ||
| 904 | */ | ||
| 905 | 32 | void WritableCatalogManager::RemoveNestedCatalog(const string &mountpoint, | |
| 906 | const bool merge) { | ||
| 907 |
1/2✓ Branch 1 taken 32 times.
✗ Branch 2 not taken.
|
32 | const string nested_root_path = MakeRelativePath(mountpoint); |
| 908 | |||
| 909 | 32 | SyncLock(); | |
| 910 | // Find the catalog which should be removed | ||
| 911 | 32 | WritableCatalog *nested_catalog = NULL; | |
| 912 |
2/4✓ Branch 1 taken 32 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 32 times.
|
32 | if (!FindCatalog(nested_root_path, &nested_catalog)) { |
| 913 | ✗ | PANIC(kLogStderr, | |
| 914 | "failed to remove nested catalog '%s': " | ||
| 915 | "mountpoint was not found in current catalog structure", | ||
| 916 | nested_root_path.c_str()); | ||
| 917 | } | ||
| 918 | |||
| 919 | // Check if the found catalog is really the nested catalog to be deleted | ||
| 920 |
6/19✓ Branch 1 taken 32 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 32 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 32 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 32 times.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 32 times.
✗ Branch 14 not taken.
✓ Branch 16 taken 32 times.
✗ Branch 17 not taken.
✗ Branch 19 not taken.
✗ Branch 20 not taken.
✗ Branch 22 not taken.
✗ Branch 23 not taken.
|
64 | assert(!nested_catalog->IsRoot() |
| 921 | && (nested_catalog->mountpoint().ToString() == nested_root_path)); | ||
| 922 | |||
| 923 |
1/2✓ Branch 0 taken 32 times.
✗ Branch 1 not taken.
|
32 | if (merge) { |
| 924 | // Merge all data from the nested catalog into it's parent | ||
| 925 |
1/2✓ Branch 1 taken 32 times.
✗ Branch 2 not taken.
|
32 | nested_catalog->MergeIntoParent(); |
| 926 | } else { | ||
| 927 | ✗ | nested_catalog->RemoveFromParent(); | |
| 928 | } | ||
| 929 | |||
| 930 | // Delete the catalog database file from the working copy | ||
| 931 |
2/6✓ Branch 1 taken 32 times.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 32 times.
|
32 | if (unlink(nested_catalog->database_path().c_str()) != 0) { |
| 932 | ✗ | PANIC(kLogStderr, | |
| 933 | "unable to delete the removed nested catalog database file '%s'", | ||
| 934 | nested_catalog->database_path().c_str()); | ||
| 935 | } | ||
| 936 | |||
| 937 | // Remove the catalog from internal data structures. | ||
| 938 | // On merge the children have been re-parented into this catalog's parent by | ||
| 939 | // MergeIntoParent (CopyCatalogsToParent), so only this catalog is detached. | ||
| 940 | // On removal (fast delete) the whole subtree is discarded together with the | ||
| 941 | // catalog, so any attached children must be detached as well - otherwise they | ||
| 942 | // would be left in the manager with a dangling parent pointer. | ||
| 943 |
1/2✓ Branch 0 taken 32 times.
✗ Branch 1 not taken.
|
32 | if (merge) { |
| 944 |
1/2✓ Branch 1 taken 32 times.
✗ Branch 2 not taken.
|
32 | DetachCatalog(nested_catalog); |
| 945 | } else { | ||
| 946 | ✗ | DetachSubtree(nested_catalog); | |
| 947 | } | ||
| 948 | 32 | SyncUnlock(); | |
| 949 | 32 | } | |
| 950 | |||
| 951 | |||
| 952 | /** | ||
| 953 | * Swap in a new nested catalog | ||
| 954 | * | ||
| 955 | * The old nested catalog must not have been already attached to the | ||
| 956 | * catalog tree. This method will not attach the new nested catalog | ||
| 957 | * to the catalog tree. | ||
| 958 | * | ||
| 959 | * @param mountpoint - the path of the nested catalog to be removed | ||
| 960 | * @param new_hash - the hash of the new nested catalog | ||
| 961 | * @param new_size - the size of the new nested catalog | ||
| 962 | */ | ||
| 963 | 87 | void WritableCatalogManager::SwapNestedCatalog(const string &mountpoint, | |
| 964 | const shash::Any &new_hash, | ||
| 965 | const uint64_t new_size) { | ||
| 966 |
1/2✓ Branch 1 taken 87 times.
✗ Branch 2 not taken.
|
87 | const string nested_root_path = MakeRelativePath(mountpoint); |
| 967 |
1/2✓ Branch 1 taken 87 times.
✗ Branch 2 not taken.
|
87 | const string parent_path = GetParentPath(nested_root_path); |
| 968 |
1/2✓ Branch 1 taken 87 times.
✗ Branch 2 not taken.
|
87 | const PathString nested_root_ps = PathString(nested_root_path); |
| 969 | |||
| 970 | 87 | SyncLock(); | |
| 971 | |||
| 972 | // Find the immediate parent catalog | ||
| 973 | 87 | WritableCatalog *parent = NULL; | |
| 974 |
3/4✓ Branch 1 taken 87 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 12 times.
✓ Branch 4 taken 75 times.
|
87 | if (!FindCatalog(parent_path, &parent)) { |
| 975 | 12 | SyncUnlock(); // this is needed for the unittest. otherwise they get stuck | |
| 976 | 12 | PANIC(kLogStderr, | |
| 977 | "failed to swap nested catalog '%s': could not find parent '%s'", | ||
| 978 | nested_root_path.c_str(), parent_path.c_str()); | ||
| 979 | } | ||
| 980 | |||
| 981 | // Get old nested catalog counters | ||
| 982 |
1/2✓ Branch 1 taken 75 times.
✗ Branch 2 not taken.
|
75 | Catalog *old_attached_catalog = parent->FindChild(nested_root_ps); |
| 983 |
1/2✓ Branch 1 taken 75 times.
✗ Branch 2 not taken.
|
75 | Counters old_counters; |
| 984 |
2/2✓ Branch 0 taken 12 times.
✓ Branch 1 taken 63 times.
|
75 | if (old_attached_catalog) { |
| 985 | // Old catalog was already attached (e.g. as a child catalog | ||
| 986 | // attached by a prior call to CreateNestedCatalog()). Ensure | ||
| 987 | // that it has not been modified, get counters, and detach it. | ||
| 988 | 12 | WritableCatalogList list; | |
| 989 |
2/4✓ Branch 1 taken 12 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 12 times.
✗ Branch 4 not taken.
|
12 | if (GetModifiedCatalogLeafsRecursively(old_attached_catalog, &list)) { |
| 990 | 12 | SyncUnlock(); | |
| 991 | 12 | PANIC(kLogStderr, | |
| 992 | "failed to swap nested catalog '%s': already modified", | ||
| 993 | nested_root_path.c_str()); | ||
| 994 | } | ||
| 995 | ✗ | old_counters = old_attached_catalog->GetCounters(); | |
| 996 | ✗ | DetachSubtree(old_attached_catalog); | |
| 997 | |||
| 998 | 12 | } else { | |
| 999 | // Old catalog was not attached. Download a freely attached | ||
| 1000 | // version and get counters. | ||
| 1001 |
1/2✓ Branch 1 taken 63 times.
✗ Branch 2 not taken.
|
63 | shash::Any old_hash; |
| 1002 | uint64_t old_size; | ||
| 1003 |
1/2✓ Branch 1 taken 63 times.
✗ Branch 2 not taken.
|
63 | const bool old_found = parent->FindNested(nested_root_ps, &old_hash, |
| 1004 | &old_size); | ||
| 1005 |
2/2✓ Branch 0 taken 12 times.
✓ Branch 1 taken 51 times.
|
63 | if (!old_found) { |
| 1006 | 12 | SyncUnlock(); | |
| 1007 | 12 | PANIC(kLogStderr, | |
| 1008 | "failed to swap nested catalog '%s': not found in parent", | ||
| 1009 | nested_root_path.c_str()); | ||
| 1010 | } | ||
| 1011 | const std::unique_ptr<Catalog> old_free_catalog( | ||
| 1012 |
1/2✓ Branch 1 taken 51 times.
✗ Branch 2 not taken.
|
51 | LoadFreeCatalog(nested_root_ps, old_hash)); |
| 1013 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 51 times.
|
51 | if (old_free_catalog.get() == nullptr) { |
| 1014 | ✗ | SyncUnlock(); | |
| 1015 | ✗ | PANIC(kLogStderr, | |
| 1016 | "failed to swap nested catalog '%s': failed to load old catalog", | ||
| 1017 | nested_root_path.c_str()); | ||
| 1018 | } | ||
| 1019 | 51 | old_counters = old_free_catalog->GetCounters(); | |
| 1020 | 51 | } | |
| 1021 | |||
| 1022 | // Load freely attached new catalog | ||
| 1023 | const std::unique_ptr<Catalog> new_catalog( | ||
| 1024 |
2/2✓ Branch 1 taken 39 times.
✓ Branch 2 taken 12 times.
|
51 | LoadFreeCatalog(nested_root_ps, new_hash)); |
| 1025 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 39 times.
|
39 | if (new_catalog.get() == nullptr) { |
| 1026 | ✗ | SyncUnlock(); | |
| 1027 | ✗ | PANIC(kLogStderr, | |
| 1028 | "failed to swap nested catalog '%s': failed to load new catalog", | ||
| 1029 | nested_root_path.c_str()); | ||
| 1030 | } | ||
| 1031 | |||
| 1032 | // Get new catalog root directory entry | ||
| 1033 |
1/2✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
|
39 | DirectoryEntry dirent; |
| 1034 | 39 | XattrList xattrs; | |
| 1035 |
1/2✓ Branch 2 taken 39 times.
✗ Branch 3 not taken.
|
39 | const bool dirent_found = new_catalog->LookupPath(nested_root_ps, &dirent); |
| 1036 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 39 times.
|
39 | if (!dirent_found) { |
| 1037 | ✗ | SyncUnlock(); | |
| 1038 | ✗ | PANIC(kLogStderr, | |
| 1039 | "failed to swap nested catalog '%s': missing dirent in new catalog", | ||
| 1040 | nested_root_path.c_str()); | ||
| 1041 | } | ||
| 1042 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 39 times.
|
39 | if (dirent.HasXattrs()) { |
| 1043 | ✗ | const bool xattrs_found = new_catalog->LookupXattrsPath(nested_root_ps, | |
| 1044 | &xattrs); | ||
| 1045 | ✗ | if (!xattrs_found) { | |
| 1046 | ✗ | SyncUnlock(); | |
| 1047 | ✗ | PANIC(kLogStderr, | |
| 1048 | "failed to swap nested catalog '%s': missing xattrs in new catalog", | ||
| 1049 | nested_root_path.c_str()); | ||
| 1050 | } | ||
| 1051 | } | ||
| 1052 | |||
| 1053 | // Swap catalogs | ||
| 1054 |
1/2✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
|
39 | parent->RemoveNestedCatalog(nested_root_path, NULL); |
| 1055 |
1/2✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
|
39 | parent->InsertNestedCatalog(nested_root_path, NULL, new_hash, new_size); |
| 1056 | |||
| 1057 | // Update parent directory entry | ||
| 1058 | 39 | dirent.set_is_nested_catalog_mountpoint(true); | |
| 1059 | 39 | dirent.set_is_nested_catalog_root(false); | |
| 1060 |
1/2✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
|
39 | parent->UpdateEntry(dirent, nested_root_path); |
| 1061 |
1/2✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
|
39 | parent->TouchEntry(dirent, xattrs, nested_root_path); |
| 1062 | |||
| 1063 | // Update counters | ||
| 1064 |
1/2✓ Branch 3 taken 39 times.
✗ Branch 4 not taken.
|
39 | const DeltaCounters delta = Counters::Diff(old_counters, |
| 1065 | new_catalog->GetCounters()); | ||
| 1066 |
1/2✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
|
39 | delta.PopulateToParent(&parent->delta_counters_); |
| 1067 | |||
| 1068 | 39 | SyncUnlock(); | |
| 1069 | 183 | } | |
| 1070 | |||
| 1071 | /** | ||
| 1072 | * Install a nested catalog (catalog hierarchy) at a new, empty directory | ||
| 1073 | * | ||
| 1074 | * The mountpoint directory must not yet exist. Its parent directory, however | ||
| 1075 | * must exist. This method combines functionality from AddDirectory(), | ||
| 1076 | * CreateNestedCatalog() and SwapNestedCatalog(). | ||
| 1077 | * The new nested catalog won't get attached. | ||
| 1078 | * | ||
| 1079 | * @param mountpoint - the path where the nested catalog should be installed | ||
| 1080 | * @param new_hash - the hash of the new nested catalog | ||
| 1081 | * @param new_size - the size of the new nested catalog | ||
| 1082 | */ | ||
| 1083 | 39 | void WritableCatalogManager::GraftNestedCatalog(const string &mountpoint, | |
| 1084 | const shash::Any &new_hash, | ||
| 1085 | const uint64_t new_size) { | ||
| 1086 |
2/2✓ Branch 1 taken 26 times.
✓ Branch 2 taken 13 times.
|
39 | if (!TryGraftNestedCatalog(mountpoint, new_hash, new_size)) { |
| 1087 | 26 | PANIC(kLogStderr, "failed to graft nested catalog '%s'", | |
| 1088 | mountpoint.c_str()); | ||
| 1089 | } | ||
| 1090 | 13 | } | |
| 1091 | |||
| 1092 | 39 | bool WritableCatalogManager::TryGraftNestedCatalog(const string &mountpoint, | |
| 1093 | const shash::Any &new_hash, | ||
| 1094 | const uint64_t new_size) { | ||
| 1095 |
1/2✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
|
39 | const string nested_root_path = MakeRelativePath(mountpoint); |
| 1096 |
1/2✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
|
39 | const string parent_path = GetParentPath(nested_root_path); |
| 1097 |
1/2✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
|
39 | const PathString nested_root_ps = PathString(nested_root_path); |
| 1098 | |||
| 1099 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 39 times.
|
39 | if (nested_root_path.empty()) { |
| 1100 | ✗ | LogCvmfs(kLogCatalog, kLogStderr, | |
| 1101 | "failed to graft nested catalog: empty mountpoint"); | ||
| 1102 | ✗ | return false; | |
| 1103 | } | ||
| 1104 | |||
| 1105 | // Load freely attached new catalog | ||
| 1106 | const std::unique_ptr<Catalog> new_catalog( | ||
| 1107 |
1/2✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
|
39 | LoadFreeCatalog(nested_root_ps, new_hash)); |
| 1108 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 39 times.
|
39 | if (new_catalog.get() == nullptr) { |
| 1109 | ✗ | LogCvmfs(kLogCatalog, kLogStderr, | |
| 1110 | "failed to graft nested catalog '%s': failed to load new catalog", | ||
| 1111 | nested_root_path.c_str()); | ||
| 1112 | ✗ | return false; | |
| 1113 | } | ||
| 1114 |
4/7✓ Branch 2 taken 39 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 39 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 13 times.
✓ Branch 9 taken 26 times.
|
39 | if (new_catalog->root_prefix() != nested_root_ps) { |
| 1115 |
1/7✗ Branch 2 not taken.
✓ Branch 3 taken 13 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
|
13 | LogCvmfs(kLogCatalog, kLogStderr, |
| 1116 | "invalid nested catalog for grafting at '%s': catalog rooted at " | ||
| 1117 | "'%s'", | ||
| 1118 | nested_root_path.c_str(), | ||
| 1119 |
2/4✓ Branch 2 taken 13 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 13 times.
✗ Branch 6 not taken.
|
26 | new_catalog->root_prefix().ToString().c_str()); |
| 1120 | 13 | return false; | |
| 1121 | } | ||
| 1122 | |||
| 1123 | // Get new catalog root directory entry | ||
| 1124 |
1/2✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
|
26 | DirectoryEntry dirent; |
| 1125 | 26 | XattrList xattrs; | |
| 1126 |
1/2✓ Branch 2 taken 26 times.
✗ Branch 3 not taken.
|
26 | const bool dirent_found = new_catalog->LookupPath(nested_root_ps, &dirent); |
| 1127 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 26 times.
|
26 | if (!dirent_found) { |
| 1128 | ✗ | LogCvmfs(kLogCatalog, kLogStderr, | |
| 1129 | "failed to graft nested catalog '%s': missing dirent in new " | ||
| 1130 | "catalog", | ||
| 1131 | nested_root_path.c_str()); | ||
| 1132 | ✗ | return false; | |
| 1133 | } | ||
| 1134 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 26 times.
|
26 | if (!dirent.IsDirectory()) { |
| 1135 | ✗ | LogCvmfs(kLogCatalog, kLogStderr, | |
| 1136 | "failed to graft nested catalog '%s': root entry is not a " | ||
| 1137 | "directory", | ||
| 1138 | nested_root_path.c_str()); | ||
| 1139 | ✗ | return false; | |
| 1140 | } | ||
| 1141 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 26 times.
|
26 | if (dirent.HasXattrs()) { |
| 1142 | ✗ | const bool xattrs_found = new_catalog->LookupXattrsPath(nested_root_ps, | |
| 1143 | &xattrs); | ||
| 1144 | ✗ | if (!xattrs_found) { | |
| 1145 | ✗ | LogCvmfs(kLogCatalog, kLogStderr, | |
| 1146 | "failed to graft nested catalog '%s': missing xattrs in new " | ||
| 1147 | "catalog", | ||
| 1148 | nested_root_path.c_str()); | ||
| 1149 | ✗ | return false; | |
| 1150 | } | ||
| 1151 | } | ||
| 1152 | // Transform the nested catalog root into a transition point to be inserted | ||
| 1153 | // in the parent catalog | ||
| 1154 | 26 | dirent.set_is_nested_catalog_root(false); | |
| 1155 | 26 | dirent.set_is_nested_catalog_mountpoint(true); | |
| 1156 | |||
| 1157 | // Add directory and nested catalog | ||
| 1158 | |||
| 1159 | 26 | SyncLock(); | |
| 1160 | WritableCatalog *parent_catalog; | ||
| 1161 |
1/2✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
|
26 | DirectoryEntry parent_entry; |
| 1162 |
2/4✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 26 times.
|
26 | if (!FindCatalog(parent_path, &parent_catalog, &parent_entry)) { |
| 1163 | ✗ | SyncUnlock(); | |
| 1164 | ✗ | LogCvmfs(kLogCatalog, kLogStderr, | |
| 1165 | "catalog for directory '%s' cannot be found", parent_path.c_str()); | ||
| 1166 | ✗ | return false; | |
| 1167 | } | ||
| 1168 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 26 times.
|
26 | if (!parent_entry.IsDirectory()) { |
| 1169 | ✗ | SyncUnlock(); | |
| 1170 | ✗ | LogCvmfs(kLogCatalog, kLogStderr, "parent path '%s' is not a directory", | |
| 1171 | parent_path.c_str()); | ||
| 1172 | ✗ | return false; | |
| 1173 | } | ||
| 1174 |
3/4✓ Branch 1 taken 26 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 13 times.
✓ Branch 4 taken 13 times.
|
26 | if (parent_catalog->LookupPath(nested_root_ps, NULL)) { |
| 1175 | 13 | SyncUnlock(); | |
| 1176 |
1/2✓ Branch 2 taken 13 times.
✗ Branch 3 not taken.
|
13 | LogCvmfs(kLogCatalog, kLogStderr, |
| 1177 | "invalid attempt to graft nested catalog into existing directory " | ||
| 1178 | "'%s'", | ||
| 1179 | nested_root_path.c_str()); | ||
| 1180 | 13 | return false; | |
| 1181 | } | ||
| 1182 |
1/2✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
|
13 | parent_catalog->AddEntry(dirent, xattrs, nested_root_path, parent_path); |
| 1183 | 13 | parent_entry.set_linkcount(parent_entry.linkcount() + 1); | |
| 1184 |
1/2✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
|
13 | parent_catalog->UpdateEntry(parent_entry, parent_path); |
| 1185 |
1/2✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
|
13 | if (parent_entry.IsNestedCatalogRoot()) { |
| 1186 | WritableCatalog *grand_parent_catalog = reinterpret_cast<WritableCatalog *>( | ||
| 1187 | 13 | parent_catalog->parent()); | |
| 1188 | 13 | parent_entry.set_is_nested_catalog_root(false); | |
| 1189 | 13 | parent_entry.set_is_nested_catalog_mountpoint(true); | |
| 1190 |
1/2✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
|
13 | grand_parent_catalog->UpdateEntry(parent_entry, parent_path); |
| 1191 | } | ||
| 1192 | |||
| 1193 |
1/2✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
|
13 | parent_catalog->InsertNestedCatalog(nested_root_path, NULL, new_hash, |
| 1194 | new_size); | ||
| 1195 | |||
| 1196 | // Fix-up counters | ||
| 1197 |
1/2✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
|
13 | const Counters counters; |
| 1198 |
1/2✓ Branch 3 taken 13 times.
✗ Branch 4 not taken.
|
13 | const DeltaCounters delta = Counters::Diff(counters, |
| 1199 | new_catalog->GetCounters()); | ||
| 1200 |
1/2✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
|
13 | delta.PopulateToParent(&parent_catalog->delta_counters_); |
| 1201 | |||
| 1202 | 13 | SyncUnlock(); | |
| 1203 | 13 | return true; | |
| 1204 | 39 | } | |
| 1205 | |||
| 1206 | /** | ||
| 1207 | * Checks if a nested catalog starts at this path. The path must be valid. | ||
| 1208 | */ | ||
| 1209 | ✗ | bool WritableCatalogManager::IsTransitionPoint(const string &mountpoint) { | |
| 1210 | ✗ | const string path = MakeRelativePath(mountpoint); | |
| 1211 | |||
| 1212 | ✗ | SyncLock(); | |
| 1213 | WritableCatalog *catalog; | ||
| 1214 | ✗ | DirectoryEntry entry; | |
| 1215 | ✗ | if (!FindCatalog(path, &catalog, &entry)) { | |
| 1216 | ✗ | PANIC(kLogStderr, "catalog for directory '%s' cannot be found", | |
| 1217 | path.c_str()); | ||
| 1218 | } | ||
| 1219 | ✗ | const bool result = entry.IsNestedCatalogRoot(); | |
| 1220 | ✗ | SyncUnlock(); | |
| 1221 | ✗ | return result; | |
| 1222 | } | ||
| 1223 | |||
| 1224 | |||
| 1225 | ✗ | void WritableCatalogManager::PrecalculateListings() { | |
| 1226 | // TODO(jblomer): meant for micro catalogs | ||
| 1227 | } | ||
| 1228 | |||
| 1229 | |||
| 1230 | ✗ | void WritableCatalogManager::SetTTL(const uint64_t new_ttl) { | |
| 1231 | ✗ | SyncLock(); | |
| 1232 | ✗ | reinterpret_cast<WritableCatalog *>(GetRootCatalog())->SetTTL(new_ttl); | |
| 1233 | ✗ | SyncUnlock(); | |
| 1234 | } | ||
| 1235 | |||
| 1236 | |||
| 1237 | ✗ | bool WritableCatalogManager::SetVOMSAuthz(const std::string &voms_authz) { | |
| 1238 | bool result; | ||
| 1239 | ✗ | SyncLock(); | |
| 1240 | ✗ | result = reinterpret_cast<WritableCatalog *>(GetRootCatalog()) | |
| 1241 | ✗ | ->SetVOMSAuthz(voms_authz); | |
| 1242 | ✗ | SyncUnlock(); | |
| 1243 | ✗ | return result; | |
| 1244 | } | ||
| 1245 | |||
| 1246 | |||
| 1247 | 326 | bool WritableCatalogManager::Commit(const bool stop_for_tweaks, | |
| 1248 | const uint64_t manual_revision, | ||
| 1249 | manifest::Manifest *manifest) { | ||
| 1250 | WritableCatalog *root_catalog = reinterpret_cast<WritableCatalog *>( | ||
| 1251 | 326 | GetRootCatalog()); | |
| 1252 |
1/2✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
|
326 | root_catalog->SetDirty(); |
| 1253 | |||
| 1254 | // set root catalog revision to manually provided number if available | ||
| 1255 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 326 times.
|
326 | if (manual_revision > 0) { |
| 1256 | ✗ | const uint64_t revision = root_catalog->GetRevision(); | |
| 1257 | ✗ | if (revision >= manual_revision) { | |
| 1258 | ✗ | LogCvmfs(kLogCatalog, kLogStderr, | |
| 1259 | "Manual revision (%" PRIu64 ") must not be " | ||
| 1260 | "smaller than the current root catalog's (%" PRIu64 | ||
| 1261 | "). Skipped!", | ||
| 1262 | manual_revision, revision); | ||
| 1263 | } else { | ||
| 1264 | // Gets incremented by FinalizeCatalog() afterwards! | ||
| 1265 | ✗ | root_catalog->SetRevision(manual_revision - 1); | |
| 1266 | } | ||
| 1267 | } | ||
| 1268 | |||
| 1269 | // do the actual catalog snapshotting and upload | ||
| 1270 |
1/2✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
|
326 | CatalogInfo root_catalog_info; |
| 1271 |
1/2✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
|
326 | if (getenv("_CVMFS_SERIALIZED_CATALOG_PROCESSING_") == NULL) |
| 1272 |
1/2✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
|
326 | root_catalog_info = SnapshotCatalogs(stop_for_tweaks); |
| 1273 | else | ||
| 1274 | ✗ | root_catalog_info = SnapshotCatalogsSerialized(stop_for_tweaks); | |
| 1275 |
2/4✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 326 times.
|
326 | if (spooler_->GetNumberOfErrors() > 0) { |
| 1276 | ✗ | LogCvmfs(kLogCatalog, kLogStderr, "failed to commit catalogs"); | |
| 1277 | ✗ | return false; | |
| 1278 | } | ||
| 1279 | |||
| 1280 | // .cvmfspublished export | ||
| 1281 |
1/2✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
|
326 | LogCvmfs(kLogCatalog, kLogVerboseMsg, "Committing repository manifest"); |
| 1282 | 326 | set_base_hash(root_catalog_info.content_hash); | |
| 1283 | |||
| 1284 | 326 | manifest->set_catalog_hash(root_catalog_info.content_hash); | |
| 1285 | 326 | manifest->set_catalog_size(root_catalog_info.size); | |
| 1286 |
2/4✓ Branch 2 taken 326 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 326 times.
✗ Branch 6 not taken.
|
326 | manifest->set_root_path(""); |
| 1287 | 326 | manifest->set_ttl(root_catalog_info.ttl); | |
| 1288 | 326 | manifest->set_revision(root_catalog_info.revision); | |
| 1289 | |||
| 1290 | 326 | return true; | |
| 1291 | } | ||
| 1292 | |||
| 1293 | |||
| 1294 | /** | ||
| 1295 | * Handles the snapshotting of dirty (i.e. modified) catalogs while trying to | ||
| 1296 | * parallelize the compression and upload as much as possible. We use a parallel | ||
| 1297 | * depth first post order tree traversal based on 'continuations'. | ||
| 1298 | * | ||
| 1299 | * The idea is as follows: | ||
| 1300 | * 1. find all leaf-catalogs (i.e. dirty catalogs with no dirty children) | ||
| 1301 | * --> these can be processed and uploaded immediately and independently | ||
| 1302 | * see WritableCatalogManager::GetModifiedCatalogLeafs() | ||
| 1303 | * 2. annotate non-leaf catalogs with their number of dirty children | ||
| 1304 | * --> a finished child will notify it's parent and decrement this number | ||
| 1305 | * see WritableCatalogManager::CatalogUploadCallback() | ||
| 1306 | * 3. if a non-leaf catalog's dirty children number reaches 0, it is scheduled | ||
| 1307 | * for processing as well (continuation) | ||
| 1308 | * --> the parallel processing walks bottom-up through the catalog tree | ||
| 1309 | * see WritableCatalogManager::CatalogUploadCallback() | ||
| 1310 | * 4. when the root catalog is reached, we notify the main thread and return | ||
| 1311 | * --> done through a Future<> in WritableCatalogManager::SnapshotCatalogs | ||
| 1312 | * | ||
| 1313 | * Note: The catalog finalisation (see WritableCatalogManager::FinalizeCatalog) | ||
| 1314 | * happens in a worker thread (i.e. the callback method) for non-leaf | ||
| 1315 | * catalogs. | ||
| 1316 | * | ||
| 1317 | * TODO(rmeusel): since all leaf catalogs are finalized in the main thread, we | ||
| 1318 | * sacrifice some potential concurrency for simplicity. | ||
| 1319 | */ | ||
| 1320 | 326 | WritableCatalogManager::CatalogInfo WritableCatalogManager::SnapshotCatalogs( | |
| 1321 | const bool stop_for_tweaks) { | ||
| 1322 | // prepare environment for parallel processing | ||
| 1323 |
1/2✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
|
326 | Future<CatalogInfo> root_catalog_info_future; |
| 1324 | CatalogUploadContext upload_context; | ||
| 1325 | 326 | upload_context.root_catalog_info = &root_catalog_info_future; | |
| 1326 | 326 | upload_context.stop_for_tweaks = stop_for_tweaks; | |
| 1327 | |||
| 1328 |
1/2✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
|
326 | spooler_->RegisterListener(&WritableCatalogManager::CatalogUploadCallback, |
| 1329 | this, upload_context); | ||
| 1330 | |||
| 1331 | // find dirty leaf catalogs and annotate non-leaf catalogs (dirty child count) | ||
| 1332 | // post-condition: the entire catalog tree is ready for concurrent processing | ||
| 1333 | 326 | WritableCatalogList leafs_to_snapshot; | |
| 1334 |
1/2✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
|
326 | GetModifiedCatalogLeafs(&leafs_to_snapshot); |
| 1335 | |||
| 1336 | // finalize and schedule the catalog processing | ||
| 1337 | 326 | WritableCatalogList::const_iterator i = leafs_to_snapshot.begin(); | |
| 1338 | 326 | const WritableCatalogList::const_iterator iend = leafs_to_snapshot.end(); | |
| 1339 |
2/2✓ Branch 2 taken 488 times.
✓ Branch 3 taken 326 times.
|
814 | for (; i != iend; ++i) { |
| 1340 |
1/2✓ Branch 2 taken 488 times.
✗ Branch 3 not taken.
|
488 | FinalizeCatalog(*i, stop_for_tweaks); |
| 1341 |
1/2✓ Branch 2 taken 488 times.
✗ Branch 3 not taken.
|
488 | ScheduleCatalogProcessing(*i); |
| 1342 | } | ||
| 1343 | |||
| 1344 |
1/2✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
|
326 | LogCvmfs(kLogCatalog, kLogVerboseMsg, "waiting for upload of catalogs"); |
| 1345 |
1/2✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
|
326 | const CatalogInfo &root_catalog_info = root_catalog_info_future.Get(); |
| 1346 |
1/2✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
|
326 | spooler_->WaitForUpload(); |
| 1347 | |||
| 1348 |
1/2✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
|
326 | spooler_->UnregisterListeners(); |
| 1349 | 326 | return root_catalog_info; | |
| 1350 | 326 | } | |
| 1351 | |||
| 1352 | |||
| 1353 | 780 | void WritableCatalogManager::FinalizeCatalog(WritableCatalog *catalog, | |
| 1354 | const bool stop_for_tweaks) { | ||
| 1355 | // update meta information of this catalog | ||
| 1356 |
1/3✓ Branch 2 taken 780 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
|
780 | LogCvmfs(kLogCatalog, kLogVerboseMsg, "creating snapshot of catalog '%s'", |
| 1357 | 1560 | catalog->mountpoint().c_str()); | |
| 1358 | |||
| 1359 | 780 | catalog->UpdateCounters(); | |
| 1360 | 780 | catalog->UpdateLastModified(); | |
| 1361 | 780 | catalog->IncrementRevision(); | |
| 1362 | |||
| 1363 | // update the previous catalog revision pointer | ||
| 1364 |
2/2✓ Branch 1 taken 326 times.
✓ Branch 2 taken 454 times.
|
780 | if (catalog->IsRoot()) { |
| 1365 |
1/4✓ Branch 2 taken 326 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
|
326 | LogCvmfs(kLogCatalog, kLogVerboseMsg, |
| 1366 | "setting '%s' as previous revision " | ||
| 1367 | "for root catalog", | ||
| 1368 | 652 | base_hash().ToStringWithSuffix().c_str()); | |
| 1369 | 326 | catalog->SetPreviousRevision(base_hash()); | |
| 1370 | } else { | ||
| 1371 | // Multiple catalogs might query the parent concurrently | ||
| 1372 | 454 | SyncLock(); | |
| 1373 |
1/2✓ Branch 1 taken 454 times.
✗ Branch 2 not taken.
|
454 | shash::Any hash_previous; |
| 1374 | uint64_t size_previous; | ||
| 1375 |
1/2✓ Branch 2 taken 454 times.
✗ Branch 3 not taken.
|
908 | const bool retval = catalog->parent()->FindNested( |
| 1376 |
1/2✓ Branch 1 taken 454 times.
✗ Branch 2 not taken.
|
908 | catalog->mountpoint(), &hash_previous, &size_previous); |
| 1377 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 454 times.
|
454 | assert(retval); |
| 1378 | 454 | SyncUnlock(); | |
| 1379 | |||
| 1380 |
1/8✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 454 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
|
908 | LogCvmfs(kLogCatalog, kLogVerboseMsg, |
| 1381 | "found '%s' as previous revision " | ||
| 1382 | "for nested catalog '%s'", | ||
| 1383 |
1/2✓ Branch 1 taken 454 times.
✗ Branch 2 not taken.
|
908 | hash_previous.ToStringWithSuffix().c_str(), |
| 1384 |
1/2✓ Branch 1 taken 454 times.
✗ Branch 2 not taken.
|
908 | catalog->mountpoint().c_str()); |
| 1385 |
1/2✓ Branch 1 taken 454 times.
✗ Branch 2 not taken.
|
454 | catalog->SetPreviousRevision(hash_previous); |
| 1386 | } | ||
| 1387 | 780 | catalog->Commit(); | |
| 1388 | |||
| 1389 | // check if catalog has too many entries | ||
| 1390 | const uint64_t catalog_limit = uint64_t(1000) | ||
| 1391 |
2/2✓ Branch 1 taken 326 times.
✓ Branch 2 taken 454 times.
|
780 | * uint64_t((catalog->IsRoot() |
| 1392 | 326 | ? root_kcatalog_limit_ | |
| 1393 | 454 | : nested_kcatalog_limit_)); | |
| 1394 | 780 | if ((catalog_limit > 0) | |
| 1395 |
2/6✗ Branch 0 not taken.
✓ Branch 1 taken 780 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 780 times.
|
780 | && (catalog->GetCounters().GetSelfEntries() > catalog_limit)) { |
| 1396 | ✗ | LogCvmfs(kLogCatalog, kLogStderr, | |
| 1397 | "%s: catalog at %s has more than %lu entries (%lu). " | ||
| 1398 | "Large catalogs stress the CernVM-FS transport infrastructure. " | ||
| 1399 | "Please split it into nested catalogs or increase the limit.", | ||
| 1400 | ✗ | enforce_limits_ ? "FATAL" : "WARNING", | |
| 1401 | ✗ | (catalog->IsRoot() ? "/" : catalog->mountpoint().c_str()), | |
| 1402 | ✗ | catalog_limit, catalog->GetCounters().GetSelfEntries()); | |
| 1403 | ✗ | if (enforce_limits_) | |
| 1404 | ✗ | PANIC(kLogStderr, "catalog at %s has more than %u entries (%u). ", | |
| 1405 | (catalog->IsRoot() ? "/" : catalog->mountpoint().c_str()), | ||
| 1406 | catalog_limit, catalog->GetCounters().GetSelfEntries()); | ||
| 1407 | } | ||
| 1408 | |||
| 1409 | // allow for manual adjustments in the catalog | ||
| 1410 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 780 times.
|
780 | if (stop_for_tweaks) { |
| 1411 | ✗ | LogCvmfs(kLogCatalog, kLogStdout, | |
| 1412 | "Allowing for tweaks in %s at %s " | ||
| 1413 | "(hit return to continue)", | ||
| 1414 | ✗ | catalog->database_path().c_str(), catalog->mountpoint().c_str()); | |
| 1415 | ✗ | const int read_char = getchar(); | |
| 1416 | ✗ | assert(read_char != EOF); | |
| 1417 | } | ||
| 1418 | |||
| 1419 | // compaction of bloated catalogs (usually after high database churn) | ||
| 1420 | 780 | catalog->VacuumDatabaseIfNecessary(); | |
| 1421 | 780 | } | |
| 1422 | |||
| 1423 | |||
| 1424 | 780 | void WritableCatalogManager::ScheduleCatalogProcessing( | |
| 1425 | WritableCatalog *catalog) { | ||
| 1426 | { | ||
| 1427 | 780 | const MutexLockGuard guard(catalog_processing_lock_); | |
| 1428 | // register catalog object for WritableCatalogManager::CatalogUploadCallback | ||
| 1429 |
2/4✓ Branch 1 taken 780 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 780 times.
✗ Branch 5 not taken.
|
780 | catalog_processing_map_[catalog->database_path()] = catalog; |
| 1430 | 780 | } | |
| 1431 |
1/2✓ Branch 2 taken 780 times.
✗ Branch 3 not taken.
|
780 | spooler_->ProcessCatalog(catalog->database_path()); |
| 1432 | 780 | } | |
| 1433 | |||
| 1434 | /** | ||
| 1435 | * Copy catalog to local cache.server | ||
| 1436 | * Must be an atomic write into the cache_dir | ||
| 1437 | * As such: create a temporary copy in cache_dir/txn and then do a | ||
| 1438 | * `rename` (which is atomic) to the actual cache path | ||
| 1439 | * | ||
| 1440 | * @returns true on success, otherwise false | ||
| 1441 | */ | ||
| 1442 | ✗ | bool WritableCatalogManager::CopyCatalogToLocalCache( | |
| 1443 | const upload::SpoolerResult &result) { | ||
| 1444 | ✗ | std::string tmp_catalog_path; | |
| 1445 | ✗ | const std::string cache_catalog_path = dir_cache_ + "/" | |
| 1446 | + result.content_hash | ||
| 1447 | ✗ | .MakePathWithoutSuffix(); | |
| 1448 | ✗ | FILE *fcatalog = CreateTempFile(dir_cache_ + "/txn/catalog", 0666, "w", | |
| 1449 | &tmp_catalog_path); | ||
| 1450 | ✗ | if (!fcatalog) { | |
| 1451 | ✗ | PANIC(kLogDebug | kLogStderr, | |
| 1452 | "Creating file for temporary catalog failed: %s", | ||
| 1453 | tmp_catalog_path.c_str()); | ||
| 1454 | } | ||
| 1455 | ✗ | CopyPath2File(result.local_path.c_str(), fcatalog); | |
| 1456 | ✗ | (void)fclose(fcatalog); | |
| 1457 | |||
| 1458 | ✗ | if (rename(tmp_catalog_path.c_str(), cache_catalog_path.c_str()) != 0) { | |
| 1459 | ✗ | PANIC(kLogDebug | kLogStderr, "Failed to copy catalog from %s to cache %s", | |
| 1460 | result.local_path.c_str(), cache_catalog_path.c_str()); | ||
| 1461 | } | ||
| 1462 | ✗ | return true; | |
| 1463 | } | ||
| 1464 | |||
| 1465 | 780 | void WritableCatalogManager::CatalogUploadCallback( | |
| 1466 | const upload::SpoolerResult &result, | ||
| 1467 | const CatalogUploadContext catalog_upload_context) { | ||
| 1468 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 780 times.
|
780 | if (result.return_code != 0) { |
| 1469 | ✗ | PANIC(kLogStderr, "failed to upload '%s' (retval: %d)", | |
| 1470 | result.local_path.c_str(), result.return_code); | ||
| 1471 | } | ||
| 1472 | |||
| 1473 | // retrieve the catalog object based on the callback information | ||
| 1474 | // see WritableCatalogManager::ScheduleCatalogProcessing() | ||
| 1475 | 780 | WritableCatalog *catalog = NULL; | |
| 1476 | { | ||
| 1477 | 780 | const MutexLockGuard guard(catalog_processing_lock_); | |
| 1478 | const std::map<std::string, WritableCatalog *>::iterator | ||
| 1479 |
1/2✓ Branch 1 taken 780 times.
✗ Branch 2 not taken.
|
780 | c = catalog_processing_map_.find(result.local_path); |
| 1480 |
1/2✗ Branch 2 not taken.
✓ Branch 3 taken 780 times.
|
780 | assert(c != catalog_processing_map_.end()); |
| 1481 | 780 | catalog = c->second; | |
| 1482 | 780 | } | |
| 1483 | |||
| 1484 | 780 | const uint64_t catalog_size = GetFileSize(result.local_path); | |
| 1485 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 780 times.
|
780 | assert(catalog_size > 0); |
| 1486 | |||
| 1487 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 780 times.
|
780 | if (UseLocalCache()) { |
| 1488 | ✗ | CopyCatalogToLocalCache(result); | |
| 1489 | } | ||
| 1490 | |||
| 1491 | 780 | SyncLock(); | |
| 1492 |
2/2✓ Branch 1 taken 454 times.
✓ Branch 2 taken 326 times.
|
780 | if (catalog->HasParent()) { |
| 1493 | // finalized nested catalogs will update their parent's pointer and schedule | ||
| 1494 | // them for processing (continuation) if the 'dirty children count' == 0 | ||
| 1495 | 454 | LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating nested catalog link"); | |
| 1496 | 454 | WritableCatalog *parent = catalog->GetWritableParent(); | |
| 1497 | |||
| 1498 |
2/4✓ Branch 1 taken 454 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 454 times.
✗ Branch 5 not taken.
|
454 | parent->UpdateNestedCatalog(catalog->mountpoint().ToString(), |
| 1499 | 454 | result.content_hash, | |
| 1500 | catalog_size, | ||
| 1501 | 454 | catalog->delta_counters_); | |
| 1502 | 454 | catalog->delta_counters_.SetZero(); | |
| 1503 | |||
| 1504 | const int remaining_dirty_children = catalog->GetWritableParent() | ||
| 1505 | 454 | ->DecrementDirtyChildren(); | |
| 1506 | |||
| 1507 | 454 | SyncUnlock(); | |
| 1508 | |||
| 1509 | // continuation of the dirty catalog tree traversal | ||
| 1510 | // see WritableCatalogManager::SnapshotCatalogs() | ||
| 1511 |
2/2✓ Branch 0 taken 292 times.
✓ Branch 1 taken 162 times.
|
454 | if (remaining_dirty_children == 0) { |
| 1512 | 292 | FinalizeCatalog(parent, catalog_upload_context.stop_for_tweaks); | |
| 1513 | 292 | ScheduleCatalogProcessing(parent); | |
| 1514 | } | ||
| 1515 | |||
| 1516 |
1/2✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
|
326 | } else if (catalog->IsRoot()) { |
| 1517 | // once the root catalog is reached, we are done with processing and report | ||
| 1518 | // back to the main via a Future<> and provide the necessary information | ||
| 1519 |
1/2✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
|
326 | CatalogInfo root_catalog_info; |
| 1520 | 326 | root_catalog_info.size = catalog_size; | |
| 1521 |
1/2✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
|
326 | root_catalog_info.ttl = catalog->GetTTL(); |
| 1522 | 326 | root_catalog_info.content_hash = result.content_hash; | |
| 1523 |
1/2✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
|
326 | root_catalog_info.revision = catalog->GetRevision(); |
| 1524 |
1/2✓ Branch 1 taken 326 times.
✗ Branch 2 not taken.
|
326 | catalog_upload_context.root_catalog_info->Set(root_catalog_info); |
| 1525 | 326 | SyncUnlock(); | |
| 1526 | } else { | ||
| 1527 | ✗ | PANIC(kLogStderr, "inconsistent state detected"); | |
| 1528 | } | ||
| 1529 | 780 | } | |
| 1530 | |||
| 1531 | |||
| 1532 | /** | ||
| 1533 | * Finds dirty catalogs that can be snapshot right away and annotates all the | ||
| 1534 | * other catalogs with their number of dirty descendants. | ||
| 1535 | * Note that there is a convenience wrapper to start the recursion: | ||
| 1536 | * WritableCatalogManager::GetModifiedCatalogLeafs() | ||
| 1537 | * | ||
| 1538 | * @param catalog the catalog for this recursion step | ||
| 1539 | * @param result the result list to be appended to | ||
| 1540 | * @return true if 'catalog' is dirty | ||
| 1541 | */ | ||
| 1542 | 792 | bool WritableCatalogManager::GetModifiedCatalogLeafsRecursively( | |
| 1543 | Catalog *catalog, WritableCatalogList *result) const { | ||
| 1544 | 792 | WritableCatalog *wr_catalog = static_cast<WritableCatalog *>(catalog); | |
| 1545 | |||
| 1546 | // Look for dirty catalogs in the descendants of *catalog | ||
| 1547 | 792 | int dirty_children = 0; | |
| 1548 |
1/2✓ Branch 1 taken 792 times.
✗ Branch 2 not taken.
|
792 | CatalogList children = wr_catalog->GetChildren(); |
| 1549 | 792 | CatalogList::const_iterator i = children.begin(); | |
| 1550 | 792 | const CatalogList::const_iterator iend = children.end(); | |
| 1551 |
2/2✓ Branch 2 taken 454 times.
✓ Branch 3 taken 792 times.
|
1246 | for (; i != iend; ++i) { |
| 1552 |
2/4✓ Branch 2 taken 454 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 454 times.
✗ Branch 5 not taken.
|
454 | if (GetModifiedCatalogLeafsRecursively(*i, result)) { |
| 1553 | 454 | ++dirty_children; | |
| 1554 | } | ||
| 1555 | } | ||
| 1556 | |||
| 1557 | // a catalog is dirty if itself or one of its children has changed | ||
| 1558 | // a leaf catalog doesn't have any dirty children | ||
| 1559 | 792 | wr_catalog->set_dirty_children(dirty_children); | |
| 1560 |
1/4✗ Branch 1 not taken.
✓ Branch 2 taken 792 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
|
792 | const bool is_dirty = wr_catalog->IsDirty() || dirty_children > 0; |
| 1561 | 792 | const bool is_leaf = dirty_children == 0; | |
| 1562 |
3/4✓ Branch 0 taken 792 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 500 times.
✓ Branch 3 taken 292 times.
|
792 | if (is_dirty && is_leaf) { |
| 1563 |
1/2✓ Branch 1 taken 500 times.
✗ Branch 2 not taken.
|
500 | result->push_back(const_cast<WritableCatalog *>(wr_catalog)); |
| 1564 | } | ||
| 1565 | |||
| 1566 | 792 | return is_dirty; | |
| 1567 | 792 | } | |
| 1568 | |||
| 1569 | |||
| 1570 | ✗ | void WritableCatalogManager::DoBalance() { | |
| 1571 | ✗ | CatalogList catalog_list = GetCatalogs(); | |
| 1572 | ✗ | reverse(catalog_list.begin(), catalog_list.end()); | |
| 1573 | ✗ | for (unsigned i = 0; i < catalog_list.size(); ++i) { | |
| 1574 | ✗ | FixWeight(static_cast<WritableCatalog *>(catalog_list[i])); | |
| 1575 | } | ||
| 1576 | } | ||
| 1577 | |||
| 1578 | ✗ | void WritableCatalogManager::FixWeight(WritableCatalog *catalog) { | |
| 1579 | // firstly check underflow because they can provoke overflows | ||
| 1580 | ✗ | if (catalog->GetNumEntries() < min_weight_ && !catalog->IsRoot() | |
| 1581 | ✗ | && catalog->IsAutogenerated()) { | |
| 1582 | ✗ | LogCvmfs(kLogCatalog, kLogStdout, | |
| 1583 | "Deleting an autogenerated catalog in '%s'", | ||
| 1584 | ✗ | catalog->mountpoint().c_str()); | |
| 1585 | // Remove the .cvmfscatalog and .cvmfsautocatalog files first | ||
| 1586 | ✗ | const string path = catalog->mountpoint().ToString(); | |
| 1587 | ✗ | catalog->RemoveEntry(path + "/.cvmfscatalog"); | |
| 1588 | ✗ | catalog->RemoveEntry(path + "/.cvmfsautocatalog"); | |
| 1589 | // Remove the actual catalog | ||
| 1590 | ✗ | const string catalog_path = catalog->mountpoint().ToString().substr(1); | |
| 1591 | ✗ | RemoveNestedCatalog(catalog_path); | |
| 1592 | ✗ | } else if (catalog->GetNumEntries() > max_weight_) { | |
| 1593 | ✗ | CatalogBalancer<WritableCatalogManager> catalog_balancer(this); | |
| 1594 | ✗ | catalog_balancer.Balance(catalog); | |
| 1595 | } | ||
| 1596 | } | ||
| 1597 | |||
| 1598 | |||
| 1599 | //**************************************************************************** | ||
| 1600 | // Workaround -- Serialized Catalog Committing | ||
| 1601 | |||
| 1602 | ✗ | int WritableCatalogManager::GetModifiedCatalogsRecursively( | |
| 1603 | const Catalog *catalog, WritableCatalogList *result) const { | ||
| 1604 | // A catalog must be snapshot, if itself or one of it's descendants is dirty. | ||
| 1605 | // So we traverse the catalog tree recursively and look for dirty catalogs | ||
| 1606 | // on the way. | ||
| 1607 | ✗ | const WritableCatalog *wr_catalog = static_cast<const WritableCatalog *>( | |
| 1608 | catalog); | ||
| 1609 | // This variable will contain the number of dirty catalogs in the sub tree | ||
| 1610 | // with *catalog as it's root. | ||
| 1611 | ✗ | int dirty_catalogs = (wr_catalog->IsDirty()) ? 1 : 0; | |
| 1612 | |||
| 1613 | // Look for dirty catalogs in the descendants of *catalog | ||
| 1614 | ✗ | CatalogList children = wr_catalog->GetChildren(); | |
| 1615 | ✗ | for (CatalogList::const_iterator i = children.begin(), iEnd = children.end(); | |
| 1616 | ✗ | i != iEnd; ++i) { | |
| 1617 | ✗ | dirty_catalogs += GetModifiedCatalogsRecursively(*i, result); | |
| 1618 | } | ||
| 1619 | |||
| 1620 | // If we found a dirty catalog in the checked sub tree, the root (*catalog) | ||
| 1621 | // must be snapshot and ends up in the result list | ||
| 1622 | ✗ | if (dirty_catalogs > 0) | |
| 1623 | ✗ | result->push_back(const_cast<WritableCatalog *>(wr_catalog)); | |
| 1624 | |||
| 1625 | // tell the upper layer about number of catalogs | ||
| 1626 | ✗ | return dirty_catalogs; | |
| 1627 | } | ||
| 1628 | |||
| 1629 | |||
| 1630 | ✗ | void WritableCatalogManager::CatalogUploadSerializedCallback( | |
| 1631 | const upload::SpoolerResult &result, const CatalogUploadContext unused) { | ||
| 1632 | ✗ | if (result.return_code != 0) { | |
| 1633 | ✗ | PANIC(kLogStderr, "failed to upload '%s' (retval: %d)", | |
| 1634 | result.local_path.c_str(), result.return_code); | ||
| 1635 | } | ||
| 1636 | |||
| 1637 | ✗ | if (UseLocalCache()) { | |
| 1638 | ✗ | CopyCatalogToLocalCache(result); | |
| 1639 | } | ||
| 1640 | |||
| 1641 | ✗ | unlink(result.local_path.c_str()); | |
| 1642 | } | ||
| 1643 | |||
| 1644 | |||
| 1645 | WritableCatalogManager::CatalogInfo | ||
| 1646 | ✗ | WritableCatalogManager::SnapshotCatalogsSerialized(const bool stop_for_tweaks) { | |
| 1647 | ✗ | LogCvmfs(kLogCvmfs, kLogStdout, "Serialized committing of file catalogs..."); | |
| 1648 | ✗ | reinterpret_cast<WritableCatalog *>(GetRootCatalog())->SetDirty(); | |
| 1649 | ✗ | WritableCatalogList catalogs_to_snapshot; | |
| 1650 | ✗ | GetModifiedCatalogs(&catalogs_to_snapshot); | |
| 1651 | CatalogUploadContext unused; | ||
| 1652 | ✗ | unused.root_catalog_info = NULL; | |
| 1653 | ✗ | unused.stop_for_tweaks = false; | |
| 1654 | ✗ | spooler_->RegisterListener( | |
| 1655 | &WritableCatalogManager::CatalogUploadSerializedCallback, this, unused); | ||
| 1656 | |||
| 1657 | ✗ | CatalogInfo root_catalog_info; | |
| 1658 | ✗ | WritableCatalogList::const_iterator i = catalogs_to_snapshot.begin(); | |
| 1659 | ✗ | const WritableCatalogList::const_iterator iend = catalogs_to_snapshot.end(); | |
| 1660 | ✗ | for (; i != iend; ++i) { | |
| 1661 | ✗ | FinalizeCatalog(*i, stop_for_tweaks); | |
| 1662 | |||
| 1663 | // Compress and upload catalog | ||
| 1664 | ✗ | shash::Any hash_catalog(spooler_->GetHashAlgorithm(), | |
| 1665 | ✗ | shash::kSuffixCatalog); | |
| 1666 | ✗ | if (!zlib::CompressPath2Null((*i)->database_path(), &hash_catalog)) { | |
| 1667 | ✗ | PANIC(kLogStderr, "could not compress catalog %s", | |
| 1668 | (*i)->mountpoint().ToString().c_str()); | ||
| 1669 | } | ||
| 1670 | |||
| 1671 | ✗ | const int64_t catalog_size = GetFileSize((*i)->database_path()); | |
| 1672 | ✗ | assert(catalog_size > 0); | |
| 1673 | |||
| 1674 | ✗ | if ((*i)->HasParent()) { | |
| 1675 | ✗ | LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating nested catalog link"); | |
| 1676 | ✗ | WritableCatalog *parent = (*i)->GetWritableParent(); | |
| 1677 | ✗ | parent->UpdateNestedCatalog((*i)->mountpoint().ToString(), hash_catalog, | |
| 1678 | ✗ | catalog_size, (*i)->delta_counters_); | |
| 1679 | ✗ | (*i)->delta_counters_.SetZero(); | |
| 1680 | ✗ | } else if ((*i)->IsRoot()) { | |
| 1681 | ✗ | root_catalog_info.size = catalog_size; | |
| 1682 | ✗ | root_catalog_info.ttl = (*i)->GetTTL(); | |
| 1683 | ✗ | root_catalog_info.content_hash = hash_catalog; | |
| 1684 | ✗ | root_catalog_info.revision = (*i)->GetRevision(); | |
| 1685 | } else { | ||
| 1686 | ✗ | PANIC(kLogStderr, "inconsistent state detected"); | |
| 1687 | } | ||
| 1688 | |||
| 1689 | ✗ | spooler_->ProcessCatalog((*i)->database_path()); | |
| 1690 | } | ||
| 1691 | ✗ | spooler_->WaitForUpload(); | |
| 1692 | |||
| 1693 | ✗ | spooler_->UnregisterListeners(); | |
| 1694 | ✗ | return root_catalog_info; | |
| 1695 | } | ||
| 1696 | |||
| 1697 | ✗ | void WritableCatalogManager::SetupSingleCatalogUploadCallback() { | |
| 1698 | ✗ | spooler_->RegisterListener( | |
| 1699 | &WritableCatalogManager::SingleCatalogUploadCallback, this); | ||
| 1700 | } | ||
| 1701 | |||
| 1702 | ✗ | void WritableCatalogManager::RemoveSingleCatalogUploadCallback() { | |
| 1703 | ✗ | spooler_->WaitForUpload(); // wait for all outstanding jobs to finish before | |
| 1704 | // tearing it down | ||
| 1705 | // parents that were waiting on children can be uploaded now | ||
| 1706 | ✗ | ScheduleReadyCatalogs(); | |
| 1707 | ✗ | spooler_->WaitForUpload(); | |
| 1708 | ✗ | spooler_->UnregisterListeners(); | |
| 1709 | ✗ | pending_catalogs_ = | |
| 1710 | ✗ | {}; // whatever we couldn't process, leave it to the Commit | |
| 1711 | } | ||
| 1712 | |||
| 1713 | ✗ | void WritableCatalogManager::AddCatalogToQueue(const std::string &path) { | |
| 1714 | ✗ | SyncLock(); | |
| 1715 | ✗ | WritableCatalog *catalog = NULL; | |
| 1716 | ✗ | bool const retval = FindCatalog(MakeRelativePath(path), &catalog, NULL); | |
| 1717 | ✗ | assert(retval); | |
| 1718 | ✗ | assert(catalog); | |
| 1719 | ✗ | catalog->SetDirty(); // ensure it's dirty so its parent will wait for it | |
| 1720 | // the parent must not be finalized before this catalog's upload callback | ||
| 1721 | // has written the new nested catalog link into it | ||
| 1722 | ✗ | if (catalog->HasParent()) | |
| 1723 | ✗ | catalog->GetWritableParent()->IncrementDirtyChildren(); | |
| 1724 | ✗ | SyncUnlock(); | |
| 1725 | ✗ | pending_catalogs_.push_back(catalog); | |
| 1726 | } | ||
| 1727 | |||
| 1728 | ✗ | void WritableCatalogManager::ScheduleReadyCatalogs() { | |
| 1729 | // best effort to schedule as many catalogs for upload as possible | ||
| 1730 | ✗ | for (auto it = pending_catalogs_.begin(); it != pending_catalogs_.end();) { | |
| 1731 | ✗ | if ((*it)->dirty_children() == 0) { | |
| 1732 | ✗ | FinalizeCatalog(*it, false /* stop_for_tweaks */); | |
| 1733 | ✗ | ScheduleCatalogProcessing(*it); | |
| 1734 | ✗ | LogCvmfs(kLogCatalog, kLogVerboseMsg, "scheduled %s for processing", | |
| 1735 | ✗ | (*it)->mountpoint().c_str()); | |
| 1736 | ✗ | it = pending_catalogs_.erase(it); | |
| 1737 | } else { | ||
| 1738 | ✗ | ++it; | |
| 1739 | } | ||
| 1740 | } | ||
| 1741 | } | ||
| 1742 | |||
| 1743 | // Callback for uploading a single catalog, similar to CatalogUploadCallback. | ||
| 1744 | // The main difference is that this callback would not trigger processing of the | ||
| 1745 | // parent | ||
| 1746 | ✗ | void WritableCatalogManager::SingleCatalogUploadCallback( | |
| 1747 | const upload::SpoolerResult &result) { | ||
| 1748 | ✗ | if (result.return_code != 0) { | |
| 1749 | ✗ | PANIC(kLogStderr, "failed to upload '%s' (retval: %d)", | |
| 1750 | result.local_path.c_str(), result.return_code); | ||
| 1751 | } | ||
| 1752 | |||
| 1753 | // retrieve the catalog object based on the callback information | ||
| 1754 | // see WritableCatalogManager::ScheduleCatalogProcessing() | ||
| 1755 | ✗ | WritableCatalog *catalog = NULL; | |
| 1756 | { | ||
| 1757 | ✗ | MutexLockGuard const guard(catalog_processing_lock_); | |
| 1758 | std::map<std::string, WritableCatalog *>::iterator const | ||
| 1759 | ✗ | c = catalog_processing_map_.find(result.local_path); | |
| 1760 | ✗ | assert(c != catalog_processing_map_.end()); | |
| 1761 | ✗ | catalog = c->second; | |
| 1762 | } | ||
| 1763 | |||
| 1764 | ✗ | uint64_t const catalog_size = GetFileSize(result.local_path); | |
| 1765 | ✗ | assert(catalog_size > 0); | |
| 1766 | |||
| 1767 | ✗ | SyncLock(); | |
| 1768 | ✗ | if (catalog->HasParent()) { | |
| 1769 | // finalized nested catalogs will update their parent's pointer | ||
| 1770 | ✗ | LogCvmfs(kLogCatalog, kLogVerboseMsg, "updating nested catalog link"); | |
| 1771 | ✗ | WritableCatalog *parent = catalog->GetWritableParent(); | |
| 1772 | |||
| 1773 | ✗ | parent->UpdateNestedCatalog(catalog->mountpoint().ToString(), | |
| 1774 | ✗ | result.content_hash, | |
| 1775 | catalog_size, | ||
| 1776 | ✗ | catalog->delta_counters_); | |
| 1777 | ✗ | parent->DecrementDirtyChildren(); | |
| 1778 | ✗ | catalog->delta_counters_.SetZero(); | |
| 1779 | } | ||
| 1780 | // JUMP: detach the catalog after uploading to free sqlite related resources | ||
| 1781 | ✗ | DetachCatalog(catalog); | |
| 1782 | ✗ | SyncUnlock(); | |
| 1783 | } | ||
| 1784 | // using the given list of dirs, fetch all relevant catalogs | ||
| 1785 | ✗ | void WritableCatalogManager::LoadCatalogs( | |
| 1786 | const std::string &base_path, const std::unordered_set<std::string> &dirs) { | ||
| 1787 | // mount everything up to "base_path" first (this would be our lease_path | ||
| 1788 | // typically) | ||
| 1789 | Catalog *base_catalog; | ||
| 1790 | ✗ | MountSubtree(PathString(base_path), NULL /* entry_point */, | |
| 1791 | true /* is_listable */, &base_catalog); | ||
| 1792 | |||
| 1793 | // start up the downloader | ||
| 1794 | CatalogDownloadContext context; | ||
| 1795 | ✗ | context.dirs = &dirs; | |
| 1796 | ✗ | catalog_download_pipeline_ = new CatalogDownloadPipeline( | |
| 1797 | ✗ | static_cast<SimpleCatalogManager *>(this)); | |
| 1798 | ✗ | catalog_download_pipeline_->RegisterListener( | |
| 1799 | &WritableCatalogManager::CatalogDownloadCallback, this, context); | ||
| 1800 | ✗ | catalog_download_pipeline_->Spawn(); | |
| 1801 | |||
| 1802 | Catalog::NestedCatalogList nested_catalogs = base_catalog | ||
| 1803 | ✗ | ->ListNestedCatalogs(); | |
| 1804 | ✗ | for (auto it = nested_catalogs.begin(); it != nested_catalogs.end(); ++it) { | |
| 1805 | // schedule relevant child nested catalogs for download | ||
| 1806 | ✗ | std::string const mountpoint = it->mountpoint.ToString(); | |
| 1807 | ✗ | if (dirs.find(mountpoint) != dirs.end()) { | |
| 1808 | ✗ | Catalog *catalog = CreateCatalog(it->mountpoint, it->hash, | |
| 1809 | ✗ | NULL /* parent */); | |
| 1810 | { | ||
| 1811 | ✗ | MutexLockGuard const guard(catalog_download_lock_); | |
| 1812 | ✗ | catalog_download_map_.insert( | |
| 1813 | ✗ | std::make_pair(it->hash.ToString(), catalog)); | |
| 1814 | } | ||
| 1815 | ✗ | catalog_download_pipeline_->Process(it->hash); | |
| 1816 | } | ||
| 1817 | } | ||
| 1818 | |||
| 1819 | ✗ | catalog_download_pipeline_->WaitFor(); | |
| 1820 | ✗ | delete catalog_download_pipeline_; // terminate all the threads | |
| 1821 | } | ||
| 1822 | |||
| 1823 | ✗ | bool WritableCatalogManager::LookupDirEntry(const string &path, | |
| 1824 | const LookupOptions options, | ||
| 1825 | DirectoryEntry *dirent) { | ||
| 1826 | ✗ | SyncLock(); | |
| 1827 | ✗ | bool const exists = LookupPath(path, options, dirent); | |
| 1828 | ✗ | SyncUnlock(); | |
| 1829 | ✗ | return exists; | |
| 1830 | } | ||
| 1831 | |||
| 1832 | ✗ | void WritableCatalogManager::CatalogHashSerializedCallback( | |
| 1833 | const CompressHashResult &result) { | ||
| 1834 | ✗ | MutexLockGuard const guard(catalog_hash_lock_); | |
| 1835 | ✗ | catalog_hash_map_[result.path] = result.hash; | |
| 1836 | } | ||
| 1837 | |||
| 1838 | ✗ | void WritableCatalogManager::CatalogDownloadCallback( | |
| 1839 | const CatalogDownloadResult &result, CatalogDownloadContext context) { | ||
| 1840 | Catalog *downloaded_catalog; | ||
| 1841 | { | ||
| 1842 | ✗ | MutexLockGuard const guard(catalog_download_lock_); | |
| 1843 | ✗ | auto it = catalog_download_map_.find(result.hash); | |
| 1844 | ✗ | assert(it != catalog_download_map_.end()); | |
| 1845 | ✗ | downloaded_catalog = it->second; | |
| 1846 | } | ||
| 1847 | |||
| 1848 | ✗ | if (!downloaded_catalog->OpenDatabase(result.db_path)) { | |
| 1849 | ✗ | LogCvmfs(kLogCvmfs, kLogDebug, "failed to initialize catalog"); | |
| 1850 | ✗ | delete downloaded_catalog; | |
| 1851 | ✗ | return; | |
| 1852 | } | ||
| 1853 | // the temp copy is only used to find nested catalogs; the download itself | ||
| 1854 | // warmed the local cache, so drop the copy together with the object | ||
| 1855 | ✗ | downloaded_catalog->TakeDatabaseFileOwnership(); | |
| 1856 | |||
| 1857 | Catalog::NestedCatalogList nested_catalogs = downloaded_catalog | ||
| 1858 | ✗ | ->ListNestedCatalogs(); | |
| 1859 | ✗ | for (auto it = nested_catalogs.begin(); it != nested_catalogs.end(); ++it) { | |
| 1860 | // schedule relevant child nested catalogs for download | ||
| 1861 | ✗ | if (context.dirs->find(it->mountpoint.ToString()) != context.dirs->end()) { | |
| 1862 | ✗ | Catalog *child_catalog = CreateCatalog(it->mountpoint, it->hash, | |
| 1863 | ✗ | NULL /* parent */); | |
| 1864 | { | ||
| 1865 | ✗ | MutexLockGuard const guard(catalog_download_lock_); | |
| 1866 | ✗ | catalog_download_map_.insert( | |
| 1867 | ✗ | std::make_pair(it->hash.ToString(), child_catalog)); | |
| 1868 | } | ||
| 1869 | ✗ | catalog_download_pipeline_->Process(it->hash); | |
| 1870 | } | ||
| 1871 | } | ||
| 1872 | ✗ | delete downloaded_catalog; | |
| 1873 | } | ||
| 1874 | |||
| 1875 | |||
| 1876 | } // namespace catalog | ||
| 1877 |