| Directory: | cvmfs/ |
|---|---|
| File: | cvmfs/catalog_virtual.cc |
| Date: | 2024-04-28 02:33:07 |
| Exec | Total | Coverage | |
|---|---|---|---|
| Lines: | 13 | 196 | 6.6% |
| Branches: | 9 | 425 | 2.1% |
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /** | ||
| 2 | * This file is part of the CernVM File System. | ||
| 3 | */ | ||
| 4 | #include "cvmfs_config.h" | ||
| 5 | #include "catalog_virtual.h" | ||
| 6 | |||
| 7 | #include <algorithm> | ||
| 8 | #include <cassert> | ||
| 9 | #include <cstdlib> | ||
| 10 | |||
| 11 | #include "catalog_mgr_rw.h" | ||
| 12 | #include "compression.h" | ||
| 13 | #include "history.h" | ||
| 14 | #include "swissknife_history.h" | ||
| 15 | #include "swissknife_sync.h" | ||
| 16 | #include "util/logging.h" | ||
| 17 | #include "util/pointer.h" | ||
| 18 | #include "util/posix.h" | ||
| 19 | #include "util/string.h" | ||
| 20 | #include "xattr.h" | ||
| 21 | |||
| 22 | using namespace std; // NOLINT | ||
| 23 | |||
| 24 | namespace catalog { | ||
| 25 | |||
| 26 | const char *VirtualCatalog::kVirtualPath = ".cvmfs"; | ||
| 27 | const char *VirtualCatalog::kSnapshotDirectory = "snapshots"; | ||
| 28 | const int VirtualCatalog::kActionNone = 0x00; | ||
| 29 | const int VirtualCatalog::kActionGenerateSnapshots = 0x01; | ||
| 30 | const int VirtualCatalog::kActionRemove = 0x02; | ||
| 31 | |||
| 32 | |||
| 33 | ✗ | void VirtualCatalog::CreateBaseDirectory() { | |
| 34 | // Add /.cvmfs as a nested catalog | ||
| 35 | ✗ | DirectoryEntryBase entry_dir; | |
| 36 | ✗ | entry_dir.name_ = NameString(string(kVirtualPath)); | |
| 37 | ✗ | entry_dir.mode_ = S_IFDIR | | |
| 38 | S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH; | ||
| 39 | ✗ | entry_dir.uid_ = 0; | |
| 40 | ✗ | entry_dir.gid_ = 0; | |
| 41 | ✗ | entry_dir.size_ = 97; | |
| 42 | ✗ | entry_dir.mtime_ = time(NULL); | |
| 43 | ✗ | catalog_mgr_->AddDirectory(entry_dir, XattrList(), ""); | |
| 44 | WritableCatalog *parent_catalog = | ||
| 45 | ✗ | catalog_mgr_->GetHostingCatalog(kVirtualPath); | |
| 46 | ✗ | catalog_mgr_->CreateNestedCatalog(kVirtualPath); | |
| 47 | WritableCatalog *virtual_catalog = | ||
| 48 | ✗ | catalog_mgr_->GetHostingCatalog(kVirtualPath); | |
| 49 | ✗ | assert(parent_catalog != virtual_catalog); | |
| 50 | |||
| 51 | // Set hidden flag in parent catalog | ||
| 52 | ✗ | DirectoryEntry entry_parent; | |
| 53 | ✗ | bool retval = parent_catalog->LookupPath( | |
| 54 | ✗ | PathString("/" + string(kVirtualPath)), &entry_parent); | |
| 55 | ✗ | assert(retval); | |
| 56 | ✗ | entry_parent.set_is_hidden(true); | |
| 57 | ✗ | parent_catalog->UpdateEntry(entry_parent, "/" + string(kVirtualPath)); | |
| 58 | |||
| 59 | // Set hidden flag in nested catalog | ||
| 60 | ✗ | DirectoryEntry entry_virtual; | |
| 61 | ✗ | retval = virtual_catalog->LookupPath(PathString("/" + string(kVirtualPath)), | |
| 62 | &entry_virtual); | ||
| 63 | ✗ | assert(retval); | |
| 64 | ✗ | entry_virtual.set_is_hidden(true); | |
| 65 | ✗ | virtual_catalog->UpdateEntry(entry_virtual, "/" + string(kVirtualPath)); | |
| 66 | } | ||
| 67 | |||
| 68 | |||
| 69 | ✗ | void VirtualCatalog::CreateNestedCatalogMarker() { | |
| 70 | ✗ | DirectoryEntryBase entry_marker; | |
| 71 | // Note that another entity needs to ensure that the object of an empty | ||
| 72 | // file is in the repository! It is currently done by the sync_mediator. | ||
| 73 | ✗ | shash::Algorithms algorithm = catalog_mgr_->spooler_->GetHashAlgorithm(); | |
| 74 | ✗ | shash::Any file_hash(algorithm); | |
| 75 | void *empty_compressed; | ||
| 76 | uint64_t sz_empty_compressed; | ||
| 77 | ✗ | bool retval = zlib::CompressMem2Mem( | |
| 78 | NULL, 0, &empty_compressed, &sz_empty_compressed); | ||
| 79 | ✗ | assert(retval); | |
| 80 | ✗ | shash::HashMem(static_cast<unsigned char *>(empty_compressed), | |
| 81 | sz_empty_compressed, &file_hash); | ||
| 82 | ✗ | free(empty_compressed); | |
| 83 | ✗ | entry_marker.name_ = NameString(".cvmfscatalog"); | |
| 84 | ✗ | entry_marker.mode_ = S_IFREG | S_IRUSR | S_IRGRP | S_IROTH; | |
| 85 | ✗ | entry_marker.checksum_ = file_hash; | |
| 86 | ✗ | entry_marker.mtime_ = time(NULL); | |
| 87 | ✗ | entry_marker.uid_ = 0; | |
| 88 | ✗ | entry_marker.gid_ = 0; | |
| 89 | ✗ | XattrList xattrs; | |
| 90 | ✗ | catalog_mgr_->AddFile(entry_marker, xattrs, kVirtualPath); | |
| 91 | } | ||
| 92 | |||
| 93 | |||
| 94 | ✗ | void VirtualCatalog::CreateSnapshotDirectory() { | |
| 95 | ✗ | DirectoryEntryBase entry_dir; | |
| 96 | ✗ | entry_dir.name_ = NameString(string(kSnapshotDirectory)); | |
| 97 | ✗ | entry_dir.mode_ = S_IFDIR | | |
| 98 | S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH; | ||
| 99 | ✗ | entry_dir.uid_ = 0; | |
| 100 | ✗ | entry_dir.gid_ = 0; | |
| 101 | ✗ | entry_dir.size_ = 97; | |
| 102 | ✗ | entry_dir.mtime_ = time(NULL); | |
| 103 | ✗ | catalog_mgr_->AddDirectory(entry_dir, XattrList(), kVirtualPath); | |
| 104 | } | ||
| 105 | |||
| 106 | |||
| 107 | /** | ||
| 108 | * Checks for the top-level /.cvmfs directory and creates it as a nested catalog | ||
| 109 | * if necessary. | ||
| 110 | */ | ||
| 111 | ✗ | void VirtualCatalog::EnsurePresence() { | |
| 112 | ✗ | DirectoryEntry e; | |
| 113 | ✗ | bool retval = catalog_mgr_->LookupPath("/" + string(kVirtualPath), | |
| 114 | kLookupDefault, &e); | ||
| 115 | ✗ | if (!retval) { | |
| 116 | ✗ | LogCvmfs(kLogCatalog, kLogDebug, "creating new virtual catalog"); | |
| 117 | ✗ | CreateBaseDirectory(); | |
| 118 | ✗ | CreateNestedCatalogMarker(); | |
| 119 | ✗ | CreateSnapshotDirectory(); | |
| 120 | } | ||
| 121 | ✗ | assert(catalog_mgr_->IsTransitionPoint(kVirtualPath)); | |
| 122 | } | ||
| 123 | |||
| 124 | |||
| 125 | ✗ | void VirtualCatalog::Generate(int actions) { | |
| 126 | ✗ | if (actions & kActionGenerateSnapshots) { | |
| 127 | ✗ | GenerateSnapshots(); | |
| 128 | } | ||
| 129 | ✗ | if (actions & kActionRemove) { | |
| 130 | ✗ | Remove(); | |
| 131 | } | ||
| 132 | } | ||
| 133 | |||
| 134 | |||
| 135 | ✗ | void VirtualCatalog::GenerateSnapshots() { | |
| 136 | ✗ | LogCvmfs(kLogCvmfs, kLogStdout, "Creating virtual snapshots"); | |
| 137 | ✗ | EnsurePresence(); | |
| 138 | |||
| 139 | ✗ | vector<TagId> tags_history; | |
| 140 | ✗ | vector<TagId> tags_catalog; | |
| 141 | ✗ | GetSortedTagsFromHistory(&tags_history); | |
| 142 | ✗ | GetSortedTagsFromCatalog(&tags_catalog); | |
| 143 | // Add artificial end markers to both lists | ||
| 144 | ✗ | string tag_name_end = ""; | |
| 145 | ✗ | if (!tags_history.empty()) | |
| 146 | ✗ | tag_name_end = std::max(tag_name_end, tags_history.rbegin()->name); | |
| 147 | ✗ | if (!tags_catalog.empty()) | |
| 148 | ✗ | tag_name_end = std::max(tag_name_end, tags_catalog.rbegin()->name); | |
| 149 | ✗ | tag_name_end += "X"; | |
| 150 | ✗ | tags_history.push_back(TagId(tag_name_end, shash::Any())); | |
| 151 | ✗ | tags_catalog.push_back(TagId(tag_name_end, shash::Any())); | |
| 152 | |||
| 153 | // Walk through both sorted lists concurrently and determine change set | ||
| 154 | ✗ | unsigned i_history = 0, i_catalog = 0; | |
| 155 | ✗ | unsigned last_history = tags_history.size() - 1; | |
| 156 | ✗ | unsigned last_catalog = tags_catalog.size() - 1; | |
| 157 | ✗ | while ((i_history < last_history) || (i_catalog < last_catalog)) { | |
| 158 | ✗ | TagId t_history = tags_history[i_history]; | |
| 159 | ✗ | TagId t_catalog = tags_catalog[i_catalog]; | |
| 160 | |||
| 161 | // Both the same, nothing to do | ||
| 162 | ✗ | if (t_history == t_catalog) { | |
| 163 | ✗ | i_history++; | |
| 164 | ✗ | i_catalog++; | |
| 165 | ✗ | continue; | |
| 166 | } | ||
| 167 | |||
| 168 | // Same tag name for different hash, re-insert | ||
| 169 | ✗ | if (t_history.name == t_catalog.name) { | |
| 170 | ✗ | RemoveSnapshot(t_catalog); | |
| 171 | ✗ | InsertSnapshot(t_history); | |
| 172 | ✗ | i_history++; | |
| 173 | ✗ | i_catalog++; | |
| 174 | ✗ | continue; | |
| 175 | } | ||
| 176 | |||
| 177 | // New tag that's missing | ||
| 178 | ✗ | if (t_history.name < t_catalog.name) { | |
| 179 | ✗ | InsertSnapshot(t_history); | |
| 180 | ✗ | i_history++; | |
| 181 | ✗ | continue; | |
| 182 | } | ||
| 183 | |||
| 184 | // A tag was removed but it is still present in the catalog | ||
| 185 | ✗ | assert(t_history.name > t_catalog.name); | |
| 186 | ✗ | RemoveSnapshot(t_catalog); | |
| 187 | ✗ | i_catalog++; | |
| 188 | } | ||
| 189 | } | ||
| 190 | |||
| 191 | |||
| 192 | 7 | bool VirtualCatalog::ParseActions( | |
| 193 | const string &action_desc, | ||
| 194 | int *actions) | ||
| 195 | { | ||
| 196 | 7 | *actions = kActionNone; | |
| 197 |
2/2✓ Branch 1 taken 1 times.
✓ Branch 2 taken 6 times.
|
7 | if (action_desc.empty()) |
| 198 | 1 | return true; | |
| 199 | |||
| 200 |
1/2✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
|
6 | vector<string> action_tokens = SplitString(action_desc, ','); |
| 201 |
2/2✓ Branch 1 taken 10 times.
✓ Branch 2 taken 4 times.
|
14 | for (unsigned i = 0; i < action_tokens.size(); ++i) { |
| 202 |
2/2✓ Branch 2 taken 4 times.
✓ Branch 3 taken 6 times.
|
10 | if (action_tokens[i] == "snapshots") { |
| 203 | 4 | *actions |= kActionGenerateSnapshots; | |
| 204 |
2/2✓ Branch 2 taken 4 times.
✓ Branch 3 taken 2 times.
|
6 | } else if (action_tokens[i] == "remove") { |
| 205 | 4 | *actions |= kActionRemove; | |
| 206 | } else { | ||
| 207 | 2 | return false; | |
| 208 | } | ||
| 209 | } | ||
| 210 | 4 | return true; | |
| 211 | 6 | } | |
| 212 | |||
| 213 | |||
| 214 | ✗ | void VirtualCatalog::GetSortedTagsFromHistory(vector<TagId> *tags) { | |
| 215 | UniquePtr<history::History> history( | ||
| 216 | ✗ | assistant_.GetHistory(swissknife::Assistant::kOpenReadOnly)); | |
| 217 | ✗ | vector<history::History::Tag> tags_history; | |
| 218 | ✗ | bool retval = history->List(&tags_history); | |
| 219 | ✗ | assert(retval); | |
| 220 | ✗ | for (unsigned i = 0, l = tags_history.size(); i < l; ++i) { | |
| 221 | ✗ | if ((tags_history[i].name == swissknife::CommandTag::kHeadTag) || | |
| 222 | ✗ | (tags_history[i].name == swissknife::CommandTag::kPreviousHeadTag)) | |
| 223 | { | ||
| 224 | ✗ | continue; | |
| 225 | } | ||
| 226 | ✗ | tags->push_back(TagId(tags_history[i].name, tags_history[i].root_hash)); | |
| 227 | } | ||
| 228 | ✗ | std::sort(tags->begin(), tags->end()); | |
| 229 | } | ||
| 230 | |||
| 231 | |||
| 232 | ✗ | void VirtualCatalog::GetSortedTagsFromCatalog(vector<TagId> *tags) { | |
| 233 | WritableCatalog *virtual_catalog = | ||
| 234 | ✗ | catalog_mgr_->GetHostingCatalog(kVirtualPath); | |
| 235 | ✗ | assert(virtual_catalog != NULL); | |
| 236 | Catalog::NestedCatalogList nested_catalogs = | ||
| 237 | ✗ | virtual_catalog->ListNestedCatalogs(); | |
| 238 | ✗ | for (unsigned i = 0, l = nested_catalogs.size(); i < l; ++i) { | |
| 239 | ✗ | tags->push_back(TagId(GetFileName(nested_catalogs[i].mountpoint).ToString(), | |
| 240 | ✗ | nested_catalogs[i].hash)); | |
| 241 | } | ||
| 242 | ✗ | std::sort(tags->begin(), tags->end()); | |
| 243 | } | ||
| 244 | |||
| 245 | |||
| 246 | ✗ | void VirtualCatalog::InsertSnapshot(TagId tag) { | |
| 247 | ✗ | LogCvmfs(kLogCatalog, kLogDebug, "add snapshot %s (%s) to virtual catalog", | |
| 248 | ✗ | tag.name.c_str(), tag.hash.ToString().c_str()); | |
| 249 | ✗ | UniquePtr<Catalog> catalog(assistant_.GetCatalog(tag.hash, | |
| 250 | ✗ | swissknife::Assistant::kOpenReadOnly)); | |
| 251 | ✗ | assert(catalog.IsValid()); | |
| 252 | ✗ | assert(catalog->root_prefix().IsEmpty()); | |
| 253 | ✗ | DirectoryEntry entry_root; | |
| 254 | ✗ | bool retval = catalog->LookupPath(PathString(""), &entry_root); | |
| 255 | ✗ | assert(retval); | |
| 256 | |||
| 257 | // Add directory entry | ||
| 258 | ✗ | DirectoryEntryBase entry_dir = entry_root; | |
| 259 | ✗ | entry_dir.name_ = NameString(tag.name); | |
| 260 | ✗ | catalog_mgr_->AddDirectory(entry_dir, XattrList(), | |
| 261 | ✗ | string(kVirtualPath) + "/" + string(kSnapshotDirectory)); | |
| 262 | |||
| 263 | // Set "bind mount" flag | ||
| 264 | WritableCatalog *virtual_catalog = | ||
| 265 | ✗ | catalog_mgr_->GetHostingCatalog(kVirtualPath); | |
| 266 | ✗ | assert(virtual_catalog != NULL); | |
| 267 | string mountpoint = | ||
| 268 | ✗ | "/" + string(kVirtualPath) + "/" + string(kSnapshotDirectory) + "/" + | |
| 269 | ✗ | tag.name; | |
| 270 | ✗ | DirectoryEntry entry_bind_mountpoint(entry_dir); | |
| 271 | ✗ | entry_bind_mountpoint.set_is_bind_mountpoint(true); | |
| 272 | ✗ | virtual_catalog->UpdateEntry(entry_bind_mountpoint, mountpoint); | |
| 273 | |||
| 274 | // Register nested catalog | ||
| 275 | ✗ | uint64_t catalog_size = GetFileSize(catalog->database_path()); | |
| 276 | ✗ | assert(catalog_size > 0); | |
| 277 | ✗ | virtual_catalog->InsertBindMountpoint(mountpoint, tag.hash, catalog_size); | |
| 278 | } | ||
| 279 | |||
| 280 | |||
| 281 | ✗ | void VirtualCatalog::Remove() { | |
| 282 | ✗ | LogCvmfs(kLogCvmfs, kLogStdout, "Removing .cvmfs virtual catalog"); | |
| 283 | |||
| 284 | // Safety check, make sure we don't remove the entire repository | ||
| 285 | WritableCatalog *virtual_catalog = | ||
| 286 | ✗ | catalog_mgr_->GetHostingCatalog(kVirtualPath); | |
| 287 | ✗ | assert(!virtual_catalog->IsRoot()); | |
| 288 | ✗ | DirectoryEntry entry_virtual; | |
| 289 | ✗ | bool retval = catalog_mgr_->LookupPath( | |
| 290 | ✗ | PathString("/" + string(kVirtualPath)), kLookupDefault, &entry_virtual); | |
| 291 | ✗ | assert(retval); | |
| 292 | ✗ | assert(entry_virtual.IsHidden()); | |
| 293 | |||
| 294 | ✗ | RemoveRecursively(kVirtualPath); | |
| 295 | ✗ | catalog_mgr_->RemoveNestedCatalog(kVirtualPath); | |
| 296 | ✗ | catalog_mgr_->RemoveDirectory(kVirtualPath); | |
| 297 | } | ||
| 298 | |||
| 299 | |||
| 300 | ✗ | void VirtualCatalog::RemoveRecursively(const string &directory) { | |
| 301 | ✗ | DirectoryEntryList listing; | |
| 302 | ✗ | bool retval = catalog_mgr_->Listing(PathString("/" + directory), &listing); | |
| 303 | ✗ | assert(retval); | |
| 304 | ✗ | for (unsigned i = 0; i < listing.size(); ++i) { | |
| 305 | ✗ | string this_path = directory + "/" + listing[i].name().ToString(); | |
| 306 | ✗ | if (listing[i].IsDirectory()) { | |
| 307 | ✗ | if (!listing[i].IsBindMountpoint()) | |
| 308 | ✗ | RemoveRecursively(this_path); | |
| 309 | ✗ | catalog_mgr_->RemoveDirectory(this_path); | |
| 310 | ✗ | } else if (listing[i].IsRegular()) { | |
| 311 | ✗ | assert(listing[i].name().ToString() == ".cvmfscatalog"); | |
| 312 | ✗ | catalog_mgr_->RemoveFile(this_path); | |
| 313 | } else { | ||
| 314 | ✗ | abort(); | |
| 315 | } | ||
| 316 | } | ||
| 317 | } | ||
| 318 | |||
| 319 | |||
| 320 | ✗ | void VirtualCatalog::RemoveSnapshot(TagId tag) { | |
| 321 | ✗ | LogCvmfs(kLogCatalog, kLogDebug, | |
| 322 | "remove snapshot %s (%s) from virtual catalog", | ||
| 323 | ✗ | tag.name.c_str(), tag.hash.ToString().c_str()); | |
| 324 | string tag_dir = | ||
| 325 | ✗ | string(kVirtualPath) + "/" + string(kSnapshotDirectory) + "/" + tag.name; | |
| 326 | ✗ | catalog_mgr_->RemoveDirectory(tag_dir); | |
| 327 | |||
| 328 | WritableCatalog *virtual_catalog = | ||
| 329 | ✗ | catalog_mgr_->GetHostingCatalog(kVirtualPath); | |
| 330 | ✗ | assert(virtual_catalog != NULL); | |
| 331 | ✗ | virtual_catalog->RemoveBindMountpoint("/" + tag_dir); | |
| 332 | } | ||
| 333 | |||
| 334 | |||
| 335 | ✗ | VirtualCatalog::VirtualCatalog( | |
| 336 | manifest::Manifest *m, | ||
| 337 | download::DownloadManager *d, | ||
| 338 | catalog::WritableCatalogManager *c, | ||
| 339 | ✗ | SyncParameters *p) | |
| 340 | ✗ | : catalog_mgr_(c) | |
| 341 | ✗ | , assistant_(d, m, p->stratum0, p->dir_temp) | |
| 342 | ✗ | { } | |
| 343 | |||
| 344 | } // namespace catalog | ||
| 345 |