| Directory: | cvmfs/ |
|---|---|
| File: | cvmfs/catalog_mgr_rw.h |
| Date: | 2026-04-26 02:35:59 |
| Exec | Total | Coverage | |
|---|---|---|---|
| Lines: | 10 | 19 | 52.6% |
| Branches: | 2 | 8 | 25.0% |
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /** | ||
| 2 | * This file is part of the CernVM File System. | ||
| 3 | * | ||
| 4 | * A WritableCatalogManager adds write functionality to the catalog | ||
| 5 | * manager. It is used for the server side of CVMFS | ||
| 6 | * All nested catalog loading functionality is inherited from | ||
| 7 | * AbstractCatalogManager. | ||
| 8 | * | ||
| 9 | * The WritableCatalogManager is provided with DirectoryEntryBase objects from | ||
| 10 | * the underlying sync infrastructure (namely SyncMediator) on the server side | ||
| 11 | * of CVMFS. In contrast to a full DirectoryEntry object DirectoryEntryBase con- | ||
| 12 | * tains only pure file system specific meta data (i.e. mtime, mode, filename). | ||
| 13 | * WritableCatalogManager is responsible for the addition and management of all | ||
| 14 | * CVMFS-specific meta data in a full DirectoryEntry, which is then saved into | ||
| 15 | * the actual Catalog databases. | ||
| 16 | * | ||
| 17 | * The inode assignment is based on the fact that the number of entries in a | ||
| 18 | * catalog do not change (except on reload). As we do exactly that with the | ||
| 19 | * WritableCatalogManager here, inode numbers derived from WritableCatalogs | ||
| 20 | * and the WritableCatalogManager may (and will) be screwed. This is not an | ||
| 21 | * issue in the current implementation, as they are not used in the syncing | ||
| 22 | * process. Just keep in mind. | ||
| 23 | * | ||
| 24 | * The WritableCatalogManager starts with a base repository (given by the | ||
| 25 | * root hash), and downloads and uncompresses all required catalogs into | ||
| 26 | * temporary storage. | ||
| 27 | */ | ||
| 28 | |||
| 29 | #ifndef CVMFS_CATALOG_MGR_RW_H_ | ||
| 30 | #define CVMFS_CATALOG_MGR_RW_H_ | ||
| 31 | |||
| 32 | #include <pthread.h> | ||
| 33 | #include <stdint.h> | ||
| 34 | |||
| 35 | #include <list> | ||
| 36 | #include <map> | ||
| 37 | #include <string> | ||
| 38 | #include <unordered_map> | ||
| 39 | #include <unordered_set> | ||
| 40 | |||
| 41 | #include "catalog_downloader.h" | ||
| 42 | #include "catalog_mgr_ro.h" | ||
| 43 | #include "catalog_rw.h" | ||
| 44 | #include "file_chunk.h" | ||
| 45 | #include "ingestion/pipeline.h" | ||
| 46 | #include "upload_spooler_result.h" | ||
| 47 | #include "util/future.h" | ||
| 48 | #include "xattr.h" | ||
| 49 | |||
| 50 | class XattrList; | ||
| 51 | namespace upload { | ||
| 52 | class Spooler; | ||
| 53 | } | ||
| 54 | |||
| 55 | namespace download { | ||
| 56 | class DownloadManager; | ||
| 57 | } | ||
| 58 | |||
| 59 | namespace manifest { | ||
| 60 | class Manifest; | ||
| 61 | } | ||
| 62 | |||
| 63 | namespace perf { | ||
| 64 | class Statistics; | ||
| 65 | } | ||
| 66 | |||
| 67 | namespace catalog { | ||
| 68 | template<class CatalogMgrT> | ||
| 69 | class CatalogBalancer; | ||
| 70 | } | ||
| 71 | |||
| 72 | namespace catalog { | ||
| 73 | |||
| 74 | class WritableCatalogManager : public SimpleCatalogManager { | ||
| 75 | friend class CatalogBalancer<WritableCatalogManager>; | ||
| 76 | // TODO(jblomer): only needed to get Spooler's hash algorithm. Remove me | ||
| 77 | // after refactoring of the swissknife utility. | ||
| 78 | friend class VirtualCatalog; | ||
| 79 | |||
| 80 | public: | ||
| 81 | WritableCatalogManager(const shash::Any &base_hash, | ||
| 82 | const std::string &stratum0, | ||
| 83 | const std::string &dir_temp, | ||
| 84 | upload::Spooler *spooler, | ||
| 85 | download::DownloadManager *download_manager, | ||
| 86 | bool enforce_limits, | ||
| 87 | const unsigned nested_kcatalog_limit, | ||
| 88 | const unsigned root_kcatalog_limit, | ||
| 89 | const unsigned file_mbyte_limit, | ||
| 90 | perf::Statistics *statistics, | ||
| 91 | bool is_balanceable, | ||
| 92 | unsigned max_weight, | ||
| 93 | unsigned min_weight, | ||
| 94 | const std::string &dir_cache = ""); | ||
| 95 | ~WritableCatalogManager(); | ||
| 96 | static manifest::Manifest *CreateRepository(const std::string &dir_temp, | ||
| 97 | const bool volatile_content, | ||
| 98 | const std::string &voms_authz, | ||
| 99 | upload::Spooler *spooler); | ||
| 100 | |||
| 101 | // DirectoryEntry handling | ||
| 102 | 3556 | void AddFile(const DirectoryEntryBase &entry, | |
| 103 | const XattrList &xattrs, | ||
| 104 | const std::string &parent_directory) { | ||
| 105 |
1/2✓ Branch 2 taken 3556 times.
✗ Branch 3 not taken.
|
3556 | AddFile(DirectoryEntry(entry), xattrs, parent_directory); |
| 106 | 3556 | } | |
| 107 | void AddChunkedFile(const DirectoryEntryBase &entry, | ||
| 108 | const XattrList &xattrs, | ||
| 109 | const std::string &parent_directory, | ||
| 110 | const FileChunkList &file_chunks); | ||
| 111 | void RemoveFile(const std::string &file_path); | ||
| 112 | |||
| 113 | void AddDirectory(const DirectoryEntryBase &entry, | ||
| 114 | const XattrList &xattrs, | ||
| 115 | const std::string &parent_directory); | ||
| 116 | void TouchDirectory(const DirectoryEntryBase &entry, | ||
| 117 | const XattrList &xattrs, | ||
| 118 | const std::string &directory_path); | ||
| 119 | void RemoveDirectory(const std::string &directory_path); | ||
| 120 | |||
| 121 | void Clone(const std::string from, const std::string to); | ||
| 122 | void CloneTree(const std::string &from_dir, const std::string &to_dir); | ||
| 123 | |||
| 124 | // Hardlink group handling | ||
| 125 | void AddHardlinkGroup(const DirectoryEntryBaseList &entries, | ||
| 126 | const XattrList &xattrs, | ||
| 127 | const std::string &parent_directory, | ||
| 128 | const FileChunkList &file_chunks); | ||
| 129 | void ShrinkHardlinkGroup(const std::string &remove_path); | ||
| 130 | |||
| 131 | // Nested catalog handling | ||
| 132 | void CreateNestedCatalog(const std::string &mountpoint); | ||
| 133 | void RemoveNestedCatalog(const std::string &mountpoint, | ||
| 134 | const bool merge = true); | ||
| 135 | void SwapNestedCatalog(const string &mountpoint, const shash::Any &new_hash, | ||
| 136 | const uint64_t new_size); | ||
| 137 | void GraftNestedCatalog(const string &mountpoint, const shash::Any &new_hash, | ||
| 138 | const uint64_t new_size); | ||
| 139 | bool IsTransitionPoint(const std::string &mountpoint); | ||
| 140 | WritableCatalog *GetHostingCatalog(const std::string &path); | ||
| 141 | |||
| 142 | ✗ | inline bool IsBalanceable() const { return is_balanceable_; } | |
| 143 | /** | ||
| 144 | * TODO | ||
| 145 | */ | ||
| 146 | void PrecalculateListings(); | ||
| 147 | |||
| 148 | void SetTTL(const uint64_t new_ttl); | ||
| 149 | bool SetVOMSAuthz(const std::string &voms_authz); | ||
| 150 | bool Commit(const bool stop_for_tweaks, | ||
| 151 | const uint64_t manual_revision, | ||
| 152 | manifest::Manifest *manifest); | ||
| 153 | |||
| 154 | ✗ | void Balance() { | |
| 155 | ✗ | if (IsBalanceable()) { | |
| 156 | ✗ | DoBalance(); | |
| 157 | } else { | ||
| 158 | ✗ | LogCvmfs(kLogCatalog, kLogVerboseMsg, | |
| 159 | "Not balancing the catalog " | ||
| 160 | "manager because it is not balanceable"); | ||
| 161 | } | ||
| 162 | } | ||
| 163 | |||
| 164 | void LoadCatalogs(const std::string &base_path, | ||
| 165 | const std::unordered_set<std::string> &dirs); | ||
| 166 | void SetupSingleCatalogUploadCallback(); | ||
| 167 | void RemoveSingleCatalogUploadCallback(); | ||
| 168 | void AddCatalogToQueue(const std::string &path); | ||
| 169 | void ScheduleReadyCatalogs(); | ||
| 170 | bool LookupDirEntry(const std::string &path, | ||
| 171 | const LookupOptions options, | ||
| 172 | DirectoryEntry *dirent); | ||
| 173 | |||
| 174 | |||
| 175 | protected: | ||
| 176 | 775 | void EnforceSqliteMemLimit() { } | |
| 177 | |||
| 178 | Catalog *CreateCatalog(const PathString &mountpoint, | ||
| 179 | const shash::Any &catalog_hash, | ||
| 180 | Catalog *parent_catalog); | ||
| 181 | void ActivateCatalog(Catalog *catalog); | ||
| 182 | |||
| 183 | void AddFile(const DirectoryEntry &entry, | ||
| 184 | const XattrList &xattrs, | ||
| 185 | const std::string &parent_directory); | ||
| 186 | |||
| 187 | private: | ||
| 188 | bool FindCatalog(const std::string &path, | ||
| 189 | WritableCatalog **result, | ||
| 190 | DirectoryEntry *dirent = NULL); | ||
| 191 | void DoBalance(); | ||
| 192 | void FixWeight(WritableCatalog *catalog); | ||
| 193 | |||
| 194 | void CloneTreeImpl(const PathString &source_dir, | ||
| 195 | const std::string &dest_parent_dir, | ||
| 196 | const NameString &dest_name); | ||
| 197 | |||
| 198 | struct CatalogInfo { | ||
| 199 | uint64_t ttl; | ||
| 200 | size_t size; | ||
| 201 | shash::Any content_hash; | ||
| 202 | uint64_t revision; | ||
| 203 | }; | ||
| 204 | |||
| 205 | struct CatalogUploadContext { | ||
| 206 | Future<CatalogInfo> *root_catalog_info; | ||
| 207 | bool stop_for_tweaks; | ||
| 208 | }; | ||
| 209 | |||
| 210 | struct CatalogDownloadContext { | ||
| 211 | const std::unordered_set<std::string> *dirs; | ||
| 212 | }; | ||
| 213 | |||
| 214 | void CatalogDownloadCallback(const CatalogDownloadResult &result, | ||
| 215 | const CatalogDownloadContext context); | ||
| 216 | void SingleCatalogUploadCallback(const upload::SpoolerResult &result); | ||
| 217 | |||
| 218 | |||
| 219 | CatalogInfo SnapshotCatalogs(const bool stop_for_tweaks); | ||
| 220 | void FinalizeCatalog(WritableCatalog *catalog, const bool stop_for_tweaks); | ||
| 221 | void ScheduleCatalogProcessing(WritableCatalog *catalog); | ||
| 222 | |||
| 223 | 663 | void GetModifiedCatalogLeafs(WritableCatalogList *result) const { | |
| 224 | 663 | const bool dirty = GetModifiedCatalogLeafsRecursively(GetRootCatalog(), | |
| 225 | result); | ||
| 226 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 663 times.
|
663 | assert(dirty); |
| 227 | 663 | } | |
| 228 | bool GetModifiedCatalogLeafsRecursively(Catalog *catalog, | ||
| 229 | WritableCatalogList *result) const; | ||
| 230 | |||
| 231 | void CatalogUploadCallback(const upload::SpoolerResult &result, | ||
| 232 | const CatalogUploadContext clg_upload_context); | ||
| 233 | |||
| 234 | bool CopyCatalogToLocalCache(const upload::SpoolerResult &result); | ||
| 235 | |||
| 236 | private: | ||
| 237 | 9154 | inline void SyncLock() { pthread_mutex_lock(sync_lock_); } | |
| 238 | 9150 | inline void SyncUnlock() { pthread_mutex_unlock(sync_lock_); } | |
| 239 | |||
| 240 | //**************************************************************************** | ||
| 241 | // Workaround -- Serialized Catalog Committing | ||
| 242 | ✗ | void GetModifiedCatalogs(WritableCatalogList *result) const { | |
| 243 | const unsigned int | ||
| 244 | ✗ | number_of_dirty_catalogs = GetModifiedCatalogsRecursively( | |
| 245 | ✗ | GetRootCatalog(), result); | |
| 246 | ✗ | assert(number_of_dirty_catalogs <= result->size()); | |
| 247 | } | ||
| 248 | int GetModifiedCatalogsRecursively(const Catalog *catalog, | ||
| 249 | WritableCatalogList *result) const; | ||
| 250 | void CatalogUploadSerializedCallback(const upload::SpoolerResult &result, | ||
| 251 | const CatalogUploadContext unused); | ||
| 252 | CatalogInfo SnapshotCatalogsSerialized(const bool stop_for_tweaks); | ||
| 253 | void CatalogHashSerializedCallback(const CompressHashResult &result); | ||
| 254 | //**************************************************************************** | ||
| 255 | |||
| 256 | // defined in catalog_mgr_rw.cc | ||
| 257 | static const std::string kCatalogFilename; | ||
| 258 | |||
| 259 | // private lock of WritableCatalogManager | ||
| 260 | pthread_mutex_t *sync_lock_; | ||
| 261 | upload::Spooler *spooler_; | ||
| 262 | |||
| 263 | pthread_mutex_t *catalog_processing_lock_; | ||
| 264 | std::map<std::string, WritableCatalog *> catalog_processing_map_; | ||
| 265 | |||
| 266 | // ingestsql | ||
| 267 | std::list<WritableCatalog *> pending_catalogs_; | ||
| 268 | CatalogDownloadPipeline *catalog_download_pipeline_; | ||
| 269 | pthread_mutex_t *catalog_download_lock_; | ||
| 270 | std::unordered_map<std::string, Catalog *> catalog_download_map_; | ||
| 271 | pthread_mutex_t *catalog_hash_lock_; | ||
| 272 | std::map<std::string, shash::Any> catalog_hash_map_; | ||
| 273 | |||
| 274 | // TODO(jblomer): catalog limits should become its own struct | ||
| 275 | bool enforce_limits_; | ||
| 276 | unsigned nested_kcatalog_limit_; | ||
| 277 | unsigned root_kcatalog_limit_; | ||
| 278 | unsigned file_mbyte_limit_; | ||
| 279 | |||
| 280 | /** | ||
| 281 | * Directories don't have extended attributes at this point. | ||
| 282 | */ | ||
| 283 | XattrList empty_xattrs; | ||
| 284 | |||
| 285 | /** | ||
| 286 | * It indicates whether this catalog manager supports balancing operations | ||
| 287 | */ | ||
| 288 | const bool is_balanceable_; | ||
| 289 | |||
| 290 | /** | ||
| 291 | * Defines the maximum weight an autogenerated catalog can have. If after a | ||
| 292 | * publishing operation the catalog's weight is greater than this threshold it | ||
| 293 | * will be considered overflowed and will automatically be split in different | ||
| 294 | * sub-catalogs. | ||
| 295 | */ | ||
| 296 | const unsigned max_weight_; | ||
| 297 | |||
| 298 | /** | ||
| 299 | * Defines the minimum weight an autogenerated catalog can have. If after a | ||
| 300 | * publishing operation the catalog's weight is lesser than this threshold it | ||
| 301 | * will be considered underflowed and will automatically be merged with its | ||
| 302 | * parent. | ||
| 303 | * This last operation can provoke an overflow in the parent, though. | ||
| 304 | */ | ||
| 305 | const unsigned min_weight_; | ||
| 306 | |||
| 307 | /** | ||
| 308 | * Defines the threshold that will be used to balance a catalog that has been | ||
| 309 | * overflowed. Its value should be lesser than max_weight_ and greater than | ||
| 310 | * min_weight. By default it is set to max_weight / 2. | ||
| 311 | */ | ||
| 312 | const unsigned balance_weight_; | ||
| 313 | }; // class WritableCatalogManager | ||
| 314 | |||
| 315 | } // namespace catalog | ||
| 316 | |||
| 317 | #endif // CVMFS_CATALOG_MGR_RW_H_ | ||
| 318 |