| Directory: | cvmfs/ |
|---|---|
| File: | cvmfs/catalog_mgr_rw.h |
| Date: | 2025-07-13 02:35:07 |
| Exec | Total | Coverage | |
|---|---|---|---|
| Lines: | 10 | 19 | 52.6% |
| Branches: | 2 | 8 | 25.0% |
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /** | ||
| 2 | * This file is part of the CernVM File System. | ||
| 3 | * | ||
| 4 | * A WritableCatalogManager adds write functionality to the catalog | ||
| 5 | * manager. It is used for the server side of CVMFS | ||
| 6 | * All nested catalog loading functionality is inherited from | ||
| 7 | * AbstractCatalogManager. | ||
| 8 | * | ||
| 9 | * The WritableCatalogManager is provided with DirectoryEntryBase objects from | ||
| 10 | * the underlying sync infrastructure (namely SyncMediator) on the server side | ||
| 11 | * of CVMFS. In contrast to a full DirectoryEntry object DirectoryEntryBase con- | ||
| 12 | * tains only pure file system specific meta data (i.e. mtime, mode, filename). | ||
| 13 | * WritableCatalogManager is responsible for the addition and management of all | ||
| 14 | * CVMFS-specific meta data in a full DirectoryEntry, which is then saved into | ||
| 15 | * the actual Catalog databases. | ||
| 16 | * | ||
| 17 | * The inode assignment is based on the fact that the number of entries in a | ||
| 18 | * catalog do not change (except on reload). As we do exactly that with the | ||
| 19 | * WritableCatalogManager here, inode numbers derived from WritableCatalogs | ||
| 20 | * and the WritableCatalogManager may (and will) be screwed. This is not an | ||
| 21 | * issue in the current implementation, as they are not used in the syncing | ||
| 22 | * process. Just keep in mind. | ||
| 23 | * | ||
| 24 | * The WritableCatalogManager starts with a base repository (given by the | ||
| 25 | * root hash), and downloads and uncompresses all required catalogs into | ||
| 26 | * temporary storage. | ||
| 27 | */ | ||
| 28 | |||
| 29 | #ifndef CVMFS_CATALOG_MGR_RW_H_ | ||
| 30 | #define CVMFS_CATALOG_MGR_RW_H_ | ||
| 31 | |||
| 32 | #include <pthread.h> | ||
| 33 | #include <stdint.h> | ||
| 34 | |||
| 35 | #include <list> | ||
| 36 | #include <map> | ||
| 37 | #include <set> | ||
| 38 | #include <string> | ||
| 39 | #include <unordered_map> | ||
| 40 | #include <unordered_set> | ||
| 41 | |||
| 42 | #include "catalog_downloader.h" | ||
| 43 | #include "catalog_mgr_ro.h" | ||
| 44 | #include "catalog_rw.h" | ||
| 45 | #include "file_chunk.h" | ||
| 46 | #include "ingestion/pipeline.h" | ||
| 47 | #include "upload_spooler_result.h" | ||
| 48 | #include "util/future.h" | ||
| 49 | #include "xattr.h" | ||
| 50 | |||
| 51 | class XattrList; | ||
| 52 | namespace upload { | ||
| 53 | class Spooler; | ||
| 54 | } | ||
| 55 | |||
| 56 | namespace download { | ||
| 57 | class DownloadManager; | ||
| 58 | } | ||
| 59 | |||
| 60 | namespace manifest { | ||
| 61 | class Manifest; | ||
| 62 | } | ||
| 63 | |||
| 64 | namespace perf { | ||
| 65 | class Statistics; | ||
| 66 | } | ||
| 67 | |||
| 68 | namespace catalog { | ||
| 69 | template<class CatalogMgrT> | ||
| 70 | class CatalogBalancer; | ||
| 71 | } | ||
| 72 | |||
| 73 | namespace catalog { | ||
| 74 | |||
| 75 | class WritableCatalogManager : public SimpleCatalogManager { | ||
| 76 | friend class CatalogBalancer<WritableCatalogManager>; | ||
| 77 | // TODO(jblomer): only needed to get Spooler's hash algorithm. Remove me | ||
| 78 | // after refactoring of the swissknife utility. | ||
| 79 | friend class VirtualCatalog; | ||
| 80 | |||
| 81 | public: | ||
| 82 | WritableCatalogManager(const shash::Any &base_hash, | ||
| 83 | const std::string &stratum0, | ||
| 84 | const std::string &dir_temp, | ||
| 85 | upload::Spooler *spooler, | ||
| 86 | download::DownloadManager *download_manager, | ||
| 87 | bool enforce_limits, | ||
| 88 | const unsigned nested_kcatalog_limit, | ||
| 89 | const unsigned root_kcatalog_limit, | ||
| 90 | const unsigned file_mbyte_limit, | ||
| 91 | perf::Statistics *statistics, | ||
| 92 | bool is_balanceable, | ||
| 93 | unsigned max_weight, | ||
| 94 | unsigned min_weight, | ||
| 95 | const std::string &dir_cache = ""); | ||
| 96 | ~WritableCatalogManager(); | ||
| 97 | static manifest::Manifest *CreateRepository(const std::string &dir_temp, | ||
| 98 | const bool volatile_content, | ||
| 99 | const std::string &voms_authz, | ||
| 100 | upload::Spooler *spooler); | ||
| 101 | |||
| 102 | // DirectoryEntry handling | ||
| 103 | 5323 | void AddFile(const DirectoryEntryBase &entry, | |
| 104 | const XattrList &xattrs, | ||
| 105 | const std::string &parent_directory) { | ||
| 106 |
1/2✓ Branch 2 taken 5323 times.
✗ Branch 3 not taken.
|
5323 | AddFile(DirectoryEntry(entry), xattrs, parent_directory); |
| 107 | 5323 | } | |
| 108 | void AddChunkedFile(const DirectoryEntryBase &entry, | ||
| 109 | const XattrList &xattrs, | ||
| 110 | const std::string &parent_directory, | ||
| 111 | const FileChunkList &file_chunks); | ||
| 112 | void RemoveFile(const std::string &file_path); | ||
| 113 | |||
| 114 | void AddDirectory(const DirectoryEntryBase &entry, | ||
| 115 | const XattrList &xattrs, | ||
| 116 | const std::string &parent_directory); | ||
| 117 | void TouchDirectory(const DirectoryEntryBase &entry, | ||
| 118 | const XattrList &xattrs, | ||
| 119 | const std::string &directory_path); | ||
| 120 | void RemoveDirectory(const std::string &directory_path); | ||
| 121 | |||
| 122 | void Clone(const std::string from, const std::string to); | ||
| 123 | void CloneTree(const std::string &from_dir, const std::string &to_dir); | ||
| 124 | |||
| 125 | // Hardlink group handling | ||
| 126 | void AddHardlinkGroup(const DirectoryEntryBaseList &entries, | ||
| 127 | const XattrList &xattrs, | ||
| 128 | const std::string &parent_directory, | ||
| 129 | const FileChunkList &file_chunks); | ||
| 130 | void ShrinkHardlinkGroup(const std::string &remove_path); | ||
| 131 | |||
| 132 | // Nested catalog handling | ||
| 133 | void CreateNestedCatalog(const std::string &mountpoint); | ||
| 134 | void RemoveNestedCatalog(const std::string &mountpoint, | ||
| 135 | const bool merge = true); | ||
| 136 | void SwapNestedCatalog(const string &mountpoint, const shash::Any &new_hash, | ||
| 137 | const uint64_t new_size); | ||
| 138 | void GraftNestedCatalog(const string &mountpoint, const shash::Any &new_hash, | ||
| 139 | const uint64_t new_size); | ||
| 140 | bool IsTransitionPoint(const std::string &mountpoint); | ||
| 141 | WritableCatalog *GetHostingCatalog(const std::string &path); | ||
| 142 | |||
| 143 | ✗ | inline bool IsBalanceable() const { return is_balanceable_; } | |
| 144 | /** | ||
| 145 | * TODO | ||
| 146 | */ | ||
| 147 | void PrecalculateListings(); | ||
| 148 | |||
| 149 | void SetTTL(const uint64_t new_ttl); | ||
| 150 | bool SetVOMSAuthz(const std::string &voms_authz); | ||
| 151 | bool Commit(const bool stop_for_tweaks, | ||
| 152 | const uint64_t manual_revision, | ||
| 153 | manifest::Manifest *manifest); | ||
| 154 | |||
| 155 | ✗ | void Balance() { | |
| 156 | ✗ | if (IsBalanceable()) { | |
| 157 | ✗ | DoBalance(); | |
| 158 | } else { | ||
| 159 | ✗ | LogCvmfs(kLogCatalog, kLogVerboseMsg, | |
| 160 | "Not balancing the catalog " | ||
| 161 | "manager because it is not balanceable"); | ||
| 162 | } | ||
| 163 | } | ||
| 164 | |||
| 165 | void LoadCatalogs(const std::string &base_path, | ||
| 166 | const std::unordered_set<std::string> &dirs); | ||
| 167 | void SetupSingleCatalogUploadCallback(); | ||
| 168 | void RemoveSingleCatalogUploadCallback(); | ||
| 169 | void AddCatalogToQueue(const std::string &path); | ||
| 170 | void ScheduleReadyCatalogs(); | ||
| 171 | bool LookupDirEntry(const std::string &path, | ||
| 172 | const LookupOptions options, | ||
| 173 | DirectoryEntry *dirent); | ||
| 174 | |||
| 175 | |||
| 176 | protected: | ||
| 177 | 2258 | void EnforceSqliteMemLimit() { } | |
| 178 | |||
| 179 | Catalog *CreateCatalog(const PathString &mountpoint, | ||
| 180 | const shash::Any &catalog_hash, | ||
| 181 | Catalog *parent_catalog); | ||
| 182 | void ActivateCatalog(Catalog *catalog); | ||
| 183 | |||
| 184 | void AddFile(const DirectoryEntry &entry, | ||
| 185 | const XattrList &xattrs, | ||
| 186 | const std::string &parent_directory); | ||
| 187 | |||
| 188 | private: | ||
| 189 | bool FindCatalog(const std::string &path, | ||
| 190 | WritableCatalog **result, | ||
| 191 | DirectoryEntry *dirent = NULL); | ||
| 192 | void DoBalance(); | ||
| 193 | void FixWeight(WritableCatalog *catalog); | ||
| 194 | |||
| 195 | void CloneTreeImpl(const PathString &source_dir, | ||
| 196 | const std::string &dest_parent_dir, | ||
| 197 | const NameString &dest_name); | ||
| 198 | |||
| 199 | struct CatalogInfo { | ||
| 200 | uint64_t ttl; | ||
| 201 | size_t size; | ||
| 202 | shash::Any content_hash; | ||
| 203 | uint64_t revision; | ||
| 204 | }; | ||
| 205 | |||
| 206 | struct CatalogUploadContext { | ||
| 207 | Future<CatalogInfo> *root_catalog_info; | ||
| 208 | bool stop_for_tweaks; | ||
| 209 | }; | ||
| 210 | |||
| 211 | struct CatalogDownloadContext { | ||
| 212 | const std::unordered_set<std::string> *dirs; | ||
| 213 | }; | ||
| 214 | |||
| 215 | void CatalogDownloadCallback(const CatalogDownloadResult &result, | ||
| 216 | const CatalogDownloadContext context); | ||
| 217 | void SingleCatalogUploadCallback(const upload::SpoolerResult &result); | ||
| 218 | |||
| 219 | |||
| 220 | CatalogInfo SnapshotCatalogs(const bool stop_for_tweaks); | ||
| 221 | void FinalizeCatalog(WritableCatalog *catalog, const bool stop_for_tweaks); | ||
| 222 | void ScheduleCatalogProcessing(WritableCatalog *catalog); | ||
| 223 | |||
| 224 | 612 | void GetModifiedCatalogLeafs(WritableCatalogList *result) const { | |
| 225 | 612 | const bool dirty = GetModifiedCatalogLeafsRecursively(GetRootCatalog(), | |
| 226 | result); | ||
| 227 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 612 times.
|
612 | assert(dirty); |
| 228 | 612 | } | |
| 229 | bool GetModifiedCatalogLeafsRecursively(Catalog *catalog, | ||
| 230 | WritableCatalogList *result) const; | ||
| 231 | |||
| 232 | void CatalogUploadCallback(const upload::SpoolerResult &result, | ||
| 233 | const CatalogUploadContext clg_upload_context); | ||
| 234 | |||
| 235 | bool CopyCatalogToLocalCache(const upload::SpoolerResult &result); | ||
| 236 | |||
| 237 | private: | ||
| 238 | 15360 | inline void SyncLock() { pthread_mutex_lock(sync_lock_); } | |
| 239 | 15311 | inline void SyncUnlock() { pthread_mutex_unlock(sync_lock_); } | |
| 240 | |||
| 241 | //**************************************************************************** | ||
| 242 | // Workaround -- Serialized Catalog Committing | ||
| 243 | ✗ | void GetModifiedCatalogs(WritableCatalogList *result) const { | |
| 244 | const unsigned int | ||
| 245 | ✗ | number_of_dirty_catalogs = GetModifiedCatalogsRecursively( | |
| 246 | ✗ | GetRootCatalog(), result); | |
| 247 | ✗ | assert(number_of_dirty_catalogs <= result->size()); | |
| 248 | } | ||
| 249 | int GetModifiedCatalogsRecursively(const Catalog *catalog, | ||
| 250 | WritableCatalogList *result) const; | ||
| 251 | void CatalogUploadSerializedCallback(const upload::SpoolerResult &result, | ||
| 252 | const CatalogUploadContext unused); | ||
| 253 | CatalogInfo SnapshotCatalogsSerialized(const bool stop_for_tweaks); | ||
| 254 | void CatalogHashSerializedCallback(const CompressHashResult &result); | ||
| 255 | //**************************************************************************** | ||
| 256 | |||
| 257 | // defined in catalog_mgr_rw.cc | ||
| 258 | static const std::string kCatalogFilename; | ||
| 259 | |||
| 260 | // private lock of WritableCatalogManager | ||
| 261 | pthread_mutex_t *sync_lock_; | ||
| 262 | upload::Spooler *spooler_; | ||
| 263 | |||
| 264 | pthread_mutex_t *catalog_processing_lock_; | ||
| 265 | std::map<std::string, WritableCatalog *> catalog_processing_map_; | ||
| 266 | |||
| 267 | // ingestsql | ||
| 268 | std::list<WritableCatalog *> pending_catalogs_; | ||
| 269 | CatalogDownloadPipeline *catalog_download_pipeline_; | ||
| 270 | pthread_mutex_t *catalog_download_lock_; | ||
| 271 | std::unordered_map<std::string, Catalog *> catalog_download_map_; | ||
| 272 | pthread_mutex_t *catalog_hash_lock_; | ||
| 273 | std::map<std::string, shash::Any> catalog_hash_map_; | ||
| 274 | |||
| 275 | // TODO(jblomer): catalog limits should become its own struct | ||
| 276 | bool enforce_limits_; | ||
| 277 | unsigned nested_kcatalog_limit_; | ||
| 278 | unsigned root_kcatalog_limit_; | ||
| 279 | unsigned file_mbyte_limit_; | ||
| 280 | |||
| 281 | /** | ||
| 282 | * Directories don't have extended attributes at this point. | ||
| 283 | */ | ||
| 284 | XattrList empty_xattrs; | ||
| 285 | |||
| 286 | /** | ||
| 287 | * It indicates whether this catalog manager supports balancing operations | ||
| 288 | */ | ||
| 289 | const bool is_balanceable_; | ||
| 290 | |||
| 291 | /** | ||
| 292 | * Defines the maximum weight an autogenerated catalog can have. If after a | ||
| 293 | * publishing operation the catalog's weight is greater than this threshold it | ||
| 294 | * will be considered overflowed and will automatically be split in different | ||
| 295 | * sub-catalogs. | ||
| 296 | */ | ||
| 297 | const unsigned max_weight_; | ||
| 298 | |||
| 299 | /** | ||
| 300 | * Defines the minimum weight an autogenerated catalog can have. If after a | ||
| 301 | * publishing operation the catalog's weight is lesser than this threshold it | ||
| 302 | * will be considered underflowed and will automatically be merged with its | ||
| 303 | * parent. | ||
| 304 | * This last operation can provoke an overflow in the parent, though. | ||
| 305 | */ | ||
| 306 | const unsigned min_weight_; | ||
| 307 | |||
| 308 | /** | ||
| 309 | * Defines the threshold that will be used to balance a catalog that has been | ||
| 310 | * overflowed. Its value should be lesser than max_weight_ and greater than | ||
| 311 | * min_weight. By default it is set to max_weight / 2. | ||
| 312 | */ | ||
| 313 | const unsigned balance_weight_; | ||
| 314 | }; // class WritableCatalogManager | ||
| 315 | |||
| 316 | } // namespace catalog | ||
| 317 | |||
| 318 | #endif // CVMFS_CATALOG_MGR_RW_H_ | ||
| 319 |