GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/catalog_mgr_rw.h
Date: 2026-06-28 02:36:10
Exec Total Coverage
Lines: 10 19 52.6%
Branches: 2 8 25.0%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * A WritableCatalogManager adds write functionality to the catalog
5 * manager. It is used for the server side of CVMFS
6 * All nested catalog loading functionality is inherited from
7 * AbstractCatalogManager.
8 *
9 * The WritableCatalogManager is provided with DirectoryEntryBase objects from
10 * the underlying sync infrastructure (namely SyncMediator) on the server side
11 * of CVMFS. In contrast to a full DirectoryEntry object DirectoryEntryBase con-
12 * tains only pure file system specific meta data (i.e. mtime, mode, filename).
13 * WritableCatalogManager is responsible for the addition and management of all
14 * CVMFS-specific meta data in a full DirectoryEntry, which is then saved into
15 * the actual Catalog databases.
16 *
17 * The inode assignment is based on the fact that the number of entries in a
18 * catalog do not change (except on reload). As we do exactly that with the
19 * WritableCatalogManager here, inode numbers derived from WritableCatalogs
20 * and the WritableCatalogManager may (and will) be screwed. This is not an
21 * issue in the current implementation, as they are not used in the syncing
22 * process. Just keep in mind.
23 *
24 * The WritableCatalogManager starts with a base repository (given by the
25 * root hash), and downloads and uncompresses all required catalogs into
26 * temporary storage.
27 */
28
29 #ifndef CVMFS_CATALOG_MGR_RW_H_
30 #define CVMFS_CATALOG_MGR_RW_H_
31
32 #include <pthread.h>
33 #include <stdint.h>
34
35 #include <list>
36 #include <map>
37 #include <string>
38 #include <unordered_map>
39 #include <unordered_set>
40
41 #include "catalog_downloader.h"
42 #include "catalog_mgr_ro.h"
43 #include "catalog_rw.h"
44 #include "file_chunk.h"
45 #include "ingestion/pipeline.h"
46 #include "upload_spooler_result.h"
47 #include "util/future.h"
48 #include "xattr.h"
49
50 class XattrList;
51 namespace upload {
52 class Spooler;
53 }
54
55 namespace download {
56 class DownloadManager;
57 }
58
59 namespace manifest {
60 class Manifest;
61 }
62
63 namespace perf {
64 class Statistics;
65 }
66
67 namespace catalog {
68 template<class CatalogMgrT>
69 class CatalogBalancer;
70 }
71
72 namespace catalog {
73
74 class WritableCatalogManager : public SimpleCatalogManager {
75 friend class CatalogBalancer<WritableCatalogManager>;
76 // TODO(jblomer): only needed to get Spooler's hash algorithm. Remove me
77 // after refactoring of the swissknife utility.
78 friend class VirtualCatalog;
79
80 public:
81 WritableCatalogManager(const shash::Any &base_hash,
82 const std::string &stratum0,
83 const std::string &dir_temp,
84 upload::Spooler *spooler,
85 download::DownloadManager *download_manager,
86 bool enforce_limits,
87 const unsigned nested_kcatalog_limit,
88 const unsigned root_kcatalog_limit,
89 const unsigned file_mbyte_limit,
90 perf::Statistics *statistics,
91 bool is_balanceable,
92 unsigned max_weight,
93 unsigned min_weight,
94 const std::string &dir_cache = "");
95 ~WritableCatalogManager();
96 static manifest::Manifest *CreateRepository(const std::string &dir_temp,
97 const bool volatile_content,
98 const std::string &voms_authz,
99 upload::Spooler *spooler);
100
101 // DirectoryEntry handling
102 5959 void AddFile(const DirectoryEntryBase &entry,
103 const XattrList &xattrs,
104 const std::string &parent_directory) {
105
1/2
✓ Branch 2 taken 5959 times.
✗ Branch 3 not taken.
 5959 AddFile(DirectoryEntry(entry), xattrs, parent_directory);
106 5959 }
107 void AddChunkedFile(const DirectoryEntryBase &entry,
108 const XattrList &xattrs,
109 const std::string &parent_directory,
110 const FileChunkList &file_chunks);
111 void RemoveFile(const std::string &file_path);
112
113 void UpdateBundleTrigger(const std::string &file_path, bool new_value);
114
115 void AddDirectory(const DirectoryEntryBase &entry,
116 const XattrList &xattrs,
117 const std::string &parent_directory);
118 void TouchDirectory(const DirectoryEntryBase &entry,
119 const XattrList &xattrs,
120 const std::string &directory_path);
121 void RemoveDirectory(const std::string &directory_path);
122
123 bool Clone(const std::string from, const std::string to,
124 const bool fail_if_source_missing = true);
125 void CloneTree(const std::string &from_dir, const std::string &to_dir);
126
127 // Hardlink group handling
128 void AddHardlinkGroup(const DirectoryEntryBaseList &entries,
129 const XattrList &xattrs,
130 const std::string &parent_directory,
131 const FileChunkList &file_chunks);
132 void ShrinkHardlinkGroup(const std::string &remove_path);
133
134 // Nested catalog handling
135 void CreateNestedCatalog(const std::string &mountpoint);
136 void RemoveNestedCatalog(const std::string &mountpoint,
137 const bool merge = true);
138 void SwapNestedCatalog(const string &mountpoint, const shash::Any &new_hash,
139 const uint64_t new_size);
140 void GraftNestedCatalog(const string &mountpoint, const shash::Any &new_hash,
141 const uint64_t new_size);
142 // Non-panicking variant used by the experimental DirectGraft gateway
143 // endpoint. Returns false for expected request validation failures (wrong
144 // catalog root, existing target directory, missing parent, etc.) so malformed
145 // experimental requests do not abort the receiver worker.
146 bool TryGraftNestedCatalog(const string &mountpoint,
147 const shash::Any &new_hash,
148 const uint64_t new_size);
149 bool IsTransitionPoint(const std::string &mountpoint);
150 WritableCatalog *GetHostingCatalog(const std::string &path);
151
152 ✗ inline bool IsBalanceable() const { return is_balanceable_; }
153 /**
154 * TODO
155 */
156 void PrecalculateListings();
157
158 void SetTTL(const uint64_t new_ttl);
159 bool SetVOMSAuthz(const std::string &voms_authz);
160 bool Commit(const bool stop_for_tweaks,
161 const uint64_t manual_revision,
162 manifest::Manifest *manifest);
163
164 ✗ void Balance() {
165 ✗ if (IsBalanceable()) {
166 ✗ DoBalance();
167 } else {
168 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg,
169 "Not balancing the catalog "
170 "manager because it is not balanceable");
171 }
172 }
173
174 void LoadCatalogs(const std::string &base_path,
175 const std::unordered_set<std::string> &dirs);
176 void SetupSingleCatalogUploadCallback();
177 void RemoveSingleCatalogUploadCallback();
178 void AddCatalogToQueue(const std::string &path);
179 void ScheduleReadyCatalogs();
180 bool LookupDirEntry(const std::string &path,
181 const LookupOptions options,
182 DirectoryEntry *dirent);
183
184
185 protected:
186 2408 void EnforceSqliteMemLimit() { }
187
188 Catalog *CreateCatalog(const PathString &mountpoint,
189 const shash::Any &catalog_hash,
190 Catalog *parent_catalog);
191 void ActivateCatalog(Catalog *catalog);
192
193 void AddFile(const DirectoryEntry &entry,
194 const XattrList &xattrs,
195 const std::string &parent_directory);
196
197 private:
198 bool FindCatalog(const std::string &path,
199 WritableCatalog **result,
200 DirectoryEntry *dirent = NULL);
201 void DoBalance();
202 void FixWeight(WritableCatalog *catalog);
203
204 void CloneTreeImpl(const PathString &source_dir,
205 const std::string &dest_parent_dir,
206 const NameString &dest_name);
207
208 struct CatalogInfo {
209 uint64_t ttl;
210 size_t size;
211 shash::Any content_hash;
212 uint64_t revision;
213 };
214
215 struct CatalogUploadContext {
216 Future<CatalogInfo> *root_catalog_info;
217 bool stop_for_tweaks;
218 };
219
220 struct CatalogDownloadContext {
221 const std::unordered_set<std::string> *dirs;
222 };
223
224 void CatalogDownloadCallback(const CatalogDownloadResult &result,
225 const CatalogDownloadContext context);
226 void SingleCatalogUploadCallback(const upload::SpoolerResult &result);
227
228
229 CatalogInfo SnapshotCatalogs(const bool stop_for_tweaks);
230 void FinalizeCatalog(WritableCatalog *catalog, const bool stop_for_tweaks);
231 void ScheduleCatalogProcessing(WritableCatalog *catalog);
232
233 875 void GetModifiedCatalogLeafs(WritableCatalogList *result) const {
234 875 const bool dirty = GetModifiedCatalogLeafsRecursively(GetRootCatalog(),
235 result);
236
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 875 times.
 875 assert(dirty);
237 875 }
238 bool GetModifiedCatalogLeafsRecursively(Catalog *catalog,
239 WritableCatalogList *result) const;
240
241 void CatalogUploadCallback(const upload::SpoolerResult &result,
242 const CatalogUploadContext clg_upload_context);
243
244 bool CopyCatalogToLocalCache(const upload::SpoolerResult &result);
245
246 private:
247 16978 inline void SyncLock() { pthread_mutex_lock(sync_lock_); }
248 16935 inline void SyncUnlock() { pthread_mutex_unlock(sync_lock_); }
249
250 //****************************************************************************
251 // Workaround -- Serialized Catalog Committing
252 ✗ void GetModifiedCatalogs(WritableCatalogList *result) const {
253 const unsigned int
254 ✗ number_of_dirty_catalogs = GetModifiedCatalogsRecursively(
255 ✗ GetRootCatalog(), result);
256 ✗ assert(number_of_dirty_catalogs <= result->size());
257 }
258 int GetModifiedCatalogsRecursively(const Catalog *catalog,
259 WritableCatalogList *result) const;
260 void CatalogUploadSerializedCallback(const upload::SpoolerResult &result,
261 const CatalogUploadContext unused);
262 CatalogInfo SnapshotCatalogsSerialized(const bool stop_for_tweaks);
263 void CatalogHashSerializedCallback(const CompressHashResult &result);
264 //****************************************************************************
265
266 // defined in catalog_mgr_rw.cc
267 static const std::string kCatalogFilename;
268
269 // private lock of WritableCatalogManager
270 pthread_mutex_t *sync_lock_;
271 upload::Spooler *spooler_;
272
273 pthread_mutex_t *catalog_processing_lock_;
274 std::map<std::string, WritableCatalog *> catalog_processing_map_;
275
276 // ingestsql
277 std::list<WritableCatalog *> pending_catalogs_;
278 CatalogDownloadPipeline *catalog_download_pipeline_;
279 pthread_mutex_t *catalog_download_lock_;
280 std::unordered_map<std::string, Catalog *> catalog_download_map_;
281 pthread_mutex_t *catalog_hash_lock_;
282 std::map<std::string, shash::Any> catalog_hash_map_;
283
284 // TODO(jblomer): catalog limits should become its own struct
285 bool enforce_limits_;
286 unsigned nested_kcatalog_limit_;
287 unsigned root_kcatalog_limit_;
288 unsigned file_mbyte_limit_;
289
290 /**
291 * Directories don't have extended attributes at this point.
292 */
293 XattrList empty_xattrs;
294
295 /**
296 * It indicates whether this catalog manager supports balancing operations
297 */
298 const bool is_balanceable_;
299
300 /**
301 * Defines the maximum weight an autogenerated catalog can have. If after a
302 * publishing operation the catalog's weight is greater than this threshold it
303 * will be considered overflowed and will automatically be split in different
304 * sub-catalogs.
305 */
306 const unsigned max_weight_;
307
308 /**
309 * Defines the minimum weight an autogenerated catalog can have. If after a
310 * publishing operation the catalog's weight is lesser than this threshold it
311 * will be considered underflowed and will automatically be merged with its
312 * parent.
313 * This last operation can provoke an overflow in the parent, though.
314 */
315 const unsigned min_weight_;
316
317 /**
318 * Defines the threshold that will be used to balance a catalog that has been
319 * overflowed. Its value should be lesser than max_weight_ and greater than
320 * min_weight. By default it is set to max_weight / 2.
321 */
322 const unsigned balance_weight_;
323 }; // class WritableCatalogManager
324
325 } // namespace catalog
326
327 #endif // CVMFS_CATALOG_MGR_RW_H_
328