GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/catalog_mgr_rw.h
Date: 2025-06-22 02:36:02
Exec Total Coverage
Lines: 10 19 52.6%
Branches: 2 8 25.0%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * A WritableCatalogManager adds write functionality to the catalog
5 * manager. It is used for the server side of CVMFS
6 * All nested catalog loading functionality is inherited from
7 * AbstractCatalogManager.
8 *
9 * The WritableCatalogManager is provided with DirectoryEntryBase objects from
10 * the underlying sync infrastructure (namely SyncMediator) on the server side
11 * of CVMFS. In contrast to a full DirectoryEntry object DirectoryEntryBase con-
12 * tains only pure file system specific meta data (i.e. mtime, mode, filename).
13 * WritableCatalogManager is responsible for the addition and management of all
14 * CVMFS-specific meta data in a full DirectoryEntry, which is then saved into
15 * the actual Catalog databases.
16 *
17 * The inode assignment is based on the fact that the number of entries in a
18 * catalog do not change (except on reload). As we do exactly that with the
19 * WritableCatalogManager here, inode numbers derived from WritableCatalogs
20 * and the WritableCatalogManager may (and will) be screwed. This is not an
21 * issue in the current implementation, as they are not used in the syncing
22 * process. Just keep in mind.
23 *
24 * The WritableCatalogManager starts with a base repository (given by the
25 * root hash), and downloads and uncompresses all required catalogs into
26 * temporary storage.
27 */
28
29 #ifndef CVMFS_CATALOG_MGR_RW_H_
30 #define CVMFS_CATALOG_MGR_RW_H_
31
32 #include <pthread.h>
33 #include <stdint.h>
34
35 #include <list>
36 #include <map>
37 #include <set>
38 #include <string>
39 #include <unordered_set>
40 #include <unordered_map>
41
42 #include "catalog_mgr_ro.h"
43 #include "catalog_rw.h"
44 #include "file_chunk.h"
45 #include "upload_spooler_result.h"
46 #include "util/future.h"
47 #include "xattr.h"
48 #include "catalog_downloader.h"
49 #include "ingestion/pipeline.h"
50
51 class XattrList;
52 namespace upload {
53 class Spooler;
54 }
55
56 namespace download {
57 class DownloadManager;
58 }
59
60 namespace manifest {
61 class Manifest;
62 }
63
64 namespace perf {
65 class Statistics;
66 }
67
68 namespace catalog {
69 template<class CatalogMgrT>
70 class CatalogBalancer;
71 }
72
73 namespace catalog {
74
75 class WritableCatalogManager : public SimpleCatalogManager {
76 friend class CatalogBalancer<WritableCatalogManager>;
77 // TODO(jblomer): only needed to get Spooler's hash algorithm. Remove me
78 // after refactoring of the swissknife utility.
79 friend class VirtualCatalog;
80
81 public:
82 WritableCatalogManager(const shash::Any &base_hash,
83 const std::string &stratum0,
84 const std::string &dir_temp,
85 upload::Spooler *spooler,
86 download::DownloadManager *download_manager,
87 bool enforce_limits,
88 const unsigned nested_kcatalog_limit,
89 const unsigned root_kcatalog_limit,
90 const unsigned file_mbyte_limit,
91 perf::Statistics *statistics,
92 bool is_balanceable,
93 unsigned max_weight,
94 unsigned min_weight,
95 const std::string &dir_cache = "");
96 ~WritableCatalogManager();
97 static manifest::Manifest *CreateRepository(const std::string &dir_temp,
98 const bool volatile_content,
99 const std::string &voms_authz,
100 upload::Spooler *spooler);
101
102 // DirectoryEntry handling
103 5355 void AddFile(const DirectoryEntryBase &entry,
104 const XattrList &xattrs,
105 const std::string &parent_directory) {
106
1/2
✓ Branch 2 taken 5355 times.
✗ Branch 3 not taken.
 5355 AddFile(DirectoryEntry(entry), xattrs, parent_directory);
107 5355 }
108 void AddChunkedFile(const DirectoryEntryBase &entry,
109 const XattrList &xattrs,
110 const std::string &parent_directory,
111 const FileChunkList &file_chunks);
112 void RemoveFile(const std::string &file_path);
113
114 void AddDirectory(const DirectoryEntryBase &entry,
115 const XattrList &xattrs,
116 const std::string &parent_directory);
117 void TouchDirectory(const DirectoryEntryBase &entry,
118 const XattrList &xattrs,
119 const std::string &directory_path);
120 void RemoveDirectory(const std::string &directory_path);
121
122 void Clone(const std::string from, const std::string to);
123 void CloneTree(const std::string &from_dir, const std::string &to_dir);
124
125 // Hardlink group handling
126 void AddHardlinkGroup(const DirectoryEntryBaseList &entries,
127 const XattrList &xattrs,
128 const std::string &parent_directory,
129 const FileChunkList &file_chunks);
130 void ShrinkHardlinkGroup(const std::string &remove_path);
131
132 // Nested catalog handling
133 void CreateNestedCatalog(const std::string &mountpoint);
134 void RemoveNestedCatalog(const std::string &mountpoint,
135 const bool merge = true);
136 void SwapNestedCatalog(const string &mountpoint, const shash::Any &new_hash,
137 const uint64_t new_size);
138 void GraftNestedCatalog(const string &mountpoint, const shash::Any &new_hash,
139 const uint64_t new_size);
140 bool IsTransitionPoint(const std::string &mountpoint);
141 WritableCatalog *GetHostingCatalog(const std::string &path);
142
143 ✗ inline bool IsBalanceable() const { return is_balanceable_; }
144 /**
145 * TODO
146 */
147 void PrecalculateListings();
148
149 void SetTTL(const uint64_t new_ttl);
150 bool SetVOMSAuthz(const std::string &voms_authz);
151 bool Commit(const bool stop_for_tweaks,
152 const uint64_t manual_revision,
153 manifest::Manifest *manifest);
154
155 ✗ void Balance() {
156 ✗ if (IsBalanceable()) {
157 ✗ DoBalance();
158 } else {
159 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg,
160 "Not balancing the catalog "
161 "manager because it is not balanceable");
162 }
163 }
164
165 void LoadCatalogs(const std::string &base_path, const std::unordered_set<std::string> &dirs);
166 void SetupSingleCatalogUploadCallback();
167 void RemoveSingleCatalogUploadCallback();
168 void AddCatalogToQueue(const std::string &path);
169 void ScheduleReadyCatalogs();
170 bool LookupDirEntry(const std::string &path,
171 const LookupOptions options,
172 DirectoryEntry *dirent);
173
174
175
176 protected:
177 1272 void EnforceSqliteMemLimit() { }
178
179 Catalog *CreateCatalog(const PathString &mountpoint,
180 const shash::Any &catalog_hash,
181 Catalog *parent_catalog);
182 void ActivateCatalog(Catalog *catalog);
183
184 void AddFile(const DirectoryEntry &entry,
185 const XattrList &xattrs,
186 const std::string &parent_directory);
187
188 private:
189 bool FindCatalog(const std::string &path,
190 WritableCatalog **result,
191 DirectoryEntry *dirent = NULL);
192 void DoBalance();
193 void FixWeight(WritableCatalog *catalog);
194
195 void CloneTreeImpl(const PathString &source_dir,
196 const std::string &dest_parent_dir,
197 const NameString &dest_name);
198
199 struct CatalogInfo {
200 uint64_t ttl;
201 size_t size;
202 shash::Any content_hash;
203 uint64_t revision;
204 };
205
206 struct CatalogUploadContext {
207 Future<CatalogInfo> *root_catalog_info;
208 bool stop_for_tweaks;
209 };
210
211 struct CatalogDownloadContext {
212 const std::unordered_set<std::string> * dirs;
213 };
214
215 void CatalogDownloadCallback(const CatalogDownloadResult &result,
216 const CatalogDownloadContext context);
217 void SingleCatalogUploadCallback(const upload::SpoolerResult &result);
218
219
220
221 CatalogInfo SnapshotCatalogs(const bool stop_for_tweaks);
222 void FinalizeCatalog(WritableCatalog *catalog, const bool stop_for_tweaks);
223 void ScheduleCatalogProcessing(WritableCatalog *catalog);
224
225 863 void GetModifiedCatalogLeafs(WritableCatalogList *result) const {
226 863 const bool dirty = GetModifiedCatalogLeafsRecursively(GetRootCatalog(),
227 result);
228
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 863 times.
 863 assert(dirty);
229 863 }
230 bool GetModifiedCatalogLeafsRecursively(Catalog *catalog,
231 WritableCatalogList *result) const;
232
233 void CatalogUploadCallback(const upload::SpoolerResult &result,
234 const CatalogUploadContext clg_upload_context);
235
236 bool CopyCatalogToLocalCache(const upload::SpoolerResult &result);
237
238 private:
239 13974 inline void SyncLock() { pthread_mutex_lock(sync_lock_); }
240 13959 inline void SyncUnlock() { pthread_mutex_unlock(sync_lock_); }
241
242 //****************************************************************************
243 // Workaround -- Serialized Catalog Committing
244 ✗ void GetModifiedCatalogs(WritableCatalogList *result) const {
245 const unsigned int
246 ✗ number_of_dirty_catalogs = GetModifiedCatalogsRecursively(
247 ✗ GetRootCatalog(), result);
248 ✗ assert(number_of_dirty_catalogs <= result->size());
249 }
250 int GetModifiedCatalogsRecursively(const Catalog *catalog,
251 WritableCatalogList *result) const;
252 void CatalogUploadSerializedCallback(const upload::SpoolerResult &result,
253 const CatalogUploadContext unused);
254 CatalogInfo SnapshotCatalogsSerialized(const bool stop_for_tweaks);
255 void CatalogHashSerializedCallback(
256 const CompressHashResult &result);
257 //****************************************************************************
258
259 // defined in catalog_mgr_rw.cc
260 static const std::string kCatalogFilename;
261
262 // private lock of WritableCatalogManager
263 pthread_mutex_t *sync_lock_;
264 upload::Spooler *spooler_;
265
266 pthread_mutex_t *catalog_processing_lock_;
267 std::map<std::string, WritableCatalog *> catalog_processing_map_;
268
269 // ingestsql
270 std::list<WritableCatalog*> pending_catalogs_;
271 CatalogDownloadPipeline *catalog_download_pipeline_;
272 pthread_mutex_t *catalog_download_lock_;
273 std::unordered_map<std::string, Catalog*> catalog_download_map_;
274 pthread_mutex_t *catalog_hash_lock_;
275 std::map<std::string, shash::Any> catalog_hash_map_;
276
277 // TODO(jblomer): catalog limits should become its own struct
278 bool enforce_limits_;
279 unsigned nested_kcatalog_limit_;
280 unsigned root_kcatalog_limit_;
281 unsigned file_mbyte_limit_;
282
283 /**
284 * Directories don't have extended attributes at this point.
285 */
286 XattrList empty_xattrs;
287
288 /**
289 * It indicates whether this catalog manager supports balancing operations
290 */
291 const bool is_balanceable_;
292
293 /**
294 * Defines the maximum weight an autogenerated catalog can have. If after a
295 * publishing operation the catalog's weight is greater than this threshold it
296 * will be considered overflowed and will automatically be split in different
297 * sub-catalogs.
298 */
299 const unsigned max_weight_;
300
301 /**
302 * Defines the minimum weight an autogenerated catalog can have. If after a
303 * publishing operation the catalog's weight is lesser than this threshold it
304 * will be considered underflowed and will automatically be merged with its
305 * parent.
306 * This last operation can provoke an overflow in the parent, though.
307 */
308 const unsigned min_weight_;
309
310 /**
311 * Defines the threshold that will be used to balance a catalog that has been
312 * overflowed. Its value should be lesser than max_weight_ and greater than
313 * min_weight. By default it is set to max_weight / 2.
314 */
315 const unsigned balance_weight_;
316 }; // class WritableCatalogManager
317
318 } // namespace catalog
319
320 #endif // CVMFS_CATALOG_MGR_RW_H_
321