GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/catalog_mgr_rw.h
Date: 2024-04-28 02:33:07
Exec Total Coverage
Lines: 10 18 55.6%
Branches: 2 8 25.0%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * A WritableCatalogManager adds write functionality to the catalog
5 * manager. It is used for the server side of CVMFS
6 * All nested catalog loading functionality is inherited from
7 * AbstractCatalogManager.
8 *
9 * The WritableCatalogManager is provided with DirectoryEntryBase objects from
10 * the underlying sync infrastructure (namely SyncMediator) on the server side
11 * of CVMFS. In contrast to a full DirectoryEntry object DirectoryEntryBase con-
12 * tains only pure file system specific meta data (i.e. mtime, mode, filename).
13 * WritableCatalogManager is responsible for the addition and management of all
14 * CVMFS-specific meta data in a full DirectoryEntry, which is then saved into
15 * the actual Catalog databases.
16 *
17 * The inode assignment is based on the fact that the number of entries in a
18 * catalog do not change (expect on reload). As we do exactly that with the
19 * WritableCatalogManager here, inode numbers derived from WritableCatalogs
20 * and the WritableCatalogManager may (and will) be screwed. This is not an
21 * issue in the current implementation, as they are not used in the syncing
22 * process. Just keep in mind.
23 *
24 * The WritableCatalogManager starts with a base repository (given by the
25 * root hash), and downloads and uncompresses all required catalogs into
26 * temporary storage.
27 */
28
29 #ifndef CVMFS_CATALOG_MGR_RW_H_
30 #define CVMFS_CATALOG_MGR_RW_H_
31
32 #include <pthread.h>
33 #include <stdint.h>
34
35 #include <map>
36 #include <set>
37 #include <string>
38
39 #include "catalog_mgr_ro.h"
40 #include "catalog_rw.h"
41 #include "file_chunk.h"
42 #include "upload_spooler_result.h"
43 #include "util/future.h"
44 #include "xattr.h"
45
46 class XattrList;
47 namespace upload {
48 class Spooler;
49 }
50
51 namespace download {
52 class DownloadManager;
53 }
54
55 namespace manifest {
56 class Manifest;
57 }
58
59 namespace perf {
60 class Statistics;
61 }
62
63 namespace catalog {
64 template <class CatalogMgrT>
65 class CatalogBalancer;
66 }
67
68 namespace catalog {
69
70 class WritableCatalogManager : public SimpleCatalogManager {
71 friend class CatalogBalancer<WritableCatalogManager>;
72 // TODO(jblomer): only needed to get Spooler's hash algorithm. Remove me
73 // after refactoring of the swissknife utility.
74 friend class VirtualCatalog;
75
76 public:
77 WritableCatalogManager(const shash::Any &base_hash,
78 const std::string &stratum0,
79 const std::string &dir_temp,
80 upload::Spooler *spooler,
81 download::DownloadManager *download_manager,
82 bool enforce_limits,
83 const unsigned nested_kcatalog_limit,
84 const unsigned root_kcatalog_limit,
85 const unsigned file_mbyte_limit,
86 perf::Statistics *statistics,
87 bool is_balanceable,
88 unsigned max_weight,
89 unsigned min_weight);
90 ~WritableCatalogManager();
91 static manifest::Manifest *CreateRepository(const std::string &dir_temp,
92 const bool volatile_content,
93 const std::string &voms_authz,
94 upload::Spooler *spooler);
95
96 // DirectoryEntry handling
97 135 void AddFile(const DirectoryEntryBase &entry,
98 const XattrList &xattrs,
99 const std::string &parent_directory)
100 {
101
1/2
✓ Branch 2 taken 135 times.
✗ Branch 3 not taken.
 135 AddFile(DirectoryEntry(entry), xattrs, parent_directory);
102 135 }
103 void AddChunkedFile(const DirectoryEntryBase &entry,
104 const XattrList &xattrs,
105 const std::string &parent_directory,
106 const FileChunkList &file_chunks);
107 void RemoveFile(const std::string &file_path);
108
109 void AddDirectory(const DirectoryEntryBase &entry,
110 const XattrList &xattrs,
111 const std::string &parent_directory);
112 void TouchDirectory(const DirectoryEntryBase &entry,
113 const XattrList &xattrs,
114 const std::string &directory_path);
115 void RemoveDirectory(const std::string &directory_path);
116
117 void Clone(const std::string from, const std::string to);
118 void CloneTree(const std::string &from_dir, const std::string &to_dir);
119
120 // Hardlink group handling
121 void AddHardlinkGroup(const DirectoryEntryBaseList &entries,
122 const XattrList &xattrs,
123 const std::string &parent_directory,
124 const FileChunkList &file_chunks);
125 void ShrinkHardlinkGroup(const std::string &remove_path);
126
127 // Nested catalog handling
128 void CreateNestedCatalog(const std::string &mountpoint);
129 void RemoveNestedCatalog(const std::string &mountpoint,
130 const bool merge = true);
131 void SwapNestedCatalog(const string &mountpoint, const shash::Any &new_hash,
132 const uint64_t new_size);
133 bool IsTransitionPoint(const std::string &mountpoint);
134 WritableCatalog *GetHostingCatalog(const std::string &path);
135
136 ✗ inline bool IsBalanceable() const { return is_balanceable_; }
137 /**
138 * TODO
139 */
140 void PrecalculateListings();
141
142 void SetTTL(const uint64_t new_ttl);
143 bool SetVOMSAuthz(const std::string &voms_authz);
144 bool Commit(const bool stop_for_tweaks,
145 const uint64_t manual_revision,
146 manifest::Manifest *manifest);
147
148 ✗ void Balance() {
149 ✗ if (IsBalanceable()) {
150 ✗ DoBalance();
151 } else {
152 ✗ LogCvmfs(kLogCatalog, kLogVerboseMsg, "Not balancing the catalog "
153 "manager because it is not balanceable");
154 }
155 }
156
157 protected:
158 47 void EnforceSqliteMemLimit() { }
159
160 Catalog *CreateCatalog(const PathString &mountpoint,
161 const shash::Any &catalog_hash,
162 Catalog *parent_catalog);
163 void ActivateCatalog(Catalog *catalog);
164
165 void AddFile(const DirectoryEntry &entry,
166 const XattrList &xattrs,
167 const std::string &parent_directory);
168
169 private:
170 bool FindCatalog(const std::string &path,
171 WritableCatalog **result,
172 DirectoryEntry *dirent = NULL);
173 void DoBalance();
174 void FixWeight(WritableCatalog *catalog);
175
176 void CloneTreeImpl(const PathString &source_dir,
177 const std::string &dest_parent_dir,
178 const NameString &dest_name);
179
180 struct CatalogInfo {
181 uint64_t ttl;
182 size_t size;
183 shash::Any content_hash;
184 uint64_t revision;
185 };
186
187 struct CatalogUploadContext {
188 Future<CatalogInfo>* root_catalog_info;
189 bool stop_for_tweaks;
190 };
191
192 CatalogInfo SnapshotCatalogs(const bool stop_for_tweaks);
193 void FinalizeCatalog(WritableCatalog *catalog,
194 const bool stop_for_tweaks);
195 void ScheduleCatalogProcessing(WritableCatalog *catalog);
196
197 20 void GetModifiedCatalogLeafs(WritableCatalogList *result) const {
198 20 const bool dirty = GetModifiedCatalogLeafsRecursively(GetRootCatalog(),
199 result);
200
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 20 times.
 20 assert(dirty);
201 20 }
202 bool GetModifiedCatalogLeafsRecursively(Catalog *catalog,
203 WritableCatalogList *result) const;
204
205 void CatalogUploadCallback(const upload::SpoolerResult &result,
206 const CatalogUploadContext clg_upload_context);
207
208 private:
209 371 inline void SyncLock() { pthread_mutex_lock(sync_lock_); }
210 370 inline void SyncUnlock() { pthread_mutex_unlock(sync_lock_); }
211
212 //****************************************************************************
213 // Workaround -- Serialized Catalog Committing
214 ✗ void GetModifiedCatalogs(WritableCatalogList *result) const {
215 const unsigned int number_of_dirty_catalogs =
216 ✗ GetModifiedCatalogsRecursively(GetRootCatalog(), result);
217 ✗ assert(number_of_dirty_catalogs <= result->size());
218 }
219 int GetModifiedCatalogsRecursively(const Catalog *catalog,
220 WritableCatalogList *result) const;
221 void CatalogUploadSerializedCallback(
222 const upload::SpoolerResult &result,
223 const CatalogUploadContext unused);
224 CatalogInfo SnapshotCatalogsSerialized(const bool stop_for_tweaks);
225 //****************************************************************************
226
227 // defined in catalog_mgr_rw.cc
228 static const std::string kCatalogFilename;
229
230 // private lock of WritableCatalogManager
231 pthread_mutex_t *sync_lock_;
232 upload::Spooler *spooler_;
233
234 pthread_mutex_t *catalog_processing_lock_;
235 std::map<std::string, WritableCatalog*> catalog_processing_map_;
236
237 // TODO(jblomer): catalog limits should become its own struct
238 bool enforce_limits_;
239 unsigned nested_kcatalog_limit_;
240 unsigned root_kcatalog_limit_;
241 unsigned file_mbyte_limit_;
242
243 /**
244 * Directories don't have extended attributes at this point.
245 */
246 XattrList empty_xattrs;
247
248 /**
249 * It indicates whether this catalog manager supports balancing operations
250 */
251 const bool is_balanceable_;
252
253 /**
254 * Defines the maximum weight an autogenerated catalog can have. If after a
255 * publishing operation the catalog's weight is greater than this threshold it
256 * will be considered overflowed and will automatically be split in different
257 * sub-catalogs.
258 */
259 const unsigned max_weight_;
260
261 /**
262 * Defines the minimum weight an autogenerated catalog can have. If after a
263 * publishing operation the catalog's weight is lesser than this threshold it
264 * will be considered underflowed and will automatically be merged with its
265 * parent.
266 * This last operation can provoke an overflow in the parent, though.
267 */
268 const unsigned min_weight_;
269
270 /**
271 * Defines the threshold that will be used to balance a catalog that has been
272 * overflowed. Its value should be lesser than max_weight_ and greater than
273 * min_weight. By default it is set to max_weight / 2.
274 */
275 const unsigned balance_weight_;
276 }; // class WritableCatalogManager
277
278 } // namespace catalog
279
280 #endif // CVMFS_CATALOG_MGR_RW_H_
281