GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/swissknife_sync.h
Date: 2026-06-28 02:36:10
Exec Total Coverage
Lines: 0 170 0.0%
Branches: 0 10 0.0%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 */
4
5 #ifndef CVMFS_SWISSKNIFE_SYNC_H_
6 #define CVMFS_SWISSKNIFE_SYNC_H_
7
8 #include <string>
9 #include <vector>
10
11 #include "compression/compression.h"
12 #include "repository_tag.h"
13 #include "swissknife.h"
14 #include "upload.h"
15
16 struct SyncParameters {
17 static const unsigned kDefaultMaxWeight = 100000;
18 static const unsigned kDefaultMinWeight = 1000;
19 static const size_t kDefaultMinFileChunkSize = 4 * 1024 * 1024;
20 static const size_t kDefaultAvgFileChunkSize = 8 * 1024 * 1024;
21 static const size_t kDefaultMaxFileChunkSize = 16 * 1024 * 1024;
22 static const unsigned kDefaultNestedKcatalogLimit = 500;
23 static const unsigned kDefaultRootKcatalogLimit = 200;
24 static const unsigned kDefaultFileMbyteLimit = 1024;
25
26 ✗ SyncParameters()
27 ✗ : spooler(NULL)
28 ✗ , union_fs_type("aufs")
29 ✗ , to_delete("")
30 ✗ , cache_dir("")
31 ✗ , print_changeset(false)
32 ✗ , dry_run(false)
33 ✗ , mucatalogs(false)
34 ✗ , use_file_chunking(false)
35 ✗ , generate_legacy_bulk_chunks(false)
36 ✗ , ignore_xdir_hardlinks(false)
37 ✗ , stop_for_catalog_tweaks(false)
38 ✗ , include_xattrs(false)
39 ✗ , enable_mtime_ns(false)
40 ✗ , fast_delete(false)
41 ✗ , tolerate_missing_hardlinks(false)
42 ✗ , external_data(false)
43 ✗ , direct_io(false)
44 ✗ , voms_authz(false)
45 ✗ , virtual_dir_actions(0)
46 ✗ , ignore_special_files(false)
47 ✗ , branched_catalog(false)
48 ✗ , compression_alg(zlib::kZlibDefault)
49 ✗ , enforce_limits(false)
50 ✗ , nested_kcatalog_limit(0)
51 ✗ , root_kcatalog_limit(0)
52 ✗ , file_mbyte_limit(0)
53 ✗ , min_file_chunk_size(kDefaultMinFileChunkSize)
54 ✗ , avg_file_chunk_size(kDefaultAvgFileChunkSize)
55 ✗ , max_file_chunk_size(kDefaultMaxFileChunkSize)
56 ✗ , manual_revision(0)
57 ✗ , ttl_seconds(0)
58 ✗ , max_concurrent_write_jobs(0)
59 ✗ , num_upload_tasks(1)
60 ✗ , is_balanced(false)
61 ✗ , max_weight(kDefaultMaxWeight)
62 ✗ , min_weight(kDefaultMinWeight)
63 ✗ , gid(-1u)
64 ✗ , uid(-1u)
65 ✗ , session_token_file()
66 ✗ , key_file()
67 ✗ , repo_tag() { }
68
69 upload::Spooler *spooler;
70 std::string repo_name;
71 std::string dir_union;
72 std::string dir_scratch;
73 std::string dir_rdonly;
74 std::string dir_temp;
75 shash::Any base_hash;
76 std::string stratum0;
77 std::string manifest_path;
78 std::string spooler_definition;
79 std::string union_fs_type;
80 std::string public_keys;
81 std::string authz_file;
82 std::string tar_file;
83 std::string base_directory;
84 std::string to_delete;
85 std::string cache_dir;
86 bool print_changeset;
87 bool dry_run;
88 bool mucatalogs;
89 bool use_file_chunking;
90 bool generate_legacy_bulk_chunks;
91 bool ignore_xdir_hardlinks;
92 bool stop_for_catalog_tweaks;
93 bool include_xattrs;
94 bool enable_mtime_ns;
95 bool fast_delete;
96 bool tolerate_missing_hardlinks;
97 bool external_data;
98 bool direct_io;
99 bool voms_authz;
100 int virtual_dir_actions; // bit field
101 bool ignore_special_files;
102 bool branched_catalog;
103 zlib::Algorithms compression_alg;
104 bool enforce_limits;
105 unsigned nested_kcatalog_limit;
106 unsigned root_kcatalog_limit;
107 unsigned file_mbyte_limit;
108 size_t min_file_chunk_size;
109 size_t avg_file_chunk_size;
110 size_t max_file_chunk_size;
111 uint64_t manual_revision;
112 uint64_t ttl_seconds;
113 uint64_t max_concurrent_write_jobs;
114 unsigned num_upload_tasks;
115 bool is_balanced;
116 unsigned max_weight;
117 unsigned min_weight;
118 gid_t gid;
119 uid_t uid;
120
121 // Parameters for when upstream type is HTTP
122 std::string session_token_file;
123 std::string key_file;
124 RepositoryTag repo_tag;
125 };
126
127 namespace catalog {
128 class Dirtab;
129 class SimpleCatalogManager;
130 } // namespace catalog
131
132 namespace swissknife {
133
134 class CommandCreate : public Command {
135 public:
136 ✗ ~CommandCreate() { }
137 ✗ virtual std::string GetName() const { return "create"; }
138 ✗ virtual std::string GetDescription() const {
139 ✗ return "Bootstraps a fresh repository.";
140 }
141 ✗ virtual ParameterList GetParams() const {
142 ✗ ParameterList r;
143 ✗ r.push_back(Parameter::Mandatory('o', "manifest output file"));
144 ✗ r.push_back(Parameter::Mandatory('t', "directory for temporary storage"));
145 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
146 ✗ r.push_back(Parameter::Mandatory('n', "repository name"));
147 ✗ r.push_back(Parameter::Mandatory('R', "path to reflog.chksum file"));
148 ✗ r.push_back(Parameter::Optional('l', "log level (0-4, default: 2)"));
149 ✗ r.push_back(Parameter::Optional('a', "hash algorithm (default: SHA-1)"));
150 ✗ r.push_back(Parameter::Optional('V',
151 "VOMS authz requirement "
152 "(default: none)"));
153 ✗ r.push_back(Parameter::Switch('v', "repository containing volatile files"));
154 ✗ r.push_back(
155 ✗ Parameter::Switch('z', "mark new repository as garbage collectable"));
156 ✗ r.push_back(Parameter::Optional('V',
157 "VOMS authz requirement "
158 "(default: none)"));
159 ✗ return r;
160 }
161 int Main(const ArgumentList &args);
162 };
163
164 class CommandUpload : public Command {
165 public:
166 ✗ ~CommandUpload() { }
167 ✗ virtual std::string GetName() const { return "upload"; }
168 ✗ virtual std::string GetDescription() const {
169 ✗ return "Uploads a local file to the repository.";
170 }
171 ✗ virtual ParameterList GetParams() const {
172 ✗ ParameterList r;
173 ✗ r.push_back(Parameter::Mandatory('i', "local file"));
174 ✗ r.push_back(Parameter::Mandatory('o', "destination path"));
175 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
176 ✗ r.push_back(Parameter::Optional('a', "hash algorithm (default: SHA-1)"));
177 ✗ return r;
178 }
179 int Main(const ArgumentList &args);
180 };
181
182 class CommandPeek : public Command {
183 public:
184 ✗ ~CommandPeek() { }
185 ✗ virtual std::string GetName() const { return "peek"; }
186 ✗ virtual std::string GetDescription() const {
187 ✗ return "Checks whether a file exists in the repository.";
188 }
189 ✗ virtual ParameterList GetParams() const {
190 ✗ ParameterList r;
191 ✗ r.push_back(Parameter::Mandatory('d', "destination path"));
192 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
193 ✗ return r;
194 }
195 int Main(const ArgumentList &args);
196 };
197
198 class CommandRemove : public Command {
199 public:
200 ✗ ~CommandRemove() { }
201 ✗ virtual std::string GetName() const { return "remove"; }
202 ✗ virtual std::string GetDescription() const {
203 ✗ return "Removes a file in the repository storage.";
204 }
205 ✗ virtual ParameterList GetParams() const {
206 ✗ ParameterList r;
207 ✗ r.push_back(Parameter::Mandatory('o', "path to file"));
208 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
209 ✗ return r;
210 }
211 int Main(const ArgumentList &args);
212 };
213
214 class CommandApplyDirtab : public Command {
215 public:
216 ✗ CommandApplyDirtab() : verbose_(false) { }
217 ✗ ~CommandApplyDirtab() { }
218 ✗ virtual std::string GetName() const { return "dirtab"; }
219 ✗ virtual std::string GetDescription() const {
220 ✗ return "Parses the dirtab file and produces nested catalog markers.";
221 }
222 ✗ virtual ParameterList GetParams() const {
223 ✗ ParameterList r;
224 ✗ r.push_back(Parameter::Mandatory('d', "path to dirtab file"));
225 ✗ r.push_back(Parameter::Mandatory('u', "union volume"));
226 ✗ r.push_back(Parameter::Mandatory('s', "scratch directory"));
227 ✗ r.push_back(Parameter::Mandatory('b', "base hash"));
228 ✗ r.push_back(Parameter::Mandatory('w', "stratum 0 base url"));
229 ✗ r.push_back(Parameter::Mandatory('t', "directory for temporary storage"));
230 ✗ r.push_back(Parameter::Optional('@', "proxy url"));
231 ✗ r.push_back(Parameter::Switch('x', "verbose mode"));
232 ✗ return r;
233 }
234 int Main(const ArgumentList &args);
235
236 protected:
237 void DetermineNestedCatalogCandidates(
238 const catalog::Dirtab &dirtab,
239 catalog::SimpleCatalogManager *catalog_manager,
240 std::vector<std::string> *nested_catalog_candidates);
241 void FilterCandidatesFromGlobResult(
242 const catalog::Dirtab &dirtab, char **paths, const size_t npaths,
243 catalog::SimpleCatalogManager *catalog_manager,
244 std::vector<std::string> *nested_catalog_candidates);
245 bool CreateCatalogMarkers(
246 const std::vector<std::string> &new_nested_catalogs);
247
248 private:
249 std::string union_dir_;
250 std::string scratch_dir_;
251 bool verbose_;
252 };
253
254 class CommandSync : public Command {
255 public:
256 ✗ ~CommandSync() { }
257 ✗ virtual std::string GetName() const { return "sync"; }
258 ✗ virtual std::string GetDescription() const {
259 ✗ return "Pushes changes from scratch area back to the repository.";
260 }
261 ✗ virtual ParameterList GetParams() const {
262 // unused characters: 1-9, all special characters but @
263 ✗ ParameterList r;
264 ✗ r.push_back(Parameter::Mandatory('b', "base hash"));
265 ✗ r.push_back(Parameter::Mandatory('c', "r/o volume"));
266 ✗ r.push_back(Parameter::Mandatory('o', "manifest output file"));
267 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
268 ✗ r.push_back(Parameter::Mandatory('s', "scratch directory"));
269 ✗ r.push_back(Parameter::Mandatory('t', "directory for tee"));
270 ✗ r.push_back(Parameter::Mandatory('u', "union volume"));
271 ✗ r.push_back(Parameter::Mandatory('w', "stratum 0 base url"));
272 ✗ r.push_back(Parameter::Mandatory('K', "public key(s) for repo"));
273 ✗ r.push_back(Parameter::Mandatory('N', "fully qualified repository name"));
274
275 ✗ r.push_back(Parameter::Optional('a', "desired average chunk size (bytes)"));
276 ✗ r.push_back(Parameter::Optional('e', "hash algorithm (default: SHA-1)"));
277 ✗ r.push_back(Parameter::Optional('f', "union filesystem type"));
278 ✗ r.push_back(Parameter::Optional('h', "maximal file chunk size in bytes"));
279 ✗ r.push_back(Parameter::Optional('l', "minimal file chunk size in bytes"));
280 ✗ r.push_back(Parameter::Optional('q', "number of concurrent write jobs"));
281 ✗ r.push_back(Parameter::Optional('0', "number of upload tasks"));
282 ✗ r.push_back(Parameter::Optional('v', "manual revision number"));
283 ✗ r.push_back(Parameter::Optional('z', "log level (0-4, default: 2)"));
284 ✗ r.push_back(Parameter::Optional('F', "Authz file listing (default: none)"));
285 ✗ r.push_back(Parameter::Optional('M', "minimum weight of the autocatalogs"));
286 ✗ r.push_back(
287 ✗ Parameter::Optional('Q', "nested catalog limit in kilo-entries"));
288 ✗ r.push_back(Parameter::Optional('R', "root catalog limit in kilo-entries"));
289 ✗ r.push_back(Parameter::Optional('T', "Root catalog TTL in seconds"));
290 ✗ r.push_back(Parameter::Optional('U', "file size limit in megabytes"));
291 ✗ r.push_back(
292 ✗ Parameter::Optional('D', "tag name (only used when upstream is GW)"));
293 ✗ r.push_back(Parameter::Optional(
294 'J', "tag description (only used when upstream is GW)"));
295 ✗ r.push_back(Parameter::Optional('X', "maximum weight of the autocatalogs"));
296 ✗ r.push_back(Parameter::Optional('Z',
297 "compression algorithm "
298 "(default: zlib)"));
299 ✗ r.push_back(Parameter::Optional('S',
300 "virtual directory options "
301 "[snapshots, remove]"));
302
303
304 ✗ r.push_back(
305 ✗ Parameter::Switch('G', "Use persistent caching for all catalogs "
306 "used during the publishing process"
307 " Warning: No automatic garbage collection!"));
308 ✗ r.push_back(Parameter::Switch('d',
309 "pause publishing to allow for catalog "
310 "tweaks"));
311 ✗ r.push_back(Parameter::Switch('i', "ignore x-directory hardlinks"));
312 ✗ r.push_back(Parameter::Switch('g', "ignore special files"));
313 ✗ r.push_back(Parameter::Switch('k', "include extended attributes"));
314 ✗ r.push_back(Parameter::Switch('j', "enable nanosecond timestamps"));
315 ✗ r.push_back(Parameter::Switch('m', "create micro catalogs"));
316 ✗ r.push_back(Parameter::Switch('n', "create new repository"));
317 ✗ r.push_back(Parameter::Switch('p', "enable file chunking"));
318 ✗ r.push_back(Parameter::Switch('O', "generate legacy bulk chunks"));
319 ✗ r.push_back(Parameter::Switch('x', "print change set"));
320 ✗ r.push_back(Parameter::Switch('y', "dry run"));
321 ✗ r.push_back(Parameter::Switch('A', "autocatalog enabled/disabled"));
322 ✗ r.push_back(Parameter::Switch('E', "enforce limits instead of warning"));
323 ✗ r.push_back(Parameter::Switch('L', "enable HTTP redirects"));
324 ✗ r.push_back(Parameter::Switch('V',
325 "Publish format compatible with "
326 "authenticated repos"));
327 ✗ r.push_back(Parameter::Switch('Y', "enable external data"));
328 ✗ r.push_back(Parameter::Switch('W', "set direct I/O for regular files"));
329 ✗ r.push_back(Parameter::Switch('B', "branched catalog (no manifest)"));
330 ✗ r.push_back(Parameter::Switch('I', "upload updated statistics DB file"));
331
332 ✗ r.push_back(Parameter::Optional('P', "session_token_file"));
333 ✗ r.push_back(Parameter::Optional('H', "key file for HTTP API"));
334 ✗ r.push_back(Parameter::Optional('@', "proxy URL"));
335 ✗ r.push_back(Parameter::Optional('C',
336 "auto tag cleanup threshold as a Unix timestamp; auto tags older than "
337 "this are removed (only used when upstream is GW)"));
338
339 ✗ return r;
340 }
341 int Main(const ArgumentList &args);
342
343 protected:
344 bool ReadFileChunkingArgs(const swissknife::ArgumentList &args,
345 SyncParameters *params);
346 bool CheckParams(const SyncParameters &p);
347 };
348
349 } // namespace swissknife
350
351 #endif // CVMFS_SWISSKNIFE_SYNC_H_
352