GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/swissknife_sync.h
Date: 2025-11-09 02:35:23
Exec Total Coverage
Lines: 0 167 0.0%
Branches: 0 10 0.0%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 */
4
5 #ifndef CVMFS_SWISSKNIFE_SYNC_H_
6 #define CVMFS_SWISSKNIFE_SYNC_H_
7
8 #include <string>
9 #include <vector>
10
11 #include "compression/compression.h"
12 #include "repository_tag.h"
13 #include "swissknife.h"
14 #include "upload.h"
15
16 struct SyncParameters {
17 static const unsigned kDefaultMaxWeight = 100000;
18 static const unsigned kDefaultMinWeight = 1000;
19 static const size_t kDefaultMinFileChunkSize = 4 * 1024 * 1024;
20 static const size_t kDefaultAvgFileChunkSize = 8 * 1024 * 1024;
21 static const size_t kDefaultMaxFileChunkSize = 16 * 1024 * 1024;
22 static const unsigned kDefaultNestedKcatalogLimit = 500;
23 static const unsigned kDefaultRootKcatalogLimit = 200;
24 static const unsigned kDefaultFileMbyteLimit = 1024;
25
26 ✗ SyncParameters()
27 ✗ : spooler(NULL)
28 ✗ , union_fs_type("aufs")
29 ✗ , to_delete("")
30 ✗ , cache_dir("")
31 ✗ , print_changeset(false)
32 ✗ , dry_run(false)
33 ✗ , mucatalogs(false)
34 ✗ , use_file_chunking(false)
35 ✗ , generate_legacy_bulk_chunks(false)
36 ✗ , ignore_xdir_hardlinks(false)
37 ✗ , stop_for_catalog_tweaks(false)
38 ✗ , include_xattrs(false)
39 ✗ , enable_mtime_ns(false)
40 ✗ , external_data(false)
41 ✗ , direct_io(false)
42 ✗ , voms_authz(false)
43 ✗ , virtual_dir_actions(0)
44 ✗ , ignore_special_files(false)
45 ✗ , branched_catalog(false)
46 ✗ , compression_alg(zlib::kZlibDefault)
47 ✗ , enforce_limits(false)
48 ✗ , nested_kcatalog_limit(0)
49 ✗ , root_kcatalog_limit(0)
50 ✗ , file_mbyte_limit(0)
51 ✗ , min_file_chunk_size(kDefaultMinFileChunkSize)
52 ✗ , avg_file_chunk_size(kDefaultAvgFileChunkSize)
53 ✗ , max_file_chunk_size(kDefaultMaxFileChunkSize)
54 ✗ , manual_revision(0)
55 ✗ , ttl_seconds(0)
56 ✗ , max_concurrent_write_jobs(0)
57 ✗ , num_upload_tasks(1)
58 ✗ , is_balanced(false)
59 ✗ , max_weight(kDefaultMaxWeight)
60 ✗ , min_weight(kDefaultMinWeight)
61 ✗ , gid(-1u)
62 ✗ , uid(-1u)
63 ✗ , session_token_file()
64 ✗ , key_file()
65 ✗ , repo_tag() { }
66
67 upload::Spooler *spooler;
68 std::string repo_name;
69 std::string dir_union;
70 std::string dir_scratch;
71 std::string dir_rdonly;
72 std::string dir_temp;
73 shash::Any base_hash;
74 std::string stratum0;
75 std::string manifest_path;
76 std::string spooler_definition;
77 std::string union_fs_type;
78 std::string public_keys;
79 std::string authz_file;
80 std::string tar_file;
81 std::string base_directory;
82 std::string to_delete;
83 std::string cache_dir;
84 bool print_changeset;
85 bool dry_run;
86 bool mucatalogs;
87 bool use_file_chunking;
88 bool generate_legacy_bulk_chunks;
89 bool ignore_xdir_hardlinks;
90 bool stop_for_catalog_tweaks;
91 bool include_xattrs;
92 bool enable_mtime_ns;
93 bool external_data;
94 bool direct_io;
95 bool voms_authz;
96 int virtual_dir_actions; // bit field
97 bool ignore_special_files;
98 bool branched_catalog;
99 zlib::Algorithms compression_alg;
100 bool enforce_limits;
101 unsigned nested_kcatalog_limit;
102 unsigned root_kcatalog_limit;
103 unsigned file_mbyte_limit;
104 size_t min_file_chunk_size;
105 size_t avg_file_chunk_size;
106 size_t max_file_chunk_size;
107 uint64_t manual_revision;
108 uint64_t ttl_seconds;
109 uint64_t max_concurrent_write_jobs;
110 unsigned num_upload_tasks;
111 bool is_balanced;
112 unsigned max_weight;
113 unsigned min_weight;
114 gid_t gid;
115 uid_t uid;
116
117 // Parameters for when upstream type is HTTP
118 std::string session_token_file;
119 std::string key_file;
120 RepositoryTag repo_tag;
121 };
122
123 namespace catalog {
124 class Dirtab;
125 class SimpleCatalogManager;
126 } // namespace catalog
127
128 namespace swissknife {
129
130 class CommandCreate : public Command {
131 public:
132 ✗ ~CommandCreate() { }
133 ✗ virtual std::string GetName() const { return "create"; }
134 ✗ virtual std::string GetDescription() const {
135 ✗ return "Bootstraps a fresh repository.";
136 }
137 ✗ virtual ParameterList GetParams() const {
138 ✗ ParameterList r;
139 ✗ r.push_back(Parameter::Mandatory('o', "manifest output file"));
140 ✗ r.push_back(Parameter::Mandatory('t', "directory for temporary storage"));
141 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
142 ✗ r.push_back(Parameter::Mandatory('n', "repository name"));
143 ✗ r.push_back(Parameter::Mandatory('R', "path to reflog.chksum file"));
144 ✗ r.push_back(Parameter::Optional('l', "log level (0-4, default: 2)"));
145 ✗ r.push_back(Parameter::Optional('a', "hash algorithm (default: SHA-1)"));
146 ✗ r.push_back(Parameter::Optional('V',
147 "VOMS authz requirement "
148 "(default: none)"));
149 ✗ r.push_back(Parameter::Switch('v', "repository containing volatile files"));
150 ✗ r.push_back(
151 ✗ Parameter::Switch('z', "mark new repository as garbage collectable"));
152 ✗ r.push_back(Parameter::Optional('V',
153 "VOMS authz requirement "
154 "(default: none)"));
155 ✗ return r;
156 }
157 int Main(const ArgumentList &args);
158 };
159
160 class CommandUpload : public Command {
161 public:
162 ✗ ~CommandUpload() { }
163 ✗ virtual std::string GetName() const { return "upload"; }
164 ✗ virtual std::string GetDescription() const {
165 ✗ return "Uploads a local file to the repository.";
166 }
167 ✗ virtual ParameterList GetParams() const {
168 ✗ ParameterList r;
169 ✗ r.push_back(Parameter::Mandatory('i', "local file"));
170 ✗ r.push_back(Parameter::Mandatory('o', "destination path"));
171 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
172 ✗ r.push_back(Parameter::Optional('a', "hash algorithm (default: SHA-1)"));
173 ✗ return r;
174 }
175 int Main(const ArgumentList &args);
176 };
177
178 class CommandPeek : public Command {
179 public:
180 ✗ ~CommandPeek() { }
181 ✗ virtual std::string GetName() const { return "peek"; }
182 ✗ virtual std::string GetDescription() const {
183 ✗ return "Checks whether a file exists in the repository.";
184 }
185 ✗ virtual ParameterList GetParams() const {
186 ✗ ParameterList r;
187 ✗ r.push_back(Parameter::Mandatory('d', "destination path"));
188 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
189 ✗ return r;
190 }
191 int Main(const ArgumentList &args);
192 };
193
194 class CommandRemove : public Command {
195 public:
196 ✗ ~CommandRemove() { }
197 ✗ virtual std::string GetName() const { return "remove"; }
198 ✗ virtual std::string GetDescription() const {
199 ✗ return "Removes a file in the repository storage.";
200 }
201 ✗ virtual ParameterList GetParams() const {
202 ✗ ParameterList r;
203 ✗ r.push_back(Parameter::Mandatory('o', "path to file"));
204 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
205 ✗ return r;
206 }
207 int Main(const ArgumentList &args);
208 };
209
210 class CommandApplyDirtab : public Command {
211 public:
212 ✗ CommandApplyDirtab() : verbose_(false) { }
213 ✗ ~CommandApplyDirtab() { }
214 ✗ virtual std::string GetName() const { return "dirtab"; }
215 ✗ virtual std::string GetDescription() const {
216 ✗ return "Parses the dirtab file and produces nested catalog markers.";
217 }
218 ✗ virtual ParameterList GetParams() const {
219 ✗ ParameterList r;
220 ✗ r.push_back(Parameter::Mandatory('d', "path to dirtab file"));
221 ✗ r.push_back(Parameter::Mandatory('u', "union volume"));
222 ✗ r.push_back(Parameter::Mandatory('s', "scratch directory"));
223 ✗ r.push_back(Parameter::Mandatory('b', "base hash"));
224 ✗ r.push_back(Parameter::Mandatory('w', "stratum 0 base url"));
225 ✗ r.push_back(Parameter::Mandatory('t', "directory for temporary storage"));
226 ✗ r.push_back(Parameter::Optional('@', "proxy url"));
227 ✗ r.push_back(Parameter::Switch('x', "verbose mode"));
228 ✗ return r;
229 }
230 int Main(const ArgumentList &args);
231
232 protected:
233 void DetermineNestedCatalogCandidates(
234 const catalog::Dirtab &dirtab,
235 catalog::SimpleCatalogManager *catalog_manager,
236 std::vector<std::string> *nested_catalog_candidates);
237 void FilterCandidatesFromGlobResult(
238 const catalog::Dirtab &dirtab, char **paths, const size_t npaths,
239 catalog::SimpleCatalogManager *catalog_manager,
240 std::vector<std::string> *nested_catalog_candidates);
241 bool CreateCatalogMarkers(
242 const std::vector<std::string> &new_nested_catalogs);
243
244 private:
245 std::string union_dir_;
246 std::string scratch_dir_;
247 bool verbose_;
248 };
249
250 class CommandSync : public Command {
251 public:
252 ✗ ~CommandSync() { }
253 ✗ virtual std::string GetName() const { return "sync"; }
254 ✗ virtual std::string GetDescription() const {
255 ✗ return "Pushes changes from scratch area back to the repository.";
256 }
257 ✗ virtual ParameterList GetParams() const {
258 // unused characters: J, 1-9, all special characters but @
259 ✗ ParameterList r;
260 ✗ r.push_back(Parameter::Mandatory('b', "base hash"));
261 ✗ r.push_back(Parameter::Mandatory('c', "r/o volume"));
262 ✗ r.push_back(Parameter::Mandatory('o', "manifest output file"));
263 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
264 ✗ r.push_back(Parameter::Mandatory('s', "scratch directory"));
265 ✗ r.push_back(Parameter::Mandatory('t', "directory for tee"));
266 ✗ r.push_back(Parameter::Mandatory('u', "union volume"));
267 ✗ r.push_back(Parameter::Mandatory('w', "stratum 0 base url"));
268 ✗ r.push_back(Parameter::Mandatory('K', "public key(s) for repo"));
269 ✗ r.push_back(Parameter::Mandatory('N', "fully qualified repository name"));
270
271 ✗ r.push_back(Parameter::Optional('a', "desired average chunk size (bytes)"));
272 ✗ r.push_back(Parameter::Optional('e', "hash algorithm (default: SHA-1)"));
273 ✗ r.push_back(Parameter::Optional('f', "union filesystem type"));
274 ✗ r.push_back(Parameter::Optional('h', "maximal file chunk size in bytes"));
275 ✗ r.push_back(Parameter::Optional('l', "minimal file chunk size in bytes"));
276 ✗ r.push_back(Parameter::Optional('q', "number of concurrent write jobs"));
277 ✗ r.push_back(Parameter::Optional('0', "number of upload tasks"));
278 ✗ r.push_back(Parameter::Optional('v', "manual revision number"));
279 ✗ r.push_back(Parameter::Optional('z', "log level (0-4, default: 2)"));
280 ✗ r.push_back(Parameter::Optional('F', "Authz file listing (default: none)"));
281 ✗ r.push_back(Parameter::Optional('M', "minimum weight of the autocatalogs"));
282 ✗ r.push_back(
283 ✗ Parameter::Optional('Q', "nested catalog limit in kilo-entries"));
284 ✗ r.push_back(Parameter::Optional('R', "root catalog limit in kilo-entries"));
285 ✗ r.push_back(Parameter::Optional('T', "Root catalog TTL in seconds"));
286 ✗ r.push_back(Parameter::Optional('U', "file size limit in megabytes"));
287 ✗ r.push_back(
288 ✗ Parameter::Optional('D', "tag name (only used when upstream is GW)"));
289 ✗ r.push_back(Parameter::Optional(
290 'J', "tag description (only used when upstream is GW)"));
291 ✗ r.push_back(Parameter::Optional('X', "maximum weight of the autocatalogs"));
292 ✗ r.push_back(Parameter::Optional('Z',
293 "compression algorithm "
294 "(default: zlib)"));
295 ✗ r.push_back(Parameter::Optional('S',
296 "virtual directory options "
297 "[snapshots, remove]"));
298
299
300 ✗ r.push_back(
301 ✗ Parameter::Switch('G', "Use persistent caching for all catalogs "
302 "used during the publishing process"
303 " Warning: No automatic garbage collection!"));
304 ✗ r.push_back(Parameter::Switch('d',
305 "pause publishing to allow for catalog "
306 "tweaks"));
307 ✗ r.push_back(Parameter::Switch('i', "ignore x-directory hardlinks"));
308 ✗ r.push_back(Parameter::Switch('g', "ignore special files"));
309 ✗ r.push_back(Parameter::Switch('k', "include extended attributes"));
310 ✗ r.push_back(Parameter::Switch('j', "enable nanosecond timestamps"));
311 ✗ r.push_back(Parameter::Switch('m', "create micro catalogs"));
312 ✗ r.push_back(Parameter::Switch('n', "create new repository"));
313 ✗ r.push_back(Parameter::Switch('p', "enable file chunking"));
314 ✗ r.push_back(Parameter::Switch('O', "generate legacy bulk chunks"));
315 ✗ r.push_back(Parameter::Switch('x', "print change set"));
316 ✗ r.push_back(Parameter::Switch('y', "dry run"));
317 ✗ r.push_back(Parameter::Switch('A', "autocatalog enabled/disabled"));
318 ✗ r.push_back(Parameter::Switch('E', "enforce limits instead of warning"));
319 ✗ r.push_back(Parameter::Switch('L', "enable HTTP redirects"));
320 ✗ r.push_back(Parameter::Switch('V',
321 "Publish format compatible with "
322 "authenticated repos"));
323 ✗ r.push_back(Parameter::Switch('Y', "enable external data"));
324 ✗ r.push_back(Parameter::Switch('W', "set direct I/O for regular files"));
325 ✗ r.push_back(Parameter::Switch('B', "branched catalog (no manifest)"));
326 ✗ r.push_back(Parameter::Switch('I', "upload updated statistics DB file"));
327
328 ✗ r.push_back(Parameter::Optional('P', "session_token_file"));
329 ✗ r.push_back(Parameter::Optional('H', "key file for HTTP API"));
330 ✗ r.push_back(Parameter::Optional('@', "proxy URL"));
331
332 ✗ return r;
333 }
334 int Main(const ArgumentList &args);
335
336 protected:
337 bool ReadFileChunkingArgs(const swissknife::ArgumentList &args,
338 SyncParameters *params);
339 bool CheckParams(const SyncParameters &p);
340 };
341
342 } // namespace swissknife
343
344 #endif // CVMFS_SWISSKNIFE_SYNC_H_
345