GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/swissknife_sync.h
Date: 2026-03-15 02:35:27
Exec Total Coverage
Lines: 0 168 0.0%
Branches: 0 10 0.0%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 */
4
5 #ifndef CVMFS_SWISSKNIFE_SYNC_H_
6 #define CVMFS_SWISSKNIFE_SYNC_H_
7
8 #include <string>
9 #include <vector>
10
11 #include "compression/compression.h"
12 #include "repository_tag.h"
13 #include "swissknife.h"
14 #include "upload.h"
15
16 struct SyncParameters {
17 static const unsigned kDefaultMaxWeight = 100000;
18 static const unsigned kDefaultMinWeight = 1000;
19 static const size_t kDefaultMinFileChunkSize = 4 * 1024 * 1024;
20 static const size_t kDefaultAvgFileChunkSize = 8 * 1024 * 1024;
21 static const size_t kDefaultMaxFileChunkSize = 16 * 1024 * 1024;
22 static const unsigned kDefaultNestedKcatalogLimit = 500;
23 static const unsigned kDefaultRootKcatalogLimit = 200;
24 static const unsigned kDefaultFileMbyteLimit = 1024;
25
26 ✗ SyncParameters()
27 ✗ : spooler(NULL)
28 ✗ , union_fs_type("aufs")
29 ✗ , to_delete("")
30 ✗ , cache_dir("")
31 ✗ , print_changeset(false)
32 ✗ , dry_run(false)
33 ✗ , mucatalogs(false)
34 ✗ , use_file_chunking(false)
35 ✗ , generate_legacy_bulk_chunks(false)
36 ✗ , ignore_xdir_hardlinks(false)
37 ✗ , stop_for_catalog_tweaks(false)
38 ✗ , include_xattrs(false)
39 ✗ , enable_mtime_ns(false)
40 ✗ , fast_delete(false)
41 ✗ , external_data(false)
42 ✗ , direct_io(false)
43 ✗ , voms_authz(false)
44 ✗ , virtual_dir_actions(0)
45 ✗ , ignore_special_files(false)
46 ✗ , branched_catalog(false)
47 ✗ , compression_alg(zlib::kZlibDefault)
48 ✗ , enforce_limits(false)
49 ✗ , nested_kcatalog_limit(0)
50 ✗ , root_kcatalog_limit(0)
51 ✗ , file_mbyte_limit(0)
52 ✗ , min_file_chunk_size(kDefaultMinFileChunkSize)
53 ✗ , avg_file_chunk_size(kDefaultAvgFileChunkSize)
54 ✗ , max_file_chunk_size(kDefaultMaxFileChunkSize)
55 ✗ , manual_revision(0)
56 ✗ , ttl_seconds(0)
57 ✗ , max_concurrent_write_jobs(0)
58 ✗ , num_upload_tasks(1)
59 ✗ , is_balanced(false)
60 ✗ , max_weight(kDefaultMaxWeight)
61 ✗ , min_weight(kDefaultMinWeight)
62 ✗ , gid(-1u)
63 ✗ , uid(-1u)
64 ✗ , session_token_file()
65 ✗ , key_file()
66 ✗ , repo_tag() { }
67
68 upload::Spooler *spooler;
69 std::string repo_name;
70 std::string dir_union;
71 std::string dir_scratch;
72 std::string dir_rdonly;
73 std::string dir_temp;
74 shash::Any base_hash;
75 std::string stratum0;
76 std::string manifest_path;
77 std::string spooler_definition;
78 std::string union_fs_type;
79 std::string public_keys;
80 std::string authz_file;
81 std::string tar_file;
82 std::string base_directory;
83 std::string to_delete;
84 std::string cache_dir;
85 bool print_changeset;
86 bool dry_run;
87 bool mucatalogs;
88 bool use_file_chunking;
89 bool generate_legacy_bulk_chunks;
90 bool ignore_xdir_hardlinks;
91 bool stop_for_catalog_tweaks;
92 bool include_xattrs;
93 bool enable_mtime_ns;
94 bool fast_delete;
95 bool external_data;
96 bool direct_io;
97 bool voms_authz;
98 int virtual_dir_actions; // bit field
99 bool ignore_special_files;
100 bool branched_catalog;
101 zlib::Algorithms compression_alg;
102 bool enforce_limits;
103 unsigned nested_kcatalog_limit;
104 unsigned root_kcatalog_limit;
105 unsigned file_mbyte_limit;
106 size_t min_file_chunk_size;
107 size_t avg_file_chunk_size;
108 size_t max_file_chunk_size;
109 uint64_t manual_revision;
110 uint64_t ttl_seconds;
111 uint64_t max_concurrent_write_jobs;
112 unsigned num_upload_tasks;
113 bool is_balanced;
114 unsigned max_weight;
115 unsigned min_weight;
116 gid_t gid;
117 uid_t uid;
118
119 // Parameters for when upstream type is HTTP
120 std::string session_token_file;
121 std::string key_file;
122 RepositoryTag repo_tag;
123 };
124
125 namespace catalog {
126 class Dirtab;
127 class SimpleCatalogManager;
128 } // namespace catalog
129
130 namespace swissknife {
131
132 class CommandCreate : public Command {
133 public:
134 ✗ ~CommandCreate() { }
135 ✗ virtual std::string GetName() const { return "create"; }
136 ✗ virtual std::string GetDescription() const {
137 ✗ return "Bootstraps a fresh repository.";
138 }
139 ✗ virtual ParameterList GetParams() const {
140 ✗ ParameterList r;
141 ✗ r.push_back(Parameter::Mandatory('o', "manifest output file"));
142 ✗ r.push_back(Parameter::Mandatory('t', "directory for temporary storage"));
143 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
144 ✗ r.push_back(Parameter::Mandatory('n', "repository name"));
145 ✗ r.push_back(Parameter::Mandatory('R', "path to reflog.chksum file"));
146 ✗ r.push_back(Parameter::Optional('l', "log level (0-4, default: 2)"));
147 ✗ r.push_back(Parameter::Optional('a', "hash algorithm (default: SHA-1)"));
148 ✗ r.push_back(Parameter::Optional('V',
149 "VOMS authz requirement "
150 "(default: none)"));
151 ✗ r.push_back(Parameter::Switch('v', "repository containing volatile files"));
152 ✗ r.push_back(
153 ✗ Parameter::Switch('z', "mark new repository as garbage collectable"));
154 ✗ r.push_back(Parameter::Optional('V',
155 "VOMS authz requirement "
156 "(default: none)"));
157 ✗ return r;
158 }
159 int Main(const ArgumentList &args);
160 };
161
162 class CommandUpload : public Command {
163 public:
164 ✗ ~CommandUpload() { }
165 ✗ virtual std::string GetName() const { return "upload"; }
166 ✗ virtual std::string GetDescription() const {
167 ✗ return "Uploads a local file to the repository.";
168 }
169 ✗ virtual ParameterList GetParams() const {
170 ✗ ParameterList r;
171 ✗ r.push_back(Parameter::Mandatory('i', "local file"));
172 ✗ r.push_back(Parameter::Mandatory('o', "destination path"));
173 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
174 ✗ r.push_back(Parameter::Optional('a', "hash algorithm (default: SHA-1)"));
175 ✗ return r;
176 }
177 int Main(const ArgumentList &args);
178 };
179
180 class CommandPeek : public Command {
181 public:
182 ✗ ~CommandPeek() { }
183 ✗ virtual std::string GetName() const { return "peek"; }
184 ✗ virtual std::string GetDescription() const {
185 ✗ return "Checks whether a file exists in the repository.";
186 }
187 ✗ virtual ParameterList GetParams() const {
188 ✗ ParameterList r;
189 ✗ r.push_back(Parameter::Mandatory('d', "destination path"));
190 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
191 ✗ return r;
192 }
193 int Main(const ArgumentList &args);
194 };
195
196 class CommandRemove : public Command {
197 public:
198 ✗ ~CommandRemove() { }
199 ✗ virtual std::string GetName() const { return "remove"; }
200 ✗ virtual std::string GetDescription() const {
201 ✗ return "Removes a file in the repository storage.";
202 }
203 ✗ virtual ParameterList GetParams() const {
204 ✗ ParameterList r;
205 ✗ r.push_back(Parameter::Mandatory('o', "path to file"));
206 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
207 ✗ return r;
208 }
209 int Main(const ArgumentList &args);
210 };
211
212 class CommandApplyDirtab : public Command {
213 public:
214 ✗ CommandApplyDirtab() : verbose_(false) { }
215 ✗ ~CommandApplyDirtab() { }
216 ✗ virtual std::string GetName() const { return "dirtab"; }
217 ✗ virtual std::string GetDescription() const {
218 ✗ return "Parses the dirtab file and produces nested catalog markers.";
219 }
220 ✗ virtual ParameterList GetParams() const {
221 ✗ ParameterList r;
222 ✗ r.push_back(Parameter::Mandatory('d', "path to dirtab file"));
223 ✗ r.push_back(Parameter::Mandatory('u', "union volume"));
224 ✗ r.push_back(Parameter::Mandatory('s', "scratch directory"));
225 ✗ r.push_back(Parameter::Mandatory('b', "base hash"));
226 ✗ r.push_back(Parameter::Mandatory('w', "stratum 0 base url"));
227 ✗ r.push_back(Parameter::Mandatory('t', "directory for temporary storage"));
228 ✗ r.push_back(Parameter::Optional('@', "proxy url"));
229 ✗ r.push_back(Parameter::Switch('x', "verbose mode"));
230 ✗ return r;
231 }
232 int Main(const ArgumentList &args);
233
234 protected:
235 void DetermineNestedCatalogCandidates(
236 const catalog::Dirtab &dirtab,
237 catalog::SimpleCatalogManager *catalog_manager,
238 std::vector<std::string> *nested_catalog_candidates);
239 void FilterCandidatesFromGlobResult(
240 const catalog::Dirtab &dirtab, char **paths, const size_t npaths,
241 catalog::SimpleCatalogManager *catalog_manager,
242 std::vector<std::string> *nested_catalog_candidates);
243 bool CreateCatalogMarkers(
244 const std::vector<std::string> &new_nested_catalogs);
245
246 private:
247 std::string union_dir_;
248 std::string scratch_dir_;
249 bool verbose_;
250 };
251
252 class CommandSync : public Command {
253 public:
254 ✗ ~CommandSync() { }
255 ✗ virtual std::string GetName() const { return "sync"; }
256 ✗ virtual std::string GetDescription() const {
257 ✗ return "Pushes changes from scratch area back to the repository.";
258 }
259 ✗ virtual ParameterList GetParams() const {
260 // unused characters: J, 1-9, all special characters but @
261 ✗ ParameterList r;
262 ✗ r.push_back(Parameter::Mandatory('b', "base hash"));
263 ✗ r.push_back(Parameter::Mandatory('c', "r/o volume"));
264 ✗ r.push_back(Parameter::Mandatory('o', "manifest output file"));
265 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
266 ✗ r.push_back(Parameter::Mandatory('s', "scratch directory"));
267 ✗ r.push_back(Parameter::Mandatory('t', "directory for tee"));
268 ✗ r.push_back(Parameter::Mandatory('u', "union volume"));
269 ✗ r.push_back(Parameter::Mandatory('w', "stratum 0 base url"));
270 ✗ r.push_back(Parameter::Mandatory('K', "public key(s) for repo"));
271 ✗ r.push_back(Parameter::Mandatory('N', "fully qualified repository name"));
272
273 ✗ r.push_back(Parameter::Optional('a', "desired average chunk size (bytes)"));
274 ✗ r.push_back(Parameter::Optional('e', "hash algorithm (default: SHA-1)"));
275 ✗ r.push_back(Parameter::Optional('f', "union filesystem type"));
276 ✗ r.push_back(Parameter::Optional('h', "maximal file chunk size in bytes"));
277 ✗ r.push_back(Parameter::Optional('l', "minimal file chunk size in bytes"));
278 ✗ r.push_back(Parameter::Optional('q', "number of concurrent write jobs"));
279 ✗ r.push_back(Parameter::Optional('0', "number of upload tasks"));
280 ✗ r.push_back(Parameter::Optional('v', "manual revision number"));
281 ✗ r.push_back(Parameter::Optional('z', "log level (0-4, default: 2)"));
282 ✗ r.push_back(Parameter::Optional('F', "Authz file listing (default: none)"));
283 ✗ r.push_back(Parameter::Optional('M', "minimum weight of the autocatalogs"));
284 ✗ r.push_back(
285 ✗ Parameter::Optional('Q', "nested catalog limit in kilo-entries"));
286 ✗ r.push_back(Parameter::Optional('R', "root catalog limit in kilo-entries"));
287 ✗ r.push_back(Parameter::Optional('T', "Root catalog TTL in seconds"));
288 ✗ r.push_back(Parameter::Optional('U', "file size limit in megabytes"));
289 ✗ r.push_back(
290 ✗ Parameter::Optional('D', "tag name (only used when upstream is GW)"));
291 ✗ r.push_back(Parameter::Optional(
292 'J', "tag description (only used when upstream is GW)"));
293 ✗ r.push_back(Parameter::Optional('X', "maximum weight of the autocatalogs"));
294 ✗ r.push_back(Parameter::Optional('Z',
295 "compression algorithm "
296 "(default: zlib)"));
297 ✗ r.push_back(Parameter::Optional('S',
298 "virtual directory options "
299 "[snapshots, remove]"));
300
301
302 ✗ r.push_back(
303 ✗ Parameter::Switch('G', "Use persistent caching for all catalogs "
304 "used during the publishing process"
305 " Warning: No automatic garbage collection!"));
306 ✗ r.push_back(Parameter::Switch('d',
307 "pause publishing to allow for catalog "
308 "tweaks"));
309 ✗ r.push_back(Parameter::Switch('i', "ignore x-directory hardlinks"));
310 ✗ r.push_back(Parameter::Switch('g', "ignore special files"));
311 ✗ r.push_back(Parameter::Switch('k', "include extended attributes"));
312 ✗ r.push_back(Parameter::Switch('j', "enable nanosecond timestamps"));
313 ✗ r.push_back(Parameter::Switch('m', "create micro catalogs"));
314 ✗ r.push_back(Parameter::Switch('n', "create new repository"));
315 ✗ r.push_back(Parameter::Switch('p', "enable file chunking"));
316 ✗ r.push_back(Parameter::Switch('O', "generate legacy bulk chunks"));
317 ✗ r.push_back(Parameter::Switch('x', "print change set"));
318 ✗ r.push_back(Parameter::Switch('y', "dry run"));
319 ✗ r.push_back(Parameter::Switch('A', "autocatalog enabled/disabled"));
320 ✗ r.push_back(Parameter::Switch('E', "enforce limits instead of warning"));
321 ✗ r.push_back(Parameter::Switch('L', "enable HTTP redirects"));
322 ✗ r.push_back(Parameter::Switch('V',
323 "Publish format compatible with "
324 "authenticated repos"));
325 ✗ r.push_back(Parameter::Switch('Y', "enable external data"));
326 ✗ r.push_back(Parameter::Switch('W', "set direct I/O for regular files"));
327 ✗ r.push_back(Parameter::Switch('B', "branched catalog (no manifest)"));
328 ✗ r.push_back(Parameter::Switch('I', "upload updated statistics DB file"));
329
330 ✗ r.push_back(Parameter::Optional('P', "session_token_file"));
331 ✗ r.push_back(Parameter::Optional('H', "key file for HTTP API"));
332 ✗ r.push_back(Parameter::Optional('@', "proxy URL"));
333
334 ✗ return r;
335 }
336 int Main(const ArgumentList &args);
337
338 protected:
339 bool ReadFileChunkingArgs(const swissknife::ArgumentList &args,
340 SyncParameters *params);
341 bool CheckParams(const SyncParameters &p);
342 };
343
344 } // namespace swissknife
345
346 #endif // CVMFS_SWISSKNIFE_SYNC_H_
347