GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/swissknife_sync.h
Date: 2025-02-09 02:34:19
Exec Total Coverage
Lines: 0 167 0.0%
Branches: 0 10 0.0%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 */
4
5 #ifndef CVMFS_SWISSKNIFE_SYNC_H_
6 #define CVMFS_SWISSKNIFE_SYNC_H_
7
8 #include <string>
9 #include <vector>
10
11 #include "compression/compression.h"
12 #include "repository_tag.h"
13 #include "swissknife.h"
14 #include "upload.h"
15
16 struct SyncParameters {
17 static const unsigned kDefaultMaxWeight = 100000;
18 static const unsigned kDefaultMinWeight = 1000;
19 static const size_t kDefaultMinFileChunkSize = 4 * 1024 * 1024;
20 static const size_t kDefaultAvgFileChunkSize = 8 * 1024 * 1024;
21 static const size_t kDefaultMaxFileChunkSize = 16 * 1024 * 1024;
22 static const unsigned kDefaultNestedKcatalogLimit = 500;
23 static const unsigned kDefaultRootKcatalogLimit = 200;
24 static const unsigned kDefaultFileMbyteLimit = 1024;
25
26 ✗ SyncParameters()
27 ✗ : spooler(NULL),
28 ✗ union_fs_type("aufs"),
29 ✗ to_delete(""),
30 ✗ cache_dir(""),
31 ✗ print_changeset(false),
32 ✗ dry_run(false),
33 ✗ mucatalogs(false),
34 ✗ use_file_chunking(false),
35 ✗ generate_legacy_bulk_chunks(false),
36 ✗ ignore_xdir_hardlinks(false),
37 ✗ stop_for_catalog_tweaks(false),
38 ✗ include_xattrs(false),
39 ✗ enable_mtime_ns(false),
40 ✗ external_data(false),
41 ✗ direct_io(false),
42 ✗ voms_authz(false),
43 ✗ virtual_dir_actions(0),
44 ✗ ignore_special_files(false),
45 ✗ branched_catalog(false),
46 ✗ compression_alg(zlib::kZlibDefault),
47 ✗ enforce_limits(false),
48 ✗ nested_kcatalog_limit(0),
49 ✗ root_kcatalog_limit(0),
50 ✗ file_mbyte_limit(0),
51 ✗ min_file_chunk_size(kDefaultMinFileChunkSize),
52 ✗ avg_file_chunk_size(kDefaultAvgFileChunkSize),
53 ✗ max_file_chunk_size(kDefaultMaxFileChunkSize),
54 ✗ manual_revision(0),
55 ✗ ttl_seconds(0),
56 ✗ max_concurrent_write_jobs(0),
57 ✗ num_upload_tasks(1),
58 ✗ is_balanced(false),
59 ✗ max_weight(kDefaultMaxWeight),
60 ✗ min_weight(kDefaultMinWeight),
61 ✗ gid(-1u),
62 ✗ uid(-1u),
63 ✗ session_token_file(),
64 ✗ key_file(),
65 ✗ repo_tag() {}
66
67 upload::Spooler *spooler;
68 std::string repo_name;
69 std::string dir_union;
70 std::string dir_scratch;
71 std::string dir_rdonly;
72 std::string dir_temp;
73 shash::Any base_hash;
74 std::string stratum0;
75 std::string manifest_path;
76 std::string spooler_definition;
77 std::string union_fs_type;
78 std::string public_keys;
79 std::string authz_file;
80 std::string tar_file;
81 std::string base_directory;
82 std::string to_delete;
83 std::string cache_dir;
84 bool print_changeset;
85 bool dry_run;
86 bool mucatalogs;
87 bool use_file_chunking;
88 bool generate_legacy_bulk_chunks;
89 bool ignore_xdir_hardlinks;
90 bool stop_for_catalog_tweaks;
91 bool include_xattrs;
92 bool enable_mtime_ns;
93 bool external_data;
94 bool direct_io;
95 bool voms_authz;
96 int virtual_dir_actions; // bit field
97 bool ignore_special_files;
98 bool branched_catalog;
99 zlib::Algorithms compression_alg;
100 bool enforce_limits;
101 unsigned nested_kcatalog_limit;
102 unsigned root_kcatalog_limit;
103 unsigned file_mbyte_limit;
104 size_t min_file_chunk_size;
105 size_t avg_file_chunk_size;
106 size_t max_file_chunk_size;
107 uint64_t manual_revision;
108 uint64_t ttl_seconds;
109 uint64_t max_concurrent_write_jobs;
110 unsigned num_upload_tasks;
111 bool is_balanced;
112 unsigned max_weight;
113 unsigned min_weight;
114 gid_t gid;
115 uid_t uid;
116
117 // Parameters for when upstream type is HTTP
118 std::string session_token_file;
119 std::string key_file;
120 RepositoryTag repo_tag;
121 };
122
123 namespace catalog {
124 class Dirtab;
125 class SimpleCatalogManager;
126 } // namespace catalog
127
128 namespace swissknife {
129
130 class CommandCreate : public Command {
131 public:
132 ✗ ~CommandCreate() {}
133 ✗ virtual std::string GetName() const { return "create"; }
134 ✗ virtual std::string GetDescription() const {
135 ✗ return "Bootstraps a fresh repository.";
136 }
137 ✗ virtual ParameterList GetParams() const {
138 ✗ ParameterList r;
139 ✗ r.push_back(Parameter::Mandatory('o', "manifest output file"));
140 ✗ r.push_back(Parameter::Mandatory('t', "directory for temporary storage"));
141 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
142 ✗ r.push_back(Parameter::Mandatory('n', "repository name"));
143 ✗ r.push_back(Parameter::Mandatory('R', "path to reflog.chksum file"));
144 ✗ r.push_back(Parameter::Optional('l', "log level (0-4, default: 2)"));
145 ✗ r.push_back(Parameter::Optional('a', "hash algorithm (default: SHA-1)"));
146 ✗ r.push_back(Parameter::Optional('V',
147 "VOMS authz requirement "
148 "(default: none)"));
149 ✗ r.push_back(Parameter::Switch('v', "repository containing volatile files"));
150 ✗ r.push_back(
151 ✗ Parameter::Switch('z', "mark new repository as garbage collectable"));
152 ✗ r.push_back(Parameter::Optional('V',
153 "VOMS authz requirement "
154 "(default: none)"));
155 ✗ return r;
156 }
157 int Main(const ArgumentList &args);
158 };
159
160 class CommandUpload : public Command {
161 public:
162 ✗ ~CommandUpload() {}
163 ✗ virtual std::string GetName() const { return "upload"; }
164 ✗ virtual std::string GetDescription() const {
165 ✗ return "Uploads a local file to the repository.";
166 }
167 ✗ virtual ParameterList GetParams() const {
168 ✗ ParameterList r;
169 ✗ r.push_back(Parameter::Mandatory('i', "local file"));
170 ✗ r.push_back(Parameter::Mandatory('o', "destination path"));
171 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
172 ✗ r.push_back(Parameter::Optional('a', "hash algorithm (default: SHA-1)"));
173 ✗ return r;
174 }
175 int Main(const ArgumentList &args);
176 };
177
178 class CommandPeek : public Command {
179 public:
180 ✗ ~CommandPeek() {}
181 ✗ virtual std::string GetName() const { return "peek"; }
182 ✗ virtual std::string GetDescription() const {
183 ✗ return "Checks whether a file exists in the repository.";
184 }
185 ✗ virtual ParameterList GetParams() const {
186 ✗ ParameterList r;
187 ✗ r.push_back(Parameter::Mandatory('d', "destination path"));
188 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
189 ✗ return r;
190 }
191 int Main(const ArgumentList &args);
192 };
193
194 class CommandRemove : public Command {
195 public:
196 ✗ ~CommandRemove() {}
197 ✗ virtual std::string GetName() const { return "remove"; }
198 ✗ virtual std::string GetDescription() const {
199 ✗ return "Removes a file in the repository storage.";
200 }
201 ✗ virtual ParameterList GetParams() const {
202 ✗ ParameterList r;
203 ✗ r.push_back(Parameter::Mandatory('o', "path to file"));
204 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
205 ✗ return r;
206 }
207 int Main(const ArgumentList &args);
208 };
209
210 class CommandApplyDirtab : public Command {
211 public:
212 ✗ CommandApplyDirtab() : verbose_(false) {}
213 ✗ ~CommandApplyDirtab() {}
214 ✗ virtual std::string GetName() const { return "dirtab"; }
215 ✗ virtual std::string GetDescription() const {
216 ✗ return "Parses the dirtab file and produces nested catalog markers.";
217 }
218 ✗ virtual ParameterList GetParams() const {
219 ✗ ParameterList r;
220 ✗ r.push_back(Parameter::Mandatory('d', "path to dirtab file"));
221 ✗ r.push_back(Parameter::Mandatory('u', "union volume"));
222 ✗ r.push_back(Parameter::Mandatory('s', "scratch directory"));
223 ✗ r.push_back(Parameter::Mandatory('b', "base hash"));
224 ✗ r.push_back(Parameter::Mandatory('w', "stratum 0 base url"));
225 ✗ r.push_back(Parameter::Mandatory('t', "directory for temporary storage"));
226 ✗ r.push_back(Parameter::Optional('@', "proxy url"));
227 ✗ r.push_back(Parameter::Switch('x', "verbose mode"));
228 ✗ return r;
229 }
230 int Main(const ArgumentList &args);
231
232 protected:
233 void DetermineNestedCatalogCandidates(
234 const catalog::Dirtab &dirtab,
235 catalog::SimpleCatalogManager *catalog_manager,
236 std::vector<std::string> *nested_catalog_candidates);
237 void FilterCandidatesFromGlobResult(
238 const catalog::Dirtab &dirtab, char **paths, const size_t npaths,
239 catalog::SimpleCatalogManager *catalog_manager,
240 std::vector<std::string> *nested_catalog_candidates);
241 bool CreateCatalogMarkers(
242 const std::vector<std::string> &new_nested_catalogs);
243
244 private:
245 std::string union_dir_;
246 std::string scratch_dir_;
247 bool verbose_;
248 };
249
250 class CommandSync : public Command {
251 public:
252 ✗ ~CommandSync() {}
253 ✗ virtual std::string GetName() const { return "sync"; }
254 ✗ virtual std::string GetDescription() const {
255 ✗ return "Pushes changes from scratch area back to the repository.";
256 }
257 ✗ virtual ParameterList GetParams() const {
258 // unused characters: J, 1-9, all special characters but @
259 ✗ ParameterList r;
260 ✗ r.push_back(Parameter::Mandatory('b', "base hash"));
261 ✗ r.push_back(Parameter::Mandatory('c', "r/o volume"));
262 ✗ r.push_back(Parameter::Mandatory('o', "manifest output file"));
263 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
264 ✗ r.push_back(Parameter::Mandatory('s', "scratch directory"));
265 ✗ r.push_back(Parameter::Mandatory('t', "directory for tee"));
266 ✗ r.push_back(Parameter::Mandatory('u', "union volume"));
267 ✗ r.push_back(Parameter::Mandatory('w', "stratum 0 base url"));
268 ✗ r.push_back(Parameter::Mandatory('K', "public key(s) for repo"));
269 ✗ r.push_back(Parameter::Mandatory('N', "fully qualified repository name"));
270
271 ✗ r.push_back(Parameter::Optional('a', "desired average chunk size (bytes)"));
272 ✗ r.push_back(Parameter::Optional('e', "hash algorithm (default: SHA-1)"));
273 ✗ r.push_back(Parameter::Optional('f', "union filesystem type"));
274 ✗ r.push_back(Parameter::Optional('h', "maximal file chunk size in bytes"));
275 ✗ r.push_back(Parameter::Optional('l', "minimal file chunk size in bytes"));
276 ✗ r.push_back(Parameter::Optional('q', "number of concurrent write jobs"));
277 ✗ r.push_back(Parameter::Optional('0', "number of upload tasks"));
278 ✗ r.push_back(Parameter::Optional('v', "manual revision number"));
279 ✗ r.push_back(Parameter::Optional('z', "log level (0-4, default: 2)"));
280 ✗ r.push_back(Parameter::Optional('F', "Authz file listing (default: none)"));
281 ✗ r.push_back(Parameter::Optional('M', "minimum weight of the autocatalogs"));
282 ✗ r.push_back(
283 ✗ Parameter::Optional('Q', "nested catalog limit in kilo-entries"));
284 ✗ r.push_back(Parameter::Optional('R', "root catalog limit in kilo-entries"));
285 ✗ r.push_back(Parameter::Optional('T', "Root catalog TTL in seconds"));
286 ✗ r.push_back(Parameter::Optional('U', "file size limit in megabytes"));
287 ✗ r.push_back(
288 ✗ Parameter::Optional('D', "tag name (only used when upstream is GW)"));
289 ✗ r.push_back(Parameter::Optional(
290 'J', "tag description (only used when upstream is GW)"));
291 ✗ r.push_back(Parameter::Optional('X', "maximum weight of the autocatalogs"));
292 ✗ r.push_back(Parameter::Optional('Z',
293 "compression algorithm "
294 "(default: zlib)"));
295 ✗ r.push_back(Parameter::Optional('S',
296 "virtual directory options "
297 "[snapshots, remove]"));
298
299
300 ✗ r.push_back(
301 ✗ Parameter::Switch('G', "Use persistent caching for all catalogs "
302 "used during the publishing process"
303 " Warning: No automatic garbage collection!"));
304 ✗ r.push_back(Parameter::Switch('d',
305 "pause publishing to allow for catalog "
306 "tweaks"));
307 ✗ r.push_back(Parameter::Switch('i', "ignore x-directory hardlinks"));
308 ✗ r.push_back(Parameter::Switch('g', "ignore special files"));
309 ✗ r.push_back(Parameter::Switch('k', "include extended attributes"));
310 ✗ r.push_back(Parameter::Switch('j', "enable nanosecond timestamps"));
311 ✗ r.push_back(Parameter::Switch('m', "create micro catalogs"));
312 ✗ r.push_back(Parameter::Switch('n', "create new repository"));
313 ✗ r.push_back(Parameter::Switch('p', "enable file chunking"));
314 ✗ r.push_back(Parameter::Switch('O', "generate legacy bulk chunks"));
315 ✗ r.push_back(Parameter::Switch('x', "print change set"));
316 ✗ r.push_back(Parameter::Switch('y', "dry run"));
317 ✗ r.push_back(Parameter::Switch('A', "autocatalog enabled/disabled"));
318 ✗ r.push_back(Parameter::Switch('E', "enforce limits instead of warning"));
319 ✗ r.push_back(Parameter::Switch('L', "enable HTTP redirects"));
320 ✗ r.push_back(Parameter::Switch('V',
321 "Publish format compatible with "
322 "authenticated repos"));
323 ✗ r.push_back(Parameter::Switch('Y', "enable external data"));
324 ✗ r.push_back(Parameter::Switch('W', "set direct I/O for regular files"));
325 ✗ r.push_back(Parameter::Switch('B', "branched catalog (no manifest)"));
326 ✗ r.push_back(Parameter::Switch('I', "upload updated statistics DB file"));
327
328 ✗ r.push_back(Parameter::Optional('P', "session_token_file"));
329 ✗ r.push_back(Parameter::Optional('H', "key file for HTTP API"));
330 ✗ r.push_back(Parameter::Optional('@', "proxy URL"));
331
332 ✗ return r;
333 }
334 int Main(const ArgumentList &args);
335
336 protected:
337 bool ReadFileChunkingArgs(const swissknife::ArgumentList &args,
338 SyncParameters *params);
339 bool CheckParams(const SyncParameters &p);
340 };
341
342 } // namespace swissknife
343
344 #endif // CVMFS_SWISSKNIFE_SYNC_H_
345