GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/swissknife_sync.h
Date: 2024-04-21 02:33:16
Exec Total Coverage
Lines: 0 162 0.0%
Branches: 0 8 0.0%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 */
4
5 #ifndef CVMFS_SWISSKNIFE_SYNC_H_
6 #define CVMFS_SWISSKNIFE_SYNC_H_
7
8 #include <string>
9 #include <vector>
10
11 #include "compression.h"
12 #include "repository_tag.h"
13 #include "swissknife.h"
14 #include "upload.h"
15
16 struct SyncParameters {
17 static const unsigned kDefaultMaxWeight = 100000;
18 static const unsigned kDefaultMinWeight = 1000;
19 static const size_t kDefaultMinFileChunkSize = 4 * 1024 * 1024;
20 static const size_t kDefaultAvgFileChunkSize = 8 * 1024 * 1024;
21 static const size_t kDefaultMaxFileChunkSize = 16 * 1024 * 1024;
22 static const unsigned kDefaultNestedKcatalogLimit = 500;
23 static const unsigned kDefaultRootKcatalogLimit = 200;
24 static const unsigned kDefaultFileMbyteLimit = 1024;
25
26 ✗ SyncParameters()
27 ✗ : spooler(NULL),
28 ✗ union_fs_type("aufs"),
29 ✗ to_delete(""),
30 ✗ print_changeset(false),
31 ✗ dry_run(false),
32 ✗ mucatalogs(false),
33 ✗ use_file_chunking(false),
34 ✗ generate_legacy_bulk_chunks(false),
35 ✗ ignore_xdir_hardlinks(false),
36 ✗ stop_for_catalog_tweaks(false),
37 ✗ include_xattrs(false),
38 ✗ external_data(false),
39 ✗ direct_io(false),
40 ✗ voms_authz(false),
41 ✗ virtual_dir_actions(0),
42 ✗ ignore_special_files(false),
43 ✗ branched_catalog(false),
44 ✗ compression_alg(zlib::kZlibDefault),
45 ✗ enforce_limits(false),
46 ✗ nested_kcatalog_limit(0),
47 ✗ root_kcatalog_limit(0),
48 ✗ file_mbyte_limit(0),
49 ✗ min_file_chunk_size(kDefaultMinFileChunkSize),
50 ✗ avg_file_chunk_size(kDefaultAvgFileChunkSize),
51 ✗ max_file_chunk_size(kDefaultMaxFileChunkSize),
52 ✗ manual_revision(0),
53 ✗ ttl_seconds(0),
54 ✗ max_concurrent_write_jobs(0),
55 ✗ num_upload_tasks(1),
56 ✗ is_balanced(false),
57 ✗ max_weight(kDefaultMaxWeight),
58 ✗ min_weight(kDefaultMinWeight),
59 ✗ gid(-1u),
60 ✗ uid(-1u),
61 ✗ session_token_file(),
62 ✗ key_file(),
63 ✗ repo_tag() {}
64
65 upload::Spooler *spooler;
66 std::string repo_name;
67 std::string dir_union;
68 std::string dir_scratch;
69 std::string dir_rdonly;
70 std::string dir_temp;
71 shash::Any base_hash;
72 std::string stratum0;
73 std::string manifest_path;
74 std::string spooler_definition;
75 std::string union_fs_type;
76 std::string public_keys;
77 std::string authz_file;
78 std::string tar_file;
79 std::string base_directory;
80 std::string to_delete;
81 bool print_changeset;
82 bool dry_run;
83 bool mucatalogs;
84 bool use_file_chunking;
85 bool generate_legacy_bulk_chunks;
86 bool ignore_xdir_hardlinks;
87 bool stop_for_catalog_tweaks;
88 bool include_xattrs;
89 bool external_data;
90 bool direct_io;
91 bool voms_authz;
92 int virtual_dir_actions; // bit field
93 bool ignore_special_files;
94 bool branched_catalog;
95 zlib::Algorithms compression_alg;
96 bool enforce_limits;
97 unsigned nested_kcatalog_limit;
98 unsigned root_kcatalog_limit;
99 unsigned file_mbyte_limit;
100 size_t min_file_chunk_size;
101 size_t avg_file_chunk_size;
102 size_t max_file_chunk_size;
103 uint64_t manual_revision;
104 uint64_t ttl_seconds;
105 uint64_t max_concurrent_write_jobs;
106 unsigned num_upload_tasks;
107 bool is_balanced;
108 unsigned max_weight;
109 unsigned min_weight;
110 gid_t gid;
111 uid_t uid;
112
113 // Parameters for when upstream type is HTTP
114 std::string session_token_file;
115 std::string key_file;
116 RepositoryTag repo_tag;
117 };
118
119 namespace catalog {
120 class Dirtab;
121 class SimpleCatalogManager;
122 } // namespace catalog
123
124 namespace swissknife {
125
126 class CommandCreate : public Command {
127 public:
128 ✗ ~CommandCreate() {}
129 ✗ virtual std::string GetName() const { return "create"; }
130 ✗ virtual std::string GetDescription() const {
131 ✗ return "Bootstraps a fresh repository.";
132 }
133 ✗ virtual ParameterList GetParams() const {
134 ✗ ParameterList r;
135 ✗ r.push_back(Parameter::Mandatory('o', "manifest output file"));
136 ✗ r.push_back(Parameter::Mandatory('t', "directory for temporary storage"));
137 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
138 ✗ r.push_back(Parameter::Mandatory('n', "repository name"));
139 ✗ r.push_back(Parameter::Mandatory('R', "path to reflog.chksum file"));
140 ✗ r.push_back(Parameter::Optional('l', "log level (0-4, default: 2)"));
141 ✗ r.push_back(Parameter::Optional('a', "hash algorithm (default: SHA-1)"));
142 ✗ r.push_back(Parameter::Optional('V',
143 "VOMS authz requirement "
144 "(default: none)"));
145 ✗ r.push_back(Parameter::Switch('v', "repository containing volatile files"));
146 ✗ r.push_back(
147 ✗ Parameter::Switch('z', "mark new repository as garbage collectable"));
148 ✗ r.push_back(Parameter::Optional('V',
149 "VOMS authz requirement "
150 "(default: none)"));
151 ✗ return r;
152 }
153 int Main(const ArgumentList &args);
154 };
155
156 class CommandUpload : public Command {
157 public:
158 ✗ ~CommandUpload() {}
159 ✗ virtual std::string GetName() const { return "upload"; }
160 ✗ virtual std::string GetDescription() const {
161 ✗ return "Uploads a local file to the repository.";
162 }
163 ✗ virtual ParameterList GetParams() const {
164 ✗ ParameterList r;
165 ✗ r.push_back(Parameter::Mandatory('i', "local file"));
166 ✗ r.push_back(Parameter::Mandatory('o', "destination path"));
167 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
168 ✗ r.push_back(Parameter::Optional('a', "hash algorithm (default: SHA-1)"));
169 ✗ return r;
170 }
171 int Main(const ArgumentList &args);
172 };
173
174 class CommandPeek : public Command {
175 public:
176 ✗ ~CommandPeek() {}
177 ✗ virtual std::string GetName() const { return "peek"; }
178 ✗ virtual std::string GetDescription() const {
179 ✗ return "Checks whether a file exists in the repository.";
180 }
181 ✗ virtual ParameterList GetParams() const {
182 ✗ ParameterList r;
183 ✗ r.push_back(Parameter::Mandatory('d', "destination path"));
184 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
185 ✗ return r;
186 }
187 int Main(const ArgumentList &args);
188 };
189
190 class CommandRemove : public Command {
191 public:
192 ✗ ~CommandRemove() {}
193 ✗ virtual std::string GetName() const { return "remove"; }
194 ✗ virtual std::string GetDescription() const {
195 ✗ return "Removes a file in the repository storage.";
196 }
197 ✗ virtual ParameterList GetParams() const {
198 ✗ ParameterList r;
199 ✗ r.push_back(Parameter::Mandatory('o', "path to file"));
200 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
201 ✗ return r;
202 }
203 int Main(const ArgumentList &args);
204 };
205
206 class CommandApplyDirtab : public Command {
207 public:
208 ✗ CommandApplyDirtab() : verbose_(false) {}
209 ✗ ~CommandApplyDirtab() {}
210 ✗ virtual std::string GetName() const { return "dirtab"; }
211 ✗ virtual std::string GetDescription() const {
212 ✗ return "Parses the dirtab file and produces nested catalog markers.";
213 }
214 ✗ virtual ParameterList GetParams() const {
215 ✗ ParameterList r;
216 ✗ r.push_back(Parameter::Mandatory('d', "path to dirtab file"));
217 ✗ r.push_back(Parameter::Mandatory('u', "union volume"));
218 ✗ r.push_back(Parameter::Mandatory('s', "scratch directory"));
219 ✗ r.push_back(Parameter::Mandatory('b', "base hash"));
220 ✗ r.push_back(Parameter::Mandatory('w', "stratum 0 base url"));
221 ✗ r.push_back(Parameter::Mandatory('t', "directory for temporary storage"));
222 ✗ r.push_back(Parameter::Optional('@', "proxy url"));
223 ✗ r.push_back(Parameter::Switch('x', "verbose mode"));
224 ✗ return r;
225 }
226 int Main(const ArgumentList &args);
227
228 protected:
229 void DetermineNestedCatalogCandidates(
230 const catalog::Dirtab &dirtab,
231 catalog::SimpleCatalogManager *catalog_manager,
232 std::vector<std::string> *nested_catalog_candidates);
233 void FilterCandidatesFromGlobResult(
234 const catalog::Dirtab &dirtab, char **paths, const size_t npaths,
235 catalog::SimpleCatalogManager *catalog_manager,
236 std::vector<std::string> *nested_catalog_candidates);
237 bool CreateCatalogMarkers(
238 const std::vector<std::string> &new_nested_catalogs);
239
240 private:
241 std::string union_dir_;
242 std::string scratch_dir_;
243 bool verbose_;
244 };
245
246 class CommandSync : public Command {
247 public:
248 ✗ ~CommandSync() {}
249 ✗ virtual std::string GetName() const { return "sync"; }
250 ✗ virtual std::string GetDescription() const {
251 ✗ return "Pushes changes from scratch area back to the repository.";
252 }
253 ✗ virtual ParameterList GetParams() const {
254 ✗ ParameterList r;
255 ✗ r.push_back(Parameter::Mandatory('b', "base hash"));
256 ✗ r.push_back(Parameter::Mandatory('c', "r/o volume"));
257 ✗ r.push_back(Parameter::Mandatory('o', "manifest output file"));
258 ✗ r.push_back(Parameter::Mandatory('r', "spooler definition"));
259 ✗ r.push_back(Parameter::Mandatory('s', "scratch directory"));
260 ✗ r.push_back(Parameter::Mandatory('t', "directory for tee"));
261 ✗ r.push_back(Parameter::Mandatory('u', "union volume"));
262 ✗ r.push_back(Parameter::Mandatory('w', "stratum 0 base url"));
263 ✗ r.push_back(Parameter::Mandatory('K', "public key(s) for repo"));
264 ✗ r.push_back(Parameter::Mandatory('N', "fully qualified repository name"));
265
266 ✗ r.push_back(Parameter::Optional('a', "desired average chunk size (bytes)"));
267 ✗ r.push_back(Parameter::Optional('e', "hash algorithm (default: SHA-1)"));
268 ✗ r.push_back(Parameter::Optional('f', "union filesystem type"));
269 ✗ r.push_back(Parameter::Optional('h', "maximal file chunk size in bytes"));
270 ✗ r.push_back(Parameter::Optional('l', "minimal file chunk size in bytes"));
271 ✗ r.push_back(Parameter::Optional('q', "number of concurrent write jobs"));
272 ✗ r.push_back(Parameter::Optional('0', "number of upload tasks"));
273 ✗ r.push_back(Parameter::Optional('v', "manual revision number"));
274 ✗ r.push_back(Parameter::Optional('z', "log level (0-4, default: 2)"));
275 ✗ r.push_back(Parameter::Optional('F', "Authz file listing (default: none)"));
276 ✗ r.push_back(Parameter::Optional('M', "minimum weight of the autocatalogs"));
277 ✗ r.push_back(
278 ✗ Parameter::Optional('Q', "nested catalog limit in kilo-entries"));
279 ✗ r.push_back(Parameter::Optional('R', "root catalog limit in kilo-entries"));
280 ✗ r.push_back(Parameter::Optional('T', "Root catalog TTL in seconds"));
281 ✗ r.push_back(Parameter::Optional('U', "file size limit in megabytes"));
282 ✗ r.push_back(
283 ✗ Parameter::Optional('D', "tag name (only used when upstream is GW)"));
284 ✗ r.push_back(Parameter::Optional(
285 'J', "tag description (only used when upstream is GW)"));
286 ✗ r.push_back(Parameter::Optional('X', "maximum weight of the autocatalogs"));
287 ✗ r.push_back(Parameter::Optional('Z',
288 "compression algorithm "
289 "(default: zlib)"));
290 ✗ r.push_back(Parameter::Optional('S',
291 "virtual directory options "
292 "[snapshots, remove]"));
293
294 ✗ r.push_back(Parameter::Switch('d',
295 "pause publishing to allow for catalog "
296 "tweaks"));
297 ✗ r.push_back(Parameter::Switch('i', "ignore x-directory hardlinks"));
298 ✗ r.push_back(Parameter::Switch('g', "ignore special files"));
299 ✗ r.push_back(Parameter::Switch('k', "include extended attributes"));
300 ✗ r.push_back(Parameter::Switch('m', "create micro catalogs"));
301 ✗ r.push_back(Parameter::Switch('n', "create new repository"));
302 ✗ r.push_back(Parameter::Switch('p', "enable file chunking"));
303 ✗ r.push_back(Parameter::Switch('O', "generate legacy bulk chunks"));
304 ✗ r.push_back(Parameter::Switch('x', "print change set"));
305 ✗ r.push_back(Parameter::Switch('y', "dry run"));
306 ✗ r.push_back(Parameter::Switch('A', "autocatalog enabled/disabled"));
307 ✗ r.push_back(Parameter::Switch('E', "enforce limits instead of warning"));
308 ✗ r.push_back(Parameter::Switch('L', "enable HTTP redirects"));
309 ✗ r.push_back(Parameter::Switch('V',
310 "Publish format compatible with "
311 "authenticated repos"));
312 ✗ r.push_back(Parameter::Switch('Y', "enable external data"));
313 ✗ r.push_back(Parameter::Switch('W', "set direct I/O for regular files"));
314 ✗ r.push_back(Parameter::Switch('B', "branched catalog (no manifest)"));
315 ✗ r.push_back(Parameter::Switch('I', "upload updated statistics DB file"));
316
317 ✗ r.push_back(Parameter::Optional('P', "session_token_file"));
318 ✗ r.push_back(Parameter::Optional('H', "key file for HTTP API"));
319 ✗ r.push_back(Parameter::Optional('@', "proxy URL"));
320
321 ✗ return r;
322 }
323 int Main(const ArgumentList &args);
324
325 protected:
326 bool ReadFileChunkingArgs(const swissknife::ArgumentList &args,
327 SyncParameters *params);
328 bool CheckParams(const SyncParameters &p);
329 };
330
331 } // namespace swissknife
332
333 #endif // CVMFS_SWISSKNIFE_SYNC_H_
334