1 |
|
|
/** |
2 |
|
|
* This file is part of the CernVM File System. |
3 |
|
|
*/ |
4 |
|
|
|
5 |
|
|
#ifndef CVMFS_SWISSKNIFE_SYNC_H_ |
6 |
|
|
#define CVMFS_SWISSKNIFE_SYNC_H_ |
7 |
|
|
|
8 |
|
|
#include <string> |
9 |
|
|
#include <vector> |
10 |
|
|
|
11 |
|
|
#include "compression.h" |
12 |
|
|
#include "repository_tag.h" |
13 |
|
|
#include "swissknife.h" |
14 |
|
|
#include "upload.h" |
15 |
|
|
|
16 |
|
|
struct SyncParameters { |
17 |
|
|
static const unsigned kDefaultMaxWeight = 100000; |
18 |
|
|
static const unsigned kDefaultMinWeight = 1000; |
19 |
|
|
static const size_t kDefaultMinFileChunkSize = 4 * 1024 * 1024; |
20 |
|
|
static const size_t kDefaultAvgFileChunkSize = 8 * 1024 * 1024; |
21 |
|
|
static const size_t kDefaultMaxFileChunkSize = 16 * 1024 * 1024; |
22 |
|
|
static const unsigned kDefaultNestedKcatalogLimit = 500; |
23 |
|
|
static const unsigned kDefaultRootKcatalogLimit = 200; |
24 |
|
|
static const unsigned kDefaultFileMbyteLimit = 1024; |
25 |
|
|
|
26 |
|
|
SyncParameters() |
27 |
|
|
: spooler(NULL), |
28 |
|
|
union_fs_type("aufs"), |
29 |
|
|
to_delete(""), |
30 |
|
|
print_changeset(false), |
31 |
|
|
dry_run(false), |
32 |
|
|
mucatalogs(false), |
33 |
|
|
use_file_chunking(false), |
34 |
|
|
generate_legacy_bulk_chunks(false), |
35 |
|
|
ignore_xdir_hardlinks(false), |
36 |
|
|
stop_for_catalog_tweaks(false), |
37 |
|
|
include_xattrs(false), |
38 |
|
|
external_data(false), |
39 |
|
|
voms_authz(false), |
40 |
|
|
virtual_dir_actions(0), |
41 |
|
|
ignore_special_files(false), |
42 |
|
|
branched_catalog(false), |
43 |
|
|
compression_alg(zlib::kZlibDefault), |
44 |
|
|
enforce_limits(false), |
45 |
|
|
nested_kcatalog_limit(0), |
46 |
|
|
root_kcatalog_limit(0), |
47 |
|
|
file_mbyte_limit(0), |
48 |
|
|
min_file_chunk_size(kDefaultMinFileChunkSize), |
49 |
|
|
avg_file_chunk_size(kDefaultAvgFileChunkSize), |
50 |
|
|
max_file_chunk_size(kDefaultMaxFileChunkSize), |
51 |
|
|
manual_revision(0), |
52 |
|
|
ttl_seconds(0), |
53 |
|
|
max_concurrent_write_jobs(0), |
54 |
|
|
num_upload_tasks(1), |
55 |
|
|
is_balanced(false), |
56 |
|
|
max_weight(kDefaultMaxWeight), |
57 |
|
|
min_weight(kDefaultMinWeight), |
58 |
|
|
session_token_file(), |
59 |
|
|
key_file(), |
60 |
|
|
repo_tag() {} |
61 |
|
|
|
62 |
|
|
upload::Spooler *spooler; |
63 |
|
|
std::string repo_name; |
64 |
|
|
std::string dir_union; |
65 |
|
|
std::string dir_scratch; |
66 |
|
|
std::string dir_rdonly; |
67 |
|
|
std::string dir_temp; |
68 |
|
|
shash::Any base_hash; |
69 |
|
|
std::string stratum0; |
70 |
|
|
std::string manifest_path; |
71 |
|
|
std::string spooler_definition; |
72 |
|
|
std::string union_fs_type; |
73 |
|
|
std::string public_keys; |
74 |
|
|
std::string trusted_certs; |
75 |
|
|
std::string authz_file; |
76 |
|
|
std::string tar_file; |
77 |
|
|
std::string base_directory; |
78 |
|
|
std::string to_delete; |
79 |
|
|
bool print_changeset; |
80 |
|
|
bool dry_run; |
81 |
|
|
bool mucatalogs; |
82 |
|
|
bool use_file_chunking; |
83 |
|
|
bool generate_legacy_bulk_chunks; |
84 |
|
|
bool ignore_xdir_hardlinks; |
85 |
|
|
bool stop_for_catalog_tweaks; |
86 |
|
|
bool include_xattrs; |
87 |
|
|
bool external_data; |
88 |
|
|
bool voms_authz; |
89 |
|
|
unsigned virtual_dir_actions; // bit field |
90 |
|
|
bool ignore_special_files; |
91 |
|
|
bool branched_catalog; |
92 |
|
|
zlib::Algorithms compression_alg; |
93 |
|
|
bool enforce_limits; |
94 |
|
|
unsigned nested_kcatalog_limit; |
95 |
|
|
unsigned root_kcatalog_limit; |
96 |
|
|
unsigned file_mbyte_limit; |
97 |
|
|
size_t min_file_chunk_size; |
98 |
|
|
size_t avg_file_chunk_size; |
99 |
|
|
size_t max_file_chunk_size; |
100 |
|
|
uint64_t manual_revision; |
101 |
|
|
uint64_t ttl_seconds; |
102 |
|
|
uint64_t max_concurrent_write_jobs; |
103 |
|
|
unsigned num_upload_tasks; |
104 |
|
|
bool is_balanced; |
105 |
|
|
unsigned max_weight; |
106 |
|
|
unsigned min_weight; |
107 |
|
|
|
108 |
|
|
// Parameters for when upstream type is HTTP |
109 |
|
|
std::string session_token_file; |
110 |
|
|
std::string key_file; |
111 |
|
|
RepositoryTag repo_tag; |
112 |
|
|
}; |
113 |
|
|
|
114 |
|
|
namespace catalog { |
115 |
|
|
class Dirtab; |
116 |
|
|
class SimpleCatalogManager; |
117 |
|
|
} // namespace catalog |
118 |
|
|
|
119 |
|
|
namespace swissknife { |
120 |
|
|
|
121 |
|
|
class CommandCreate : public Command { |
122 |
|
|
public: |
123 |
|
|
~CommandCreate() {} |
124 |
|
|
virtual std::string GetName() const { return "create"; } |
125 |
|
|
virtual std::string GetDescription() const { |
126 |
|
|
return "Bootstraps a fresh repository."; |
127 |
|
|
} |
128 |
|
|
virtual ParameterList GetParams() const { |
129 |
|
|
ParameterList r; |
130 |
|
|
r.push_back(Parameter::Mandatory('o', "manifest output file")); |
131 |
|
|
r.push_back(Parameter::Mandatory('t', "directory for temporary storage")); |
132 |
|
|
r.push_back(Parameter::Mandatory('r', "spooler definition")); |
133 |
|
|
r.push_back(Parameter::Mandatory('n', "repository name")); |
134 |
|
|
r.push_back(Parameter::Mandatory('R', "path to reflog.chksum file")); |
135 |
|
|
r.push_back(Parameter::Optional('l', "log level (0-4, default: 2)")); |
136 |
|
|
r.push_back(Parameter::Optional('a', "hash algorithm (default: SHA-1)")); |
137 |
|
|
r.push_back(Parameter::Optional('V', |
138 |
|
|
"VOMS authz requirement " |
139 |
|
|
"(default: none)")); |
140 |
|
|
r.push_back(Parameter::Switch('v', "repository containing volatile files")); |
141 |
|
|
r.push_back( |
142 |
|
|
Parameter::Switch('z', "mark new repository as garbage collectable")); |
143 |
|
|
r.push_back(Parameter::Optional('V', |
144 |
|
|
"VOMS authz requirement " |
145 |
|
|
"(default: none)")); |
146 |
|
|
return r; |
147 |
|
|
} |
148 |
|
|
int Main(const ArgumentList &args); |
149 |
|
|
}; |
150 |
|
|
|
151 |
|
|
class CommandUpload : public Command { |
152 |
|
|
public: |
153 |
|
|
~CommandUpload() {} |
154 |
|
|
virtual std::string GetName() const { return "upload"; } |
155 |
|
|
virtual std::string GetDescription() const { |
156 |
|
|
return "Uploads a local file to the repository."; |
157 |
|
|
} |
158 |
|
|
virtual ParameterList GetParams() const { |
159 |
|
|
ParameterList r; |
160 |
|
|
r.push_back(Parameter::Mandatory('i', "local file")); |
161 |
|
|
r.push_back(Parameter::Mandatory('o', "destination path")); |
162 |
|
|
r.push_back(Parameter::Mandatory('r', "spooler definition")); |
163 |
|
|
r.push_back(Parameter::Optional('a', "hash algorithm (default: SHA-1)")); |
164 |
|
|
return r; |
165 |
|
|
} |
166 |
|
|
int Main(const ArgumentList &args); |
167 |
|
|
}; |
168 |
|
|
|
169 |
|
|
class CommandPeek : public Command { |
170 |
|
|
public: |
171 |
|
|
~CommandPeek() {} |
172 |
|
|
virtual std::string GetName() const { return "peek"; } |
173 |
|
|
virtual std::string GetDescription() const { |
174 |
|
|
return "Checks whether a file exists in the repository."; |
175 |
|
|
} |
176 |
|
|
virtual ParameterList GetParams() const { |
177 |
|
|
ParameterList r; |
178 |
|
|
r.push_back(Parameter::Mandatory('d', "destination path")); |
179 |
|
|
r.push_back(Parameter::Mandatory('r', "spooler definition")); |
180 |
|
|
return r; |
181 |
|
|
} |
182 |
|
|
int Main(const ArgumentList &args); |
183 |
|
|
}; |
184 |
|
|
|
185 |
|
|
class CommandRemove : public Command { |
186 |
|
|
public: |
187 |
|
|
~CommandRemove() {} |
188 |
|
|
virtual std::string GetName() const { return "remove"; } |
189 |
|
|
virtual std::string GetDescription() const { |
190 |
|
|
return "Removes a file in the repository storage."; |
191 |
|
|
} |
192 |
|
|
virtual ParameterList GetParams() const { |
193 |
|
|
ParameterList r; |
194 |
|
|
r.push_back(Parameter::Mandatory('o', "path to file")); |
195 |
|
|
r.push_back(Parameter::Mandatory('r', "spooler definition")); |
196 |
|
|
return r; |
197 |
|
|
} |
198 |
|
|
int Main(const ArgumentList &args); |
199 |
|
|
}; |
200 |
|
|
|
201 |
|
|
class CommandApplyDirtab : public Command { |
202 |
|
|
public: |
203 |
|
|
CommandApplyDirtab() : verbose_(false) {} |
204 |
|
|
~CommandApplyDirtab() {} |
205 |
|
|
virtual std::string GetName() const { return "dirtab"; } |
206 |
|
|
virtual std::string GetDescription() const { |
207 |
|
|
return "Parses the dirtab file and produces nested catalog markers."; |
208 |
|
|
} |
209 |
|
|
virtual ParameterList GetParams() const { |
210 |
|
|
ParameterList r; |
211 |
|
|
r.push_back(Parameter::Mandatory('d', "path to dirtab file")); |
212 |
|
|
r.push_back(Parameter::Mandatory('u', "union volume")); |
213 |
|
|
r.push_back(Parameter::Mandatory('s', "scratch directory")); |
214 |
|
|
r.push_back(Parameter::Mandatory('b', "base hash")); |
215 |
|
|
r.push_back(Parameter::Mandatory('w', "stratum 0 base url")); |
216 |
|
|
r.push_back(Parameter::Mandatory('t', "directory for temporary storage")); |
217 |
|
|
r.push_back(Parameter::Switch('x', "verbose mode")); |
218 |
|
|
return r; |
219 |
|
|
} |
220 |
|
|
int Main(const ArgumentList &args); |
221 |
|
|
|
222 |
|
|
protected: |
223 |
|
|
void DetermineNestedCatalogCandidates( |
224 |
|
|
const catalog::Dirtab &dirtab, |
225 |
|
|
catalog::SimpleCatalogManager *catalog_manager, |
226 |
|
|
std::vector<std::string> *nested_catalog_candidates); |
227 |
|
|
void FilterCandidatesFromGlobResult( |
228 |
|
|
const catalog::Dirtab &dirtab, char **paths, const size_t npaths, |
229 |
|
|
catalog::SimpleCatalogManager *catalog_manager, |
230 |
|
|
std::vector<std::string> *nested_catalog_candidates); |
231 |
|
|
bool CreateCatalogMarkers( |
232 |
|
|
const std::vector<std::string> &new_nested_catalogs); |
233 |
|
|
|
234 |
|
|
private: |
235 |
|
|
std::string union_dir_; |
236 |
|
|
std::string scratch_dir_; |
237 |
|
|
bool verbose_; |
238 |
|
|
}; |
239 |
|
|
|
240 |
|
|
class CommandSync : public Command { |
241 |
|
|
public: |
242 |
|
|
~CommandSync() {} |
243 |
|
|
virtual std::string GetName() const { return "sync"; } |
244 |
|
|
virtual std::string GetDescription() const { |
245 |
|
|
return "Pushes changes from scratch area back to the repository."; |
246 |
|
|
} |
247 |
|
|
virtual ParameterList GetParams() const { |
248 |
|
|
ParameterList r; |
249 |
|
|
r.push_back(Parameter::Mandatory('b', "base hash")); |
250 |
|
|
r.push_back(Parameter::Mandatory('c', "r/o volume")); |
251 |
|
|
r.push_back(Parameter::Mandatory('o', "manifest output file")); |
252 |
|
|
r.push_back(Parameter::Mandatory('r', "spooler definition")); |
253 |
|
|
r.push_back(Parameter::Mandatory('s', "scratch directory")); |
254 |
|
|
r.push_back(Parameter::Mandatory('t', "directory for tee")); |
255 |
|
|
r.push_back(Parameter::Mandatory('u', "union volume")); |
256 |
|
|
r.push_back(Parameter::Mandatory('w', "stratum 0 base url")); |
257 |
|
|
r.push_back(Parameter::Mandatory('K', "public key(s) for repo")); |
258 |
|
|
r.push_back(Parameter::Mandatory('N', "fully qualified repository name")); |
259 |
|
|
|
260 |
|
|
r.push_back(Parameter::Optional('a', "desired average chunk size (bytes)")); |
261 |
|
|
r.push_back(Parameter::Optional('e', "hash algorithm (default: SHA-1)")); |
262 |
|
|
r.push_back(Parameter::Optional('f', "union filesystem type")); |
263 |
|
|
r.push_back(Parameter::Optional('h', "maximal file chunk size in bytes")); |
264 |
|
|
r.push_back(Parameter::Optional('l', "minimal file chunk size in bytes")); |
265 |
|
|
r.push_back(Parameter::Optional('q', "number of concurrent write jobs")); |
266 |
|
|
r.push_back(Parameter::Optional('0', "number of upload tasks")); |
267 |
|
|
r.push_back(Parameter::Optional('v', "manual revision number")); |
268 |
|
|
r.push_back(Parameter::Optional('z', "log level (0-4, default: 2)")); |
269 |
|
|
r.push_back(Parameter::Optional('C', "trusted certificates")); |
270 |
|
|
r.push_back(Parameter::Optional('F', "Authz file listing (default: none)")); |
271 |
|
|
r.push_back(Parameter::Optional('M', "minimum weight of the autocatalogs")); |
272 |
|
|
r.push_back( |
273 |
|
|
Parameter::Optional('Q', "nested catalog limit in kilo-entries")); |
274 |
|
|
r.push_back(Parameter::Optional('R', "root catalog limit in kilo-entries")); |
275 |
|
|
r.push_back(Parameter::Optional('T', "Root catalog TTL in seconds")); |
276 |
|
|
r.push_back(Parameter::Optional('U', "file size limit in megabytes")); |
277 |
|
|
r.push_back( |
278 |
|
|
Parameter::Optional('D', "tag name (only used when upstream is GW)")); |
279 |
|
|
r.push_back(Parameter::Optional( |
280 |
|
|
'G', "tag channel (only used when upstream is GW)")); |
281 |
|
|
r.push_back(Parameter::Optional( |
282 |
|
|
'J', "tag description (only used when upstream is GW)")); |
283 |
|
|
r.push_back(Parameter::Optional('X', "maximum weight of the autocatalogs")); |
284 |
|
|
r.push_back(Parameter::Optional('Z', |
285 |
|
|
"compression algorithm " |
286 |
|
|
"(default: zlib)")); |
287 |
|
|
r.push_back(Parameter::Optional('S', |
288 |
|
|
"virtual directory options " |
289 |
|
|
"[snapshots, remove]")); |
290 |
|
|
|
291 |
|
|
r.push_back(Parameter::Switch('d', |
292 |
|
|
"pause publishing to allow for catalog " |
293 |
|
|
"tweaks")); |
294 |
|
|
r.push_back(Parameter::Switch('i', "ignore x-directory hardlinks")); |
295 |
|
|
r.push_back(Parameter::Switch('g', "ignore special files")); |
296 |
|
|
r.push_back(Parameter::Switch('k', "include extended attributes")); |
297 |
|
|
r.push_back(Parameter::Switch('m', "create micro catalogs")); |
298 |
|
|
r.push_back(Parameter::Switch('n', "create new repository")); |
299 |
|
|
r.push_back(Parameter::Switch('p', "enable file chunking")); |
300 |
|
|
r.push_back(Parameter::Switch('O', "generate legacy bulk chunks")); |
301 |
|
|
r.push_back(Parameter::Switch('x', "print change set")); |
302 |
|
|
r.push_back(Parameter::Switch('y', "dry run")); |
303 |
|
|
r.push_back(Parameter::Switch('A', "autocatalog enabled/disabled")); |
304 |
|
|
r.push_back(Parameter::Switch('E', "enforce limits instead of warning")); |
305 |
|
|
r.push_back(Parameter::Switch('L', "enable HTTP redirects")); |
306 |
|
|
r.push_back(Parameter::Switch('V', |
307 |
|
|
"Publish format compatible with " |
308 |
|
|
"authenticated repos")); |
309 |
|
|
r.push_back(Parameter::Switch('Y', "enable external data")); |
310 |
|
|
r.push_back(Parameter::Switch('B', "branched catalog (no manifest)")); |
311 |
|
|
|
312 |
|
|
r.push_back(Parameter::Optional('P', "session_token_file")); |
313 |
|
|
r.push_back(Parameter::Optional('H', "key file for HTTP API")); |
314 |
|
|
|
315 |
|
|
return r; |
316 |
|
|
} |
317 |
|
|
int Main(const ArgumentList &args); |
318 |
|
|
|
319 |
|
|
protected: |
320 |
|
|
bool ReadFileChunkingArgs(const swissknife::ArgumentList &args, |
321 |
|
|
SyncParameters *params); |
322 |
|
|
bool CheckParams(const SyncParameters &p); |
323 |
|
|
bool ObtainDacReadSearchCapability(); |
324 |
|
|
}; |
325 |
|
|
|
326 |
|
|
} // namespace swissknife |
327 |
|
|
|
328 |
|
|
#endif // CVMFS_SWISSKNIFE_SYNC_H_ |