CernVM-FS  2.12.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
swissknife_sync.h
Go to the documentation of this file.
1 
5 #ifndef CVMFS_SWISSKNIFE_SYNC_H_
6 #define CVMFS_SWISSKNIFE_SYNC_H_
7 
8 #include <string>
9 #include <vector>
10 
12 #include "repository_tag.h"
13 #include "swissknife.h"
14 #include "upload.h"
15 
17  static const unsigned kDefaultMaxWeight = 100000;
18  static const unsigned kDefaultMinWeight = 1000;
19  static const size_t kDefaultMinFileChunkSize = 4 * 1024 * 1024;
20  static const size_t kDefaultAvgFileChunkSize = 8 * 1024 * 1024;
21  static const size_t kDefaultMaxFileChunkSize = 16 * 1024 * 1024;
22  static const unsigned kDefaultNestedKcatalogLimit = 500;
23  static const unsigned kDefaultRootKcatalogLimit = 200;
24  static const unsigned kDefaultFileMbyteLimit = 1024;
25 
27  : spooler(NULL),
28  union_fs_type("aufs"),
29  to_delete(""),
30  cache_dir(""),
31  print_changeset(false),
32  dry_run(false),
33  mucatalogs(false),
34  use_file_chunking(false),
36  ignore_xdir_hardlinks(false),
38  include_xattrs(false),
39  enable_mtime_ns(false),
40  external_data(false),
41  direct_io(false),
42  voms_authz(false),
44  ignore_special_files(false),
45  branched_catalog(false),
47  enforce_limits(false),
54  manual_revision(0),
55  ttl_seconds(0),
58  is_balanced(false),
61  gid(-1u),
62  uid(-1u),
64  key_file(),
65  repo_tag() {}
66 
67  upload::Spooler *spooler;
68  std::string repo_name;
69  std::string dir_union;
70  std::string dir_scratch;
71  std::string dir_rdonly;
72  std::string dir_temp;
74  std::string stratum0;
75  std::string manifest_path;
76  std::string spooler_definition;
77  std::string union_fs_type;
78  std::string public_keys;
79  std::string authz_file;
80  std::string tar_file;
81  std::string base_directory;
82  std::string to_delete;
83  std::string cache_dir;
85  bool dry_run;
86  bool mucatalogs;
94  bool direct_io;
95  bool voms_authz;
96  int virtual_dir_actions; // bit field
107  uint64_t manual_revision;
108  uint64_t ttl_seconds;
112  unsigned max_weight;
113  unsigned min_weight;
114  gid_t gid;
115  uid_t uid;
116 
117  // Parameters for when upstream type is HTTP
118  std::string session_token_file;
119  std::string key_file;
121 };
122 
123 namespace catalog {
124 class Dirtab;
125 class SimpleCatalogManager;
126 } // namespace catalog
127 
128 namespace swissknife {
129 
130 class CommandCreate : public Command {
131  public:
133  virtual std::string GetName() const { return "create"; }
134  virtual std::string GetDescription() const {
135  return "Bootstraps a fresh repository.";
136  }
137  virtual ParameterList GetParams() const {
138  ParameterList r;
139  r.push_back(Parameter::Mandatory('o', "manifest output file"));
140  r.push_back(Parameter::Mandatory('t', "directory for temporary storage"));
141  r.push_back(Parameter::Mandatory('r', "spooler definition"));
142  r.push_back(Parameter::Mandatory('n', "repository name"));
143  r.push_back(Parameter::Mandatory('R', "path to reflog.chksum file"));
144  r.push_back(Parameter::Optional('l', "log level (0-4, default: 2)"));
145  r.push_back(Parameter::Optional('a', "hash algorithm (default: SHA-1)"));
146  r.push_back(Parameter::Optional('V',
147  "VOMS authz requirement "
148  "(default: none)"));
149  r.push_back(Parameter::Switch('v', "repository containing volatile files"));
150  r.push_back(
151  Parameter::Switch('z', "mark new repository as garbage collectable"));
152  r.push_back(Parameter::Optional('V',
153  "VOMS authz requirement "
154  "(default: none)"));
155  return r;
156  }
157  int Main(const ArgumentList &args);
158 };
159 
160 class CommandUpload : public Command {
161  public:
163  virtual std::string GetName() const { return "upload"; }
164  virtual std::string GetDescription() const {
165  return "Uploads a local file to the repository.";
166  }
167  virtual ParameterList GetParams() const {
168  ParameterList r;
169  r.push_back(Parameter::Mandatory('i', "local file"));
170  r.push_back(Parameter::Mandatory('o', "destination path"));
171  r.push_back(Parameter::Mandatory('r', "spooler definition"));
172  r.push_back(Parameter::Optional('a', "hash algorithm (default: SHA-1)"));
173  return r;
174  }
175  int Main(const ArgumentList &args);
176 };
177 
178 class CommandPeek : public Command {
179  public:
181  virtual std::string GetName() const { return "peek"; }
182  virtual std::string GetDescription() const {
183  return "Checks whether a file exists in the repository.";
184  }
185  virtual ParameterList GetParams() const {
186  ParameterList r;
187  r.push_back(Parameter::Mandatory('d', "destination path"));
188  r.push_back(Parameter::Mandatory('r', "spooler definition"));
189  return r;
190  }
191  int Main(const ArgumentList &args);
192 };
193 
194 class CommandRemove : public Command {
195  public:
197  virtual std::string GetName() const { return "remove"; }
198  virtual std::string GetDescription() const {
199  return "Removes a file in the repository storage.";
200  }
201  virtual ParameterList GetParams() const {
202  ParameterList r;
203  r.push_back(Parameter::Mandatory('o', "path to file"));
204  r.push_back(Parameter::Mandatory('r', "spooler definition"));
205  return r;
206  }
207  int Main(const ArgumentList &args);
208 };
209 
210 class CommandApplyDirtab : public Command {
211  public:
214  virtual std::string GetName() const { return "dirtab"; }
215  virtual std::string GetDescription() const {
216  return "Parses the dirtab file and produces nested catalog markers.";
217  }
218  virtual ParameterList GetParams() const {
219  ParameterList r;
220  r.push_back(Parameter::Mandatory('d', "path to dirtab file"));
221  r.push_back(Parameter::Mandatory('u', "union volume"));
222  r.push_back(Parameter::Mandatory('s', "scratch directory"));
223  r.push_back(Parameter::Mandatory('b', "base hash"));
224  r.push_back(Parameter::Mandatory('w', "stratum 0 base url"));
225  r.push_back(Parameter::Mandatory('t', "directory for temporary storage"));
226  r.push_back(Parameter::Optional('@', "proxy url"));
227  r.push_back(Parameter::Switch('x', "verbose mode"));
228  return r;
229  }
230  int Main(const ArgumentList &args);
231 
232  protected:
234  const catalog::Dirtab &dirtab,
235  catalog::SimpleCatalogManager *catalog_manager,
236  std::vector<std::string> *nested_catalog_candidates);
238  const catalog::Dirtab &dirtab, char **paths, const size_t npaths,
239  catalog::SimpleCatalogManager *catalog_manager,
240  std::vector<std::string> *nested_catalog_candidates);
242  const std::vector<std::string> &new_nested_catalogs);
243 
244  private:
245  std::string union_dir_;
246  std::string scratch_dir_;
247  bool verbose_;
248 };
249 
250 class CommandSync : public Command {
251  public:
253  virtual std::string GetName() const { return "sync"; }
254  virtual std::string GetDescription() const {
255  return "Pushes changes from scratch area back to the repository.";
256  }
257  virtual ParameterList GetParams() const {
258  // unused characters: J, 1-9, all special characters but @
259  ParameterList r;
260  r.push_back(Parameter::Mandatory('b', "base hash"));
261  r.push_back(Parameter::Mandatory('c', "r/o volume"));
262  r.push_back(Parameter::Mandatory('o', "manifest output file"));
263  r.push_back(Parameter::Mandatory('r', "spooler definition"));
264  r.push_back(Parameter::Mandatory('s', "scratch directory"));
265  r.push_back(Parameter::Mandatory('t', "directory for tee"));
266  r.push_back(Parameter::Mandatory('u', "union volume"));
267  r.push_back(Parameter::Mandatory('w', "stratum 0 base url"));
268  r.push_back(Parameter::Mandatory('K', "public key(s) for repo"));
269  r.push_back(Parameter::Mandatory('N', "fully qualified repository name"));
270 
271  r.push_back(Parameter::Optional('a', "desired average chunk size (bytes)"));
272  r.push_back(Parameter::Optional('e', "hash algorithm (default: SHA-1)"));
273  r.push_back(Parameter::Optional('f', "union filesystem type"));
274  r.push_back(Parameter::Optional('h', "maximal file chunk size in bytes"));
275  r.push_back(Parameter::Optional('l', "minimal file chunk size in bytes"));
276  r.push_back(Parameter::Optional('q', "number of concurrent write jobs"));
277  r.push_back(Parameter::Optional('0', "number of upload tasks"));
278  r.push_back(Parameter::Optional('v', "manual revision number"));
279  r.push_back(Parameter::Optional('z', "log level (0-4, default: 2)"));
280  r.push_back(Parameter::Optional('F', "Authz file listing (default: none)"));
281  r.push_back(Parameter::Optional('M', "minimum weight of the autocatalogs"));
282  r.push_back(
283  Parameter::Optional('Q', "nested catalog limit in kilo-entries"));
284  r.push_back(Parameter::Optional('R', "root catalog limit in kilo-entries"));
285  r.push_back(Parameter::Optional('T', "Root catalog TTL in seconds"));
286  r.push_back(Parameter::Optional('U', "file size limit in megabytes"));
287  r.push_back(
288  Parameter::Optional('D', "tag name (only used when upstream is GW)"));
289  r.push_back(Parameter::Optional(
290  'J', "tag description (only used when upstream is GW)"));
291  r.push_back(Parameter::Optional('X', "maximum weight of the autocatalogs"));
292  r.push_back(Parameter::Optional('Z',
293  "compression algorithm "
294  "(default: zlib)"));
295  r.push_back(Parameter::Optional('S',
296  "virtual directory options "
297  "[snapshots, remove]"));
298 
299 
300  r.push_back(
301  Parameter::Switch('G', "Use persistent caching for all catalogs "
302  "used during the publishing process"
303  " Warning: No automatic garbage collection!"));
304  r.push_back(Parameter::Switch('d',
305  "pause publishing to allow for catalog "
306  "tweaks"));
307  r.push_back(Parameter::Switch('i', "ignore x-directory hardlinks"));
308  r.push_back(Parameter::Switch('g', "ignore special files"));
309  r.push_back(Parameter::Switch('k', "include extended attributes"));
310  r.push_back(Parameter::Switch('j', "enable nanosecond timestamps"));
311  r.push_back(Parameter::Switch('m', "create micro catalogs"));
312  r.push_back(Parameter::Switch('n', "create new repository"));
313  r.push_back(Parameter::Switch('p', "enable file chunking"));
314  r.push_back(Parameter::Switch('O', "generate legacy bulk chunks"));
315  r.push_back(Parameter::Switch('x', "print change set"));
316  r.push_back(Parameter::Switch('y', "dry run"));
317  r.push_back(Parameter::Switch('A', "autocatalog enabled/disabled"));
318  r.push_back(Parameter::Switch('E', "enforce limits instead of warning"));
319  r.push_back(Parameter::Switch('L', "enable HTTP redirects"));
320  r.push_back(Parameter::Switch('V',
321  "Publish format compatible with "
322  "authenticated repos"));
323  r.push_back(Parameter::Switch('Y', "enable external data"));
324  r.push_back(Parameter::Switch('W', "set direct I/O for regular files"));
325  r.push_back(Parameter::Switch('B', "branched catalog (no manifest)"));
326  r.push_back(Parameter::Switch('I', "upload updated statistics DB file"));
327 
328  r.push_back(Parameter::Optional('P', "session_token_file"));
329  r.push_back(Parameter::Optional('H', "key file for HTTP API"));
330  r.push_back(Parameter::Optional('@', "proxy URL"));
331 
332  return r;
333  }
334  int Main(const ArgumentList &args);
335 
336  protected:
338  SyncParameters *params);
339  bool CheckParams(const SyncParameters &p);
340 };
341 
342 } // namespace swissknife
343 
344 #endif // CVMFS_SWISSKNIFE_SYNC_H_
static Parameter Optional(const char key, const std::string &desc)
Definition: swissknife.h:41
virtual std::string GetName() const
static const size_t kDefaultMaxFileChunkSize
size_t avg_file_chunk_size
virtual std::string GetName() const
virtual ParameterList GetParams() const
std::string stratum0
static Parameter Switch(const char key, const std::string &desc)
Definition: swissknife.h:44
int Main(const ArgumentList &args)
std::string cache_dir
virtual ParameterList GetParams() const
virtual std::string GetDescription() const
static const unsigned kDefaultMaxWeight
bool generate_legacy_bulk_chunks
unsigned file_mbyte_limit
virtual std::string GetName() const
uint64_t max_concurrent_write_jobs
static const unsigned kDefaultFileMbyteLimit
std::vector< Parameter > ParameterList
Definition: swissknife.h:71
std::string to_delete
virtual std::string GetDescription() const
static const unsigned kDefaultNestedKcatalogLimit
static const unsigned kDefaultRootKcatalogLimit
int Main(const ArgumentList &args)
size_t min_file_chunk_size
unsigned num_upload_tasks
virtual std::string GetDescription() const
std::string spooler_definition
std::string manifest_path
bool ReadFileChunkingArgs(const swissknife::ArgumentList &args, SyncParameters *params)
bool CheckParams(const SyncParameters &p)
virtual std::string GetName() const
zlib::Algorithms compression_alg
unsigned nested_kcatalog_limit
std::string repo_name
std::string union_fs_type
std::string dir_scratch
static const size_t kDefaultMinFileChunkSize
int Main(const ArgumentList &args)
virtual std::string GetDescription() const
std::string key_file
std::string dir_temp
int Main(const ArgumentList &args)
virtual std::string GetName() const
Algorithms
Definition: compression.h:44
std::string base_directory
virtual std::string GetDescription() const
void FilterCandidatesFromGlobResult(const catalog::Dirtab &dirtab, char **paths, const size_t npaths, catalog::SimpleCatalogManager *catalog_manager, std::vector< std::string > *nested_catalog_candidates)
std::string dir_rdonly
static Parameter Mandatory(const char key, const std::string &desc)
Definition: swissknife.h:38
static const unsigned kDefaultMinWeight
shash::Any base_hash
upload::Spooler * spooler
virtual std::string GetName() const
int Main(const ArgumentList &args)
std::string dir_union
virtual ParameterList GetParams() const
virtual ParameterList GetParams() const
int Main(const ArgumentList &args)
bool stop_for_catalog_tweaks
unsigned root_kcatalog_limit
size_t max_file_chunk_size
uint64_t manual_revision
std::string session_token_file
std::map< char, SharedPtr< std::string > > ArgumentList
Definition: swissknife.h:72
uint64_t ttl_seconds
static const size_t kDefaultAvgFileChunkSize
std::string authz_file
virtual std::string GetDescription() const
void DetermineNestedCatalogCandidates(const catalog::Dirtab &dirtab, catalog::SimpleCatalogManager *catalog_manager, std::vector< std::string > *nested_catalog_candidates)
RepositoryTag repo_tag
bool ignore_xdir_hardlinks
std::string tar_file
virtual ParameterList GetParams() const
bool CreateCatalogMarkers(const std::vector< std::string > &new_nested_catalogs)
virtual ParameterList GetParams() const
std::string public_keys