CernVM-FS  2.12.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
swissknife_sync.h
Go to the documentation of this file.
1 
5 #ifndef CVMFS_SWISSKNIFE_SYNC_H_
6 #define CVMFS_SWISSKNIFE_SYNC_H_
7 
8 #include <string>
9 #include <vector>
10 
11 #include "compression.h"
12 #include "repository_tag.h"
13 #include "swissknife.h"
14 #include "upload.h"
15 
17  static const unsigned kDefaultMaxWeight = 100000;
18  static const unsigned kDefaultMinWeight = 1000;
19  static const size_t kDefaultMinFileChunkSize = 4 * 1024 * 1024;
20  static const size_t kDefaultAvgFileChunkSize = 8 * 1024 * 1024;
21  static const size_t kDefaultMaxFileChunkSize = 16 * 1024 * 1024;
22  static const unsigned kDefaultNestedKcatalogLimit = 500;
23  static const unsigned kDefaultRootKcatalogLimit = 200;
24  static const unsigned kDefaultFileMbyteLimit = 1024;
25 
27  : spooler(NULL),
28  union_fs_type("aufs"),
29  to_delete(""),
30  print_changeset(false),
31  dry_run(false),
32  mucatalogs(false),
33  use_file_chunking(false),
35  ignore_xdir_hardlinks(false),
37  include_xattrs(false),
38  external_data(false),
39  direct_io(false),
40  voms_authz(false),
42  ignore_special_files(false),
43  branched_catalog(false),
45  enforce_limits(false),
52  manual_revision(0),
53  ttl_seconds(0),
56  is_balanced(false),
60  key_file(),
61  repo_tag() {}
62 
63  upload::Spooler *spooler;
64  std::string repo_name;
65  std::string dir_union;
66  std::string dir_scratch;
67  std::string dir_rdonly;
68  std::string dir_temp;
70  std::string stratum0;
71  std::string manifest_path;
72  std::string spooler_definition;
73  std::string union_fs_type;
74  std::string public_keys;
75  std::string trusted_certs;
76  std::string authz_file;
77  std::string tar_file;
78  std::string base_directory;
79  std::string to_delete;
81  bool dry_run;
82  bool mucatalogs;
89  bool direct_io;
90  bool voms_authz;
91  int virtual_dir_actions; // bit field
98  unsigned file_mbyte_limit;
102  uint64_t manual_revision;
103  uint64_t ttl_seconds;
107  unsigned max_weight;
108  unsigned min_weight;
109 
110  // Parameters for when upstream type is HTTP
111  std::string session_token_file;
112  std::string key_file;
114 };
115 
116 namespace catalog {
117 class Dirtab;
118 class SimpleCatalogManager;
119 } // namespace catalog
120 
121 namespace swissknife {
122 
123 class CommandCreate : public Command {
124  public:
126  virtual std::string GetName() const { return "create"; }
127  virtual std::string GetDescription() const {
128  return "Bootstraps a fresh repository.";
129  }
130  virtual ParameterList GetParams() const {
131  ParameterList r;
132  r.push_back(Parameter::Mandatory('o', "manifest output file"));
133  r.push_back(Parameter::Mandatory('t', "directory for temporary storage"));
134  r.push_back(Parameter::Mandatory('r', "spooler definition"));
135  r.push_back(Parameter::Mandatory('n', "repository name"));
136  r.push_back(Parameter::Mandatory('R', "path to reflog.chksum file"));
137  r.push_back(Parameter::Optional('l', "log level (0-4, default: 2)"));
138  r.push_back(Parameter::Optional('a', "hash algorithm (default: SHA-1)"));
139  r.push_back(Parameter::Optional('V',
140  "VOMS authz requirement "
141  "(default: none)"));
142  r.push_back(Parameter::Switch('v', "repository containing volatile files"));
143  r.push_back(
144  Parameter::Switch('z', "mark new repository as garbage collectable"));
145  r.push_back(Parameter::Optional('V',
146  "VOMS authz requirement "
147  "(default: none)"));
148  return r;
149  }
150  int Main(const ArgumentList &args);
151 };
152 
153 class CommandUpload : public Command {
154  public:
156  virtual std::string GetName() const { return "upload"; }
157  virtual std::string GetDescription() const {
158  return "Uploads a local file to the repository.";
159  }
160  virtual ParameterList GetParams() const {
161  ParameterList r;
162  r.push_back(Parameter::Mandatory('i', "local file"));
163  r.push_back(Parameter::Mandatory('o', "destination path"));
164  r.push_back(Parameter::Mandatory('r', "spooler definition"));
165  r.push_back(Parameter::Optional('a', "hash algorithm (default: SHA-1)"));
166  return r;
167  }
168  int Main(const ArgumentList &args);
169 };
170 
171 class CommandPeek : public Command {
172  public:
174  virtual std::string GetName() const { return "peek"; }
175  virtual std::string GetDescription() const {
176  return "Checks whether a file exists in the repository.";
177  }
178  virtual ParameterList GetParams() const {
179  ParameterList r;
180  r.push_back(Parameter::Mandatory('d', "destination path"));
181  r.push_back(Parameter::Mandatory('r', "spooler definition"));
182  return r;
183  }
184  int Main(const ArgumentList &args);
185 };
186 
187 class CommandRemove : public Command {
188  public:
190  virtual std::string GetName() const { return "remove"; }
191  virtual std::string GetDescription() const {
192  return "Removes a file in the repository storage.";
193  }
194  virtual ParameterList GetParams() const {
195  ParameterList r;
196  r.push_back(Parameter::Mandatory('o', "path to file"));
197  r.push_back(Parameter::Mandatory('r', "spooler definition"));
198  return r;
199  }
200  int Main(const ArgumentList &args);
201 };
202 
203 class CommandApplyDirtab : public Command {
204  public:
207  virtual std::string GetName() const { return "dirtab"; }
208  virtual std::string GetDescription() const {
209  return "Parses the dirtab file and produces nested catalog markers.";
210  }
211  virtual ParameterList GetParams() const {
212  ParameterList r;
213  r.push_back(Parameter::Mandatory('d', "path to dirtab file"));
214  r.push_back(Parameter::Mandatory('u', "union volume"));
215  r.push_back(Parameter::Mandatory('s', "scratch directory"));
216  r.push_back(Parameter::Mandatory('b', "base hash"));
217  r.push_back(Parameter::Mandatory('w', "stratum 0 base url"));
218  r.push_back(Parameter::Mandatory('t', "directory for temporary storage"));
219  r.push_back(Parameter::Optional('@', "proxy url"));
220  r.push_back(Parameter::Switch('x', "verbose mode"));
221  return r;
222  }
223  int Main(const ArgumentList &args);
224 
225  protected:
227  const catalog::Dirtab &dirtab,
228  catalog::SimpleCatalogManager *catalog_manager,
229  std::vector<std::string> *nested_catalog_candidates);
231  const catalog::Dirtab &dirtab, char **paths, const size_t npaths,
232  catalog::SimpleCatalogManager *catalog_manager,
233  std::vector<std::string> *nested_catalog_candidates);
235  const std::vector<std::string> &new_nested_catalogs);
236 
237  private:
238  std::string union_dir_;
239  std::string scratch_dir_;
240  bool verbose_;
241 };
242 
243 class CommandSync : public Command {
244  public:
246  virtual std::string GetName() const { return "sync"; }
247  virtual std::string GetDescription() const {
248  return "Pushes changes from scratch area back to the repository.";
249  }
250  virtual ParameterList GetParams() const {
251  ParameterList r;
252  r.push_back(Parameter::Mandatory('b', "base hash"));
253  r.push_back(Parameter::Mandatory('c', "r/o volume"));
254  r.push_back(Parameter::Mandatory('o', "manifest output file"));
255  r.push_back(Parameter::Mandatory('r', "spooler definition"));
256  r.push_back(Parameter::Mandatory('s', "scratch directory"));
257  r.push_back(Parameter::Mandatory('t', "directory for tee"));
258  r.push_back(Parameter::Mandatory('u', "union volume"));
259  r.push_back(Parameter::Mandatory('w', "stratum 0 base url"));
260  r.push_back(Parameter::Mandatory('K', "public key(s) for repo"));
261  r.push_back(Parameter::Mandatory('N', "fully qualified repository name"));
262 
263  r.push_back(Parameter::Optional('a', "desired average chunk size (bytes)"));
264  r.push_back(Parameter::Optional('e', "hash algorithm (default: SHA-1)"));
265  r.push_back(Parameter::Optional('f', "union filesystem type"));
266  r.push_back(Parameter::Optional('h', "maximal file chunk size in bytes"));
267  r.push_back(Parameter::Optional('l', "minimal file chunk size in bytes"));
268  r.push_back(Parameter::Optional('q', "number of concurrent write jobs"));
269  r.push_back(Parameter::Optional('0', "number of upload tasks"));
270  r.push_back(Parameter::Optional('v', "manual revision number"));
271  r.push_back(Parameter::Optional('z', "log level (0-4, default: 2)"));
272  r.push_back(Parameter::Optional('C', "trusted certificates"));
273  r.push_back(Parameter::Optional('F', "Authz file listing (default: none)"));
274  r.push_back(Parameter::Optional('M', "minimum weight of the autocatalogs"));
275  r.push_back(
276  Parameter::Optional('Q', "nested catalog limit in kilo-entries"));
277  r.push_back(Parameter::Optional('R', "root catalog limit in kilo-entries"));
278  r.push_back(Parameter::Optional('T', "Root catalog TTL in seconds"));
279  r.push_back(Parameter::Optional('U', "file size limit in megabytes"));
280  r.push_back(
281  Parameter::Optional('D', "tag name (only used when upstream is GW)"));
282  r.push_back(Parameter::Optional(
283  'J', "tag description (only used when upstream is GW)"));
284  r.push_back(Parameter::Optional('X', "maximum weight of the autocatalogs"));
285  r.push_back(Parameter::Optional('Z',
286  "compression algorithm "
287  "(default: zlib)"));
288  r.push_back(Parameter::Optional('S',
289  "virtual directory options "
290  "[snapshots, remove]"));
291 
292  r.push_back(Parameter::Switch('d',
293  "pause publishing to allow for catalog "
294  "tweaks"));
295  r.push_back(Parameter::Switch('i', "ignore x-directory hardlinks"));
296  r.push_back(Parameter::Switch('g', "ignore special files"));
297  r.push_back(Parameter::Switch('k', "include extended attributes"));
298  r.push_back(Parameter::Switch('m', "create micro catalogs"));
299  r.push_back(Parameter::Switch('n', "create new repository"));
300  r.push_back(Parameter::Switch('p', "enable file chunking"));
301  r.push_back(Parameter::Switch('O', "generate legacy bulk chunks"));
302  r.push_back(Parameter::Switch('x', "print change set"));
303  r.push_back(Parameter::Switch('y', "dry run"));
304  r.push_back(Parameter::Switch('A', "autocatalog enabled/disabled"));
305  r.push_back(Parameter::Switch('E', "enforce limits instead of warning"));
306  r.push_back(Parameter::Switch('L', "enable HTTP redirects"));
307  r.push_back(Parameter::Switch('V',
308  "Publish format compatible with "
309  "authenticated repos"));
310  r.push_back(Parameter::Switch('Y', "enable external data"));
311  r.push_back(Parameter::Switch('W', "set direct I/O for regular files"));
312  r.push_back(Parameter::Switch('B', "branched catalog (no manifest)"));
313  r.push_back(Parameter::Switch('I', "upload updated statistics DB file"));
314 
315  r.push_back(Parameter::Optional('P', "session_token_file"));
316  r.push_back(Parameter::Optional('H', "key file for HTTP API"));
317  r.push_back(Parameter::Optional('@', "proxy URL"));
318 
319  return r;
320  }
321  int Main(const ArgumentList &args);
322 
323  protected:
325  SyncParameters *params);
326  bool CheckParams(const SyncParameters &p);
327 };
328 
329 } // namespace swissknife
330 
331 #endif // CVMFS_SWISSKNIFE_SYNC_H_
static Parameter Optional(const char key, const std::string &desc)
Definition: swissknife.h:41
virtual std::string GetName() const
static const size_t kDefaultMaxFileChunkSize
size_t avg_file_chunk_size
virtual std::string GetName() const
virtual ParameterList GetParams() const
std::string stratum0
static Parameter Switch(const char key, const std::string &desc)
Definition: swissknife.h:44
int Main(const ArgumentList &args)
virtual ParameterList GetParams() const
virtual std::string GetDescription() const
static const unsigned kDefaultMaxWeight
bool generate_legacy_bulk_chunks
unsigned file_mbyte_limit
virtual std::string GetName() const
uint64_t max_concurrent_write_jobs
static const unsigned kDefaultFileMbyteLimit
std::vector< Parameter > ParameterList
Definition: swissknife.h:71
std::string to_delete
virtual std::string GetDescription() const
static const unsigned kDefaultNestedKcatalogLimit
static const unsigned kDefaultRootKcatalogLimit
int Main(const ArgumentList &args)
size_t min_file_chunk_size
unsigned num_upload_tasks
virtual std::string GetDescription() const
std::string spooler_definition
std::string manifest_path
bool ReadFileChunkingArgs(const swissknife::ArgumentList &args, SyncParameters *params)
bool CheckParams(const SyncParameters &p)
virtual std::string GetName() const
zlib::Algorithms compression_alg
unsigned nested_kcatalog_limit
std::string repo_name
std::string union_fs_type
std::string dir_scratch
static const size_t kDefaultMinFileChunkSize
int Main(const ArgumentList &args)
virtual std::string GetDescription() const
std::string key_file
std::string dir_temp
int Main(const ArgumentList &args)
virtual std::string GetName() const
Algorithms
Definition: compression.h:44
std::string base_directory
virtual std::string GetDescription() const
void FilterCandidatesFromGlobResult(const catalog::Dirtab &dirtab, char **paths, const size_t npaths, catalog::SimpleCatalogManager *catalog_manager, std::vector< std::string > *nested_catalog_candidates)
std::string dir_rdonly
static Parameter Mandatory(const char key, const std::string &desc)
Definition: swissknife.h:38
static const unsigned kDefaultMinWeight
shash::Any base_hash
upload::Spooler * spooler
virtual std::string GetName() const
int Main(const ArgumentList &args)
std::string dir_union
virtual ParameterList GetParams() const
virtual ParameterList GetParams() const
int Main(const ArgumentList &args)
bool stop_for_catalog_tweaks
unsigned root_kcatalog_limit
size_t max_file_chunk_size
uint64_t manual_revision
std::string session_token_file
std::string trusted_certs
std::map< char, SharedPtr< std::string > > ArgumentList
Definition: swissknife.h:72
uint64_t ttl_seconds
static const size_t kDefaultAvgFileChunkSize
std::string authz_file
virtual std::string GetDescription() const
void DetermineNestedCatalogCandidates(const catalog::Dirtab &dirtab, catalog::SimpleCatalogManager *catalog_manager, std::vector< std::string > *nested_catalog_candidates)
RepositoryTag repo_tag
bool ignore_xdir_hardlinks
std::string tar_file
virtual ParameterList GetParams() const
bool CreateCatalogMarkers(const std::vector< std::string > &new_nested_catalogs)
virtual ParameterList GetParams() const
std::string public_keys