GCC Code Coverage Report
Directory: cvmfs/ Exec Total Coverage
File: cvmfs/swissknife_sync.h Lines: 0 127 0.0 %
Date: 2019-02-03 02:48:13 Branches: 0 12 0.0 %

Line Branch Exec Source
1
/**
2
 * This file is part of the CernVM File System.
3
 */
4
5
#ifndef CVMFS_SWISSKNIFE_SYNC_H_
6
#define CVMFS_SWISSKNIFE_SYNC_H_
7
8
#include <string>
9
#include <vector>
10
11
#include "compression.h"
12
#include "repository_tag.h"
13
#include "swissknife.h"
14
#include "upload.h"
15
16
struct SyncParameters {
17
  static const unsigned kDefaultMaxWeight = 100000;
18
  static const unsigned kDefaultMinWeight = 1000;
19
  static const size_t kDefaultMinFileChunkSize = 4 * 1024 * 1024;
20
  static const size_t kDefaultAvgFileChunkSize = 8 * 1024 * 1024;
21
  static const size_t kDefaultMaxFileChunkSize = 16 * 1024 * 1024;
22
  static const unsigned kDefaultNestedKcatalogLimit = 500;
23
  static const unsigned kDefaultRootKcatalogLimit = 200;
24
  static const unsigned kDefaultFileMbyteLimit = 1024;
25
26
  SyncParameters()
27
      : spooler(NULL),
28
        union_fs_type("aufs"),
29
        to_delete(""),
30
        print_changeset(false),
31
        dry_run(false),
32
        mucatalogs(false),
33
        use_file_chunking(false),
34
        generate_legacy_bulk_chunks(false),
35
        ignore_xdir_hardlinks(false),
36
        stop_for_catalog_tweaks(false),
37
        include_xattrs(false),
38
        external_data(false),
39
        voms_authz(false),
40
        virtual_dir_actions(0),
41
        ignore_special_files(false),
42
        branched_catalog(false),
43
        compression_alg(zlib::kZlibDefault),
44
        enforce_limits(false),
45
        nested_kcatalog_limit(0),
46
        root_kcatalog_limit(0),
47
        file_mbyte_limit(0),
48
        min_file_chunk_size(kDefaultMinFileChunkSize),
49
        avg_file_chunk_size(kDefaultAvgFileChunkSize),
50
        max_file_chunk_size(kDefaultMaxFileChunkSize),
51
        manual_revision(0),
52
        ttl_seconds(0),
53
        max_concurrent_write_jobs(0),
54
        num_upload_tasks(1),
55
        is_balanced(false),
56
        max_weight(kDefaultMaxWeight),
57
        min_weight(kDefaultMinWeight),
58
        session_token_file(),
59
        key_file(),
60
        repo_tag() {}
61
62
  upload::Spooler *spooler;
63
  std::string repo_name;
64
  std::string dir_union;
65
  std::string dir_scratch;
66
  std::string dir_rdonly;
67
  std::string dir_temp;
68
  shash::Any base_hash;
69
  std::string stratum0;
70
  std::string manifest_path;
71
  std::string spooler_definition;
72
  std::string union_fs_type;
73
  std::string public_keys;
74
  std::string trusted_certs;
75
  std::string authz_file;
76
  std::string tar_file;
77
  std::string base_directory;
78
  std::string to_delete;
79
  bool print_changeset;
80
  bool dry_run;
81
  bool mucatalogs;
82
  bool use_file_chunking;
83
  bool generate_legacy_bulk_chunks;
84
  bool ignore_xdir_hardlinks;
85
  bool stop_for_catalog_tweaks;
86
  bool include_xattrs;
87
  bool external_data;
88
  bool voms_authz;
89
  unsigned virtual_dir_actions;  // bit field
90
  bool ignore_special_files;
91
  bool branched_catalog;
92
  zlib::Algorithms compression_alg;
93
  bool enforce_limits;
94
  unsigned nested_kcatalog_limit;
95
  unsigned root_kcatalog_limit;
96
  unsigned file_mbyte_limit;
97
  size_t min_file_chunk_size;
98
  size_t avg_file_chunk_size;
99
  size_t max_file_chunk_size;
100
  uint64_t manual_revision;
101
  uint64_t ttl_seconds;
102
  uint64_t max_concurrent_write_jobs;
103
  unsigned num_upload_tasks;
104
  bool is_balanced;
105
  unsigned max_weight;
106
  unsigned min_weight;
107
108
  // Parameters for when upstream type is HTTP
109
  std::string session_token_file;
110
  std::string key_file;
111
  RepositoryTag repo_tag;
112
};
113
114
namespace catalog {
115
class Dirtab;
116
class SimpleCatalogManager;
117
}  // namespace catalog
118
119
namespace swissknife {
120
121
class CommandCreate : public Command {
122
 public:
123
  ~CommandCreate() {}
124
  virtual std::string GetName() const { return "create"; }
125
  virtual std::string GetDescription() const {
126
    return "Bootstraps a fresh repository.";
127
  }
128
  virtual ParameterList GetParams() const {
129
    ParameterList r;
130
    r.push_back(Parameter::Mandatory('o', "manifest output file"));
131
    r.push_back(Parameter::Mandatory('t', "directory for temporary storage"));
132
    r.push_back(Parameter::Mandatory('r', "spooler definition"));
133
    r.push_back(Parameter::Mandatory('n', "repository name"));
134
    r.push_back(Parameter::Mandatory('R', "path to reflog.chksum file"));
135
    r.push_back(Parameter::Optional('l', "log level (0-4, default: 2)"));
136
    r.push_back(Parameter::Optional('a', "hash algorithm (default: SHA-1)"));
137
    r.push_back(Parameter::Optional('V',
138
                                    "VOMS authz requirement "
139
                                    "(default: none)"));
140
    r.push_back(Parameter::Switch('v', "repository containing volatile files"));
141
    r.push_back(
142
        Parameter::Switch('z', "mark new repository as garbage collectable"));
143
    r.push_back(Parameter::Optional('V',
144
                                    "VOMS authz requirement "
145
                                    "(default: none)"));
146
    return r;
147
  }
148
  int Main(const ArgumentList &args);
149
};
150
151
class CommandUpload : public Command {
152
 public:
153
  ~CommandUpload() {}
154
  virtual std::string GetName() const { return "upload"; }
155
  virtual std::string GetDescription() const {
156
    return "Uploads a local file to the repository.";
157
  }
158
  virtual ParameterList GetParams() const {
159
    ParameterList r;
160
    r.push_back(Parameter::Mandatory('i', "local file"));
161
    r.push_back(Parameter::Mandatory('o', "destination path"));
162
    r.push_back(Parameter::Mandatory('r', "spooler definition"));
163
    r.push_back(Parameter::Optional('a', "hash algorithm (default: SHA-1)"));
164
    return r;
165
  }
166
  int Main(const ArgumentList &args);
167
};
168
169
class CommandPeek : public Command {
170
 public:
171
  ~CommandPeek() {}
172
  virtual std::string GetName() const { return "peek"; }
173
  virtual std::string GetDescription() const {
174
    return "Checks whether a file exists in the repository.";
175
  }
176
  virtual ParameterList GetParams() const {
177
    ParameterList r;
178
    r.push_back(Parameter::Mandatory('d', "destination path"));
179
    r.push_back(Parameter::Mandatory('r', "spooler definition"));
180
    return r;
181
  }
182
  int Main(const ArgumentList &args);
183
};
184
185
class CommandRemove : public Command {
186
 public:
187
  ~CommandRemove() {}
188
  virtual std::string GetName() const { return "remove"; }
189
  virtual std::string GetDescription() const {
190
    return "Removes a file in the repository storage.";
191
  }
192
  virtual ParameterList GetParams() const {
193
    ParameterList r;
194
    r.push_back(Parameter::Mandatory('o', "path to file"));
195
    r.push_back(Parameter::Mandatory('r', "spooler definition"));
196
    return r;
197
  }
198
  int Main(const ArgumentList &args);
199
};
200
201
class CommandApplyDirtab : public Command {
202
 public:
203
  CommandApplyDirtab() : verbose_(false) {}
204
  ~CommandApplyDirtab() {}
205
  virtual std::string GetName() const { return "dirtab"; }
206
  virtual std::string GetDescription() const {
207
    return "Parses the dirtab file and produces nested catalog markers.";
208
  }
209
  virtual ParameterList GetParams() const {
210
    ParameterList r;
211
    r.push_back(Parameter::Mandatory('d', "path to dirtab file"));
212
    r.push_back(Parameter::Mandatory('u', "union volume"));
213
    r.push_back(Parameter::Mandatory('s', "scratch directory"));
214
    r.push_back(Parameter::Mandatory('b', "base hash"));
215
    r.push_back(Parameter::Mandatory('w', "stratum 0 base url"));
216
    r.push_back(Parameter::Mandatory('t', "directory for temporary storage"));
217
    r.push_back(Parameter::Switch('x', "verbose mode"));
218
    return r;
219
  }
220
  int Main(const ArgumentList &args);
221
222
 protected:
223
  void DetermineNestedCatalogCandidates(
224
      const catalog::Dirtab &dirtab,
225
      catalog::SimpleCatalogManager *catalog_manager,
226
      std::vector<std::string> *nested_catalog_candidates);
227
  void FilterCandidatesFromGlobResult(
228
      const catalog::Dirtab &dirtab, char **paths, const size_t npaths,
229
      catalog::SimpleCatalogManager *catalog_manager,
230
      std::vector<std::string> *nested_catalog_candidates);
231
  bool CreateCatalogMarkers(
232
      const std::vector<std::string> &new_nested_catalogs);
233
234
 private:
235
  std::string union_dir_;
236
  std::string scratch_dir_;
237
  bool verbose_;
238
};
239
240
class CommandSync : public Command {
241
 public:
242
  ~CommandSync() {}
243
  virtual std::string GetName() const { return "sync"; }
244
  virtual std::string GetDescription() const {
245
    return "Pushes changes from scratch area back to the repository.";
246
  }
247
  virtual ParameterList GetParams() const {
248
    ParameterList r;
249
    r.push_back(Parameter::Mandatory('b', "base hash"));
250
    r.push_back(Parameter::Mandatory('c', "r/o volume"));
251
    r.push_back(Parameter::Mandatory('o', "manifest output file"));
252
    r.push_back(Parameter::Mandatory('r', "spooler definition"));
253
    r.push_back(Parameter::Mandatory('s', "scratch directory"));
254
    r.push_back(Parameter::Mandatory('t', "directory for tee"));
255
    r.push_back(Parameter::Mandatory('u', "union volume"));
256
    r.push_back(Parameter::Mandatory('w', "stratum 0 base url"));
257
    r.push_back(Parameter::Mandatory('K', "public key(s) for repo"));
258
    r.push_back(Parameter::Mandatory('N', "fully qualified repository name"));
259
260
    r.push_back(Parameter::Optional('a', "desired average chunk size (bytes)"));
261
    r.push_back(Parameter::Optional('e', "hash algorithm (default: SHA-1)"));
262
    r.push_back(Parameter::Optional('f', "union filesystem type"));
263
    r.push_back(Parameter::Optional('h', "maximal file chunk size in bytes"));
264
    r.push_back(Parameter::Optional('l', "minimal file chunk size in bytes"));
265
    r.push_back(Parameter::Optional('q', "number of concurrent write jobs"));
266
    r.push_back(Parameter::Optional('0', "number of upload tasks"));
267
    r.push_back(Parameter::Optional('v', "manual revision number"));
268
    r.push_back(Parameter::Optional('z', "log level (0-4, default: 2)"));
269
    r.push_back(Parameter::Optional('C', "trusted certificates"));
270
    r.push_back(Parameter::Optional('F', "Authz file listing (default: none)"));
271
    r.push_back(Parameter::Optional('M', "minimum weight of the autocatalogs"));
272
    r.push_back(
273
        Parameter::Optional('Q', "nested catalog limit in kilo-entries"));
274
    r.push_back(Parameter::Optional('R', "root catalog limit in kilo-entries"));
275
    r.push_back(Parameter::Optional('T', "Root catalog TTL in seconds"));
276
    r.push_back(Parameter::Optional('U', "file size limit in megabytes"));
277
    r.push_back(
278
        Parameter::Optional('D', "tag name (only used when upstream is GW)"));
279
    r.push_back(Parameter::Optional(
280
        'G', "tag channel (only used when upstream is GW)"));
281
    r.push_back(Parameter::Optional(
282
        'J', "tag description (only used when upstream is GW)"));
283
    r.push_back(Parameter::Optional('X', "maximum weight of the autocatalogs"));
284
    r.push_back(Parameter::Optional('Z',
285
                                    "compression algorithm "
286
                                    "(default: zlib)"));
287
    r.push_back(Parameter::Optional('S',
288
                                    "virtual directory options "
289
                                    "[snapshots, remove]"));
290
291
    r.push_back(Parameter::Switch('d',
292
                                  "pause publishing to allow for catalog "
293
                                  "tweaks"));
294
    r.push_back(Parameter::Switch('i', "ignore x-directory hardlinks"));
295
    r.push_back(Parameter::Switch('g', "ignore special files"));
296
    r.push_back(Parameter::Switch('k', "include extended attributes"));
297
    r.push_back(Parameter::Switch('m', "create micro catalogs"));
298
    r.push_back(Parameter::Switch('n', "create new repository"));
299
    r.push_back(Parameter::Switch('p', "enable file chunking"));
300
    r.push_back(Parameter::Switch('O', "generate legacy bulk chunks"));
301
    r.push_back(Parameter::Switch('x', "print change set"));
302
    r.push_back(Parameter::Switch('y', "dry run"));
303
    r.push_back(Parameter::Switch('A', "autocatalog enabled/disabled"));
304
    r.push_back(Parameter::Switch('E', "enforce limits instead of warning"));
305
    r.push_back(Parameter::Switch('L', "enable HTTP redirects"));
306
    r.push_back(Parameter::Switch('V',
307
                                  "Publish format compatible with "
308
                                  "authenticated repos"));
309
    r.push_back(Parameter::Switch('Y', "enable external data"));
310
    r.push_back(Parameter::Switch('B', "branched catalog (no manifest)"));
311
312
    r.push_back(Parameter::Optional('P', "session_token_file"));
313
    r.push_back(Parameter::Optional('H', "key file for HTTP API"));
314
315
    return r;
316
  }
317
  int Main(const ArgumentList &args);
318
319
 protected:
320
  bool ReadFileChunkingArgs(const swissknife::ArgumentList &args,
321
                            SyncParameters *params);
322
  bool CheckParams(const SyncParameters &p);
323
  bool ObtainDacReadSearchCapability();
324
};
325
326
}  // namespace swissknife
327
328
#endif  // CVMFS_SWISSKNIFE_SYNC_H_