| 1 |  |  | /** | 
    
    | 2 |  |  |  * This file is part of the CernVM File System. | 
    
    | 3 |  |  |  */ | 
    
    | 4 |  |  |  | 
    
    | 5 |  |  | #ifndef CVMFS_SWISSKNIFE_SYNC_H_ | 
    
    | 6 |  |  | #define CVMFS_SWISSKNIFE_SYNC_H_ | 
    
    | 7 |  |  |  | 
    
    | 8 |  |  | #include <string> | 
    
    | 9 |  |  | #include <vector> | 
    
    | 10 |  |  |  | 
    
    | 11 |  |  | #include "compression.h" | 
    
    | 12 |  |  | #include "repository_tag.h" | 
    
    | 13 |  |  | #include "swissknife.h" | 
    
    | 14 |  |  | #include "upload.h" | 
    
    | 15 |  |  |  | 
    
    | 16 |  |  | struct SyncParameters { | 
    
    | 17 |  |  |   static const unsigned kDefaultMaxWeight = 100000; | 
    
    | 18 |  |  |   static const unsigned kDefaultMinWeight = 1000; | 
    
    | 19 |  |  |   static const size_t kDefaultMinFileChunkSize = 4 * 1024 * 1024; | 
    
    | 20 |  |  |   static const size_t kDefaultAvgFileChunkSize = 8 * 1024 * 1024; | 
    
    | 21 |  |  |   static const size_t kDefaultMaxFileChunkSize = 16 * 1024 * 1024; | 
    
    | 22 |  |  |   static const unsigned kDefaultNestedKcatalogLimit = 500; | 
    
    | 23 |  |  |   static const unsigned kDefaultRootKcatalogLimit = 200; | 
    
    | 24 |  |  |   static const unsigned kDefaultFileMbyteLimit = 1024; | 
    
    | 25 |  |  |  | 
    
    | 26 |  |  |   SyncParameters() | 
    
    | 27 |  |  |       : spooler(NULL), | 
    
    | 28 |  |  |         union_fs_type("aufs"), | 
    
    | 29 |  |  |         to_delete(""), | 
    
    | 30 |  |  |         print_changeset(false), | 
    
    | 31 |  |  |         dry_run(false), | 
    
    | 32 |  |  |         mucatalogs(false), | 
    
    | 33 |  |  |         use_file_chunking(false), | 
    
    | 34 |  |  |         generate_legacy_bulk_chunks(false), | 
    
    | 35 |  |  |         ignore_xdir_hardlinks(false), | 
    
    | 36 |  |  |         stop_for_catalog_tweaks(false), | 
    
    | 37 |  |  |         include_xattrs(false), | 
    
    | 38 |  |  |         external_data(false), | 
    
    | 39 |  |  |         voms_authz(false), | 
    
    | 40 |  |  |         virtual_dir_actions(0), | 
    
    | 41 |  |  |         ignore_special_files(false), | 
    
    | 42 |  |  |         branched_catalog(false), | 
    
    | 43 |  |  |         compression_alg(zlib::kZlibDefault), | 
    
    | 44 |  |  |         enforce_limits(false), | 
    
    | 45 |  |  |         nested_kcatalog_limit(0), | 
    
    | 46 |  |  |         root_kcatalog_limit(0), | 
    
    | 47 |  |  |         file_mbyte_limit(0), | 
    
    | 48 |  |  |         min_file_chunk_size(kDefaultMinFileChunkSize), | 
    
    | 49 |  |  |         avg_file_chunk_size(kDefaultAvgFileChunkSize), | 
    
    | 50 |  |  |         max_file_chunk_size(kDefaultMaxFileChunkSize), | 
    
    | 51 |  |  |         manual_revision(0), | 
    
    | 52 |  |  |         ttl_seconds(0), | 
    
    | 53 |  |  |         max_concurrent_write_jobs(0), | 
    
    | 54 |  |  |         num_upload_tasks(1), | 
    
    | 55 |  |  |         is_balanced(false), | 
    
    | 56 |  |  |         max_weight(kDefaultMaxWeight), | 
    
    | 57 |  |  |         min_weight(kDefaultMinWeight), | 
    
    | 58 |  |  |         session_token_file(), | 
    
    | 59 |  |  |         key_file(), | 
    
    | 60 |  |  |         repo_tag() {} | 
    
    | 61 |  |  |  | 
    
    | 62 |  |  |   upload::Spooler *spooler; | 
    
    | 63 |  |  |   std::string repo_name; | 
    
    | 64 |  |  |   std::string dir_union; | 
    
    | 65 |  |  |   std::string dir_scratch; | 
    
    | 66 |  |  |   std::string dir_rdonly; | 
    
    | 67 |  |  |   std::string dir_temp; | 
    
    | 68 |  |  |   shash::Any base_hash; | 
    
    | 69 |  |  |   std::string stratum0; | 
    
    | 70 |  |  |   std::string manifest_path; | 
    
    | 71 |  |  |   std::string spooler_definition; | 
    
    | 72 |  |  |   std::string union_fs_type; | 
    
    | 73 |  |  |   std::string public_keys; | 
    
    | 74 |  |  |   std::string trusted_certs; | 
    
    | 75 |  |  |   std::string authz_file; | 
    
    | 76 |  |  |   std::string tar_file; | 
    
    | 77 |  |  |   std::string base_directory; | 
    
    | 78 |  |  |   std::string to_delete; | 
    
    | 79 |  |  |   bool print_changeset; | 
    
    | 80 |  |  |   bool dry_run; | 
    
    | 81 |  |  |   bool mucatalogs; | 
    
    | 82 |  |  |   bool use_file_chunking; | 
    
    | 83 |  |  |   bool generate_legacy_bulk_chunks; | 
    
    | 84 |  |  |   bool ignore_xdir_hardlinks; | 
    
    | 85 |  |  |   bool stop_for_catalog_tweaks; | 
    
    | 86 |  |  |   bool include_xattrs; | 
    
    | 87 |  |  |   bool external_data; | 
    
    | 88 |  |  |   bool voms_authz; | 
    
    | 89 |  |  |   unsigned virtual_dir_actions;  // bit field | 
    
    | 90 |  |  |   bool ignore_special_files; | 
    
    | 91 |  |  |   bool branched_catalog; | 
    
    | 92 |  |  |   zlib::Algorithms compression_alg; | 
    
    | 93 |  |  |   bool enforce_limits; | 
    
    | 94 |  |  |   unsigned nested_kcatalog_limit; | 
    
    | 95 |  |  |   unsigned root_kcatalog_limit; | 
    
    | 96 |  |  |   unsigned file_mbyte_limit; | 
    
    | 97 |  |  |   size_t min_file_chunk_size; | 
    
    | 98 |  |  |   size_t avg_file_chunk_size; | 
    
    | 99 |  |  |   size_t max_file_chunk_size; | 
    
    | 100 |  |  |   uint64_t manual_revision; | 
    
    | 101 |  |  |   uint64_t ttl_seconds; | 
    
    | 102 |  |  |   uint64_t max_concurrent_write_jobs; | 
    
    | 103 |  |  |   unsigned num_upload_tasks; | 
    
    | 104 |  |  |   bool is_balanced; | 
    
    | 105 |  |  |   unsigned max_weight; | 
    
    | 106 |  |  |   unsigned min_weight; | 
    
    | 107 |  |  |  | 
    
    | 108 |  |  |   // Parameters for when upstream type is HTTP | 
    
    | 109 |  |  |   std::string session_token_file; | 
    
    | 110 |  |  |   std::string key_file; | 
    
    | 111 |  |  |   RepositoryTag repo_tag; | 
    
    | 112 |  |  | }; | 
    
    | 113 |  |  |  | 
    
    | 114 |  |  | namespace catalog { | 
    
    | 115 |  |  | class Dirtab; | 
    
    | 116 |  |  | class SimpleCatalogManager; | 
    
    | 117 |  |  | }  // namespace catalog | 
    
    | 118 |  |  |  | 
    
    | 119 |  |  | namespace swissknife { | 
    
    | 120 |  |  |  | 
    
    | 121 |  |  | class CommandCreate : public Command { | 
    
    | 122 |  |  |  public: | 
    
    | 123 |  |  |   ~CommandCreate() {} | 
    
    | 124 |  |  |   virtual std::string GetName() const { return "create"; } | 
    
    | 125 |  |  |   virtual std::string GetDescription() const { | 
    
    | 126 |  |  |     return "Bootstraps a fresh repository."; | 
    
    | 127 |  |  |   } | 
    
    | 128 |  |  |   virtual ParameterList GetParams() const { | 
    
    | 129 |  |  |     ParameterList r; | 
    
    | 130 |  |  |     r.push_back(Parameter::Mandatory('o', "manifest output file")); | 
    
    | 131 |  |  |     r.push_back(Parameter::Mandatory('t', "directory for temporary storage")); | 
    
    | 132 |  |  |     r.push_back(Parameter::Mandatory('r', "spooler definition")); | 
    
    | 133 |  |  |     r.push_back(Parameter::Mandatory('n', "repository name")); | 
    
    | 134 |  |  |     r.push_back(Parameter::Mandatory('R', "path to reflog.chksum file")); | 
    
    | 135 |  |  |     r.push_back(Parameter::Optional('l', "log level (0-4, default: 2)")); | 
    
    | 136 |  |  |     r.push_back(Parameter::Optional('a', "hash algorithm (default: SHA-1)")); | 
    
    | 137 |  |  |     r.push_back(Parameter::Optional('V', | 
    
    | 138 |  |  |                                     "VOMS authz requirement " | 
    
    | 139 |  |  |                                     "(default: none)")); | 
    
    | 140 |  |  |     r.push_back(Parameter::Switch('v', "repository containing volatile files")); | 
    
    | 141 |  |  |     r.push_back( | 
    
    | 142 |  |  |         Parameter::Switch('z', "mark new repository as garbage collectable")); | 
    
    | 143 |  |  |     r.push_back(Parameter::Optional('V', | 
    
    | 144 |  |  |                                     "VOMS authz requirement " | 
    
    | 145 |  |  |                                     "(default: none)")); | 
    
    | 146 |  |  |     return r; | 
    
    | 147 |  |  |   } | 
    
    | 148 |  |  |   int Main(const ArgumentList &args); | 
    
    | 149 |  |  | }; | 
    
    | 150 |  |  |  | 
    
    | 151 |  |  | class CommandUpload : public Command { | 
    
    | 152 |  |  |  public: | 
    
    | 153 |  |  |   ~CommandUpload() {} | 
    
    | 154 |  |  |   virtual std::string GetName() const { return "upload"; } | 
    
    | 155 |  |  |   virtual std::string GetDescription() const { | 
    
    | 156 |  |  |     return "Uploads a local file to the repository."; | 
    
    | 157 |  |  |   } | 
    
    | 158 |  |  |   virtual ParameterList GetParams() const { | 
    
    | 159 |  |  |     ParameterList r; | 
    
    | 160 |  |  |     r.push_back(Parameter::Mandatory('i', "local file")); | 
    
    | 161 |  |  |     r.push_back(Parameter::Mandatory('o', "destination path")); | 
    
    | 162 |  |  |     r.push_back(Parameter::Mandatory('r', "spooler definition")); | 
    
    | 163 |  |  |     r.push_back(Parameter::Optional('a', "hash algorithm (default: SHA-1)")); | 
    
    | 164 |  |  |     return r; | 
    
    | 165 |  |  |   } | 
    
    | 166 |  |  |   int Main(const ArgumentList &args); | 
    
    | 167 |  |  | }; | 
    
    | 168 |  |  |  | 
    
    | 169 |  |  | class CommandPeek : public Command { | 
    
    | 170 |  |  |  public: | 
    
    | 171 |  |  |   ~CommandPeek() {} | 
    
    | 172 |  |  |   virtual std::string GetName() const { return "peek"; } | 
    
    | 173 |  |  |   virtual std::string GetDescription() const { | 
    
    | 174 |  |  |     return "Checks whether a file exists in the repository."; | 
    
    | 175 |  |  |   } | 
    
    | 176 |  |  |   virtual ParameterList GetParams() const { | 
    
    | 177 |  |  |     ParameterList r; | 
    
    | 178 |  |  |     r.push_back(Parameter::Mandatory('d', "destination path")); | 
    
    | 179 |  |  |     r.push_back(Parameter::Mandatory('r', "spooler definition")); | 
    
    | 180 |  |  |     return r; | 
    
    | 181 |  |  |   } | 
    
    | 182 |  |  |   int Main(const ArgumentList &args); | 
    
    | 183 |  |  | }; | 
    
    | 184 |  |  |  | 
    
    | 185 |  |  | class CommandRemove : public Command { | 
    
    | 186 |  |  |  public: | 
    
    | 187 |  |  |   ~CommandRemove() {} | 
    
    | 188 |  |  |   virtual std::string GetName() const { return "remove"; } | 
    
    | 189 |  |  |   virtual std::string GetDescription() const { | 
    
    | 190 |  |  |     return "Removes a file in the repository storage."; | 
    
    | 191 |  |  |   } | 
    
    | 192 |  |  |   virtual ParameterList GetParams() const { | 
    
    | 193 |  |  |     ParameterList r; | 
    
    | 194 |  |  |     r.push_back(Parameter::Mandatory('o', "path to file")); | 
    
    | 195 |  |  |     r.push_back(Parameter::Mandatory('r', "spooler definition")); | 
    
    | 196 |  |  |     return r; | 
    
    | 197 |  |  |   } | 
    
    | 198 |  |  |   int Main(const ArgumentList &args); | 
    
    | 199 |  |  | }; | 
    
    | 200 |  |  |  | 
    
    | 201 |  |  | class CommandApplyDirtab : public Command { | 
    
    | 202 |  |  |  public: | 
    
    | 203 |  |  |   CommandApplyDirtab() : verbose_(false) {} | 
    
    | 204 |  |  |   ~CommandApplyDirtab() {} | 
    
    | 205 |  |  |   virtual std::string GetName() const { return "dirtab"; } | 
    
    | 206 |  |  |   virtual std::string GetDescription() const { | 
    
    | 207 |  |  |     return "Parses the dirtab file and produces nested catalog markers."; | 
    
    | 208 |  |  |   } | 
    
    | 209 |  |  |   virtual ParameterList GetParams() const { | 
    
    | 210 |  |  |     ParameterList r; | 
    
    | 211 |  |  |     r.push_back(Parameter::Mandatory('d', "path to dirtab file")); | 
    
    | 212 |  |  |     r.push_back(Parameter::Mandatory('u', "union volume")); | 
    
    | 213 |  |  |     r.push_back(Parameter::Mandatory('s', "scratch directory")); | 
    
    | 214 |  |  |     r.push_back(Parameter::Mandatory('b', "base hash")); | 
    
    | 215 |  |  |     r.push_back(Parameter::Mandatory('w', "stratum 0 base url")); | 
    
    | 216 |  |  |     r.push_back(Parameter::Mandatory('t', "directory for temporary storage")); | 
    
    | 217 |  |  |     r.push_back(Parameter::Switch('x', "verbose mode")); | 
    
    | 218 |  |  |     return r; | 
    
    | 219 |  |  |   } | 
    
    | 220 |  |  |   int Main(const ArgumentList &args); | 
    
    | 221 |  |  |  | 
    
    | 222 |  |  |  protected: | 
    
    | 223 |  |  |   void DetermineNestedCatalogCandidates( | 
    
    | 224 |  |  |       const catalog::Dirtab &dirtab, | 
    
    | 225 |  |  |       catalog::SimpleCatalogManager *catalog_manager, | 
    
    | 226 |  |  |       std::vector<std::string> *nested_catalog_candidates); | 
    
    | 227 |  |  |   void FilterCandidatesFromGlobResult( | 
    
    | 228 |  |  |       const catalog::Dirtab &dirtab, char **paths, const size_t npaths, | 
    
    | 229 |  |  |       catalog::SimpleCatalogManager *catalog_manager, | 
    
    | 230 |  |  |       std::vector<std::string> *nested_catalog_candidates); | 
    
    | 231 |  |  |   bool CreateCatalogMarkers( | 
    
    | 232 |  |  |       const std::vector<std::string> &new_nested_catalogs); | 
    
    | 233 |  |  |  | 
    
    | 234 |  |  |  private: | 
    
    | 235 |  |  |   std::string union_dir_; | 
    
    | 236 |  |  |   std::string scratch_dir_; | 
    
    | 237 |  |  |   bool verbose_; | 
    
    | 238 |  |  | }; | 
    
    | 239 |  |  |  | 
    
    | 240 |  |  | class CommandSync : public Command { | 
    
    | 241 |  |  |  public: | 
    
    | 242 |  |  |   ~CommandSync() {} | 
    
    | 243 |  |  |   virtual std::string GetName() const { return "sync"; } | 
    
    | 244 |  |  |   virtual std::string GetDescription() const { | 
    
    | 245 |  |  |     return "Pushes changes from scratch area back to the repository."; | 
    
    | 246 |  |  |   } | 
    
    | 247 |  |  |   virtual ParameterList GetParams() const { | 
    
    | 248 |  |  |     ParameterList r; | 
    
    | 249 |  |  |     r.push_back(Parameter::Mandatory('b', "base hash")); | 
    
    | 250 |  |  |     r.push_back(Parameter::Mandatory('c', "r/o volume")); | 
    
    | 251 |  |  |     r.push_back(Parameter::Mandatory('o', "manifest output file")); | 
    
    | 252 |  |  |     r.push_back(Parameter::Mandatory('r', "spooler definition")); | 
    
    | 253 |  |  |     r.push_back(Parameter::Mandatory('s', "scratch directory")); | 
    
    | 254 |  |  |     r.push_back(Parameter::Mandatory('t', "directory for tee")); | 
    
    | 255 |  |  |     r.push_back(Parameter::Mandatory('u', "union volume")); | 
    
    | 256 |  |  |     r.push_back(Parameter::Mandatory('w', "stratum 0 base url")); | 
    
    | 257 |  |  |     r.push_back(Parameter::Mandatory('K', "public key(s) for repo")); | 
    
    | 258 |  |  |     r.push_back(Parameter::Mandatory('N', "fully qualified repository name")); | 
    
    | 259 |  |  |  | 
    
    | 260 |  |  |     r.push_back(Parameter::Optional('a', "desired average chunk size (bytes)")); | 
    
    | 261 |  |  |     r.push_back(Parameter::Optional('e', "hash algorithm (default: SHA-1)")); | 
    
    | 262 |  |  |     r.push_back(Parameter::Optional('f', "union filesystem type")); | 
    
    | 263 |  |  |     r.push_back(Parameter::Optional('h', "maximal file chunk size in bytes")); | 
    
    | 264 |  |  |     r.push_back(Parameter::Optional('l', "minimal file chunk size in bytes")); | 
    
    | 265 |  |  |     r.push_back(Parameter::Optional('q', "number of concurrent write jobs")); | 
    
    | 266 |  |  |     r.push_back(Parameter::Optional('0', "number of upload tasks")); | 
    
    | 267 |  |  |     r.push_back(Parameter::Optional('v', "manual revision number")); | 
    
    | 268 |  |  |     r.push_back(Parameter::Optional('z', "log level (0-4, default: 2)")); | 
    
    | 269 |  |  |     r.push_back(Parameter::Optional('C', "trusted certificates")); | 
    
    | 270 |  |  |     r.push_back(Parameter::Optional('F', "Authz file listing (default: none)")); | 
    
    | 271 |  |  |     r.push_back(Parameter::Optional('M', "minimum weight of the autocatalogs")); | 
    
    | 272 |  |  |     r.push_back( | 
    
    | 273 |  |  |         Parameter::Optional('Q', "nested catalog limit in kilo-entries")); | 
    
    | 274 |  |  |     r.push_back(Parameter::Optional('R', "root catalog limit in kilo-entries")); | 
    
    | 275 |  |  |     r.push_back(Parameter::Optional('T', "Root catalog TTL in seconds")); | 
    
    | 276 |  |  |     r.push_back(Parameter::Optional('U', "file size limit in megabytes")); | 
    
    | 277 |  |  |     r.push_back( | 
    
    | 278 |  |  |         Parameter::Optional('D', "tag name (only used when upstream is GW)")); | 
    
    | 279 |  |  |     r.push_back(Parameter::Optional( | 
    
    | 280 |  |  |         'G', "tag channel (only used when upstream is GW)")); | 
    
    | 281 |  |  |     r.push_back(Parameter::Optional( | 
    
    | 282 |  |  |         'J', "tag description (only used when upstream is GW)")); | 
    
    | 283 |  |  |     r.push_back(Parameter::Optional('X', "maximum weight of the autocatalogs")); | 
    
    | 284 |  |  |     r.push_back(Parameter::Optional('Z', | 
    
    | 285 |  |  |                                     "compression algorithm " | 
    
    | 286 |  |  |                                     "(default: zlib)")); | 
    
    | 287 |  |  |     r.push_back(Parameter::Optional('S', | 
    
    | 288 |  |  |                                     "virtual directory options " | 
    
    | 289 |  |  |                                     "[snapshots, remove]")); | 
    
    | 290 |  |  |  | 
    
    | 291 |  |  |     r.push_back(Parameter::Switch('d', | 
    
    | 292 |  |  |                                   "pause publishing to allow for catalog " | 
    
    | 293 |  |  |                                   "tweaks")); | 
    
    | 294 |  |  |     r.push_back(Parameter::Switch('i', "ignore x-directory hardlinks")); | 
    
    | 295 |  |  |     r.push_back(Parameter::Switch('g', "ignore special files")); | 
    
    | 296 |  |  |     r.push_back(Parameter::Switch('k', "include extended attributes")); | 
    
    | 297 |  |  |     r.push_back(Parameter::Switch('m', "create micro catalogs")); | 
    
    | 298 |  |  |     r.push_back(Parameter::Switch('n', "create new repository")); | 
    
    | 299 |  |  |     r.push_back(Parameter::Switch('p', "enable file chunking")); | 
    
    | 300 |  |  |     r.push_back(Parameter::Switch('O', "generate legacy bulk chunks")); | 
    
    | 301 |  |  |     r.push_back(Parameter::Switch('x', "print change set")); | 
    
    | 302 |  |  |     r.push_back(Parameter::Switch('y', "dry run")); | 
    
    | 303 |  |  |     r.push_back(Parameter::Switch('A', "autocatalog enabled/disabled")); | 
    
    | 304 |  |  |     r.push_back(Parameter::Switch('E', "enforce limits instead of warning")); | 
    
    | 305 |  |  |     r.push_back(Parameter::Switch('L', "enable HTTP redirects")); | 
    
    | 306 |  |  |     r.push_back(Parameter::Switch('V', | 
    
    | 307 |  |  |                                   "Publish format compatible with " | 
    
    | 308 |  |  |                                   "authenticated repos")); | 
    
    | 309 |  |  |     r.push_back(Parameter::Switch('Y', "enable external data")); | 
    
    | 310 |  |  |     r.push_back(Parameter::Switch('B', "branched catalog (no manifest)")); | 
    
    | 311 |  |  |  | 
    
    | 312 |  |  |     r.push_back(Parameter::Optional('P', "session_token_file")); | 
    
    | 313 |  |  |     r.push_back(Parameter::Optional('H', "key file for HTTP API")); | 
    
    | 314 |  |  |  | 
    
    | 315 |  |  |     return r; | 
    
    | 316 |  |  |   } | 
    
    | 317 |  |  |   int Main(const ArgumentList &args); | 
    
    | 318 |  |  |  | 
    
    | 319 |  |  |  protected: | 
    
    | 320 |  |  |   bool ReadFileChunkingArgs(const swissknife::ArgumentList &args, | 
    
    | 321 |  |  |                             SyncParameters *params); | 
    
    | 322 |  |  |   bool CheckParams(const SyncParameters &p); | 
    
    | 323 |  |  |   bool ObtainDacReadSearchCapability(); | 
    
    | 324 |  |  | }; | 
    
    | 325 |  |  |  | 
    
    | 326 |  |  | }  // namespace swissknife | 
    
    | 327 |  |  |  | 
    
    | 328 |  |  | #endif  // CVMFS_SWISSKNIFE_SYNC_H_ |