CernVM-FS  2.13.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
swissknife_history.cc
Go to the documentation of this file.
1 
5 #include "swissknife_history.h"
6 
7 #include <algorithm>
8 #include <cassert>
9 #include <ctime>
10 
11 #include "catalog_rw.h"
12 #include "crypto/hash.h"
13 #include "crypto/signature.h"
14 #include "manifest_fetch.h"
15 #include "network/download.h"
16 #include "upload.h"
17 
18 using namespace std; // NOLINT
19 using namespace swissknife; // NOLINT
20 
21 const std::string CommandTag::kHeadTag = "trunk";
22 const std::string CommandTag::kPreviousHeadTag = "trunk-previous";
23 
24 const std::string CommandTag::kHeadTagDescription = "current HEAD";
25 const std::string
26  CommandTag::kPreviousHeadTagDescription = "default undo target";
27 
29  r->push_back(Parameter::Mandatory('w', "repository directory / url"));
30  r->push_back(Parameter::Mandatory('t', "temporary scratch directory"));
31  r->push_back(Parameter::Optional('p', "public key of the repository"));
32  r->push_back(Parameter::Optional('f', "fully qualified repository name"));
33  r->push_back(Parameter::Optional('r', "spooler definition string"));
34  r->push_back(Parameter::Optional('m', "(unsigned) manifest file to edit"));
35  r->push_back(Parameter::Optional('b', "mounted repository base hash"));
36  r->push_back(
37  Parameter::Optional('e', "hash algorithm to use (default SHA1)"));
38  r->push_back(Parameter::Switch('L', "follow HTTP redirects"));
39  r->push_back(Parameter::Optional('P', "session_token_file"));
40  r->push_back(Parameter::Optional('@', "proxy url"));
41 }
42 
43 CommandTag::Environment *CommandTag::InitializeEnvironment(
44  const ArgumentList &args, const bool read_write) {
45  const string repository_url = MakeCanonicalPath(*args.find('w')->second);
46  const string tmp_path = MakeCanonicalPath(*args.find('t')->second);
47  const string spl_definition = (args.find('r') == args.end())
48  ? ""
50  *args.find('r')->second);
51  const string manifest_path = (args.find('m') == args.end())
52  ? ""
53  : MakeCanonicalPath(*args.find('m')->second);
54  const shash::Algorithms hash_algo = (args.find('e') == args.end())
55  ? shash::kSha1
57  *args.find('e')->second);
58  const string pubkey_path = (args.find('p') == args.end())
59  ? ""
60  : MakeCanonicalPath(*args.find('p')->second);
61  const shash::Any base_hash = (args.find('b') == args.end())
62  ? shash::Any()
64  shash::HexPtr(*args.find('b')->second),
66  const string repo_name = (args.find('f') == args.end())
67  ? ""
68  : *args.find('f')->second;
69 
70  string session_token_file;
71  if (args.find('P') != args.end()) {
72  session_token_file = *args.find('P')->second;
73  }
74 
75  // Sanity checks
76  if (hash_algo == shash::kAny) {
77  LogCvmfs(kLogCvmfs, kLogStderr, "failed to parse hash algorithm to use");
78  return NULL;
79  }
80 
81  if (read_write && spl_definition.empty()) {
82  LogCvmfs(kLogCvmfs, kLogStderr, "no upstream storage provided (-r)");
83  return NULL;
84  }
85 
86  if (read_write && manifest_path.empty()) {
87  LogCvmfs(kLogCvmfs, kLogStderr, "no (unsigned) manifest provided (-m)");
88  return NULL;
89  }
90 
91  if (!read_write && pubkey_path.empty()) {
92  LogCvmfs(kLogCvmfs, kLogStderr, "no public key provided (-p)");
93  return NULL;
94  }
95 
96  if (!read_write && repo_name.empty()) {
97  LogCvmfs(kLogCvmfs, kLogStderr, "no repository name provided (-f)");
98  return NULL;
99  }
100 
101  if (HasPrefix(spl_definition, "gw", false)) {
102  if (session_token_file.empty()) {
103  PrintError("Session token file has to be provided "
104  "when upstream type is gw.");
105  return NULL;
106  }
107  }
108 
109  // create new environment
110  // Note: We use this encapsulation because we cannot be sure that the
111  // Command object gets deleted properly. With the Environment object at
112  // hand we have full control and can make heavy and safe use of RAII
113  UniquePtr<Environment> env(new Environment(repository_url, tmp_path));
114  env->manifest_path.Set(manifest_path);
115  env->history_path.Set(CreateTempPath(tmp_path + "/history", 0600));
116 
117  // initialize the (swissknife global) download manager
118  const bool follow_redirects = (args.count('L') > 0);
119  const std::string &proxy = (args.count('@') > 0) ? *args.find('@')->second
120  : "";
121  if (!this->InitDownloadManager(follow_redirects, proxy)) {
122  return NULL;
123  }
124 
125  // initialize the (swissknife global) signature manager (if possible)
126  if (!pubkey_path.empty() && !this->InitSignatureManager(pubkey_path)) {
127  return NULL;
128  }
129 
130  // open the (yet unsigned) manifest file if it is there, otherwise load the
131  // latest manifest from the server
132  env->manifest = (FileExists(env->manifest_path.path()))
133  ? OpenLocalManifest(env->manifest_path.path())
134  : FetchRemoteManifest(env->repository_url, repo_name,
135  base_hash);
136 
137  if (!env->manifest.IsValid()) {
138  LogCvmfs(kLogCvmfs, kLogStderr, "failed to load manifest file");
139  return NULL;
140  }
141 
142  // figure out the hash of the history from the previous revision if needed
143  if (read_write && env->manifest->history().IsNull() && !base_hash.IsNull()) {
144  env->previous_manifest = FetchRemoteManifest(env->repository_url, repo_name,
145  base_hash);
146  if (!env->previous_manifest.IsValid()) {
147  LogCvmfs(kLogCvmfs, kLogStderr, "failed to load previous manifest");
148  return NULL;
149  }
150 
152  "using history database '%s' from previous "
153  "manifest (%s) as basis",
154  env->previous_manifest->history().ToString().c_str(),
155  env->previous_manifest->repository_name().c_str());
156  env->manifest->set_history(env->previous_manifest->history());
157  env->manifest->set_repository_name(
158  env->previous_manifest->repository_name());
159  }
160 
161  // download the history database referenced in the manifest
162  env->history = GetHistory(env->manifest.weak_ref(), env->repository_url,
163  env->history_path.path(), read_write);
164  if (!env->history.IsValid()) {
165  return NULL;
166  }
167 
168  // if the using Command is expected to change the history database, we
169  // need
170  // to initialize the upload spooler for potential later history upload
171  if (read_write) {
172  const bool use_file_chunking = false;
173  const bool generate_legacy_bulk_chunks = false;
174  const upload::SpoolerDefinition sd(
175  spl_definition, hash_algo, zlib::kZlibDefault,
176  generate_legacy_bulk_chunks, use_file_chunking, 0, 0, 0,
177  session_token_file);
178  env->spooler = upload::Spooler::Construct(sd);
179  if (!env->spooler.IsValid()) {
180  LogCvmfs(kLogCvmfs, kLogStderr, "failed to initialize upload spooler");
181  return NULL;
182  }
183  }
184 
185  // return the pointer of the Environment (passing the ownership along)
186  return env.Release();
187 }
188 
189 bool CommandTag::CloseAndPublishHistory(Environment *env) {
190  assert(env->spooler.IsValid());
191 
192  // set the previous revision pointer of the history database
193  env->history->SetPreviousRevision(env->manifest->history());
194 
195  // close the history database
196  history::History *weak_history = env->history.Release();
197  delete weak_history;
198 
199  // compress and upload the new history database
200  Future<shash::Any> history_hash;
201  upload::Spooler::CallbackPtr callback = env->spooler->RegisterListener(
202  &CommandTag::UploadClosure, this, &history_hash);
203  env->spooler->ProcessHistory(env->history_path.path());
204  env->spooler->WaitForUpload();
205  const shash::Any new_history_hash = history_hash.Get();
206  env->spooler->UnregisterListener(callback);
207 
208  // retrieve the (async) uploader result
209  if (new_history_hash.IsNull()) {
210  return false;
211  }
212 
213  // update the (yet unsigned) manifest file
214  env->manifest->set_history(new_history_hash);
215  if (!env->manifest->Export(env->manifest_path.path())) {
216  LogCvmfs(kLogCvmfs, kLogStderr, "failed to export the new manifest '%s'",
217  env->manifest_path.path().c_str());
218  return false;
219  }
220 
221  // disable the unlink guard in order to keep the newly exported manifest file
222  env->manifest_path.Disable();
224  "exported manifest (%" PRIu64 ") with new history '%s'",
225  env->manifest->revision(), new_history_hash.ToString().c_str());
226 
227  return true;
228 }
229 
230 
231 bool CommandTag::UploadCatalogAndUpdateManifest(
233  assert(env->spooler.IsValid());
234 
235  // gather information about catalog to be uploaded and update manifest
236  UniquePtr<catalog::WritableCatalog> wr_catalog(catalog);
237  const std::string catalog_path = wr_catalog->database_path();
238  env->manifest->set_ttl(wr_catalog->GetTTL());
239  env->manifest->set_revision(wr_catalog->GetRevision());
240  env->manifest->set_publish_timestamp(wr_catalog->GetLastModified());
241 
242  // close the catalog
243  catalog::WritableCatalog *weak_catalog = wr_catalog.Release();
244  delete weak_catalog;
245 
246  // upload the catalog
247  Future<shash::Any> catalog_hash;
248  upload::Spooler::CallbackPtr callback = env->spooler->RegisterListener(
249  &CommandTag::UploadClosure, this, &catalog_hash);
250  env->spooler->ProcessCatalog(catalog_path);
251  env->spooler->WaitForUpload();
252  const shash::Any new_catalog_hash = catalog_hash.Get();
253  env->spooler->UnregisterListener(callback);
254 
255  // check if the upload succeeded
256  if (new_catalog_hash.IsNull()) {
257  LogCvmfs(kLogCvmfs, kLogStderr, "failed to upload catalog '%s'",
258  catalog_path.c_str());
259  return false;
260  }
261 
262  // update the catalog size and hash in the manifest
263  const size_t catalog_size = GetFileSize(catalog_path);
264  env->manifest->set_catalog_size(catalog_size);
265  env->manifest->set_catalog_hash(new_catalog_hash);
266 
267  LogCvmfs(kLogCvmfs, kLogVerboseMsg, "uploaded new catalog (%lu bytes) '%s'",
268  catalog_size, new_catalog_hash.ToString().c_str());
269 
270  return true;
271 }
272 
273 void CommandTag::UploadClosure(const upload::SpoolerResult &result,
274  Future<shash::Any> *hash) {
275  assert(!result.IsChunked());
276  if (result.return_code != 0) {
277  LogCvmfs(kLogCvmfs, kLogStderr, "failed to upload history database (%d)",
278  result.return_code);
279  hash->Set(shash::Any());
280  } else {
281  hash->Set(result.content_hash);
282  }
283 }
284 
285 bool CommandTag::UpdateUndoTags(
286  Environment *env, const history::History::Tag &current_head_template,
287  const bool undo_rollback) {
288  assert(env->history.IsValid());
289 
290  history::History::Tag current_head;
291  history::History::Tag current_old_head;
292 
293  // remove previous HEAD tag
294  if (!env->history->Remove(CommandTag::kPreviousHeadTag)) {
295  LogCvmfs(kLogCvmfs, kLogVerboseMsg, "didn't find a previous HEAD tag");
296  }
297 
298  // check if we have a current HEAD tag that needs to renamed to previous
299  // HEAD
300  if (env->history->GetByName(CommandTag::kHeadTag, &current_head)) {
301  // remove current HEAD tag
302  if (!env->history->Remove(CommandTag::kHeadTag)) {
303  LogCvmfs(kLogCvmfs, kLogStderr, "failed to remove current HEAD tag");
304  return false;
305  }
306 
307  // set previous HEAD tag where current HEAD used to be
308  if (!undo_rollback) {
309  current_old_head = current_head;
310  current_old_head.name = CommandTag::kPreviousHeadTag;
311  current_old_head.description = CommandTag::kPreviousHeadTagDescription;
312  if (!env->history->Insert(current_old_head)) {
313  LogCvmfs(kLogCvmfs, kLogStderr, "failed to set previous HEAD tag");
314  return false;
315  }
316  }
317  }
318 
319  // set the current HEAD to the catalog provided by the template HEAD
320  current_head = current_head_template;
321  current_head.name = CommandTag::kHeadTag;
322  current_head.description = CommandTag::kHeadTagDescription;
323  if (!env->history->Insert(current_head)) {
324  LogCvmfs(kLogCvmfs, kLogStderr, "failed to set new current HEAD");
325  return false;
326  }
327 
328  return true;
329 }
330 
331 bool CommandTag::FetchObject(const std::string &repository_url,
332  const shash::Any &object_hash,
333  const std::string &destination_path) const {
334  assert(!object_hash.IsNull());
335 
336  download::Failures dl_retval;
337  const std::string url = repository_url + "/data/" + object_hash.MakePath();
338 
339  cvmfs::PathSink pathsink(destination_path);
340  download::JobInfo download_object(&url, true, false, &object_hash, &pathsink);
341  dl_retval = download_manager()->Fetch(&download_object);
342 
343  if (dl_retval != download::kFailOk) {
344  LogCvmfs(kLogCvmfs, kLogStderr, "failed to download object '%s' (%d - %s)",
345  object_hash.ToStringWithSuffix().c_str(), dl_retval,
346  download::Code2Ascii(dl_retval));
347  return false;
348  }
349 
350  return true;
351 }
352 
353 history::History *CommandTag::GetHistory(const manifest::Manifest *manifest,
354  const std::string &repository_url,
355  const std::string &history_path,
356  const bool read_write) const {
357  const shash::Any history_hash = manifest->history();
359 
360  if (history_hash.IsNull()) {
361  history = history::SqliteHistory::Create(history_path,
362  manifest->repository_name());
363  if (NULL == history) {
364  LogCvmfs(kLogCvmfs, kLogStderr, "failed to create history database");
365  return NULL;
366  }
367  } else {
368  if (!FetchObject(repository_url, history_hash, history_path)) {
369  return NULL;
370  }
371 
372  history = (read_write) ? history::SqliteHistory::OpenWritable(history_path)
373  : history::SqliteHistory::Open(history_path);
374  if (NULL == history) {
375  LogCvmfs(kLogCvmfs, kLogStderr, "failed to open history database (%s)",
376  history_path.c_str());
377  unlink(history_path.c_str());
378  return NULL;
379  }
380 
381  assert(history->fqrn() == manifest->repository_name());
382  }
383 
384  return history;
385 }
386 
387 catalog::Catalog *CommandTag::GetCatalog(const std::string &repository_url,
388  const shash::Any &catalog_hash,
389  const std::string catalog_path,
390  const bool read_write) const {
391  assert(shash::kSuffixCatalog == catalog_hash.suffix);
392  if (!FetchObject(repository_url, catalog_hash, catalog_path)) {
393  return NULL;
394  }
395 
396  const std::string catalog_root_path = "";
397  return (read_write) ? catalog::WritableCatalog::AttachFreely(
398  catalog_root_path, catalog_path, catalog_hash)
400  catalog_root_path, catalog_path, catalog_hash);
401 }
402 
403 void CommandTag::PrintTagMachineReadable(
404  const history::History::Tag &tag) const {
405  LogCvmfs(kLogCvmfs, kLogStdout, "%s %s %" PRIu64 " %" PRIu64 " %ld %s %s",
406  tag.name.c_str(), tag.root_hash.ToString().c_str(), tag.size,
407  tag.revision, tag.timestamp,
408  (tag.branch == "") ? "(default)" : tag.branch.c_str(),
409  tag.description.c_str());
410 }
411 
412 std::string CommandTag::AddPadding(const std::string &str, const size_t padding,
413  const bool align_right,
414  const std::string &fill_char) const {
415  assert(str.size() <= padding);
416  std::string result(str);
417  result.resize(padding);
418  const size_t pos = (align_right) ? 0 : str.size();
419  const size_t padding_width = padding - str.size();
420  for (size_t i = 0; i < padding_width; ++i)
421  result.insert(pos, fill_char);
422  return result;
423 }
424 
425 bool CommandTag::IsUndoTagName(const std::string &tag_name) const {
426  return tag_name == CommandTag::kHeadTag
427  || tag_name == CommandTag::kPreviousHeadTag;
428 }
429 
430 //------------------------------------------------------------------------------
431 
432 ParameterList CommandEditTag::GetParams() const {
433  ParameterList r;
435 
436  r.push_back(Parameter::Optional('d', "space separated tags to be deleted"));
437  r.push_back(Parameter::Optional('a', "name of the new tag"));
438  r.push_back(Parameter::Optional('D', "description of the tag"));
439  r.push_back(Parameter::Optional('B', "branch of the new tag"));
440  r.push_back(Parameter::Optional('P', "predecessor branch"));
441  r.push_back(Parameter::Optional('h', "root hash of the new tag"));
442  r.push_back(Parameter::Switch('x', "maintain undo tags"));
443  return r;
444 }
445 
446 int CommandEditTag::Main(const ArgumentList &args) {
447  if ((args.find('d') == args.end()) && (args.find('a') == args.end())
448  && (args.find('x') == args.end())) {
449  LogCvmfs(kLogCvmfs, kLogStderr, "nothing to do");
450  return 1;
451  }
452 
453  // initialize the Environment (taking ownership)
454  const bool history_read_write = true;
455  UniquePtr<Environment> env(InitializeEnvironment(args, history_read_write));
456  if (!env.IsValid()) {
457  LogCvmfs(kLogCvmfs, kLogStderr, "failed to init environment");
458  return 1;
459  }
460 
461  int retval;
462  if (args.find('d') != args.end()) {
463  retval = RemoveTags(args, env.weak_ref());
464  if (retval != 0)
465  return retval;
466  }
467  if ((args.find('a') != args.end()) || (args.find('x') != args.end())) {
468  retval = AddNewTag(args, env.weak_ref());
469  if (retval != 0)
470  return retval;
471  }
472 
473  // finalize processing and upload new history database
474  if (!CloseAndPublishHistory(env.weak_ref())) {
475  return 1;
476  }
477  return 0;
478 }
479 
480 int CommandEditTag::AddNewTag(const ArgumentList &args, Environment *env) {
481  const std::string tag_name = (args.find('a') != args.end())
482  ? *args.find('a')->second
483  : "";
484  const std::string tag_description = (args.find('D') != args.end())
485  ? *args.find('D')->second
486  : "";
487  const bool undo_tags = (args.find('x') != args.end());
488  const std::string root_hash_string = (args.find('h') != args.end())
489  ? *args.find('h')->second
490  : "";
491  const std::string branch_name = (args.find('B') != args.end())
492  ? *args.find('B')->second
493  : "";
494  const std::string previous_branch_name = (args.find('P') != args.end())
495  ? *args.find('P')->second
496  : "";
497 
498  if (tag_name.find(" ") != std::string::npos) {
499  LogCvmfs(kLogCvmfs, kLogStderr, "tag names must not contain spaces");
500  return 1;
501  }
502 
503  assert(!tag_name.empty() || undo_tags);
504 
505  if (IsUndoTagName(tag_name)) {
506  LogCvmfs(kLogCvmfs, kLogStderr, "undo tags are managed internally");
507  return 1;
508  }
509 
510  // set the root hash to be tagged to the current HEAD if no other hash was
511  // given by the user
512  shash::Any root_hash = GetTagRootHash(env, root_hash_string);
513  if (root_hash.IsNull()) {
514  return 1;
515  }
516 
517  // open the catalog to be tagged (to check for existence and for meta info)
518  const UnlinkGuard catalog_path(
519  CreateTempPath(env->tmp_path + "/catalog", 0600));
520  const bool catalog_read_write = false;
521  const UniquePtr<catalog::Catalog> catalog(GetCatalog(
522  env->repository_url, root_hash, catalog_path.path(), catalog_read_write));
523  if (!catalog.IsValid()) {
524  LogCvmfs(kLogCvmfs, kLogStderr, "catalog with hash '%s' does not exist",
525  root_hash.ToString().c_str());
526  return 1;
527  }
528 
529  // check if the catalog is a root catalog
530  if (!catalog->root_prefix().IsEmpty()) {
532  "cannot tag catalog '%s' that is not a "
533  "root catalog.",
534  root_hash.ToString().c_str());
535  return 1;
536  }
537 
538  // create a template for the new tag to be created, moved or used as undo tag
539  history::History::Tag tag_template;
540  tag_template.name = "<template>";
541  tag_template.root_hash = root_hash;
542  tag_template.size = GetFileSize(catalog_path.path());
543  tag_template.revision = catalog->GetRevision();
544  tag_template.timestamp = catalog->GetLastModified();
545  tag_template.branch = branch_name;
546  tag_template.description = tag_description;
547 
548  // manipulate the tag database by creating a new tag or moving an existing one
549  if (!tag_name.empty()) {
550  tag_template.name = tag_name;
551  const bool user_provided_hash = (!root_hash_string.empty());
552 
553  if (!env->history->ExistsBranch(tag_template.branch)) {
555  tag_template.branch, previous_branch_name, tag_template.revision);
556  if (!env->history->InsertBranch(branch)) {
557  LogCvmfs(kLogCvmfs, kLogStderr, "cannot insert branch '%s'",
558  tag_template.branch.c_str());
559  return 1;
560  }
561  }
562 
563  if (!ManipulateTag(env, tag_template, user_provided_hash)) {
564  return 1;
565  }
566  }
567 
568  // handle undo tags ('trunk' and 'trunk-previous') if necessary
569  if (undo_tags && !UpdateUndoTags(env, tag_template)) {
570  return 1;
571  }
572 
573  return 0;
574 }
575 
576 shash::Any CommandEditTag::GetTagRootHash(
577  Environment *env, const std::string &root_hash_string) const {
578  shash::Any root_hash;
579 
580  if (root_hash_string.empty()) {
582  "no catalog hash provided, using hash"
583  "of current HEAD catalog (%s)",
584  env->manifest->catalog_hash().ToString().c_str());
585  root_hash = env->manifest->catalog_hash();
586  } else {
587  root_hash = shash::MkFromHexPtr(shash::HexPtr(root_hash_string),
589  if (root_hash.IsNull()) {
591  "failed to read provided catalog hash '%s'",
592  root_hash_string.c_str());
593  }
594  }
595 
596  return root_hash;
597 }
598 
599 bool CommandEditTag::ManipulateTag(Environment *env,
600  const history::History::Tag &tag_template,
601  const bool user_provided_hash) {
602  const std::string &tag_name = tag_template.name;
603 
604  // check if the tag already exists, otherwise create it and return
605  if (!env->history->Exists(tag_name)) {
606  return CreateTag(env, tag_template);
607  }
608 
609  // tag does exist already, now we need to see if we can move it
610  if (!user_provided_hash) {
612  "a tag with the name '%s' already exists. Do you want to move it? "
613  "(-h <root hash>)",
614  tag_name.c_str());
615  return false;
616  }
617 
618  // move the already existing tag and return
619  return MoveTag(env, tag_template);
620 }
621 
622 bool CommandEditTag::MoveTag(Environment *env,
623  const history::History::Tag &tag_template) {
624  const std::string &tag_name = tag_template.name;
625  history::History::Tag new_tag = tag_template;
626 
627  // get the already existent tag
628  history::History::Tag old_tag;
629  if (!env->history->GetByName(tag_name, &old_tag)) {
630  LogCvmfs(kLogCvmfs, kLogStderr, "failed to retrieve tag '%s' for moving",
631  tag_name.c_str());
632  return false;
633  }
634 
635  // check if we would move the tag to the same hash
636  if (old_tag.root_hash == new_tag.root_hash) {
637  LogCvmfs(kLogCvmfs, kLogStderr, "tag '%s' already points to '%s'",
638  tag_name.c_str(), old_tag.root_hash.ToString().c_str());
639  return false;
640  }
641 
642  // copy over old description if no new description was given
643  if (new_tag.description.empty()) {
644  new_tag.description = old_tag.description;
645  }
646  new_tag.branch = old_tag.branch;
647 
648  // remove the old tag from the database
649  if (!env->history->Remove(tag_name)) {
650  LogCvmfs(kLogCvmfs, kLogStderr, "removing old tag '%s' before move failed",
651  tag_name.c_str());
652  return false;
653  }
654  if (!env->history->PruneBranches()) {
655  LogCvmfs(kLogCvmfs, kLogStderr, "could not prune unused branches");
656  return false;
657  }
658  bool retval = env->history->Vacuum();
659  assert(retval);
660 
661  LogCvmfs(kLogCvmfs, kLogStdout, "moving tag '%s' from '%s' to '%s'",
662  tag_name.c_str(), old_tag.root_hash.ToString().c_str(),
663  tag_template.root_hash.ToString().c_str());
664 
665  // re-create the moved tag
666  return CreateTag(env, new_tag);
667 }
668 
669 bool CommandEditTag::CreateTag(Environment *env,
670  const history::History::Tag &new_tag) {
671  if (!env->history->Insert(new_tag)) {
672  LogCvmfs(kLogCvmfs, kLogStderr, "failed to insert new tag '%s'",
673  new_tag.name.c_str());
674  return false;
675  }
676 
677  return true;
678 }
679 
680 int CommandEditTag::RemoveTags(const ArgumentList &args, Environment *env) {
681  typedef std::vector<std::string> TagNames;
682  const std::string tags_to_delete = *args.find('d')->second;
683 
684  const TagNames condemned_tags = SplitString(tags_to_delete, ' ');
685 
686  // check if user tries to remove a magic undo tag
687  TagNames::const_iterator i = condemned_tags.begin();
688  const TagNames::const_iterator iend = condemned_tags.end();
689  for (; i != iend; ++i) {
690  if (IsUndoTagName(*i)) {
692  "undo tags are handled internally and cannot be deleted");
693  return 1;
694  }
695  }
696 
697  LogCvmfs(kLogCvmfs, kLogDebug, "proceeding to delete %lu tags",
698  condemned_tags.size());
699 
700  // check if the tags to be deleted exist
701  bool all_exist = true;
702  for (i = condemned_tags.begin(); i != iend; ++i) {
703  if (!env->history->Exists(*i)) {
704  LogCvmfs(kLogCvmfs, kLogStderr, "tag '%s' does not exist", i->c_str());
705  all_exist = false;
706  }
707  }
708  if (!all_exist) {
709  return 1;
710  }
711 
712  // delete the tags from the tag database and print their root hashes
713  i = condemned_tags.begin();
714  env->history->BeginTransaction();
715  for (; i != iend; ++i) {
716  // print some information about the tag to be deleted
717  history::History::Tag condemned_tag;
718  const bool found_tag = env->history->GetByName(*i, &condemned_tag);
719  assert(found_tag);
720  LogCvmfs(kLogCvmfs, kLogStdout, "deleting '%s' (%s)",
721  condemned_tag.name.c_str(),
722  condemned_tag.root_hash.ToString().c_str());
723 
724  // remove the tag
725  if (!env->history->Remove(*i)) {
726  LogCvmfs(kLogCvmfs, kLogStderr, "failed to remove tag '%s' from history",
727  i->c_str());
728  return 1;
729  }
730  }
731  bool retval = env->history->PruneBranches();
732  if (!retval) {
734  "failed to prune unused branches from history");
735  return 1;
736  }
737  env->history->CommitTransaction();
738  retval = env->history->Vacuum();
739  assert(retval);
740 
741  return 0;
742 }
743 
744 //------------------------------------------------------------------------------
745 
746 
747 ParameterList CommandListTags::GetParams() const {
748  ParameterList r;
750  r.push_back(Parameter::Switch('x', "machine readable output"));
751  r.push_back(Parameter::Switch('B', "print branch hierarchy"));
752  return r;
753 }
754 
755 void CommandListTags::PrintHumanReadableTagList(
756  const CommandListTags::TagList &tags) const {
757  // go through the list of tags and figure out the column widths
758  const std::string name_label = "Name";
759  const std::string rev_label = "Revision";
760  const std::string time_label = "Timestamp";
761  const std::string branch_label = "Branch";
762  const std::string desc_label = "Description";
763 
764  // figure out the maximal lengths of the fields in the lists
765  TagList::const_reverse_iterator i = tags.rbegin();
766  const TagList::const_reverse_iterator iend = tags.rend();
767  size_t max_name_len = name_label.size();
768  size_t max_rev_len = rev_label.size();
769  size_t max_time_len = desc_label.size();
770  size_t max_branch_len = branch_label.size();
771  for (; i != iend; ++i) {
772  max_name_len = std::max(max_name_len, i->name.size());
773  max_rev_len = std::max(max_rev_len, StringifyInt(i->revision).size());
774  max_time_len = std::max(max_time_len,
775  StringifyTime(i->timestamp, true).size());
776  max_branch_len = std::max(max_branch_len, i->branch.size());
777  }
778 
779  // print the list header
780  LogCvmfs(kLogCvmfs, kLogStdout, "%s \u2502 %s \u2502 %s \u2502 %s \u2502 %s",
781  AddPadding(name_label, max_name_len).c_str(),
782  AddPadding(rev_label, max_rev_len).c_str(),
783  AddPadding(time_label, max_time_len).c_str(),
784  AddPadding(branch_label, max_branch_len).c_str(),
785  desc_label.c_str());
787  "%s\u2500\u253C\u2500%s\u2500\u253C\u2500%s"
788  "\u2500\u253C\u2500%s\u2500\u253C\u2500%s",
789  AddPadding("", max_name_len, false, "\u2500").c_str(),
790  AddPadding("", max_rev_len, false, "\u2500").c_str(),
791  AddPadding("", max_time_len, false, "\u2500").c_str(),
792  AddPadding("", max_branch_len, false, "\u2500").c_str(),
793  AddPadding("", desc_label.size() + 1, false, "\u2500").c_str());
794 
795  // print the rows of the list
796  i = tags.rbegin();
797  for (; i != iend; ++i) {
798  LogCvmfs(
799  kLogCvmfs, kLogStdout, "%s \u2502 %s \u2502 %s \u2502 %s \u2502 %s",
800  AddPadding(i->name, max_name_len).c_str(),
801  AddPadding(StringifyInt(i->revision), max_rev_len, true).c_str(),
802  AddPadding(StringifyTime(i->timestamp, true), max_time_len).c_str(),
803  AddPadding(i->branch, max_branch_len).c_str(), i->description.c_str());
804  }
805 
806  // print the list footer
808  "%s\u2500\u2534\u2500%s\u2500\u2534\u2500%s"
809  "\u2500\u2534\u2500%s\u2500\u2534\u2500%s",
810  AddPadding("", max_name_len, false, "\u2500").c_str(),
811  AddPadding("", max_rev_len, false, "\u2500").c_str(),
812  AddPadding("", max_time_len, false, "\u2500").c_str(),
813  AddPadding("", max_branch_len, false, "\u2500").c_str(),
814  AddPadding("", desc_label.size() + 1, false, "\u2500").c_str());
815 
816  // print the number of tags listed
817  LogCvmfs(kLogCvmfs, kLogStdout, "listing contains %lu tags", tags.size());
818 }
819 
820 void CommandListTags::PrintMachineReadableTagList(const TagList &tags) const {
821  TagList::const_iterator i = tags.begin();
822  const TagList::const_iterator iend = tags.end();
823  for (; i != iend; ++i) {
824  PrintTagMachineReadable(*i);
825  }
826 }
827 
828 
829 void CommandListTags::PrintHumanReadableBranchList(
830  const BranchHierarchy &branches) const {
831  unsigned N = branches.size();
832  for (unsigned i = 0; i < N; ++i) {
833  for (unsigned l = 0; l < branches[i].level; ++l) {
835  ((l + 1) == branches[i].level) ? "\u251c " : "\u2502 ");
836  }
837  LogCvmfs(kLogCvmfs, kLogStdout, "%s @%" PRIu64,
838  branches[i].branch.branch.c_str(),
839  branches[i].branch.initial_revision);
840  }
841 }
842 
843 
844 void CommandListTags::PrintMachineReadableBranchList(
845  const BranchHierarchy &branches) const {
846  unsigned N = branches.size();
847  for (unsigned i = 0; i < N; ++i) {
848  LogCvmfs(kLogCvmfs, kLogStdout, "[%u] %s%s @%" PRIu64, branches[i].level,
849  AddPadding("", branches[i].level, false, " ").c_str(),
850  branches[i].branch.branch.c_str(),
851  branches[i].branch.initial_revision);
852  }
853 }
854 
855 
856 void CommandListTags::SortBranchesRecursively(
857  unsigned level,
858  const string &parent_branch,
859  const BranchList &branches,
860  BranchHierarchy *hierarchy) const {
861  // For large numbers of branches, this should be turned into the O(n) version
862  // using a linked list
863  unsigned N = branches.size();
864  for (unsigned i = 0; i < N; ++i) {
865  if (branches[i].branch == "")
866  continue;
867  if (branches[i].parent == parent_branch) {
868  hierarchy->push_back(BranchLevel(branches[i], level));
869  SortBranchesRecursively(level + 1, branches[i].branch, branches,
870  hierarchy);
871  }
872  }
873 }
874 
875 
876 CommandListTags::BranchHierarchy CommandListTags::SortBranches(
877  const BranchList &branches) const {
878  BranchHierarchy hierarchy;
879  hierarchy.push_back(
880  BranchLevel(history::History::Branch("(default)", "", 0), 0));
881  SortBranchesRecursively(1, "", branches, &hierarchy);
882  return hierarchy;
883 }
884 
885 
886 int CommandListTags::Main(const ArgumentList &args) {
887  const bool machine_readable = (args.find('x') != args.end());
888  const bool branch_hierarchy = (args.find('B') != args.end());
889 
890  // initialize the Environment (taking ownership)
891  const bool history_read_write = false;
892  UniquePtr<Environment> env(InitializeEnvironment(args, history_read_write));
893  if (!env.IsValid()) {
894  LogCvmfs(kLogCvmfs, kLogStderr, "failed to init environment");
895  return 1;
896  }
897 
898  if (branch_hierarchy) {
899  BranchList branch_list;
900  if (!env->history->ListBranches(&branch_list)) {
902  "failed to list branches in history database");
903  return 1;
904  }
905  BranchHierarchy branch_hierarchy = SortBranches(branch_list);
906 
907  if (machine_readable) {
908  PrintMachineReadableBranchList(branch_hierarchy);
909  } else {
910  PrintHumanReadableBranchList(branch_hierarchy);
911  }
912  } else {
913  // obtain a full list of all tags
914  TagList tags;
915  if (!env->history->List(&tags)) {
917  "failed to list tags in history database");
918  return 1;
919  }
920 
921  if (machine_readable) {
922  PrintMachineReadableTagList(tags);
923  } else {
924  PrintHumanReadableTagList(tags);
925  }
926  }
927 
928  return 0;
929 }
930 
931 //------------------------------------------------------------------------------
932 
933 ParameterList CommandInfoTag::GetParams() const {
934  ParameterList r;
936 
937  r.push_back(Parameter::Mandatory('n', "name of the tag to be inspected"));
938  r.push_back(Parameter::Switch('x', "machine readable output"));
939  return r;
940 }
941 
942 std::string CommandInfoTag::HumanReadableFilesize(const size_t filesize) const {
943  const size_t kiB = 1024;
944  const size_t MiB = kiB * 1024;
945  const size_t GiB = MiB * 1024;
946 
947  if (filesize > GiB) {
948  return StringifyDouble(static_cast<double>(filesize) / GiB) + " GiB";
949  } else if (filesize > MiB) {
950  return StringifyDouble(static_cast<double>(filesize) / MiB) + " MiB";
951  } else if (filesize > kiB) {
952  return StringifyDouble(static_cast<double>(filesize) / kiB) + " kiB";
953  } else {
954  return StringifyInt(filesize) + " Byte";
955  }
956 }
957 
958 void CommandInfoTag::PrintHumanReadableInfo(
959  const history::History::Tag &tag) const {
961  "Name: %s\n"
962  "Revision: %" PRIu64 "\n"
963  "Timestamp: %s\n"
964  "Branch: %s\n"
965  "Root Hash: %s\n"
966  "Catalog Size: %s\n"
967  "%s",
968  tag.name.c_str(), tag.revision,
969  StringifyTime(tag.timestamp, true /* utc */).c_str(),
970  tag.branch.c_str(), tag.root_hash.ToString().c_str(),
971  HumanReadableFilesize(tag.size).c_str(), tag.description.c_str());
972 }
973 
974 int CommandInfoTag::Main(const ArgumentList &args) {
975  const std::string tag_name = *args.find('n')->second;
976  const bool machine_readable = (args.find('x') != args.end());
977 
978  // initialize the Environment (taking ownership)
979  const bool history_read_write = false;
980  UniquePtr<Environment> env(InitializeEnvironment(args, history_read_write));
981  if (!env.IsValid()) {
982  LogCvmfs(kLogCvmfs, kLogStderr, "failed to init environment");
983  return 1;
984  }
985 
987  const bool found = env->history->GetByName(tag_name, &tag);
988  if (!found) {
989  LogCvmfs(kLogCvmfs, kLogStderr, "tag '%s' does not exist",
990  tag_name.c_str());
991  return 1;
992  }
993 
994  if (machine_readable) {
995  PrintTagMachineReadable(tag);
996  } else {
997  PrintHumanReadableInfo(tag);
998  }
999 
1000  return 0;
1001 }
1002 
1003 //------------------------------------------------------------------------------
1004 
1005 ParameterList CommandRollbackTag::GetParams() const {
1006  ParameterList r;
1008 
1009  r.push_back(Parameter::Optional('n', "name of the tag to be republished"));
1010  return r;
1011 }
1012 
1013 int CommandRollbackTag::Main(const ArgumentList &args) {
1014  const bool undo_rollback = (args.find('n') == args.end());
1015  const std::string tag_name = (!undo_rollback) ? *args.find('n')->second
1016  : CommandTag::kPreviousHeadTag;
1017 
1018  // initialize the Environment (taking ownership)
1019  const bool history_read_write = true;
1020  UniquePtr<Environment> env(InitializeEnvironment(args, history_read_write));
1021  if (!env.IsValid()) {
1022  LogCvmfs(kLogCvmfs, kLogStderr, "failed to init environment");
1023  return 1;
1024  }
1025 
1026  // find tag to be rolled back to
1027  history::History::Tag target_tag;
1028  const bool found = env->history->GetByName(tag_name, &target_tag);
1029  if (!found) {
1030  if (undo_rollback) {
1032  "only one anonymous rollback supported - "
1033  "perhaps you want to provide a tag name?");
1034  } else {
1035  LogCvmfs(kLogCvmfs, kLogStderr, "tag '%s' does not exist",
1036  tag_name.c_str());
1037  }
1038  return 1;
1039  }
1040  if (target_tag.branch != "") {
1042  "rollback is only supported on the default branch");
1043  return 1;
1044  }
1045 
1046  // list the tags that will be deleted
1047  TagList affected_tags;
1048  if (!env->history->ListTagsAffectedByRollback(tag_name, &affected_tags)) {
1050  "failed to list condemned tags prior to rollback to '%s'",
1051  tag_name.c_str());
1052  return 1;
1053  }
1054 
1055  // check if tag is valid to be rolled back to
1056  const uint64_t current_revision = env->manifest->revision();
1057  assert(target_tag.revision <= current_revision);
1058  if (target_tag.revision == current_revision) {
1060  "not rolling back to current head (%" PRIu64 ")",
1061  current_revision);
1062  return 1;
1063  }
1064 
1065  // open the catalog to be rolled back to
1066  const UnlinkGuard catalog_path(
1067  CreateTempPath(env->tmp_path + "/catalog", 0600));
1068  const bool catalog_read_write = true;
1070  dynamic_cast<catalog::WritableCatalog *>(
1071  GetCatalog(env->repository_url, target_tag.root_hash,
1072  catalog_path.path(), catalog_read_write)));
1073  if (!catalog.IsValid()) {
1074  LogCvmfs(kLogCvmfs, kLogStderr, "failed to open catalog with hash '%s'",
1075  target_tag.root_hash.ToString().c_str());
1076  return 1;
1077  }
1078 
1079  // check if the catalog has a supported schema version
1080  if (catalog->schema() < catalog::CatalogDatabase::kLatestSupportedSchema
1083  "not rolling back to outdated and "
1084  "incompatible catalog schema (%.1f < %.1f)",
1085  catalog->schema(),
1087  return 1;
1088  }
1089 
1090  // update the catalog to be republished
1091  catalog->Transaction();
1092  catalog->UpdateLastModified();
1093  catalog->SetRevision(current_revision + 1);
1094  catalog->SetPreviousRevision(env->manifest->catalog_hash());
1095  catalog->Commit();
1096 
1097  // Upload catalog (handing over ownership of catalog pointer)
1098  if (!UploadCatalogAndUpdateManifest(env.weak_ref(), catalog.Release())) {
1099  LogCvmfs(kLogCvmfs, kLogStderr, "catalog upload failed");
1100  return 1;
1101  }
1102 
1103  // update target tag with newly published root catalog information
1104  history::History::Tag updated_target_tag(target_tag);
1105  updated_target_tag.root_hash = env->manifest->catalog_hash();
1106  updated_target_tag.size = env->manifest->catalog_size();
1107  updated_target_tag.revision = env->manifest->revision();
1108  updated_target_tag.timestamp = env->manifest->publish_timestamp();
1109  if (!env->history->Rollback(updated_target_tag)) {
1110  LogCvmfs(kLogCvmfs, kLogStderr, "failed to rollback history to '%s'",
1111  updated_target_tag.name.c_str());
1112  return 1;
1113  }
1114  bool retval = env->history->Vacuum();
1115  assert(retval);
1116 
1117  // set the magic undo tags
1118  if (!UpdateUndoTags(env.weak_ref(), updated_target_tag, undo_rollback)) {
1119  LogCvmfs(kLogCvmfs, kLogStderr, "failed to update magic undo tags");
1120  return 1;
1121  }
1122 
1123  // finalize the history and upload it
1124  if (!CloseAndPublishHistory(env.weak_ref())) {
1125  return 1;
1126  }
1127 
1128  // print the tags that have been removed by the rollback
1129  PrintDeletedTagList(affected_tags);
1130 
1131  return 0;
1132 }
1133 
1134 void CommandRollbackTag::PrintDeletedTagList(const TagList &tags) const {
1135  size_t longest_name = 0;
1136  TagList::const_iterator i = tags.begin();
1137  const TagList::const_iterator iend = tags.end();
1138  for (; i != iend; ++i) {
1139  longest_name = std::max(i->name.size(), longest_name);
1140  }
1141 
1142  i = tags.begin();
1143  for (; i != iend; ++i) {
1144  LogCvmfs(kLogCvmfs, kLogStdout, "removed tag %s (%s)",
1145  AddPadding(i->name, longest_name).c_str(),
1146  i->root_hash.ToString().c_str());
1147  }
1148 }
1149 
1150 //------------------------------------------------------------------------------
1151 
1152 ParameterList CommandEmptyRecycleBin::GetParams() const {
1153  ParameterList r;
1155  return r;
1156 }
1157 
1158 int CommandEmptyRecycleBin::Main(const ArgumentList &args) {
1159  // initialize the Environment (taking ownership)
1160  const bool history_read_write = true;
1161  UniquePtr<Environment> env(InitializeEnvironment(args, history_read_write));
1162  if (!env.IsValid()) {
1163  LogCvmfs(kLogCvmfs, kLogStderr, "failed to init environment");
1164  return 1;
1165  }
1166 
1167  if (!env->history->EmptyRecycleBin()) {
1168  LogCvmfs(kLogCvmfs, kLogStderr, "failed to empty recycle bin");
1169  return 1;
1170  }
1171 
1172  // finalize the history and upload it
1173  if (!CloseAndPublishHistory(env.weak_ref())) {
1174  return 1;
1175  }
1176 
1177  return 0;
1178 }
int return_code
the return value of the spooler operation
bool IsNull() const
Definition: hash.h:371
const manifest::Manifest * manifest() const
Definition: repository.h:125
static SqliteHistory * Open(const std::string &file_name)
std::string name
Definition: history.h:92
std::vector< Parameter > ParameterList
Definition: swissknife.h:71
UniquePtr< manifest::Manifest > manifest
std::string ToString(const bool with_suffix=false) const
Definition: hash.h:241
const history::History * history() const
std::string ToStringWithSuffix() const
Definition: hash.h:296
std::vector< BranchLevel > BranchHierarchy
std::string CreateTempPath(const std::string &path_prefix, const int mode)
Definition: posix.cc:1041
static void InsertCommonParameters(ParameterList *r)
assert((mem||(size==0))&&"Out Of Memory")
string StringifyTime(const time_t seconds, const bool utc)
Definition: string.cc:104
const std::string & fqrn() const
Definition: history.h:197
string StringifyDouble(const double value)
Definition: string.cc:95
void Set(const T &object)
Definition: future.h:53
std::vector< history::History::Tag > TagList
T & Get()
Definition: future.h:66
Algorithms
Definition: hash.h:41
static SqliteHistory * OpenWritable(const std::string &file_name)
static WritableCatalog * AttachFreely(const std::string &root_path, const std::string &file, const shash::Any &catalog_hash, Catalog *parent=NULL, const bool is_not_root=false)
Definition: catalog_rw.cc:49
std::vector< history::History::Branch > BranchList
std::string description
Definition: history.h:97
bool FileExists(const std::string &path)
Definition: posix.cc:803
uint64_t GetRevision() const
Definition: catalog.cc:517
uint64_t GetLastModified() const
Definition: catalog.cc:522
const char * Code2Ascii(const Failures error)
vector< string > SplitString(const string &str, char delim)
Definition: string.cc:306
static const float kSchemaEpsilon
Definition: sql.h:104
std::string repository_name() const
Definition: manifest.h:123
const char kSuffixCatalog
Definition: hash.h:54
string StringifyInt(const int64_t value)
Definition: string.cc:77
bool HasPrefix(const string &str, const string &prefix, const bool ignore_case)
Definition: string.cc:279
PathString root_prefix() const
Definition: catalog.h:181
bool IsEmpty() const
Definition: shortstring.h:137
std::map< char, SharedPtr< std::string > > ArgumentList
Definition: swissknife.h:72
Algorithms ParseHashAlgorithm(const string &algorithm_option)
Definition: hash.cc:71
shash::Any history() const
Definition: manifest.h:128
shash::Any root_hash
Definition: history.h:93
static Catalog * AttachFreely(const std::string &imaginary_mountpoint, const std::string &file, const shash::Any &catalog_hash, Catalog *parent=NULL, const bool is_nested=false)
Definition: catalog.cc:29
std::string branch
Definition: history.h:101
static SqliteHistory * Create(const std::string &file_name, const std::string &fqrn)
UniquePtr< history::History > history
Any MkFromHexPtr(const HexPtr hex, const char suffix)
Definition: hash.cc:82
int64_t GetFileSize(const std::string &path)
Definition: posix.cc:812
const int kLogVerboseMsg
static const float kLatestSupportedSchema
Definition: catalog_sql.h:44
Suffix suffix
Definition: hash.h:123
std::string MakePath() const
Definition: hash.h:306
std::string MakeCanonicalPath(const std::string &path)
Definition: posix.cc:98
uint64_t revision
Definition: history.h:95
void PrintError(const string &message)
Definition: logging.cc:556
UniquePtr< upload::Spooler > spooler
CVMFS_EXPORT void LogCvmfs(const LogSource source, const int mask, const char *format,...)
Definition: logging.cc:545