CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
swissknife_history.cc
Go to the documentation of this file.
1 
5 #include "swissknife_history.h"
6 #include "cvmfs_config.h"
7 
8 #include <algorithm>
9 #include <cassert>
10 #include <ctime>
11 
12 #include "catalog_rw.h"
13 #include "download.h"
14 #include "hash.h"
15 #include "manifest_fetch.h"
16 #include "signature.h"
17 #include "upload.h"
18 
19 using namespace std; // NOLINT
20 using namespace swissknife; // NOLINT
21 
22 const std::string CommandTag::kHeadTag = "trunk";
23 const std::string CommandTag::kPreviousHeadTag = "trunk-previous";
24 
25 const std::string CommandTag::kHeadTagDescription = "current HEAD";
26 const std::string CommandTag::kPreviousHeadTagDescription =
27  "default undo target";
28 
30  r->push_back(Parameter::Mandatory('w', "repository directory / url"));
31  r->push_back(Parameter::Mandatory('t', "temporary scratch directory"));
32  r->push_back(Parameter::Optional('p', "public key of the repository"));
33  r->push_back(Parameter::Optional('z', "trusted certificates"));
34  r->push_back(Parameter::Optional('f', "fully qualified repository name"));
35  r->push_back(Parameter::Optional('r', "spooler definition string"));
36  r->push_back(Parameter::Optional('m', "(unsigned) manifest file to edit"));
37  r->push_back(Parameter::Optional('b', "mounted repository base hash"));
38  r->push_back(
39  Parameter::Optional('e', "hash algorithm to use (default SHA1)"));
40  r->push_back(Parameter::Switch('L', "follow HTTP redirects"));
41  r->push_back(Parameter::Optional('P', "session_token_file"));
42 }
43 
44 CommandTag::Environment *CommandTag::InitializeEnvironment(
45  const ArgumentList &args, const bool read_write) {
46  const string repository_url = MakeCanonicalPath(*args.find('w')->second);
47  const string tmp_path = MakeCanonicalPath(*args.find('t')->second);
48  const string spl_definition =
49  (args.find('r') == args.end())
50  ? ""
51  : MakeCanonicalPath(*args.find('r')->second);
52  const string manifest_path = (args.find('m') == args.end())
53  ? ""
54  : MakeCanonicalPath(*args.find('m')->second);
55  const shash::Algorithms hash_algo =
56  (args.find('e') == args.end())
57  ? shash::kSha1
58  : shash::ParseHashAlgorithm(*args.find('e')->second);
59  const string pubkey_path = (args.find('p') == args.end())
60  ? ""
61  : MakeCanonicalPath(*args.find('p')->second);
62  const string trusted_certs = (args.find('z') == args.end())
63  ? ""
64  : MakeCanonicalPath(*args.find('z')->second);
65  const shash::Any base_hash =
66  (args.find('b') == args.end())
67  ? shash::Any()
68  : shash::MkFromHexPtr(shash::HexPtr(*args.find('b')->second),
70  const string repo_name =
71  (args.find('f') == args.end()) ? "" : *args.find('f')->second;
72 
73  string session_token_file;
74  if (args.find('P') != args.end()) {
75  session_token_file = *args.find('P')->second;
76  }
77 
78  // Sanity checks
79  if (hash_algo == shash::kAny) {
80  LogCvmfs(kLogCvmfs, kLogStderr, "failed to parse hash algorithm to use");
81  return NULL;
82  }
83 
84  if (read_write && spl_definition.empty()) {
85  LogCvmfs(kLogCvmfs, kLogStderr, "no upstream storage provided (-r)");
86  return NULL;
87  }
88 
89  if (read_write && manifest_path.empty()) {
90  LogCvmfs(kLogCvmfs, kLogStderr, "no (unsigned) manifest provided (-m)");
91  return NULL;
92  }
93 
94  if (!read_write && pubkey_path.empty()) {
95  LogCvmfs(kLogCvmfs, kLogStderr, "no public key provided (-p)");
96  return NULL;
97  }
98 
99  if (!read_write && repo_name.empty()) {
100  LogCvmfs(kLogCvmfs, kLogStderr, "no repository name provided (-f)");
101  return NULL;
102  }
103 
104  if (HasPrefix(spl_definition, "gw", false)) {
105  if (session_token_file.empty()) {
106  PrintError(
107  "Session token file has to be provided "
108  "when upstream type is gw.");
109  return NULL;
110  }
111  }
112 
113  // create new environment
114  // Note: We use this encapsulation because we cannot be sure that the
115  // Command object gets deleted properly. With the Environment object at
116  // hand we have full control and can make heavy and safe use of RAII
117  UniquePtr<Environment> env(new Environment(repository_url, tmp_path));
118  env->manifest_path.Set(manifest_path);
119  env->history_path.Set(CreateTempPath(tmp_path + "/history", 0600));
120 
121  // initialize the (swissknife global) download manager
122  const bool follow_redirects = (args.count('L') > 0);
123  if (!this->InitDownloadManager(follow_redirects)) {
124  return NULL;
125  }
126 
127  // initialize the (swissknife global) signature manager (if possible)
128  if (!pubkey_path.empty() &&
129  !this->InitVerifyingSignatureManager(pubkey_path, trusted_certs)) {
130  return NULL;
131  }
132 
133  // open the (yet unsigned) manifest file if it is there, otherwise load the
134  // latest manifest from the server
135  env->manifest =
136  (FileExists(env->manifest_path.path()))
137  ? OpenLocalManifest(env->manifest_path.path())
138  : FetchRemoteManifest(env->repository_url, repo_name, base_hash);
139 
140  if (!env->manifest.IsValid()) {
141  LogCvmfs(kLogCvmfs, kLogStderr, "failed to load manifest file");
142  return NULL;
143  }
144 
145  // figure out the hash of the history from the previous revision if needed
146  if (read_write && env->manifest->history().IsNull() && !base_hash.IsNull()) {
147  env->previous_manifest =
148  FetchRemoteManifest(env->repository_url, repo_name, base_hash);
149  if (!env->previous_manifest.IsValid()) {
150  LogCvmfs(kLogCvmfs, kLogStderr, "failed to load previous manifest");
151  return NULL;
152  }
153 
155  "using history database '%s' from previous "
156  "manifest (%s) as basis",
157  env->previous_manifest->history().ToString().c_str(),
158  env->previous_manifest->repository_name().c_str());
159  env->manifest->set_history(env->previous_manifest->history());
160  env->manifest->set_repository_name(
161  env->previous_manifest->repository_name());
162  }
163 
164  // download the history database referenced in the manifest
165  env->history = GetHistory(env->manifest.weak_ref(), env->repository_url,
166  env->history_path.path(), read_write);
167  if (!env->history.IsValid()) {
168  return NULL;
169  }
170 
171  // if the using Command is expected to change the history database, we
172  // need
173  // to initialize the upload spooler for potential later history upload
174  if (read_write) {
175  const bool use_file_chunking = false;
176  const bool generate_legacy_bulk_chunks = false;
177  const upload::SpoolerDefinition sd(spl_definition, hash_algo,
179  generate_legacy_bulk_chunks,
180  use_file_chunking, 0, 0, 0,
181  session_token_file);
182  env->spooler = upload::Spooler::Construct(sd);
183  if (!env->spooler.IsValid()) {
184  LogCvmfs(kLogCvmfs, kLogStderr, "failed to initialize upload spooler");
185  return NULL;
186  }
187  }
188 
189  // return the pointer of the Environment (passing the ownership along)
190  return env.Release();
191 }
192 
193 bool CommandTag::CloseAndPublishHistory(Environment *env) {
194  assert(env->spooler.IsValid());
195 
196  // set the previous revision pointer of the history database
197  env->history->SetPreviousRevision(env->manifest->history());
198 
199  // close the history database
200  history::History *weak_history = env->history.Release();
201  delete weak_history;
202 
203  // compress and upload the new history database
204  Future<shash::Any> history_hash;
205  upload::Spooler::CallbackPtr callback = env->spooler->RegisterListener(
206  &CommandTag::UploadClosure, this, &history_hash);
207  env->spooler->ProcessHistory(env->history_path.path());
208  env->spooler->WaitForUpload();
209  const shash::Any new_history_hash = history_hash.Get();
210  env->spooler->UnregisterListener(callback);
211 
212  // retrieve the (async) uploader result
213  if (new_history_hash.IsNull()) {
214  return false;
215  }
216 
217  // update the (yet unsigned) manifest file
218  env->manifest->set_history(new_history_hash);
219  if (!env->manifest->Export(env->manifest_path.path())) {
220  LogCvmfs(kLogCvmfs, kLogStderr, "failed to export the new manifest '%s'",
221  env->manifest_path.path().c_str());
222  return false;
223  }
224 
225  // disable the unlink guard in order to keep the newly exported manifest file
226  env->manifest_path.Disable();
228  "exported manifest (%d) with new "
229  "history '%s'",
230  env->manifest->revision(), new_history_hash.ToString().c_str());
231 
232  return true;
233 }
234 
235 
236 bool CommandTag::UploadCatalogAndUpdateManifest(
238  assert(env->spooler.IsValid());
239 
240  // gather information about catalog to be uploaded and update manifest
241  UniquePtr<catalog::WritableCatalog> wr_catalog(catalog);
242  const std::string catalog_path = wr_catalog->database_path();
243  env->manifest->set_ttl(wr_catalog->GetTTL());
244  env->manifest->set_revision(wr_catalog->GetRevision());
245  env->manifest->set_publish_timestamp(wr_catalog->GetLastModified());
246 
247  // close the catalog
248  catalog::WritableCatalog *weak_catalog = wr_catalog.Release();
249  delete weak_catalog;
250 
251  // upload the catalog
252  Future<shash::Any> catalog_hash;
253  upload::Spooler::CallbackPtr callback = env->spooler->RegisterListener(
254  &CommandTag::UploadClosure, this, &catalog_hash);
255  env->spooler->ProcessCatalog(catalog_path);
256  env->spooler->WaitForUpload();
257  const shash::Any new_catalog_hash = catalog_hash.Get();
258  env->spooler->UnregisterListener(callback);
259 
260  // check if the upload succeeded
261  if (new_catalog_hash.IsNull()) {
262  LogCvmfs(kLogCvmfs, kLogStderr, "failed to upload catalog '%s'",
263  catalog_path.c_str());
264  return false;
265  }
266 
267  // update the catalog size and hash in the manifest
268  const size_t catalog_size = GetFileSize(catalog_path);
269  env->manifest->set_catalog_size(catalog_size);
270  env->manifest->set_catalog_hash(new_catalog_hash);
271 
272  LogCvmfs(kLogCvmfs, kLogVerboseMsg, "uploaded new catalog (%d bytes) '%s'",
273  catalog_size, new_catalog_hash.ToString().c_str());
274 
275  return true;
276 }
277 
278 void CommandTag::UploadClosure(const upload::SpoolerResult &result,
279  Future<shash::Any> *hash) {
280  assert(!result.IsChunked());
281  if (result.return_code != 0) {
282  LogCvmfs(kLogCvmfs, kLogStderr, "failed to upload history database (%d)",
283  result.return_code);
284  hash->Set(shash::Any());
285  } else {
286  hash->Set(result.content_hash);
287  }
288 }
289 
290 bool CommandTag::UpdateUndoTags(
291  Environment *env, const history::History::Tag &current_head_template,
292  const bool undo_rollback) {
293  assert(env->history.IsValid());
294 
295  history::History::Tag current_head;
296  history::History::Tag current_old_head;
297 
298  // remove previous HEAD tag
299  if (!env->history->Remove(CommandTag::kPreviousHeadTag)) {
300  LogCvmfs(kLogCvmfs, kLogVerboseMsg, "didn't find a previous HEAD tag");
301  }
302 
303  // check if we have a current HEAD tag that needs to renamed to previous
304  // HEAD
305  if (env->history->GetByName(CommandTag::kHeadTag, &current_head)) {
306  // remove current HEAD tag
307  if (!env->history->Remove(CommandTag::kHeadTag)) {
308  LogCvmfs(kLogCvmfs, kLogStderr, "failed to remove current HEAD tag");
309  return false;
310  }
311 
312  // set previous HEAD tag where current HEAD used to be
313  if (!undo_rollback) {
314  current_old_head = current_head;
315  current_old_head.name = CommandTag::kPreviousHeadTag;
316  current_old_head.channel = history::History::kChannelTrunk;
317  current_old_head.description = CommandTag::kPreviousHeadTagDescription;
318  if (!env->history->Insert(current_old_head)) {
319  LogCvmfs(kLogCvmfs, kLogStderr, "failed to set previous HEAD tag");
320  return false;
321  }
322  }
323  }
324 
325  // set the current HEAD to the catalog provided by the template HEAD
326  current_head = current_head_template;
327  current_head.name = CommandTag::kHeadTag;
328  current_head.channel = history::History::kChannelTrunk;
329  current_head.description = CommandTag::kHeadTagDescription;
330  if (!env->history->Insert(current_head)) {
331  LogCvmfs(kLogCvmfs, kLogStderr, "failed to set new current HEAD");
332  return false;
333  }
334 
335  return true;
336 }
337 
338 bool CommandTag::FetchObject(const std::string &repository_url,
339  const shash::Any &object_hash,
340  const std::string &destination_path) const {
341  assert(!object_hash.IsNull());
342 
343  download::Failures dl_retval;
344  const std::string url = repository_url + "/data/" + object_hash.MakePath();
345 
346  download::JobInfo download_object(&url, true, false, &destination_path,
347  &object_hash);
348  dl_retval = download_manager()->Fetch(&download_object);
349 
350  if (dl_retval != download::kFailOk) {
351  LogCvmfs(kLogCvmfs, kLogStderr, "failed to download object '%s' (%d - %s)",
352  object_hash.ToStringWithSuffix().c_str(), dl_retval,
353  download::Code2Ascii(dl_retval));
354  return false;
355  }
356 
357  return true;
358 }
359 
360 history::History *CommandTag::GetHistory(const manifest::Manifest *manifest,
361  const std::string &repository_url,
362  const std::string &history_path,
363  const bool read_write) const {
364  const shash::Any history_hash = manifest->history();
366 
367  if (history_hash.IsNull()) {
368  history = history::SqliteHistory::Create(history_path,
369  manifest->repository_name());
370  if (NULL == history) {
371  LogCvmfs(kLogCvmfs, kLogStderr, "failed to create history database");
372  return NULL;
373  }
374  } else {
375  if (!FetchObject(repository_url, history_hash, history_path)) {
376  return NULL;
377  }
378 
379  history = (read_write) ? history::SqliteHistory::OpenWritable(history_path)
380  : history::SqliteHistory::Open(history_path);
381  if (NULL == history) {
382  LogCvmfs(kLogCvmfs, kLogStderr, "failed to open history database (%s)",
383  history_path.c_str());
384  unlink(history_path.c_str());
385  return NULL;
386  }
387 
388  assert(history->fqrn() == manifest->repository_name());
389  }
390 
391  return history;
392 }
393 
394 catalog::Catalog *CommandTag::GetCatalog(const std::string &repository_url,
395  const shash::Any &catalog_hash,
396  const std::string catalog_path,
397  const bool read_write) const {
398  assert(shash::kSuffixCatalog == catalog_hash.suffix);
399  if (!FetchObject(repository_url, catalog_hash, catalog_path)) {
400  return NULL;
401  }
402 
403  const std::string catalog_root_path = "";
404  return (read_write) ? catalog::WritableCatalog::AttachFreely(
405  catalog_root_path, catalog_path, catalog_hash)
407  catalog_root_path, catalog_path, catalog_hash);
408 }
409 
410 void CommandTag::PrintTagMachineReadable(
411  const history::History::Tag &tag) const {
412  LogCvmfs(kLogCvmfs, kLogStdout, "%s %s %d %d %d %s %s %s", tag.name.c_str(),
413  tag.root_hash.ToString().c_str(), tag.size, tag.revision,
414  tag.timestamp, tag.GetChannelName(),
415  (tag.branch == "") ? "(default)" : tag.branch.c_str(),
416  tag.description.c_str());
417 }
418 
419 std::string CommandTag::AddPadding(const std::string &str, const size_t padding,
420  const bool align_right,
421  const std::string &fill_char) const {
422  assert(str.size() <= padding);
423  std::string result(str);
424  result.resize(padding);
425  const size_t pos = (align_right) ? 0 : str.size();
426  const size_t padding_width = padding - str.size();
427  for (size_t i = 0; i < padding_width; ++i) result.insert(pos, fill_char);
428  return result;
429 }
430 
431 bool CommandTag::IsUndoTagName(const std::string &tag_name) const {
432  return tag_name == CommandTag::kHeadTag ||
433  tag_name == CommandTag::kPreviousHeadTag;
434 }
435 
436 //------------------------------------------------------------------------------
437 
438 ParameterList CommandEditTag::GetParams() const {
439  ParameterList r;
441 
442  r.push_back(Parameter::Optional('d', "space separated tags to be deleted"));
443  r.push_back(Parameter::Optional('a', "name of the new tag"));
444  r.push_back(Parameter::Optional('D', "description of the tag"));
445  r.push_back(Parameter::Optional('B', "branch of the new tag"));
446  r.push_back(Parameter::Optional('P', "predecessor branch"));
447  r.push_back(Parameter::Optional('h', "root hash of the new tag"));
448  r.push_back(Parameter::Optional('c', "channel of the new tag"));
449  r.push_back(Parameter::Switch('x', "maintain undo tags"));
450  return r;
451 }
452 
453 int CommandEditTag::Main(const ArgumentList &args) {
454  if ((args.find('d') == args.end()) && (args.find('a') == args.end()) &&
455  (args.find('x') == args.end())) {
456  LogCvmfs(kLogCvmfs, kLogStderr, "nothing to do");
457  return 1;
458  }
459 
460  // initialize the Environment (taking ownership)
461  const bool history_read_write = true;
462  UniquePtr<Environment> env(InitializeEnvironment(args, history_read_write));
463  if (!env.IsValid()) {
464  LogCvmfs(kLogCvmfs, kLogStderr, "failed to init environment");
465  return 1;
466  }
467 
468  int retval;
469  if (args.find('d') != args.end()) {
470  retval = RemoveTags(args, env.weak_ref());
471  if (retval != 0) return retval;
472  }
473  if ((args.find('a') != args.end()) || (args.find('x') != args.end())) {
474  retval = AddNewTag(args, env.weak_ref());
475  if (retval != 0) return retval;
476  }
477 
478  // finalize processing and upload new history database
479  if (!CloseAndPublishHistory(env.weak_ref())) {
480  return 1;
481  }
482  return 0;
483 }
484 
485 int CommandEditTag::AddNewTag(const ArgumentList &args, Environment *env) {
486  typedef history::History::UpdateChannel TagChannel;
487  const std::string tag_name =
488  (args.find('a') != args.end()) ? *args.find('a')->second : "";
489  const std::string tag_description =
490  (args.find('D') != args.end()) ? *args.find('D')->second : "";
491  const TagChannel tag_channel =
492  (args.find('c') != args.end())
493  ? static_cast<TagChannel>(String2Uint64(*args.find('c')->second))
495  const bool undo_tags = (args.find('x') != args.end());
496  const std::string root_hash_string =
497  (args.find('h') != args.end()) ? *args.find('h')->second : "";
498  const std::string branch_name =
499  (args.find('B') != args.end()) ? *args.find('B')->second : "";
500  const std::string previous_branch_name =
501  (args.find('P') != args.end()) ? *args.find('P')->second : "";
502 
503  if (tag_name.find(" ") != std::string::npos) {
504  LogCvmfs(kLogCvmfs, kLogStderr, "tag names must not contain spaces");
505  return 1;
506  }
507 
508  assert(!tag_name.empty() || undo_tags);
509 
510  if (IsUndoTagName(tag_name)) {
511  LogCvmfs(kLogCvmfs, kLogStderr, "undo tags are managed internally");
512  return 1;
513  }
514 
515  // set the root hash to be tagged to the current HEAD if no other hash was
516  // given by the user
517  shash::Any root_hash = GetTagRootHash(env, root_hash_string);
518  if (root_hash.IsNull()) {
519  return 1;
520  }
521 
522  // open the catalog to be tagged (to check for existance and for meta info)
523  const UnlinkGuard catalog_path(
524  CreateTempPath(env->tmp_path + "/catalog", 0600));
525  const bool catalog_read_write = false;
526  const UniquePtr<catalog::Catalog> catalog(GetCatalog(
527  env->repository_url, root_hash, catalog_path.path(), catalog_read_write));
528  if (!catalog.IsValid()) {
529  LogCvmfs(kLogCvmfs, kLogStderr, "catalog with hash '%s' does not exist",
530  root_hash.ToString().c_str());
531  return 1;
532  }
533 
534  // check if the catalog is a root catalog
535  if (!catalog->root_prefix().IsEmpty()) {
537  "cannot tag catalog '%s' that is not a "
538  "root catalog.",
539  root_hash.ToString().c_str());
540  return 1;
541  }
542 
543  // create a template for the new tag to be created, moved or used as undo tag
544  history::History::Tag tag_template;
545  tag_template.name = "<template>";
546  tag_template.root_hash = root_hash;
547  tag_template.size = GetFileSize(catalog_path.path());
548  tag_template.revision = catalog->GetRevision();
549  tag_template.timestamp = catalog->GetLastModified();
550  tag_template.branch = branch_name;
551  tag_template.channel = tag_channel;
552  tag_template.description = tag_description;
553 
554  // manipulate the tag database by creating a new tag or moving an existing one
555  if (!tag_name.empty()) {
556  tag_template.name = tag_name;
557  const bool user_provided_hash = (!root_hash_string.empty());
558 
559  if (!env->history->ExistsBranch(tag_template.branch)) {
561  tag_template.branch,
562  previous_branch_name,
563  tag_template.revision);
564  if (!env->history->InsertBranch(branch)) {
565  LogCvmfs(kLogCvmfs, kLogStderr, "cannot insert branch '%s'",
566  tag_template.branch.c_str());
567  return 1;
568  }
569  }
570 
571  if (!ManipulateTag(env, tag_template, user_provided_hash)) {
572  return 1;
573  }
574  }
575 
576  // handle undo tags ('trunk' and 'trunk-previous') if necessary
577  if (undo_tags && !UpdateUndoTags(env, tag_template)) {
578  return 1;
579  }
580 
581  return 0;
582 }
583 
584 shash::Any CommandEditTag::GetTagRootHash(
585  Environment *env, const std::string &root_hash_string) const {
586  shash::Any root_hash;
587 
588  if (root_hash_string.empty()) {
590  "no catalog hash provided, using hash"
591  "of current HEAD catalog (%s)",
592  env->manifest->catalog_hash().ToString().c_str());
593  root_hash = env->manifest->catalog_hash();
594  } else {
595  root_hash = shash::MkFromHexPtr(shash::HexPtr(root_hash_string),
597  if (root_hash.IsNull()) {
599  "failed to read provided catalog hash '%s'",
600  root_hash_string.c_str());
601  }
602  }
603 
604  return root_hash;
605 }
606 
607 bool CommandEditTag::ManipulateTag(Environment *env,
608  const history::History::Tag &tag_template,
609  const bool user_provided_hash) {
610  const std::string &tag_name = tag_template.name;
611 
612  // check if the tag already exists, otherwise create it and return
613  if (!env->history->Exists(tag_name)) {
614  return CreateTag(env, tag_template);
615  }
616 
617  // tag does exist already, now we need to see if we can move it
618  if (!user_provided_hash) {
620  "a tag with the name '%s' already exists. Do you want to move it? "
621  "(-h <root hash>)",
622  tag_name.c_str());
623  return false;
624  }
625 
626  // move the already existing tag and return
627  return MoveTag(env, tag_template);
628 }
629 
630 bool CommandEditTag::MoveTag(Environment *env,
631  const history::History::Tag &tag_template)
632 {
633  const std::string &tag_name = tag_template.name;
634  history::History::Tag new_tag = tag_template;
635 
636  // get the already existent tag
637  history::History::Tag old_tag;
638  if (!env->history->GetByName(tag_name, &old_tag)) {
639  LogCvmfs(kLogCvmfs, kLogStderr, "failed to retrieve tag '%s' for moving",
640  tag_name.c_str());
641  return false;
642  }
643 
644  // check if we would move the tag to the same hash
645  if (old_tag.root_hash == new_tag.root_hash) {
646  LogCvmfs(kLogCvmfs, kLogStderr, "tag '%s' already points to '%s'",
647  tag_name.c_str(), old_tag.root_hash.ToString().c_str());
648  return false;
649  }
650 
651  // check that tag is not moved to another channel
652  if (new_tag.channel != old_tag.channel) {
653  LogCvmfs(kLogCvmfs, kLogStderr, "cannot move tag '%s' to another channel",
654  tag_name.c_str());
655  return false;
656  }
657 
658  // copy over old description if no new description was given
659  if (new_tag.description.empty()) {
660  new_tag.description = old_tag.description;
661  }
662  new_tag.branch = old_tag.branch;
663 
664  // remove the old tag from the database
665  if (!env->history->Remove(tag_name)) {
666  LogCvmfs(kLogCvmfs, kLogStderr, "removing old tag '%s' before move failed",
667  tag_name.c_str());
668  return false;
669  }
670  if (!env->history->PruneBranches()) {
671  LogCvmfs(kLogCvmfs, kLogStderr, "could not prune unused branches");
672  return false;
673  }
674  bool retval = env->history->Vacuum();
675  assert(retval);
676 
677  LogCvmfs(kLogCvmfs, kLogStdout, "moving tag '%s' from '%s' to '%s'",
678  tag_name.c_str(), old_tag.root_hash.ToString().c_str(),
679  tag_template.root_hash.ToString().c_str());
680 
681  // re-create the moved tag
682  return CreateTag(env, new_tag);
683 }
684 
685 bool CommandEditTag::CreateTag(Environment *env,
686  const history::History::Tag &new_tag) {
687  if (!env->history->Insert(new_tag)) {
688  LogCvmfs(kLogCvmfs, kLogStderr, "failed to insert new tag '%s'",
689  new_tag.name.c_str());
690  return false;
691  }
692 
693  return true;
694 }
695 
696 int CommandEditTag::RemoveTags(const ArgumentList &args, Environment *env) {
697  typedef std::vector<std::string> TagNames;
698  const std::string tags_to_delete = *args.find('d')->second;
699 
700  const TagNames condemned_tags = SplitString(tags_to_delete, ' ');
701 
702  // check if user tries to remove a magic undo tag
703  TagNames::const_iterator i = condemned_tags.begin();
704  const TagNames::const_iterator iend = condemned_tags.end();
705  for (; i != iend; ++i) {
706  if (IsUndoTagName(*i)) {
708  "undo tags are handled internally and cannot be deleted");
709  return 1;
710  }
711  }
712 
713  LogCvmfs(kLogCvmfs, kLogDebug, "proceeding to delete %d tags",
714  condemned_tags.size());
715 
716  // check if the tags to be deleted exist
717  bool all_exist = true;
718  for (i = condemned_tags.begin(); i != iend; ++i) {
719  if (!env->history->Exists(*i)) {
720  LogCvmfs(kLogCvmfs, kLogStderr, "tag '%s' does not exist", i->c_str());
721  all_exist = false;
722  }
723  }
724  if (!all_exist) {
725  return 1;
726  }
727 
728  // delete the tags from the tag database and print their root hashes
729  i = condemned_tags.begin();
730  env->history->BeginTransaction();
731  for (; i != iend; ++i) {
732  // print some information about the tag to be deleted
733  history::History::Tag condemned_tag;
734  const bool found_tag = env->history->GetByName(*i, &condemned_tag);
735  assert(found_tag);
736  LogCvmfs(kLogCvmfs, kLogStdout, "deleting '%s' (%s)",
737  condemned_tag.name.c_str(),
738  condemned_tag.root_hash.ToString().c_str());
739 
740  // remove the tag
741  if (!env->history->Remove(*i)) {
742  LogCvmfs(kLogCvmfs, kLogStderr, "failed to remove tag '%s' from history",
743  i->c_str());
744  return 1;
745  }
746  }
747  bool retval = env->history->PruneBranches();
748  if (!retval) {
750  "failed to prune unused branches from history");
751  return 1;
752  }
753  env->history->CommitTransaction();
754  retval = env->history->Vacuum();
755  assert(retval);
756 
757  return 0;
758 }
759 
760 //------------------------------------------------------------------------------
761 
762 
763 ParameterList CommandListTags::GetParams() const {
764  ParameterList r;
766  r.push_back(Parameter::Switch('x', "machine readable output"));
767  r.push_back(Parameter::Switch('B', "print branch hierarchy"));
768  return r;
769 }
770 
771 void CommandListTags::PrintHumanReadableTagList(
772  const CommandListTags::TagList &tags) const {
773  // go through the list of tags and figure out the column widths
774  const std::string name_label = "Name";
775  const std::string rev_label = "Revision";
776  const std::string chan_label = "Channel";
777  const std::string time_label = "Timestamp";
778  const std::string branch_label = "Branch";
779  const std::string desc_label = "Description";
780 
781  // figure out the maximal lengths of the fields in the lists
782  TagList::const_reverse_iterator i = tags.rbegin();
783  const TagList::const_reverse_iterator iend = tags.rend();
784  size_t max_name_len = name_label.size();
785  size_t max_rev_len = rev_label.size();
786  size_t max_chan_len = chan_label.size();
787  size_t max_time_len = desc_label.size();
788  size_t max_branch_len = branch_label.size();
789  for (; i != iend; ++i) {
790  max_name_len = std::max(max_name_len, i->name.size());
791  max_rev_len = std::max(max_rev_len, StringifyInt(i->revision).size());
792  max_chan_len = std::max(max_chan_len, strlen(i->GetChannelName()));
793  max_time_len =
794  std::max(max_time_len, StringifyTime(i->timestamp, true).size());
795  max_branch_len = std::max(max_branch_len, i->branch.size());
796  }
797 
798  // print the list header
800  "%s \u2502 %s \u2502 %s \u2502 %s \u2502 %s \u2502 %s",
801  AddPadding(name_label, max_name_len).c_str(),
802  AddPadding(rev_label, max_rev_len).c_str(),
803  AddPadding(chan_label, max_chan_len).c_str(),
804  AddPadding(time_label, max_time_len).c_str(),
805  AddPadding(branch_label, max_branch_len).c_str(),
806  desc_label.c_str());
808  "%s\u2500\u253C\u2500%s\u2500\u253C\u2500%s"
809  "\u2500\u253C\u2500%s\u2500\u253C\u2500%s\u2500\u253C\u2500%s",
810  AddPadding("", max_name_len, false, "\u2500").c_str(),
811  AddPadding("", max_rev_len, false, "\u2500").c_str(),
812  AddPadding("", max_chan_len, false, "\u2500").c_str(),
813  AddPadding("", max_time_len, false, "\u2500").c_str(),
814  AddPadding("", max_branch_len, false, "\u2500").c_str(),
815  AddPadding("", desc_label.size() + 1, false, "\u2500").c_str());
816 
817  // print the rows of the list
818  i = tags.rbegin();
819  for (; i != iend; ++i) {
820  LogCvmfs(
822  "%s \u2502 %s \u2502 %s \u2502 %s \u2502 %s \u2502 %s",
823  AddPadding(i->name, max_name_len).c_str(),
824  AddPadding(StringifyInt(i->revision), max_rev_len, true).c_str(),
825  AddPadding(i->GetChannelName(), max_chan_len).c_str(),
826  AddPadding(StringifyTime(i->timestamp, true), max_time_len).c_str(),
827  AddPadding(i->branch, max_branch_len).c_str(),
828  i->description.c_str());
829  }
830 
831  // print the list footer
833  "%s\u2500\u2534\u2500%s\u2500\u2534\u2500%s"
834  "\u2500\u2534\u2500%s\u2500\u2534\u2500%s\u2500\u2534\u2500%s",
835  AddPadding("", max_name_len, false, "\u2500").c_str(),
836  AddPadding("", max_rev_len, false, "\u2500").c_str(),
837  AddPadding("", max_chan_len, false, "\u2500").c_str(),
838  AddPadding("", max_time_len, false, "\u2500").c_str(),
839  AddPadding("", max_branch_len, false, "\u2500").c_str(),
840  AddPadding("", desc_label.size() + 1, false, "\u2500").c_str());
841 
842  // print the number of tags listed
843  LogCvmfs(kLogCvmfs, kLogStdout, "listing contains %d tags", tags.size());
844 }
845 
846 void CommandListTags::PrintMachineReadableTagList(const TagList &tags) const {
847  TagList::const_iterator i = tags.begin();
848  const TagList::const_iterator iend = tags.end();
849  for (; i != iend; ++i) {
850  PrintTagMachineReadable(*i);
851  }
852 }
853 
854 
855 void CommandListTags::PrintHumanReadableBranchList(
856  const BranchHierarchy &branches) const
857 {
858  unsigned N = branches.size();
859  for (unsigned i = 0; i < N; ++i) {
860  for (unsigned l = 0; l < branches[i].level; ++l) {
862  ((l + 1) == branches[i].level) ? "\u251c " : "\u2502 ");
863  }
864  LogCvmfs(kLogCvmfs, kLogStdout, "%s @%u",
865  branches[i].branch.branch.c_str(),
866  branches[i].branch.initial_revision);
867  }
868 }
869 
870 
871 void CommandListTags::PrintMachineReadableBranchList(
872  const BranchHierarchy &branches) const
873 {
874  unsigned N = branches.size();
875  for (unsigned i = 0; i < N; ++i) {
876  LogCvmfs(kLogCvmfs, kLogStdout, "[%u] %s%s @%u",
877  branches[i].level,
878  AddPadding("", branches[i].level, false, " ").c_str(),
879  branches[i].branch.branch.c_str(),
880  branches[i].branch.initial_revision);
881  }
882 }
883 
884 
885 void CommandListTags::SortBranchesRecursively(
886  unsigned level,
887  const string &parent_branch,
888  const BranchList &branches,
889  BranchHierarchy *hierarchy) const
890 {
891  // For large numbers of branches, this should be turned into the O(n) version
892  // using a linked list
893  unsigned N = branches.size();
894  for (unsigned i = 0; i < N; ++i) {
895  if (branches[i].branch == "")
896  continue;
897  if (branches[i].parent == parent_branch) {
898  hierarchy->push_back(BranchLevel(branches[i], level));
899  SortBranchesRecursively(
900  level + 1, branches[i].branch, branches, hierarchy);
901  }
902  }
903 }
904 
905 
906 CommandListTags::BranchHierarchy CommandListTags::SortBranches(
907  const BranchList &branches) const
908 {
909  BranchHierarchy hierarchy;
910  hierarchy.push_back(
911  BranchLevel(history::History::Branch("(default)", "", 0), 0));
912  SortBranchesRecursively(1, "", branches, &hierarchy);
913  return hierarchy;
914 }
915 
916 
917 int CommandListTags::Main(const ArgumentList &args) {
918  const bool machine_readable = (args.find('x') != args.end());
919  const bool branch_hierarchy = (args.find('B') != args.end());
920 
921  // initialize the Environment (taking ownership)
922  const bool history_read_write = false;
923  UniquePtr<Environment> env(InitializeEnvironment(args, history_read_write));
924  if (!env.IsValid()) {
925  LogCvmfs(kLogCvmfs, kLogStderr, "failed to init environment");
926  return 1;
927  }
928 
929  if (branch_hierarchy) {
930  BranchList branch_list;
931  if (!env->history->ListBranches(&branch_list)) {
933  "failed to list branches in history database");
934  return 1;
935  }
936  BranchHierarchy branch_hierarchy = SortBranches(branch_list);
937 
938  if (machine_readable) {
939  PrintMachineReadableBranchList(branch_hierarchy);
940  } else {
941  PrintHumanReadableBranchList(branch_hierarchy);
942  }
943  } else {
944  // obtain a full list of all tags
945  TagList tags;
946  if (!env->history->List(&tags)) {
948  "failed to list tags in history database");
949  return 1;
950  }
951 
952  if (machine_readable) {
953  PrintMachineReadableTagList(tags);
954  } else {
955  PrintHumanReadableTagList(tags);
956  }
957  }
958 
959  return 0;
960 }
961 
962 //------------------------------------------------------------------------------
963 
964 ParameterList CommandInfoTag::GetParams() const {
965  ParameterList r;
967 
968  r.push_back(Parameter::Mandatory('n', "name of the tag to be inspected"));
969  r.push_back(Parameter::Switch('x', "machine readable output"));
970  return r;
971 }
972 
973 std::string CommandInfoTag::HumanReadableFilesize(const size_t filesize) const {
974  const size_t kiB = 1024;
975  const size_t MiB = kiB * 1024;
976  const size_t GiB = MiB * 1024;
977 
978  if (filesize > GiB) {
979  return StringifyDouble(static_cast<double>(filesize) / GiB) + " GiB";
980  } else if (filesize > MiB) {
981  return StringifyDouble(static_cast<double>(filesize) / MiB) + " MiB";
982  } else if (filesize > kiB) {
983  return StringifyDouble(static_cast<double>(filesize) / kiB) + " kiB";
984  } else {
985  return StringifyInt(filesize) + " Byte";
986  }
987 }
988 
989 void CommandInfoTag::PrintHumanReadableInfo(
990  const history::History::Tag &tag) const {
992  "Name: %s\n"
993  "Revision: %d\n"
994  "Channel: %s\n"
995  "Timestamp: %s\n"
996  "Branch: %s\n"
997  "Root Hash: %s\n"
998  "Catalog Size: %s\n"
999  "%s",
1000  tag.name.c_str(), tag.revision, tag.GetChannelName(),
1001  StringifyTime(tag.timestamp, true /* utc */).c_str(),
1002  tag.branch.c_str(),
1003  tag.root_hash.ToString().c_str(),
1004  HumanReadableFilesize(tag.size).c_str(),
1005  tag.description.c_str());
1006 }
1007 
1008 int CommandInfoTag::Main(const ArgumentList &args) {
1009  const std::string tag_name = *args.find('n')->second;
1010  const bool machine_readable = (args.find('x') != args.end());
1011 
1012  // initialize the Environment (taking ownership)
1013  const bool history_read_write = false;
1014  UniquePtr<Environment> env(InitializeEnvironment(args, history_read_write));
1015  if (!env.IsValid()) {
1016  LogCvmfs(kLogCvmfs, kLogStderr, "failed to init environment");
1017  return 1;
1018  }
1019 
1021  const bool found = env->history->GetByName(tag_name, &tag);
1022  if (!found) {
1023  LogCvmfs(kLogCvmfs, kLogStderr, "tag '%s' does not exist",
1024  tag_name.c_str());
1025  return 1;
1026  }
1027 
1028  if (machine_readable) {
1029  PrintTagMachineReadable(tag);
1030  } else {
1031  PrintHumanReadableInfo(tag);
1032  }
1033 
1034  return 0;
1035 }
1036 
1037 //------------------------------------------------------------------------------
1038 
1039 ParameterList CommandRollbackTag::GetParams() const {
1040  ParameterList r;
1042 
1043  r.push_back(Parameter::Optional('n', "name of the tag to be republished"));
1044  return r;
1045 }
1046 
1047 int CommandRollbackTag::Main(const ArgumentList &args) {
1048  const bool undo_rollback = (args.find('n') == args.end());
1049  const std::string tag_name =
1050  (!undo_rollback) ? *args.find('n')->second : CommandTag::kPreviousHeadTag;
1051 
1052  // initialize the Environment (taking ownership)
1053  const bool history_read_write = true;
1054  UniquePtr<Environment> env(InitializeEnvironment(args, history_read_write));
1055  if (!env.IsValid()) {
1056  LogCvmfs(kLogCvmfs, kLogStderr, "failed to init environment");
1057  return 1;
1058  }
1059 
1060  // find tag to be rolled back to
1061  history::History::Tag target_tag;
1062  const bool found = env->history->GetByName(tag_name, &target_tag);
1063  if (!found) {
1064  if (undo_rollback) {
1066  "only one anonymous rollback supported - "
1067  "perhaps you want to provide a tag name?");
1068  } else {
1069  LogCvmfs(kLogCvmfs, kLogStderr, "tag '%s' does not exist",
1070  tag_name.c_str());
1071  }
1072  return 1;
1073  }
1074  if (target_tag.branch != "") {
1076  "rollback is only supported on the default branch");
1077  return 1;
1078  }
1079 
1080  // list the tags that will be deleted
1081  TagList affected_tags;
1082  if (!env->history->ListTagsAffectedByRollback(tag_name, &affected_tags)) {
1084  "failed to list condemned tags prior to rollback to '%s'",
1085  tag_name.c_str());
1086  return 1;
1087  }
1088 
1089  // check if tag is valid to be rolled back to
1090  const uint64_t current_revision = env->manifest->revision();
1091  assert(target_tag.revision <= current_revision);
1092  if (target_tag.revision == current_revision) {
1093  LogCvmfs(kLogCvmfs, kLogStderr, "not rolling back to current head (%u)",
1094  current_revision);
1095  return 1;
1096  }
1097 
1098  // open the catalog to be rolled back to
1099  const UnlinkGuard catalog_path(
1100  CreateTempPath(env->tmp_path + "/catalog", 0600));
1101  const bool catalog_read_write = true;
1103  dynamic_cast<catalog::WritableCatalog *>(
1104  GetCatalog(env->repository_url, target_tag.root_hash,
1105  catalog_path.path(), catalog_read_write)));
1106  if (!catalog.IsValid()) {
1107  LogCvmfs(kLogCvmfs, kLogStderr, "failed to open catalog with hash '%s'",
1108  target_tag.root_hash.ToString().c_str());
1109  return 1;
1110  }
1111 
1112  // check if the catalog has a supported schema version
1113  if (catalog->schema() < catalog::CatalogDatabase::kLatestSupportedSchema -
1116  "not rolling back to outdated and "
1117  "incompatible catalog schema (%.1f < %.1f)",
1118  catalog->schema(),
1120  return 1;
1121  }
1122 
1123  // update the catalog to be republished
1124  catalog->Transaction();
1125  catalog->UpdateLastModified();
1126  catalog->SetRevision(current_revision + 1);
1127  catalog->SetPreviousRevision(env->manifest->catalog_hash());
1128  catalog->Commit();
1129 
1130  // Upload catalog (handing over ownership of catalog pointer)
1131  if (!UploadCatalogAndUpdateManifest(env.weak_ref(), catalog.Release())) {
1132  LogCvmfs(kLogCvmfs, kLogStderr, "catalog upload failed");
1133  return 1;
1134  }
1135 
1136  // update target tag with newly published root catalog information
1137  history::History::Tag updated_target_tag(target_tag);
1138  updated_target_tag.root_hash = env->manifest->catalog_hash();
1139  updated_target_tag.size = env->manifest->catalog_size();
1140  updated_target_tag.revision = env->manifest->revision();
1141  updated_target_tag.timestamp = env->manifest->publish_timestamp();
1142  if (!env->history->Rollback(updated_target_tag)) {
1143  LogCvmfs(kLogCvmfs, kLogStderr, "failed to rollback history to '%s'",
1144  updated_target_tag.name.c_str());
1145  return 1;
1146  }
1147  bool retval = env->history->Vacuum();
1148  assert(retval);
1149 
1150  // set the magic undo tags
1151  if (!UpdateUndoTags(env.weak_ref(), updated_target_tag, undo_rollback)) {
1152  LogCvmfs(kLogCvmfs, kLogStderr, "failed to update magic undo tags");
1153  return 1;
1154  }
1155 
1156  // finalize the history and upload it
1157  if (!CloseAndPublishHistory(env.weak_ref())) {
1158  return 1;
1159  }
1160 
1161  // print the tags that have been removed by the rollback
1162  PrintDeletedTagList(affected_tags);
1163 
1164  return 0;
1165 }
1166 
1167 void CommandRollbackTag::PrintDeletedTagList(const TagList &tags) const {
1168  size_t longest_name = 0;
1169  TagList::const_iterator i = tags.begin();
1170  const TagList::const_iterator iend = tags.end();
1171  for (; i != iend; ++i) {
1172  longest_name = std::max(i->name.size(), longest_name);
1173  }
1174 
1175  i = tags.begin();
1176  for (; i != iend; ++i) {
1177  LogCvmfs(kLogCvmfs, kLogStdout, "removed tag %s (%s)",
1178  AddPadding(i->name, longest_name).c_str(),
1179  i->root_hash.ToString().c_str());
1180  }
1181 }
1182 
1183 //------------------------------------------------------------------------------
1184 
1185 ParameterList CommandEmptyRecycleBin::GetParams() const {
1186  ParameterList r;
1188  return r;
1189 }
1190 
1191 int CommandEmptyRecycleBin::Main(const ArgumentList &args) {
1192  // initialize the Environment (taking ownership)
1193  const bool history_read_write = true;
1194  UniquePtr<Environment> env(InitializeEnvironment(args, history_read_write));
1195  if (!env.IsValid()) {
1196  LogCvmfs(kLogCvmfs, kLogStderr, "failed to init environment");
1197  return 1;
1198  }
1199 
1200  if (!env->history->EmptyRecycleBin()) {
1201  LogCvmfs(kLogCvmfs, kLogStderr, "failed to empty recycle bin");
1202  return 1;
1203  }
1204 
1205  // finalize the history and upload it
1206  if (!CloseAndPublishHistory(env.weak_ref())) {
1207  return 1;
1208  }
1209 
1210  return 0;
1211 }
int return_code
the return value of the spooler operation
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
UpdateChannel channel
Definition: history.h:118
bool IsNull() const
Definition: hash.h:379
const manifest::Manifest * manifest() const
Definition: repository.h:123
static SqliteHistory * Open(const std::string &file_name)
vector< string > SplitString(const string &str, const char delim, const unsigned max_chunks)
Definition: string.cc:288
std::string name
Definition: history.h:113
std::vector< Parameter > ParameterList
Definition: swissknife.h:71
UniquePtr< manifest::Manifest > manifest
std::string ToString(const bool with_suffix=false) const
Definition: hash.h:245
const history::History * history() const
std::string ToStringWithSuffix() const
Definition: hash.h:300
std::vector< BranchLevel > BranchHierarchy
std::string CreateTempPath(const std::string &path_prefix, const int mode)
Definition: posix.cc:1059
static void InsertCommonParameters(ParameterList *r)
assert((mem||(size==0))&&"Out Of Memory")
string StringifyTime(const time_t seconds, const bool utc)
Definition: string.cc:105
const std::string & fqrn() const
Definition: history.h:220
string StringifyDouble(const double value)
Definition: string.cc:96
void Set(const T &object)
std::vector< history::History::Tag > TagList
Algorithms
Definition: hash.h:39
static SqliteHistory * OpenWritable(const std::string &file_name)
static WritableCatalog * AttachFreely(const std::string &root_path, const std::string &file, const shash::Any &catalog_hash, Catalog *parent=NULL, const bool is_not_root=false)
Definition: catalog_rw.cc:49
std::vector< history::History::Branch > BranchList
std::string description
Definition: history.h:119
bool FileExists(const std::string &path)
Definition: posix.cc:816
uint64_t GetRevision() const
Definition: catalog.cc:528
uint64_t GetLastModified() const
Definition: catalog.cc:533
const char * Code2Ascii(const Failures error)
Definition: download.h:87
const char * GetChannelName() const
Definition: history.h:86
static const float kSchemaEpsilon
Definition: sql.h:105
std::string repository_name() const
Definition: manifest.h:122
const char kSuffixCatalog
Definition: hash.h:52
Any MkFromHexPtr(const HexPtr hex, const char suffix)
Definition: hash.cc:83
string StringifyInt(const int64_t value)
Definition: string.cc:78
bool HasPrefix(const string &str, const string &prefix, const bool ignore_case)
Definition: string.cc:265
PathString root_prefix() const
Definition: catalog.h:185
bool IsEmpty() const
Definition: shortstring.h:110
uint64_t String2Uint64(const string &value)
Definition: string.cc:228
std::map< char, SharedPtr< std::string > > ArgumentList
Definition: swissknife.h:72
Algorithms ParseHashAlgorithm(const string &algorithm_option)
Definition: hash.cc:72
shash::Any history() const
Definition: manifest.h:127
shash::Any root_hash
Definition: history.h:114
static Catalog * AttachFreely(const std::string &imaginary_mountpoint, const std::string &file, const shash::Any &catalog_hash, Catalog *parent=NULL, const bool is_nested=false)
Definition: catalog.cc:29
std::string branch
Definition: history.h:123
static SqliteHistory * Create(const std::string &file_name, const std::string &fqrn)
UniquePtr< history::History > history
int64_t GetFileSize(const std::string &path)
Definition: posix.cc:826
const int kLogVerboseMsg
static const float kLatestSupportedSchema
Definition: catalog_sql.h:44
Suffix suffix
Definition: hash.h:124
std::string MakePath() const
Definition: hash.h:312
std::string MakeCanonicalPath(const std::string &path)
Definition: posix.cc:96
void PrintError(const string &message)
Definition: logging.cc:465
UniquePtr< upload::Spooler > spooler