CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
swissknife_reflog.cc
Go to the documentation of this file.
1 
5 #include "swissknife_reflog.h"
6 
7 #include <cassert>
8 
9 #include <string>
10 #include <vector>
11 
12 #include "manifest.h"
13 #include "object_fetcher.h"
14 #include "upload_facility.h"
15 #include "util/exception.h"
16 
17 namespace swissknife {
18 
20 
22  public:
25 
26  public:
28  ObjectFetcher *object_fetcher,
30  : object_fetcher_(object_fetcher)
31  , reflog_(reflog)
32  , manifest_(manifest) {}
33 
35 
36  protected:
37  typedef std::vector<shash::Any> CatalogList;
38 
39  protected:
40  CatalogTN* FetchCatalog(const shash::Any catalog_hash);
41  HistoryTN* FetchHistory(const shash::Any history_hash);
42 
43  void WalkRootCatalogs(const shash::Any &root_catalog_hash);
44  void WalkHistories(const shash::Any &history_hash);
45 
47  void WalkListedCatalogs(const CatalogList &catalog_list);
48 
49  template <class DatabaseT>
50  DatabaseT* ReturnOrAbort(const ObjectFetcherFailures::Failures failure,
51  const shash::Any &content_hash,
52  DatabaseT *database);
53 
54  private:
58 };
59 
60 
62  ParameterList r;
63  r.push_back(Parameter::Mandatory('r', "repository url"));
64  r.push_back(Parameter::Mandatory('u', "spooler definition string"));
65  r.push_back(Parameter::Mandatory('n', "fully qualified repository name"));
66  r.push_back(Parameter::Mandatory('t', "temporary directory"));
67  r.push_back(Parameter::Mandatory('k', "repository keychain"));
68  r.push_back(Parameter::Mandatory('R', "path to reflog.chksum file"));
69  return r;
70 }
71 
72 
74  const std::string &repo_url = *args.find('r')->second;
75  const std::string &spooler = *args.find('u')->second;
76  const std::string &repo_name = *args.find('n')->second;
77  const std::string &tmp_dir = *args.find('t')->second;
78  const std::string &repo_keys = *args.find('k')->second;
79  const std::string &reflog_chksum_path = *args.find('R')->second;
80 
81  const bool follow_redirects = false;
82  if (!this->InitDownloadManager(follow_redirects) ||
83  !this->InitVerifyingSignatureManager(repo_keys)) {
84  LogCvmfs(kLogCvmfs, kLogStderr, "failed to init repo connection");
85  return 1;
86  }
87 
88  ObjectFetcher object_fetcher(repo_name,
89  repo_url,
90  tmp_dir,
93 
95  ObjectFetcher::Failures retval = object_fetcher.FetchManifest(&manifest);
96  if (retval != ObjectFetcher::kFailOk) {
97  LogCvmfs(kLogCvmfs, kLogStderr, "failed to load repository manifest "
98  "(%d - %s)",
99  retval, Code2Ascii(retval));
100  return 1;
101  }
102 
103  const upload::SpoolerDefinition spooler_definition(spooler, shash::kAny);
105  upload::AbstractUploader::Construct(spooler_definition));
106 
107  if (!uploader.IsValid()) {
108  LogCvmfs(kLogCvmfs, kLogStderr, "failed to initialize spooler for '%s'",
109  spooler.c_str());
110  return 1;
111  }
112 
114  reflog->TakeDatabaseFileOwnership();
115 
116  reflog->BeginTransaction();
117  AddStaticManifestObjects(reflog.weak_ref(), manifest.weak_ref());
118  RootChainWalker walker(manifest.weak_ref(),
119  &object_fetcher,
120  reflog.weak_ref());
122  reflog->CommitTransaction();
123 
124  LogCvmfs(kLogCvmfs, kLogStdout, "found %d entries", reflog->CountEntries());
125 
126  reflog->DropDatabaseFileOwnership();
127  const std::string reflog_db = reflog->database_file();
128  reflog.Destroy();
129  uploader->UploadFile(reflog_db, ".cvmfsreflog");
130  shash::Any reflog_hash(manifest->GetHashAlgorithm());
131  manifest::Reflog::HashDatabase(reflog_db, &reflog_hash);
132  uploader->WaitForUpload();
133  unlink(reflog_db.c_str());
134 
135  const int errors = uploader->GetNumberOfErrors();
136  if (errors > 0) {
137  LogCvmfs(kLogCvmfs, kLogStderr, "failed to upload generated Reflog");
138  }
139 
140  uploader->TearDown();
141 
142  manifest::Reflog::WriteChecksum(reflog_chksum_path, reflog_hash);
143 
144  return (errors == 0) ? 0 : 1;
145 }
146 
147 
150  manifest::Manifest *manifest) const {
151  const shash::Any certificate = manifest->certificate();
152  const shash::Any meta_info = manifest->meta_info();
153  assert(!certificate.IsNull());
154 
155  bool success = reflog->AddCertificate(certificate);
156  assert(success);
157  LogCvmfs(kLogCvmfs, kLogStdout, "Certificate: %s",
158  certificate.ToString().c_str());
159 
160  if (!meta_info.IsNull()) {
161  success = reflog->AddMetainfo(meta_info);
162  assert(success);
163  LogCvmfs(kLogCvmfs, kLogStdout, "Metainfo: %s",
164  meta_info.ToString().c_str());
165  }
166 }
167 
168 
170  const shash::Any root_catalog = manifest_->catalog_hash();
171  const shash::Any history = manifest_->history();
172 
173  assert(!root_catalog.IsNull());
174  WalkRootCatalogs(root_catalog);
175 
176  if (!history.IsNull()) {
177  WalkHistories(history);
178  }
179 }
180 
181 
182 void RootChainWalker::WalkRootCatalogs(const shash::Any &root_catalog_hash) {
183  shash::Any current_hash = root_catalog_hash;
184  UniquePtr<CatalogTN> current_catalog;
185 
186  while (!current_hash.IsNull() &&
187  !reflog_->ContainsCatalog(current_hash) &&
188  (current_catalog = FetchCatalog(current_hash)).IsValid()) {
189  LogCvmfs(kLogCvmfs, kLogStdout, "Catalog: %s Revision: %d",
190  current_hash.ToString().c_str(), current_catalog->GetRevision());
191 
192  const bool success = reflog_->AddCatalog(current_hash);
193  assert(success);
194 
195  current_hash = current_catalog->GetPreviousRevision();
196  }
197 }
198 
199 
200 void RootChainWalker::WalkHistories(const shash::Any &history_hash) {
201  shash::Any current_hash = history_hash;
202  UniquePtr<HistoryTN> current_history;
203 
204  while (!current_hash.IsNull() &&
205  !reflog_->ContainsHistory(current_hash) &&
206  (current_history = FetchHistory(current_hash)).IsValid()) {
207  LogCvmfs(kLogCvmfs, kLogStdout, "History: %s",
208  current_hash.ToString().c_str());
209 
210  bool cancel = WalkCatalogsInHistory(current_history);
211  const bool success = reflog_->AddHistory(current_hash);
212  assert(success);
213 
214  if (cancel) {
215  current_hash = shash::Any(current_hash.algorithm);
216  } else {
217  current_hash = current_history->previous_revision();
218  }
219  }
220 }
221 
222 
230  CatalogList tag_hashes;
231  const bool list_success = history->GetHashes(&tag_hashes);
232  assert(list_success);
233 
234  CatalogList bin_hashes;
235  const bool bin_success = history->ListRecycleBin(&bin_hashes);
236  if (!bin_success) {
237  LogCvmfs(kLogCvmfs, kLogStderr, " Warning: 'recycle bin' table missing");
238  }
239 
240  WalkListedCatalogs(tag_hashes);
241  WalkListedCatalogs(bin_hashes);
242 
243  return !bin_success;
244 }
245 
246 
248  const RootChainWalker::CatalogList &catalog_list) {
249  CatalogList::const_iterator i = catalog_list.begin();
250  CatalogList::const_iterator iend = catalog_list.end();
251  for (; i != iend; ++i) {
252  WalkRootCatalogs(*i);
253  }
254 }
255 
256 
258  const shash::Any catalog_hash) {
259  CatalogTN *catalog = NULL;
260  const char *root_path = "";
262  object_fetcher_->FetchCatalog(catalog_hash, root_path, &catalog);
263 
264  return ReturnOrAbort(failure, catalog_hash, catalog);
265 }
266 
267 
269  const shash::Any history_hash) {
270  HistoryTN *history = NULL;
272  object_fetcher_->FetchHistory(&history, history_hash);
273 
274  return ReturnOrAbort(failure, history_hash, history);
275 }
276 
277 
278 template <class DatabaseT>
280  const ObjectFetcherFailures::Failures failure,
281  const shash::Any &content_hash,
282  DatabaseT *database) {
283  switch (failure) {
285  return database;
287  return NULL;
288  default:
289  PANIC(kLogStderr, "Failed to load object '%s' (%d - %s)",
290  content_hash.ToStringWithSuffix().c_str(), failure,
291  Code2Ascii(failure));
292  }
293 }
294 
295 } // namespace swissknife
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
HistoryTN * FetchHistory(const shash::Any history_hash)
void TakeDatabaseFileOwnership()
Definition: reflog.cc:307
const char * Code2Ascii(const ObjectFetcherFailures::Failures error)
std::string database_file() const
Definition: reflog.cc:337
bool ContainsHistory(const shash::Any &history) const
Definition: reflog.cc:239
bool IsNull() const
Definition: hash.h:379
const manifest::Manifest * manifest() const
Definition: repository.h:123
bool WalkCatalogsInHistory(const HistoryTN *history)
bool AddHistory(const shash::Any &history)
Definition: reflog.cc:134
void WalkRootCatalogs(const shash::Any &root_catalog_hash)
CatalogTN * FetchCatalog(const shash::Any catalog_hash)
const manifest::Manifest * manifest_
#define PANIC(...)
Definition: exception.h:26
std::vector< Parameter > ParameterList
Definition: swissknife.h:71
std::string ToString(const bool with_suffix=false) const
Definition: hash.h:245
ObjectFetcher::CatalogTN CatalogTN
const history::History * history() const
std::string ToStringWithSuffix() const
Definition: hash.h:300
Failures FetchHistory(HistoryTN **history, const shash::Any &history_hash=shash::Any())
int Main(const ArgumentList &args)
RootChainWalker(const manifest::Manifest *manifest, ObjectFetcher *object_fetcher, manifest::Reflog *reflog)
assert((mem||(size==0))&&"Out Of Memory")
void WalkListedCatalogs(const CatalogList &catalog_list)
void WalkHistories(const shash::Any &history_hash)
Algorithms algorithm
Definition: hash.h:123
bool InitVerifyingSignatureManager(const std::string &pubkey_path, const std::string &trusted_certs="")
Definition: server_tool.cc:43
bool AddCatalog(const shash::Any &catalog)
Definition: reflog.cc:128
uint64_t CountEntries()
Definition: reflog.cc:146
signature::SignatureManager * signature_manager() const
Definition: server_tool.cc:112
virtual ParameterList GetParams() const
static AbstractUploader * Construct(const SpoolerDefinition &param)
Definition: plugin.h:188
download::DownloadManager * download_manager() const
Definition: server_tool.cc:107
object_fetcher_traits< HttpObjectFetcher< CatalogT, HistoryT, ReflogT > >::CatalogTN CatalogTN
bool AddMetainfo(const shash::Any &metainfo)
Definition: reflog.cc:140
bool InitDownloadManager(const bool follow_redirects, const unsigned max_pool_handles=1, const bool use_system_proxy=true)
Definition: server_tool.cc:21
static Parameter Mandatory(const char key, const std::string &desc)
Definition: swissknife.h:38
Failures FetchCatalog(const shash::Any &catalog_hash, const std::string &catalog_path, CatalogTN **catalog, const bool is_nested=false, CatalogTN *parent=NULL)
shash::Any certificate() const
Definition: manifest.h:126
object_fetcher_traits< HttpObjectFetcher< CatalogT, HistoryT, ReflogT > >::HistoryTN HistoryTN
void BeginTransaction()
Definition: reflog.cc:295
DatabaseT * ReturnOrAbort(const ObjectFetcherFailures::Failures failure, const shash::Any &content_hash, DatabaseT *database)
static void HashDatabase(const std::string &database_path, shash::Any *hash_reflog)
Definition: reflog.cc:322
shash::Any catalog_hash() const
Definition: manifest.h:124
bool IsValid() const
Definition: pointer.h:44
void DropDatabaseFileOwnership()
Definition: reflog.cc:313
bool AddCertificate(const shash::Any &certificate)
Definition: reflog.cc:121
bool ContainsCatalog(const shash::Any &catalog) const
Definition: reflog.cc:222
std::map< char, SharedPtr< std::string > > ArgumentList
Definition: swissknife.h:72
shash::Any history() const
Definition: manifest.h:127
void CommitTransaction()
Definition: reflog.cc:301
std::string meta_info() const
Definition: repository.h:126
HttpObjectFetcher ObjectFetcher
Failures FetchManifest(manifest::Manifest **manifest)
void AddStaticManifestObjects(manifest::Reflog *reflog, manifest::Manifest *manifest) const
static bool WriteChecksum(const std::string &path, const shash::Any &value)
Definition: reflog.cc:64
manifest::Reflog * CreateEmptyReflog(const std::string &temp_directory, const std::string &repo_name)
Definition: server_tool.cc:174
ObjectFetcher::HistoryTN HistoryTN
std::vector< shash::Any > CatalogList
shash::Any meta_info() const
Definition: manifest.h:131