CernVM-FS  2.10.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
swissknife_reflog.cc
Go to the documentation of this file.
1 
5 #include "swissknife_reflog.h"
6 
7 #include <cassert>
8 
9 #include <string>
10 #include <vector>
11 
12 #include "manifest.h"
13 #include "object_fetcher.h"
14 #include "upload_facility.h"
15 #include "util/exception.h"
16 
17 namespace swissknife {
18 
20 
22  public:
25 
26  public:
28  ObjectFetcher *object_fetcher,
30  : object_fetcher_(object_fetcher)
31  , reflog_(reflog)
32  , manifest_(manifest) {}
33 
35 
36  protected:
37  typedef std::vector<shash::Any> CatalogList;
38 
39  protected:
40  CatalogTN* FetchCatalog(const shash::Any catalog_hash);
41  HistoryTN* FetchHistory(const shash::Any history_hash);
42 
43  void WalkRootCatalogs(const shash::Any &root_catalog_hash);
44  void WalkHistories(const shash::Any &history_hash);
45 
47  void WalkListedCatalogs(const CatalogList &catalog_list);
48 
49  template <class DatabaseT>
50  DatabaseT* ReturnOrAbort(const ObjectFetcherFailures::Failures failure,
51  const shash::Any &content_hash,
52  DatabaseT *database);
53 
54  private:
58 };
59 
60 
62  ParameterList r;
63  r.push_back(Parameter::Mandatory('r', "repository url"));
64  r.push_back(Parameter::Mandatory('u', "spooler definition string"));
65  r.push_back(Parameter::Mandatory('n', "fully qualified repository name"));
66  r.push_back(Parameter::Mandatory('t', "temporary directory"));
67  r.push_back(Parameter::Mandatory('k', "repository keychain"));
68  r.push_back(Parameter::Mandatory('R', "path to reflog.chksum file"));
69  r.push_back(Parameter::Optional('@', "proxy url"));
70  return r;
71 }
72 
73 
75  const std::string &repo_url = *args.find('r')->second;
76  const std::string &spooler = *args.find('u')->second;
77  const std::string &repo_name = *args.find('n')->second;
78  const std::string &tmp_dir = *args.find('t')->second;
79  const std::string &repo_keys = *args.find('k')->second;
80  const std::string &reflog_chksum_path = *args.find('R')->second;
81 
82  const bool follow_redirects = false;
83  const std::string proxy = ((args.count('@') > 0) ?
84  *args.find('@')->second : "");
85  if (!this->InitDownloadManager(follow_redirects, proxy) ||
86  !this->InitVerifyingSignatureManager(repo_keys)) {
87  LogCvmfs(kLogCvmfs, kLogStderr, "failed to init repo connection");
88  return 1;
89  }
90 
91  ObjectFetcher object_fetcher(repo_name,
92  repo_url,
93  tmp_dir,
96 
98  ObjectFetcher::Failures retval = object_fetcher.FetchManifest(&manifest);
99  if (retval != ObjectFetcher::kFailOk) {
100  LogCvmfs(kLogCvmfs, kLogStderr, "failed to load repository manifest "
101  "(%d - %s)",
102  retval, Code2Ascii(retval));
103  return 1;
104  }
105 
106  const upload::SpoolerDefinition spooler_definition(spooler, shash::kAny);
108  upload::AbstractUploader::Construct(spooler_definition));
109 
110  if (!uploader.IsValid()) {
111  LogCvmfs(kLogCvmfs, kLogStderr, "failed to initialize spooler for '%s'",
112  spooler.c_str());
113  return 1;
114  }
115 
117  reflog->TakeDatabaseFileOwnership();
118 
119  reflog->BeginTransaction();
120  AddStaticManifestObjects(reflog.weak_ref(), manifest.weak_ref());
121  RootChainWalker walker(manifest.weak_ref(),
122  &object_fetcher,
123  reflog.weak_ref());
125  reflog->CommitTransaction();
126 
127  LogCvmfs(kLogCvmfs, kLogStdout, "found %d entries", reflog->CountEntries());
128 
129  reflog->DropDatabaseFileOwnership();
130  const std::string reflog_db = reflog->database_file();
131  reflog.Destroy();
132  uploader->UploadFile(reflog_db, ".cvmfsreflog");
133  shash::Any reflog_hash(manifest->GetHashAlgorithm());
134  manifest::Reflog::HashDatabase(reflog_db, &reflog_hash);
135  uploader->WaitForUpload();
136  unlink(reflog_db.c_str());
137 
138  const int errors = uploader->GetNumberOfErrors();
139  if (errors > 0) {
140  LogCvmfs(kLogCvmfs, kLogStderr, "failed to upload generated Reflog");
141  }
142 
143  uploader->TearDown();
144 
145  manifest::Reflog::WriteChecksum(reflog_chksum_path, reflog_hash);
146 
147  return (errors == 0) ? 0 : 1;
148 }
149 
150 
153  manifest::Manifest *manifest) const {
154  const shash::Any certificate = manifest->certificate();
155  const shash::Any meta_info = manifest->meta_info();
156  assert(!certificate.IsNull());
157 
158  bool success = reflog->AddCertificate(certificate);
159  assert(success);
160  LogCvmfs(kLogCvmfs, kLogStdout, "Certificate: %s",
161  certificate.ToString().c_str());
162 
163  if (!meta_info.IsNull()) {
164  success = reflog->AddMetainfo(meta_info);
165  assert(success);
166  LogCvmfs(kLogCvmfs, kLogStdout, "Metainfo: %s",
167  meta_info.ToString().c_str());
168  }
169 }
170 
171 
173  const shash::Any root_catalog = manifest_->catalog_hash();
174  const shash::Any history = manifest_->history();
175 
176  assert(!root_catalog.IsNull());
177  WalkRootCatalogs(root_catalog);
178 
179  if (!history.IsNull()) {
180  WalkHistories(history);
181  }
182 }
183 
184 
185 void RootChainWalker::WalkRootCatalogs(const shash::Any &root_catalog_hash) {
186  shash::Any current_hash = root_catalog_hash;
187  UniquePtr<CatalogTN> current_catalog;
188 
189  while (!current_hash.IsNull() &&
190  !reflog_->ContainsCatalog(current_hash) &&
191  (current_catalog = FetchCatalog(current_hash)).IsValid()) {
192  LogCvmfs(kLogCvmfs, kLogStdout, "Catalog: %s Revision: %d",
193  current_hash.ToString().c_str(), current_catalog->GetRevision());
194 
195  const bool success = reflog_->AddCatalog(current_hash);
196  assert(success);
197 
198  current_hash = current_catalog->GetPreviousRevision();
199  }
200 }
201 
202 
203 void RootChainWalker::WalkHistories(const shash::Any &history_hash) {
204  shash::Any current_hash = history_hash;
205  UniquePtr<HistoryTN> current_history;
206 
207  while (!current_hash.IsNull() &&
208  !reflog_->ContainsHistory(current_hash) &&
209  (current_history = FetchHistory(current_hash)).IsValid()) {
210  LogCvmfs(kLogCvmfs, kLogStdout, "History: %s",
211  current_hash.ToString().c_str());
212 
213  bool cancel = WalkCatalogsInHistory(current_history.weak_ref());
214  const bool success = reflog_->AddHistory(current_hash);
215  assert(success);
216 
217  if (cancel) {
218  current_hash = shash::Any(current_hash.algorithm);
219  } else {
220  current_hash = current_history->previous_revision();
221  }
222  }
223 }
224 
225 
233  CatalogList tag_hashes;
234  const bool list_success = history->GetHashes(&tag_hashes);
235  assert(list_success);
236 
237  CatalogList bin_hashes;
238  const bool bin_success = history->ListRecycleBin(&bin_hashes);
239  if (!bin_success) {
240  LogCvmfs(kLogCvmfs, kLogStderr, " Warning: 'recycle bin' table missing");
241  }
242 
243  WalkListedCatalogs(tag_hashes);
244  WalkListedCatalogs(bin_hashes);
245 
246  return !bin_success;
247 }
248 
249 
251  const RootChainWalker::CatalogList &catalog_list) {
252  CatalogList::const_iterator i = catalog_list.begin();
253  CatalogList::const_iterator iend = catalog_list.end();
254  for (; i != iend; ++i) {
255  WalkRootCatalogs(*i);
256  }
257 }
258 
259 
261  const shash::Any catalog_hash) {
262  CatalogTN *catalog = NULL;
263  const char *root_path = "";
265  object_fetcher_->FetchCatalog(catalog_hash, root_path, &catalog);
266 
267  return ReturnOrAbort(failure, catalog_hash, catalog);
268 }
269 
270 
272  const shash::Any history_hash) {
273  HistoryTN *history = NULL;
275  object_fetcher_->FetchHistory(&history, history_hash);
276 
277  return ReturnOrAbort(failure, history_hash, history);
278 }
279 
280 
281 template <class DatabaseT>
283  const ObjectFetcherFailures::Failures failure,
284  const shash::Any &content_hash,
285  DatabaseT *database) {
286  switch (failure) {
288  return database;
290  return NULL;
291  default:
292  PANIC(kLogStderr, "Failed to load object '%s' (%d - %s)",
293  content_hash.ToStringWithSuffix().c_str(), failure,
294  Code2Ascii(failure));
295  }
296 }
297 
298 } // namespace swissknife
static Parameter Optional(const char key, const std::string &desc)
Definition: swissknife.h:41
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
HistoryTN * FetchHistory(const shash::Any history_hash)
void TakeDatabaseFileOwnership()
Definition: reflog.cc:307
const char * Code2Ascii(const ObjectFetcherFailures::Failures error)
std::string database_file() const
Definition: reflog.cc:337
bool ContainsHistory(const shash::Any &history) const
Definition: reflog.cc:239
bool IsNull() const
Definition: hash.h:382
const manifest::Manifest * manifest() const
Definition: repository.h:125
bool WalkCatalogsInHistory(const HistoryTN *history)
bool AddHistory(const shash::Any &history)
Definition: reflog.cc:134
void WalkRootCatalogs(const shash::Any &root_catalog_hash)
CatalogTN * FetchCatalog(const shash::Any catalog_hash)
const manifest::Manifest * manifest_
#define PANIC(...)
Definition: exception.h:26
std::vector< Parameter > ParameterList
Definition: swissknife.h:71
std::string ToString(const bool with_suffix=false) const
Definition: hash.h:248
ObjectFetcher::CatalogTN CatalogTN
const history::History * history() const
std::string ToStringWithSuffix() const
Definition: hash.h:303
Failures FetchHistory(HistoryTN **history, const shash::Any &history_hash=shash::Any())
int Main(const ArgumentList &args)
RootChainWalker(const manifest::Manifest *manifest, ObjectFetcher *object_fetcher, manifest::Reflog *reflog)
assert((mem||(size==0))&&"Out Of Memory")
void WalkListedCatalogs(const CatalogList &catalog_list)
void WalkHistories(const shash::Any &history_hash)
Algorithms algorithm
Definition: hash.h:124
bool InitVerifyingSignatureManager(const std::string &pubkey_path, const std::string &trusted_certs="")
Definition: server_tool.cc:49
bool AddCatalog(const shash::Any &catalog)
Definition: reflog.cc:128
uint64_t CountEntries()
Definition: reflog.cc:146
signature::SignatureManager * signature_manager() const
Definition: server_tool.cc:118
virtual ParameterList GetParams() const
static AbstractUploader * Construct(const SpoolerDefinition &param)
Definition: plugin.h:188
download::DownloadManager * download_manager() const
Definition: server_tool.cc:113
object_fetcher_traits< HttpObjectFetcher< CatalogT, HistoryT, ReflogT > >::CatalogTN CatalogTN
bool AddMetainfo(const shash::Any &metainfo)
Definition: reflog.cc:140
static Parameter Mandatory(const char key, const std::string &desc)
Definition: swissknife.h:38
Failures FetchCatalog(const shash::Any &catalog_hash, const std::string &catalog_path, CatalogTN **catalog, const bool is_nested=false, CatalogTN *parent=NULL)
shash::Any certificate() const
Definition: manifest.h:126
object_fetcher_traits< HttpObjectFetcher< CatalogT, HistoryT, ReflogT > >::HistoryTN HistoryTN
void BeginTransaction()
Definition: reflog.cc:295
DatabaseT * ReturnOrAbort(const ObjectFetcherFailures::Failures failure, const shash::Any &content_hash, DatabaseT *database)
static void HashDatabase(const std::string &database_path, shash::Any *hash_reflog)
Definition: reflog.cc:322
shash::Any catalog_hash() const
Definition: manifest.h:124
bool IsValid() const
Definition: pointer.h:43
void DropDatabaseFileOwnership()
Definition: reflog.cc:313
bool AddCertificate(const shash::Any &certificate)
Definition: reflog.cc:121
bool ContainsCatalog(const shash::Any &catalog) const
Definition: reflog.cc:222
std::map< char, SharedPtr< std::string > > ArgumentList
Definition: swissknife.h:72
shash::Any history() const
Definition: manifest.h:127
void CommitTransaction()
Definition: reflog.cc:301
std::string meta_info() const
Definition: repository.h:128
HttpObjectFetcher ObjectFetcher
Failures FetchManifest(manifest::Manifest **manifest)
void AddStaticManifestObjects(manifest::Reflog *reflog, manifest::Manifest *manifest) const
static bool WriteChecksum(const std::string &path, const shash::Any &value)
Definition: reflog.cc:64
manifest::Reflog * CreateEmptyReflog(const std::string &temp_directory, const std::string &repo_name)
Definition: server_tool.cc:180
bool InitDownloadManager(const bool follow_redirects, const std::string &proxy, const unsigned max_pool_handles=1)
Definition: server_tool.cc:21
ObjectFetcher::HistoryTN HistoryTN
std::vector< shash::Any > CatalogList
shash::Any meta_info() const
Definition: manifest.h:131