CernVM-FS  2.10.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
commit_processor.cc
Go to the documentation of this file.
1 
5 #include "commit_processor.h"
6 
7 #include <time.h>
8 
9 #include <vector>
10 
11 #include "catalog_diff_tool.h"
12 #include "catalog_merge_tool.h"
13 #include "catalog_mgr_ro.h"
14 #include "catalog_mgr_rw.h"
15 #include "compression.h"
16 #include "download.h"
17 #include "logging.h"
18 #include "manifest.h"
19 #include "manifest_fetch.h"
20 #include "params.h"
21 #include "signing_tool.h"
22 #include "statistics.h"
23 #include "statistics_database.h"
24 #include "swissknife.h"
25 #include "swissknife_history.h"
26 #include "util/algorithm.h"
27 #include "util/pointer.h"
28 #include "util/posix.h"
29 #include "util/raii_temp_dir.h"
30 #include "util/string.h"
31 
32 namespace {
33 
34 PathString RemoveRepoName(const PathString& lease_path) {
35  std::string abs_path = lease_path.ToString();
36  std::string::const_iterator it =
37  std::find(abs_path.begin(), abs_path.end(), '/');
38  if (it != abs_path.end()) {
39  size_t idx = it - abs_path.begin() + 1;
40  return lease_path.Suffix(idx);
41  } else {
42  return lease_path;
43  }
44 }
45 
46 bool CreateNewTag(const RepositoryTag& repo_tag, const std::string& repo_name,
47  const receiver::Params& params, const std::string& temp_dir,
48  const std::string& manifest_path,
49  const std::string& public_key_path,
50  const std::string& proxy) {
52  args['r'].Reset(new std::string(params.spooler_configuration));
53  args['w'].Reset(new std::string(params.stratum0));
54  args['t'].Reset(new std::string(temp_dir));
55  args['m'].Reset(new std::string(manifest_path));
56  args['p'].Reset(new std::string(public_key_path));
57  args['f'].Reset(new std::string(repo_name));
58  args['e'].Reset(new std::string(params.hash_alg_str));
59  args['a'].Reset(new std::string(repo_tag.name()));
60  args['D'].Reset(new std::string(repo_tag.description()));
61  args['x'].Reset(new std::string());
62  args['@'].Reset(new std::string(proxy));
63 
66  const int ret = edit_cmd->Main(args);
67 
68  if (ret) {
69  LogCvmfs(kLogReceiver, kLogSyslogErr, "Error %d creating tag: %s", ret,
70  repo_tag.name().c_str());
71  return false;
72  }
73 
74  return true;
75 }
76 
77 } // namespace
78 
79 namespace receiver {
80 
81 CommitProcessor::CommitProcessor() : num_errors_(0), statistics_(NULL) {}
82 
84 
102  const std::string& lease_path, const shash::Any& old_root_hash,
103  const shash::Any& new_root_hash, const RepositoryTag& tag,
104  uint64_t *final_revision) {
105  RepositoryTag final_tag = tag;
106  // If tag_name is a generic tag, update the time stamp
107  if (final_tag.HasGenericName()) {
108  final_tag.SetGenericName();
109  }
110 
112  "CommitProcessor - lease_path: %s, old hash: %s, new hash: %s, "
113  "tag_name: %s, tag_description: %s",
114  lease_path.c_str(), old_root_hash.ToString(true).c_str(),
115  new_root_hash.ToString(true).c_str(), final_tag.name().c_str(),
116  final_tag.description().c_str());
117 
118  const std::vector<std::string> lease_path_tokens =
119  SplitString(lease_path, '/');
120 
121  const std::string repo_name = lease_path_tokens.front();
122 
123  Params params;
124  if (!GetParamsFromFile(repo_name, &params)) {
125  LogCvmfs(
127  "CommitProcessor - error: Could not get configuration parameters.");
128  return kError;
129  }
130 
131  UniquePtr<ServerTool> server_tool(new ServerTool());
132 
133  if (!server_tool->InitDownloadManager(true, params.proxy)) {
134  LogCvmfs(
136  "CommitProcessor - error: Could not initialize the download manager");
137  return kError;
138  }
139 
140  const std::string public_key = "/etc/cvmfs/keys/" + repo_name + ".pub";
141  const std::string trusted_certs =
142  "/etc/cvmfs/repositories.d/" + repo_name + "/trusted_certs";
143  if (!server_tool->InitVerifyingSignatureManager(public_key, trusted_certs)) {
144  LogCvmfs(
146  "CommitProcessor - error: Could not initialize the signature manager");
147  return kError;
148  }
149 
150  shash::Any manifest_base_hash;
152  params.stratum0, repo_name, manifest_base_hash));
153 
154  // Current catalog from the gateway machine
155  if (!manifest.IsValid()) {
157  "CommitProcessor - error: Could not open repository manifest");
158  return kError;
159  }
160 
162  "CommitProcessor - lease_path: %s, target root hash: %s",
163  lease_path.c_str(),
164  manifest->catalog_hash().ToString(false).c_str());
165 
166  const std::string spooler_temp_dir =
168  assert(!spooler_temp_dir.empty());
169  assert(MkdirDeep(spooler_temp_dir + "/receiver", 0666, true));
170  const std::string temp_dir_root =
171  spooler_temp_dir + "/receiver/commit_processor";
172 
173  const PathString relative_lease_path = RemoveRepoName(PathString(lease_path));
174 
176  "CommitProcessor - lease_path: %s, merging catalogs",
177  lease_path.c_str());
178 
181  merge_tool(params.stratum0, old_root_hash, new_root_hash,
182  relative_lease_path, temp_dir_root,
183  server_tool->download_manager(), manifest.weak_ref(),
184  statistics_);
185  if (!merge_tool.Init()) {
187  "Error: Could not initialize the catalog merge tool");
188  return kError;
189  }
190 
191  std::string new_manifest_path;
192  if (!merge_tool.Run(params, &new_manifest_path, final_revision)) {
194  "CommitProcessor - error: Catalog merge failed");
195  return kMergeFailure;
196  }
197 
198  UniquePtr<RaiiTempDir> raii_temp_dir(RaiiTempDir::Create(temp_dir_root));
199  const std::string temp_dir = raii_temp_dir->dir();
200  const std::string certificate = "/etc/cvmfs/keys/" + repo_name + ".crt";
201  const std::string private_key = "/etc/cvmfs/keys/" + repo_name + ".key";
202 
203  if (!CreateNewTag(final_tag, repo_name, params, temp_dir, new_manifest_path,
204  public_key, params.proxy)) {
205  LogCvmfs(kLogReceiver, kLogSyslogErr, "Error creating tag: %s",
206  final_tag.name().c_str());
207  return kError;
208  }
209 
210  // We need to re-initialize the ServerTool component for signing
211  server_tool.Destroy();
212  server_tool = new ServerTool();
213 
215  "CommitProcessor - lease_path: %s, signing manifest",
216  lease_path.c_str());
217 
218  // Add C_N root catalog hash to reflog through SigningTool,
219  // so garbage collector can later delete it.
220  std::vector<shash::Any> reflog_catalogs;
221  reflog_catalogs.push_back(new_root_hash);
222 
223  SigningTool signing_tool(server_tool.weak_ref());
224  SigningTool::Result res = signing_tool.Run(
225  new_manifest_path, params.stratum0, params.spooler_configuration,
226  temp_dir, certificate, private_key, repo_name, "", "",
227  "/var/spool/cvmfs/" + repo_name + "/reflog.chksum", params.proxy,
228  params.garbage_collection, false, false, reflog_catalogs);
229  switch (res) {
232  "CommitProcessor - error: missing reflog.chksum");
233  return kMissingReflog;
236  "CommitProcessor - error: missing reflog");
237  return kMissingReflog;
238  case SigningTool::kError:
241  "CommitProcessor - error: signing manifest");
242  return kError;
245  "CommitProcessor - lease_path: %s, success.",
246  lease_path.c_str());
247  }
248 
249  {
250  UniquePtr<ServerTool> server_tool(new ServerTool());
251 
252  if (!server_tool->InitDownloadManager(true, params.proxy)) {
253  LogCvmfs(
255  "CommitProcessor - error: Could not initialize the download manager");
256  return kError;
257  }
258 
259  const std::string public_key = "/etc/cvmfs/keys/" + repo_name + ".pub";
260  const std::string trusted_certs =
261  "/etc/cvmfs/repositories.d/" + repo_name + "/trusted_certs";
262  if (!server_tool->InitVerifyingSignatureManager(public_key,
263  trusted_certs)) {
265  "CommitProcessor - error: Could not initialize the signature "
266  "manager");
267  return kError;
268  }
269 
270  shash::Any manifest_base_hash;
272  params.stratum0, repo_name, manifest_base_hash));
273 
275  "CommitProcessor - lease_path: %s, new root hash: %s",
276  lease_path.c_str(),
277  manifest->catalog_hash().ToString(false).c_str());
278  }
279 
280  // Ensure CVMFS_ROOT_HASH is not set in
281  // /var/spool/cvmfs/<REPO_NAME>/client.local
282  const std::string fname = "/var/spool/cvmfs/" + repo_name + "/client.local";
283  if (truncate(fname.c_str(), 0) < 0) {
284  LogCvmfs(kLogReceiver, kLogSyslogErr, "Could not truncate %s\n",
285  fname.c_str());
286  return kError;
287  }
288 
290  if (stats_db != NULL) {
291  if (!stats_db->StorePublishStatistics(statistics_, start_time_, true)) {
293  "Could not store publish statistics");
294  }
295  if (params.upload_stats_db) {
297  upload::Spooler *spooler = upload::Spooler::Construct(sd);
298  if (!stats_db->UploadStatistics(spooler)) {
300  "Could not upload statistics DB to upstream storage");
301  }
302  delete spooler;
303  }
304  delete stats_db;
305 
306  } else {
307  LogCvmfs(kLogReceiver, kLogSyslogErr, "Could not open statistics DB");
308  }
309 
310  return kSuccess;
311 }
312 
314  const std::string &start_time)
315 {
316  statistics_ = st;
317  statistics_->Register("publish.revision", "");
318  start_time_ = start_time;
319 }
320 
321 } // namespace receiver
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
std::string name() const
Counter * Register(const std::string &name, const std::string &desc)
Definition: statistics.cc:139
const manifest::Manifest * manifest() const
Definition: repository.h:125
void SetGenericName()
Result Run(const std::string &manifest_path, const std::string &repo_url, const std::string &spooler_definition, const std::string &temp_dir, const std::string &certificate="", const std::string &priv_key="", const std::string &repo_name="", const std::string &pwd="", const std::string &meta_info="", const std::string &reflog_chksum_path="", const std::string &proxy="", const bool garbage_collectable=false, const bool bootstrap_shortcuts=false, const bool return_early=false, const std::vector< shash::Any > reflog_catalogs=std::vector< shash::Any >())
Definition: signing_tool.cc:27
std::string description() const
std::string spooler_configuration
Definition: params.h:20
bool UploadStatistics(upload::Spooler *spooler, std::string local_path="")
ShortString Suffix(const unsigned start_at) const
Definition: shortstring.h:171
vector< string > SplitString(const string &str, const char delim, const unsigned max_chunks)
Definition: string.cc:288
perf::Statistics * statistics_
std::string ToString(const bool with_suffix=false) const
Definition: hash.h:248
int Main(const ArgumentList &args)
bool GetParamsFromFile(const std::string &repo_name, Params *params)
Definition: params.cc:20
perf::Statistics * statistics_
Definition: repository.h:139
manifest::Manifest * FetchRemoteManifest(const std::string &repository_url, const std::string &repository_name, const shash::Any &base_hash=shash::Any()) const
Definition: server_tool.cc:138
assert((mem||(size==0))&&"Out Of Memory")
std::string GetSpoolerTempDir(const std::string &spooler_config)
Definition: params.cc:14
bool InitVerifyingSignatureManager(const std::string &pubkey_path, const std::string &trusted_certs="")
Definition: server_tool.cc:49
std::string hash_alg_str
Definition: params.h:22
bool CreateNewTag(const RepositoryTag &repo_tag, const std::string &repo_name, const receiver::Params &params, const std::string &temp_dir, const std::string &manifest_path, const std::string &public_key_path, const std::string &proxy)
download::DownloadManager * download_manager() const
Definition: server_tool.cc:113
std::string stratum0
Definition: params.h:18
static RaiiTempDir * Create(const std::string &prefix)
Definition: raii_temp_dir.cc:9
bool MkdirDeep(const std::string &path, const mode_t mode, bool verify_writable)
Definition: posix.cc:870
PathString RemoveRepoName(const PathString &lease_path)
shash::Any catalog_hash() const
Definition: manifest.h:124
static StatisticsDatabase * OpenStandardDB(const std::string repo_name)
bool garbage_collection
Definition: params.h:33
void SetStatistics(perf::Statistics *st, const std::string &start_time)
bool StorePublishStatistics(const perf::Statistics *statistics, const std::string &start_time, const bool success)
bool HasGenericName()
std::string ToString() const
Definition: shortstring.h:114
std::map< char, SharedPtr< std::string > > ArgumentList
Definition: swissknife.h:72
ShortString< kDefaultMaxPath, 0 > PathString
Definition: shortstring.h:190
Result Process(const std::string &lease_path, const shash::Any &old_root_hash, const shash::Any &new_root_hash, const RepositoryTag &tag, uint64_t *final_revision)
std::string proxy
Definition: params.h:19
bool upload_stats_db
Definition: params.h:37
bool InitDownloadManager(const bool follow_redirects, const std::string &proxy, const unsigned max_pool_handles=1)
Definition: server_tool.cc:21