CernVM-FS  2.13.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
catalog_mgr_client.cc
Go to the documentation of this file.
1 
6 #include "catalog_mgr_client.h"
7 
8 #include <string>
9 #include <vector>
10 
11 #include "cache_posix.h"
12 #include "crypto/signature.h"
13 #include "fetch.h"
14 #include "manifest.h"
15 #include "mountpoint.h"
16 #include "network/download.h"
17 #include "quota.h"
18 #include "statistics.h"
19 #include "util/posix.h"
20 #include "util/string.h"
21 
22 using namespace std; // NOLINT
23 
24 namespace catalog {
25 
29 void ClientCatalogManager::ActivateCatalog(Catalog *catalog) {
30  const Counters &counters = const_cast<const Catalog *>(catalog)
31  ->GetCounters();
32  if (catalog->IsRoot()) {
33  all_inodes_ = counters.GetAllEntries();
34  }
35  loaded_inodes_ += counters.GetSelfEntries();
36 }
37 
38 
39 ClientCatalogManager::ClientCatalogManager(MountPoint *mountpoint)
40  : AbstractCatalogManager<Catalog>(mountpoint->statistics())
41  , repo_name_(mountpoint->fqrn())
42  , fetcher_(mountpoint->fetcher())
43  , signature_mgr_(mountpoint->signature_mgr())
44  , workspace_(mountpoint->file_system()->workspace())
45  , offline_mode_(false)
46  , all_inodes_(0)
47  , loaded_inodes_(0)
48  , fixed_root_catalog_()
49  , fixed_alt_root_catalog_(false)
50  , root_fd_(-1) {
51  LogCvmfs(kLogCatalog, kLogDebug, "constructing client catalog manager");
52  n_certificate_hits_ = mountpoint->statistics()->Register(
53  "cache.n_certificate_hits", "Number of certificate hits");
54  n_certificate_misses_ = mountpoint->statistics()->Register(
55  "cache.n_certificate_misses", "Number of certificate misses");
56 }
57 
58 
60  LogCvmfs(kLogCache, kLogDebug, "unpinning / unloading all catalogs");
61 
62  for (map<PathString, shash::Any>::iterator i = mounted_catalogs_.begin(),
63  iend = mounted_catalogs_.end();
64  i != iend;
65  ++i) {
66  fetcher_->cache_mgr()->quota_mgr()->Unpin(i->second);
67  }
68 }
69 
70 
72  const shash::Any &catalog_hash,
73  catalog::Catalog *parent_catalog) {
74  mounted_catalogs_[mountpoint] = loaded_catalogs_[mountpoint];
75  loaded_catalogs_.erase(mountpoint);
76  return new Catalog(mountpoint, catalog_hash, parent_catalog);
77 }
78 
79 
81  ReadLock();
82  shash::Any result = mounted_catalogs_[PathString("", 0)];
83  Unlock();
84  return result;
85 }
86 
87 
95  bool alternative_path) {
96  LogCvmfs(kLogCatalog, kLogDebug, "Initialize catalog with fixed root hash %s",
97  root_hash.ToString().c_str());
98  WriteLock();
99  fixed_alt_root_catalog_ = alternative_path;
100  fixed_root_catalog_ = root_hash;
101 
102  bool attached = MountCatalog(PathString("", 0), root_hash, NULL);
103  Unlock();
104 
105  if (!attached) {
106  LogCvmfs(kLogCatalog, kLogDebug, "failed to initialize fixed root catalog");
107  }
108 
109  return attached;
110 }
111 
128  CatalogContext *result) {
129  result->SetMountpoint(PathString("", 0));
130 
131  // quick escape if we have a fixed catalog
132  if (!fixed_root_catalog_.IsNull()) {
133  result->SetHash(fixed_root_catalog_);
135 
136  // it might or might not be already mounted, but we do not care
137  // as we do no need to download and save the manifest
138  // (see LoadCatalogByHash()) as such we must set the location to this
140  offline_mode_ = false;
141 
142  // we can do this here as the very first time fixed catalog is loaded it
143  // call directly MountCatalog() and will skip the call to this function
144  // here
145  return catalog::kLoadUp2Date;
146  }
147 
148  // 1) Get alien cache root catalog (local)
149 
150  // Happens only on init/remount, i.e. quota won't delete a cached catalog
151  shash::Any local_newest_hash(shash::kSha1, shash::kSuffixCatalog);
153  uint64_t local_newest_timestamp = 0;
154  uint64_t local_newest_revision = manifest::Breadcrumb::kInvalidRevision;
155 
157  repo_name_);
158  if (breadcrumb.IsValid()) {
159  local_newest_hash = breadcrumb.catalog_hash;
160  local_newest_timestamp = breadcrumb.timestamp;
161  local_newest_revision = breadcrumb.revision;
163  "Cached copy publish date %s (hash %s, revision %" PRIu64 ")",
164  StringifyTime(static_cast<int64_t>(local_newest_timestamp), true)
165  .c_str(),
166  local_newest_hash.ToString().c_str(), breadcrumb.revision);
167  } else {
168  LogCvmfs(kLogCache, kLogDebug, "Unable to read local checksum %s",
169  breadcrumb.ToString().c_str());
170  }
171 
172  // 2) Select local newest catalog: mounted vs alien
173 
175  LoadReturn success_code = catalog::kLoadNew;
176 
177  if (mounted_catalogs_.size() > 0) {
178  const std::map<PathString, shash::Any>::iterator
179  curr_hash_itr = mounted_catalogs_.find(PathString("", 0));
180  mounted_hash = curr_hash_itr->second;
181  }
182 
183  // We only look for currently loaded catalog if the revision is newer than
184  // the breadcrumb revision and both revision numbers are valid (!= -1ul).
185  if ((local_newest_revision <= GetRevisionNoLock()
186  || local_newest_revision == manifest::Breadcrumb::kInvalidRevision)
187  && mounted_catalogs_.size() > 0) {
188  local_newest_hash = mounted_hash;
189  local_newest_revision = GetRevisionNoLock();
190  // if needed for integration test 707: breadcrumb_timestamp_newer()
191  local_newest_timestamp = GetTimestampNoLock() > local_newest_timestamp
193  : local_newest_timestamp;
195  success_code = catalog::kLoadUp2Date;
196  } else if (local_newest_revision == 0 && mounted_catalogs_.size() > 0) {
197  // breadcrumb has no revision
198  // TODO(heretherebedragons) this branch can be removed in future versions
199 
200  // revisions are better, but if we dont have any we need to compare by
201  // timestamp (you can have multiple revisions in the same timestamp)
202  if (local_newest_timestamp < GetTimestampNoLock()) {
203  local_newest_hash = mounted_hash;
204  local_newest_revision = GetRevisionNoLock();
205  local_newest_timestamp = GetTimestampNoLock();
207  success_code = catalog::kLoadUp2Date;
208  }
209  }
210 
211  // 3) Get remote root catalog (fails if remote catalog is older)
212  manifest::Failures manifest_failure;
215  manifest_failure = manifest::Fetch(
216  "", repo_name_, local_newest_timestamp, &local_newest_hash,
217  signature_mgr_, fetcher_->download_mgr(), ensemble.weak_ref());
218 
219  if (manifest_failure == manifest::kFailOk) {
220  // server has newest revision or no valid local revision
221  if (ensemble->manifest->revision() > local_newest_revision
222  || local_newest_revision == manifest::Breadcrumb::kInvalidRevision
223  // if revision is 0 both local and server, load catalog from server
224  // as local is most likely just "initialized" without valid value
225  || (ensemble->manifest->revision() == 0
226  && local_newest_revision == 0)) {
227  result->SetHash(ensemble->manifest->catalog_hash());
228  result->SetRootCtlgRevision(ensemble->manifest->revision());
230  fixed_alt_root_catalog_ = ensemble->manifest->has_alt_catalog_path();
231 
232  result->TakeManifestEnsemble(
233  static_cast<manifest::ManifestEnsemble *>(ensemble.Release()));
234  offline_mode_ = false;
235 
236  return catalog::kLoadNew;
237  }
238  }
240  "Failed fetch manifest from server: "
241  "manifest too old or server unreachable (%d - %s)",
242  manifest_failure, manifest::Code2Ascii(manifest_failure));
243 
244  // total failure: server not reachable and no valid local hash
245  if ((manifest_failure != manifest::kFailOk) && local_newest_hash.IsNull()) {
246  LogCvmfs(kLogCache, kLogDebug, "No valid root catalog found!");
247  return catalog::kLoadFail;
248  }
249 
250  if (manifest_failure == manifest::kFailOk
251  && ensemble->manifest->revision() == local_newest_revision) {
252  offline_mode_ = false;
253  } else {
254  offline_mode_ = true;
255  }
256  result->SetHash(local_newest_hash);
257  result->SetRootCtlgRevision(local_newest_revision);
258 
259  // for integration test 707: breadcrumb_revision_large()
260  if (breadcrumb.IsValid() && breadcrumb.catalog_hash == mounted_hash) {
261  success_code = catalog::kLoadUp2Date;
262  }
263 
264  return success_code;
265 }
266 
268  const PathString &mountpoint, const shash::Any &hash) {
269  return "file catalog at " + repo_name_ + ":"
270  + (mountpoint.IsEmpty()
271  ? "/"
272  : string(mountpoint.GetChars(), mountpoint.GetLength()))
273  + " (" + hash.ToString() + ")";
274 }
275 
290  CatalogContext *ctlg_context) {
291  string catalog_descr = GetCatalogDescription(ctlg_context->mountpoint(),
292  ctlg_context->hash());
293  string alt_root_catalog_path = "";
294 
295  // root catalog needs special handling because of alt_root_catalog_path
296  if (ctlg_context->IsRootCatalog() && fixed_alt_root_catalog_) {
297  alt_root_catalog_path = ctlg_context->hash().MakeAlternativePath();
298  }
299 
300  const LoadReturn load_ret = FetchCatalogByHash(
301  ctlg_context->hash(), catalog_descr, alt_root_catalog_path,
302  ctlg_context->GetSqlitePathPtr());
303  if (load_ret == catalog::kLoadNew) {
304  loaded_catalogs_[ctlg_context->mountpoint()] = ctlg_context->hash();
305 
306  if (ctlg_context->IsRootCatalog()) {
307  if (ctlg_context->root_ctlg_location() == kCtlgLocationMounted) {
308  return kLoadUp2Date;
309  }
310 
311  // if coming from server: update breadcrumb
312  if (ctlg_context->root_ctlg_location() == kCtlgLocationServer) {
313  // Store new manifest and certificate
314  CacheManager::Label label;
315  label.path = repo_name_;
317  if (ctlg_context->manifest_ensemble()->cert_size > 0) {
320  ctlg_context->manifest_ensemble()->manifest->certificate(),
321  label),
322  ctlg_context->manifest_ensemble()->cert_buf,
323  ctlg_context->manifest_ensemble()->cert_size);
324  }
326  *ctlg_context->manifest_ensemble()->manifest);
327  }
328  }
329  }
330 
331  return load_ret;
332 }
333 
345  const shash::Any &hash,
346  const string &name,
347  const std::string &alt_root_catalog_path,
348  std::string *sqlite_path) {
350  CacheManager::Label label;
351  label.path = name;
353  int fd = fetcher_->Fetch(CacheManager::LabeledObject(hash, label),
354  alt_root_catalog_path);
355  if (fd >= 0) {
356  if (root_fd_ < 0) {
357  root_fd_ = fd;
358  }
359 
360  LogCvmfs(kLogCatalog, kLogDebug, "FetchCatalogByHash filedescriptor %d",
361  fd);
362  *sqlite_path = "@" + StringifyInt(fd);
363  return kLoadNew;
364  }
365 
366  if (fd == -ENOSPC)
367  return kLoadNoSpace;
368 
369  return kLoadFail;
370 }
371 
373  const shash::Any &hash, const PathString &mountpoint) {
375 
376  CacheManager::Label label;
377  label.path = GetCatalogDescription(mountpoint, hash);
379  int fd = fetcher_->Fetch(CacheManager::LabeledObject(hash, label));
380  if (fd >= 0)
381  fetcher_->cache_mgr()->Close(fd);
382 }
383 
385  LogCvmfs(kLogCache, kLogDebug, "unloading catalog %s",
386  catalog->mountpoint().c_str());
387 
388  map<PathString, shash::Any>::iterator iter = mounted_catalogs_.find(
389  catalog->mountpoint());
390  assert(iter != mounted_catalogs_.end());
391  fetcher_->cache_mgr()->quota_mgr()->Unpin(iter->second);
392  mounted_catalogs_.erase(iter);
393  const catalog::Counters &counters = catalog->GetCounters();
394  loaded_inodes_ -= counters.GetSelfEntries();
395 }
396 
397 
407  uint64_t revision = GetRevision();
408 
410  "checking if %s revision %" PRIu64 " is blacklisted",
411  repo_name_.c_str(), revision);
412 
413  vector<string> blacklist = signature_mgr_->GetBlacklist();
414  for (unsigned i = 0; i < blacklist.size(); ++i) {
415  std::string line = blacklist[i];
416  if (line[0] != '<')
417  continue;
418  unsigned idx = repo_name_.length() + 1;
419  if (line.length() <= idx)
420  continue;
421  if ((line[idx] != ' ') && (line[idx] != '\t'))
422  continue;
423  if (line.substr(1, idx - 1) != repo_name_)
424  continue;
425  ++idx;
426  while ((line[idx] == ' ') || (line[idx] == '\t'))
427  ++idx;
428  if (idx >= line.length())
429  continue;
430  uint64_t rev;
431  if (!String2Uint64Parse(line.substr(idx), &rev))
432  continue;
433  if (revision < rev)
434  return true;
435  }
436 
437  return false;
438 }
439 
440 
441 //------------------------------------------------------------------------------
442 
443 
445  CacheManager::Label label;
447  label.path = catalog_mgr_->repo_name();
448  uint64_t size;
449  bool retval = cache_mgr_->Open2Mem(CacheManager::LabeledObject(hash, label),
450  &cert_buf, &size);
451  cert_size = size;
452  if (retval)
454  else
456 }
457 
458 } // namespace catalog
shash::Any hash() const
Definition: catalog_mgr.h:124
Counter * Register(const std::string &name, const std::string &desc)
Definition: statistics.cc:163
const Counters & GetCounters() const
Definition: catalog.h:171
bool IsNull() const
Definition: hash.h:371
bool IsRoot() const
Definition: catalog.h:189
download::DownloadManager * download_mgr()
Definition: fetch.h:120
uint64_t timestamp
Definition: manifest.h:39
std::string MakeAlternativePath() const
Definition: hash.h:313
perf::Statistics * statistics()
Definition: mountpoint.h:538
shash::Any catalog_hash
Definition: manifest.h:38
static const int kLabelCatalog
Definition: cache.h:80
virtual void Unpin(const shash::Any &hash)=0
std::string ToString(const bool with_suffix=false) const
Definition: hash.h:241
CacheManager * cache_mgr()
Definition: fetch.h:119
static const uint64_t kInvalidRevision
Definition: manifest.h:23
RootCatalogLocation root_ctlg_location() const
Definition: catalog_mgr.h:128
virtual manifest::Breadcrumb LoadBreadcrumb(const std::string &)
Definition: cache.h:215
virtual void StageNestedCatalogByHash(const shash::Any &hash, const PathString &mountpoint)
Counters_t GetSelfEntries() const
assert((mem||(size==0))&&"Out Of Memory")
virtual bool StoreBreadcrumb(const manifest::Manifest &)
Definition: cache.h:218
string StringifyTime(const time_t seconds, const bool utc)
Definition: string.cc:104
std::map< PathString, shash::Any > loaded_catalogs_
bool String2Uint64Parse(const std::string &value, uint64_t *result)
Definition: string.cc:257
manifest::ManifestEnsemble * manifest_ensemble() const
Definition: catalog_mgr.h:129
uint64_t revision
Definition: manifest.h:40
void SetMountpoint(const PathString &mountpoint)
Definition: catalog_mgr.h:134
std::string repo_name() const
void TakeManifestEnsemble(manifest::ManifestEnsemble *manifest_ensemble)
Definition: catalog_mgr.h:147
static const int kLabelCertificate
Definition: cache.h:85
LoadReturn FetchCatalogByHash(const shash::Any &hash, const std::string &name, const std::string &alt_catalog_path, std::string *catalog_path)
int Fetch(const CacheManager::LabeledObject &object, const std::string &alt_url="")
Definition: fetch.cc:82
signature::SignatureManager * signature_mgr_
Definition: repository.h:139
bool CommitFromMem(const LabeledObject &object, const unsigned char *buffer, const uint64_t size)
Definition: cache.cc:77
const char kSuffixCatalog
Definition: hash.h:54
std::map< PathString, shash::Any > mounted_catalogs_
Failures Fetch(const std::string &base_url, const std::string &repository_name, const uint64_t minimum_timestamp, const shash::Any *base_catalog, signature::SignatureManager *signature_manager, download::DownloadManager *download_manager, ManifestEnsemble *ensemble)
const signature::SignatureManager * signature_mgr() const
Definition: repository.h:121
shash::Any certificate() const
Definition: manifest.h:127
void FetchCertificate(const shash::Any &hash)
PathString mountpoint() const
Definition: catalog.h:175
virtual LoadReturn LoadCatalogByHash(CatalogContext *ctlg_context)
string StringifyInt(const int64_t value)
Definition: string.cc:77
void Inc(class Counter *counter)
Definition: statistics.h:50
ClientCatalogManager * catalog_mgr_
virtual int Close(int fd)=0
void SetRootCtlgLocation(RootCatalogLocation root_ctlg_location)
Definition: catalog_mgr.h:141
bool InitFixed(const shash::Any &root_hash, bool alternative_path)
std::string GetCatalogDescription(const PathString &mountpoint, const shash::Any &hash)
signature::SignatureManager * signature_mgr_
QuotaManager * quota_mgr()
Definition: cache.h:191
catalog::Catalog * CreateCatalog(const PathString &mountpoint, const shash::Any &catalog_hash, catalog::Catalog *parent_catalog)
std::string * GetSqlitePathPtr()
Definition: catalog_mgr.h:121
bool IsEmpty() const
Definition: shortstring.h:137
std::string ToString() const
Definition: manifest.cc:72
ShortString< kDefaultMaxPath, 0 > PathString
Definition: shortstring.h:213
PathString mountpoint() const
Definition: catalog_mgr.h:125
Catalog * MountCatalog(const PathString &mountpoint, const shash::Any &hash, Catalog *parent_catalog)
bool IsValid() const
Definition: manifest.h:33
virtual LoadReturn GetNewRootCatalogContext(CatalogContext *result)
std::string path
Definition: cache.h:133
bool Open2Mem(const LabeledObject &object, unsigned char **buffer, uint64_t *size)
Definition: cache.cc:122
void SetHash(shash::Any hash)
Definition: catalog_mgr.h:133
unsigned GetLength() const
Definition: shortstring.h:131
Suffix suffix
Definition: hash.h:123
Counters_t GetAllEntries() const
const char * c_str() const
Definition: shortstring.h:143
const char * GetChars() const
Definition: shortstring.h:123
static void size_t size
Definition: smalloc.h:54
std::vector< std::string > GetBlacklist()
Definition: signature.cc:590
void UnloadCatalog(const catalog::Catalog *catalog)
void SetRootCtlgRevision(uint64_t root_ctlg_revision)
Definition: catalog_mgr.h:138
const char * Code2Ascii(const Failures error)
CVMFS_EXPORT void LogCvmfs(const LogSource source, const int mask, const char *format,...)
Definition: logging.cc:545