CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
catalog_mgr_client.cc
Go to the documentation of this file.
1 
5 #include "cvmfs_config.h"
6 #include "catalog_mgr_client.h"
7 
8 #include <string>
9 #include <vector>
10 
11 #include "cache_posix.h"
12 #include "download.h"
13 #include "fetch.h"
14 #include "manifest.h"
15 #include "mountpoint.h"
16 #include "quota.h"
17 #include "signature.h"
18 #include "statistics.h"
19 #include "util/posix.h"
20 #include "util/string.h"
21 
22 using namespace std; // NOLINT
23 
24 namespace catalog {
25 
29 void ClientCatalogManager::ActivateCatalog(Catalog *catalog) {
30  const Counters &counters = const_cast<const Catalog*>(catalog)->GetCounters();
31  if (catalog->IsRoot()) {
32  all_inodes_ = counters.GetAllEntries();
33  }
34  loaded_inodes_ += counters.GetSelfEntries();
35 }
36 
37 
38 ClientCatalogManager::ClientCatalogManager(MountPoint *mountpoint)
39  : AbstractCatalogManager<Catalog>(mountpoint->statistics())
40  , repo_name_(mountpoint->fqrn())
41  , fetcher_(mountpoint->fetcher())
42  , signature_mgr_(mountpoint->signature_mgr())
43  , workspace_(mountpoint->file_system()->workspace())
44  , offline_mode_(false)
45  , all_inodes_(0)
46  , loaded_inodes_(0)
47  , fixed_alt_root_catalog_(false)
48 {
49  LogCvmfs(kLogCatalog, kLogDebug, "constructing client catalog manager");
50  n_certificate_hits_ = mountpoint->statistics()->Register(
51  "cache.n_certificate_hits", "Number of certificate hits");
52  n_certificate_misses_ = mountpoint->statistics()->Register(
53  "cache.n_certificate_misses", "Number of certificate misses");
54 }
55 
56 
58  LogCvmfs(kLogCache, kLogDebug, "unpinning / unloading all catalogs");
59 
60  for (map<PathString, shash::Any>::iterator i = mounted_catalogs_.begin(),
61  iend = mounted_catalogs_.end(); i != iend; ++i)
62  {
63  fetcher_->cache_mgr()->quota_mgr()->Unpin(i->second);
64  }
65 }
66 
67 
69  const PathString &mountpoint,
70  const shash::Any &catalog_hash,
71  catalog::Catalog *parent_catalog
72 ) {
73  mounted_catalogs_[mountpoint] = loaded_catalogs_[mountpoint];
74  loaded_catalogs_.erase(mountpoint);
75  return new Catalog(mountpoint, catalog_hash, parent_catalog);
76 }
77 
78 
80  ReadLock();
81  shash::Any result = mounted_catalogs_[PathString("", 0)];
82  Unlock();
83  return result;
84 }
85 
86 
91  const shash::Any &root_hash,
92  bool alternative_path)
93 {
94  LogCvmfs(kLogCatalog, kLogDebug, "Initialize catalog with root hash %s",
95  root_hash.ToString().c_str());
96  WriteLock();
97  fixed_alt_root_catalog_ = alternative_path;
98  bool attached = MountCatalog(PathString("", 0), root_hash, NULL);
99  Unlock();
100 
101  if (!attached) {
102  LogCvmfs(kLogCatalog, kLogDebug, "failed to initialize root catalog");
103  }
104 
105  return attached;
106 }
107 
108 
110  const PathString &mountpoint,
111  const shash::Any &hash,
112  std::string *catalog_path,
113  shash::Any *catalog_hash)
114 {
115  string cvmfs_path = "file catalog at " + repo_name_ + ":" +
116  (mountpoint.IsEmpty() ?
117  "/" : string(mountpoint.GetChars(), mountpoint.GetLength()));
118 
119  // send the catalog hash to a blind memory position if it zero (save some ifs)
120  shash::Any blind_hash;
121  if (catalog_hash == NULL) {
122  catalog_hash = &blind_hash;
123  }
124 
125  // Load a particular catalog
126  if (!hash.IsNull()) {
127  cvmfs_path += " (" + hash.ToString() + ")";
128  string alt_catalog_path = "";
129  if (mountpoint.IsEmpty() && fixed_alt_root_catalog_)
130  alt_catalog_path = hash.MakeAlternativePath();
131  LoadError load_error =
132  LoadCatalogCas(hash, cvmfs_path, alt_catalog_path, catalog_path);
133  if (load_error == catalog::kLoadNew)
134  loaded_catalogs_[mountpoint] = hash;
135  *catalog_hash = hash;
136  return load_error;
137  }
138 
139  // Happens only on init/remount, i.e. quota won't delete a cached catalog
141  uint64_t cache_last_modified = 0;
142 
143  manifest::Breadcrumb breadcrumb =
145  if (breadcrumb.IsValid()) {
146  cache_hash = breadcrumb.catalog_hash;
147  cache_last_modified = breadcrumb.timestamp;
148  LogCvmfs(kLogCache, kLogDebug, "cached copy publish date %s (hash %s)",
149  StringifyTime(cache_last_modified, true).c_str(),
150  cache_hash.ToString().c_str());
151  } else {
152  LogCvmfs(kLogCache, kLogDebug, "unable to read local checksum");
153  }
154 
155  // Load and verify remote checksum
156  manifest::Failures manifest_failure;
157  CachedManifestEnsemble ensemble(fetcher_->cache_mgr(), this);
158  manifest_failure = manifest::Fetch("", repo_name_, cache_last_modified,
159  &cache_hash, signature_mgr_,
161  &ensemble);
162  if (manifest_failure != manifest::kFailOk) {
163  LogCvmfs(kLogCache, kLogDebug, "failed to fetch manifest (%d - %s)",
164  manifest_failure, manifest::Code2Ascii(manifest_failure));
165 
166  LoadError success_code = catalog::kLoadUp2Date;
167 
168  // Network unavailable but cached copy updated externally?
169  std::map<PathString, shash::Any>::const_iterator iter =
170  mounted_catalogs_.find(mountpoint);
171  if (iter != mounted_catalogs_.end()) {
172  if (breadcrumb.IsValid() && (iter->second != cache_hash)) {
173  success_code = catalog::kLoadNew;
174  }
175  }
176 
177  if (catalog_path) {
178  LoadError success_code =
179  LoadCatalogCas(cache_hash, cvmfs_path, "", catalog_path);
180  if (success_code != catalog::kLoadNew)
181  return success_code;
182  loaded_catalogs_[mountpoint] = cache_hash;
183  }
184 
185  *catalog_hash = cache_hash;
186  offline_mode_ = true;
187  return success_code;
188  }
189 
190  manifest_ = new manifest::Manifest(*ensemble.manifest);
191 
192  offline_mode_ = false;
193  cvmfs_path += " (" + ensemble.manifest->catalog_hash().ToString() + ")";
194  LogCvmfs(kLogCache, kLogDebug, "remote checksum is %s",
195  ensemble.manifest->catalog_hash().ToString().c_str());
196 
197  // Short way out, use cached copy
198  if (ensemble.manifest->catalog_hash() == cache_hash) {
199  LoadError success_code = catalog::kLoadUp2Date;
200 
201  // Has the breadcrumb been updated externally?
202  std::map<PathString, shash::Any>::const_iterator iter =
203  mounted_catalogs_.find(mountpoint);
204  if (iter != mounted_catalogs_.end()) {
205  if (iter->second != cache_hash) {
206  LogCvmfs(kLogCache, kLogDebug, "updating from %s to alien cache copy",
207  iter->second.ToString().c_str());
208  success_code = catalog::kLoadNew;
209  }
210  }
211 
212  if (catalog_path) {
213  LoadError error =
214  LoadCatalogCas(cache_hash, cvmfs_path, "", catalog_path);
215  if (error == catalog::kLoadNew) {
216  loaded_catalogs_[mountpoint] = cache_hash;
217  *catalog_hash = cache_hash;
218  return success_code;
219  }
221  "unable to open catalog from local checksum, downloading");
222  } else {
223  *catalog_hash = cache_hash;
224  return success_code;
225  }
226  }
227  if (!catalog_path)
228  return catalog::kLoadNew;
229 
230  // Load new catalog
231  catalog::LoadError load_retval =
232  LoadCatalogCas(ensemble.manifest->catalog_hash(),
233  cvmfs_path,
234  ensemble.manifest->has_alt_catalog_path() ?
235  ensemble.manifest->MakeCatalogPath() : "",
236  catalog_path);
237  if (load_retval != catalog::kLoadNew)
238  return load_retval;
239  loaded_catalogs_[mountpoint] = ensemble.manifest->catalog_hash();
240  *catalog_hash = ensemble.manifest->catalog_hash();
241 
242  // Store new manifest and certificate
243  fetcher_->cache_mgr()->CommitFromMem(ensemble.manifest->certificate(),
244  ensemble.cert_buf, ensemble.cert_size,
245  "certificate for " + repo_name_);
246  fetcher_->cache_mgr()->StoreBreadcrumb(*ensemble.manifest);
247  return catalog::kLoadNew;
248 }
249 
250 
252  const shash::Any &hash,
253  const string &name,
254  const std::string &alt_catalog_path,
255  string *catalog_path)
256 {
258  int fd = fetcher_->Fetch(hash, CacheManager::kSizeUnknown, name,
260  if (fd >= 0) {
261  *catalog_path = "@" + StringifyInt(fd);
262  return kLoadNew;
263  }
264 
265  if (fd == -ENOSPC)
266  return kLoadNoSpace;
267 
268  return kLoadFail;
269 }
270 
271 
273  LogCvmfs(kLogCache, kLogDebug, "unloading catalog %s",
274  catalog->mountpoint().c_str());
275 
276  map<PathString, shash::Any>::iterator iter =
277  mounted_catalogs_.find(catalog->mountpoint());
278  assert(iter != mounted_catalogs_.end());
279  fetcher_->cache_mgr()->quota_mgr()->Unpin(iter->second);
280  mounted_catalogs_.erase(iter);
281  const catalog::Counters &counters = catalog->GetCounters();
282  loaded_inodes_ -= counters.GetSelfEntries();
283 }
284 
285 
295  uint64_t revision = GetRevision();
296 
297  LogCvmfs(kLogCache, kLogDebug, "checking if %s revision %u is blacklisted",
298  repo_name_.c_str(), revision);
299 
300  vector<string> blacklist = signature_mgr_->GetBlacklist();
301  for (unsigned i = 0; i < blacklist.size(); ++i) {
302  std::string line = blacklist[i];
303  if (line[0] != '<')
304  continue;
305  unsigned idx = repo_name_.length() + 1;
306  if (line.length() <= idx)
307  continue;
308  if ((line[idx] != ' ') && (line[idx] != '\t'))
309  continue;
310  if (line.substr(1, idx - 1) != repo_name_)
311  continue;
312  ++idx;
313  while ((line[idx] == ' ') || (line[idx] == '\t'))
314  ++idx;
315  if (idx >= line.length())
316  continue;
317  uint64_t rev;
318  if (!String2Uint64Parse(line.substr(idx), &rev))
319  continue;
320  if (revision < rev)
321  return true;
322  }
323 
324  return false;
325 }
326 
327 
328 //------------------------------------------------------------------------------
329 
330 
332  uint64_t size;
333  bool retval = cache_mgr_->Open2Mem(
334  hash, "certificate for " + catalog_mgr_->repo_name(), &cert_buf, &size);
335  cert_size = size;
336  if (retval)
338  else
340 }
341 
342 } // namespace catalog
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
Counter * Register(const std::string &name, const std::string &desc)
Definition: statistics.cc:139
const Counters & GetCounters() const
Definition: catalog.h:175
bool IsNull() const
Definition: hash.h:379
bool IsRoot() const
Definition: catalog.h:193
download::DownloadManager * download_mgr()
Definition: fetch.h:86
uint64_t timestamp
Definition: manifest.h:33
std::string MakeAlternativePath() const
Definition: hash.h:321
perf::Statistics * statistics()
Definition: mountpoint.h:438
shash::Any catalog_hash
Definition: manifest.h:32
LoadError LoadCatalog(const PathString &mountpoint, const shash::Any &hash, std::string *catalog_path, shash::Any *catalog_hash)
virtual void Unpin(const shash::Any &hash)=0
UniquePtr< manifest::Manifest > manifest_
std::string ToString(const bool with_suffix=false) const
Definition: hash.h:245
CacheManager * cache_mgr()
Definition: fetch.h:85
virtual manifest::Breadcrumb LoadBreadcrumb(const std::string &)
Definition: cache.h:219
Counters_t GetSelfEntries() const
assert((mem||(size==0))&&"Out Of Memory")
virtual bool StoreBreadcrumb(const manifest::Manifest &)
Definition: cache.h:222
string StringifyTime(const time_t seconds, const bool utc)
Definition: string.cc:105
std::map< PathString, shash::Any > loaded_catalogs_
bool String2Uint64Parse(const std::string &value, uint64_t *result)
Definition: string.cc:243
std::string repo_name() const
signature::SignatureManager * signature_mgr_
Definition: repository.h:139
const char kSuffixCatalog
Definition: hash.h:52
std::map< PathString, shash::Any > mounted_catalogs_
Failures Fetch(const std::string &base_url, const std::string &repository_name, const uint64_t minimum_timestamp, const shash::Any *base_catalog, signature::SignatureManager *signature_manager, download::DownloadManager *download_manager, ManifestEnsemble *ensemble)
const signature::SignatureManager * signature_mgr() const
Definition: repository.h:119
LoadError LoadCatalogCas(const shash::Any &hash, const std::string &name, const std::string &alt_catalog_path, std::string *catalog_path)
void FetchCertificate(const shash::Any &hash)
PathString mountpoint() const
Definition: catalog.h:179
bool CommitFromMem(const shash::Any &id, const unsigned char *buffer, const uint64_t size, const std::string &description)
Definition: cache.cc:79
bool Open2Mem(const shash::Any &id, const std::string &description, unsigned char **buffer, uint64_t *size)
Definition: cache.cc:127
string StringifyInt(const int64_t value)
Definition: string.cc:78
void Inc(class Counter *counter)
Definition: statistics.h:50
ClientCatalogManager * catalog_mgr_
bool InitFixed(const shash::Any &root_hash, bool alternative_path)
signature::SignatureManager * signature_mgr_
QuotaManager * quota_mgr()
Definition: cache.h:198
catalog::Catalog * CreateCatalog(const PathString &mountpoint, const shash::Any &catalog_hash, catalog::Catalog *parent_catalog)
bool IsEmpty() const
Definition: shortstring.h:110
ShortString< kDefaultMaxPath, 0 > PathString
Definition: shortstring.h:190
Catalog * MountCatalog(const PathString &mountpoint, const shash::Any &hash, Catalog *parent_catalog)
bool IsValid() const
Definition: manifest.h:30
unsigned GetLength() const
Definition: shortstring.h:104
Suffix suffix
Definition: hash.h:124
Counters_t GetAllEntries() const
const char * c_str() const
Definition: shortstring.h:118
const char * GetChars() const
Definition: shortstring.h:96
static void size_t size
Definition: smalloc.h:47
std::vector< std::string > GetBlacklist()
Definition: signature.cc:539
static const uint64_t kSizeUnknown
Definition: cache.h:72
void UnloadCatalog(const catalog::Catalog *catalog)
int Fetch(const shash::Any &id, const uint64_t size, const std::string &name, const zlib::Algorithms compression_algorithm, const CacheManager::ObjectType object_type, const std::string &alt_url="", off_t range_offset=-1)
Definition: fetch.cc:81
const char * Code2Ascii(const Failures error)