GCC Code Coverage Report
Directory: cvmfs/ Exec Total Coverage
File: cvmfs/catalog_mgr_client.cc Lines: 117 150 78.0 %
Date: 2019-02-03 02:48:13 Branches: 62 120 51.7 %

Line Branch Exec Source
1
/**
2
 * This file is part of the CernVM file system.
3
 */
4
5
#include "cvmfs_config.h"
6
#include "catalog_mgr_client.h"
7
8
#include <string>
9
#include <vector>
10
11
#include "cache_posix.h"
12
#include "download.h"
13
#include "fetch.h"
14
#include "manifest.h"
15
#include "mountpoint.h"
16
#include "quota.h"
17
#include "signature.h"
18
#include "statistics.h"
19
#include "util/posix.h"
20
#include "util/string.h"
21
22
using namespace std;  // NOLINT
23
24
namespace catalog {
25
26
/**
27
 * Triggered when the catalog is attached (db file opened)
28
 */
29
44
void ClientCatalogManager::ActivateCatalog(Catalog *catalog) {
30
44
  const Counters &counters = const_cast<const Catalog*>(catalog)->GetCounters();
31
44
  if (catalog->IsRoot()) {
32
40
    all_inodes_ = counters.GetAllEntries();
33
  }
34
44
  loaded_inodes_ += counters.GetSelfEntries();
35
44
}
36
37
38
57
ClientCatalogManager::ClientCatalogManager(MountPoint *mountpoint)
39
  : AbstractCatalogManager<Catalog>(mountpoint->statistics())
40
  , repo_name_(mountpoint->fqrn())
41
  , fetcher_(mountpoint->fetcher())
42
  , signature_mgr_(mountpoint->signature_mgr())
43
  , workspace_(mountpoint->file_system()->workspace())
44
  , offline_mode_(false)
45
  , all_inodes_(0)
46
  , loaded_inodes_(0)
47
57
  , fixed_alt_root_catalog_(false)
48
{
49
57
  LogCvmfs(kLogCatalog, kLogDebug, "constructing client catalog manager");
50
  n_certificate_hits_ = mountpoint->statistics()->Register(
51
57
    "cache.n_certificate_hits", "Number of certificate hits");
52
  n_certificate_misses_ = mountpoint->statistics()->Register(
53
57
    "cache.n_certificate_misses", "Number of certificate misses");
54
57
}
55
56
57
114
ClientCatalogManager::~ClientCatalogManager() {
58
57
  LogCvmfs(kLogCache, kLogDebug, "unpinning / unloading all catalogs");
59
60
158
  for (map<PathString, shash::Any>::iterator i = mounted_catalogs_.begin(),
61
57
       iend = mounted_catalogs_.end(); i != iend; ++i)
62
  {
63
44
    fetcher_->cache_mgr()->quota_mgr()->Unpin(i->second);
64
  }
65
57
}
66
67
68
44
Catalog *ClientCatalogManager::CreateCatalog(
69
  const PathString  &mountpoint,
70
  const shash::Any  &catalog_hash,
71
  catalog::Catalog  *parent_catalog
72
) {
73
44
  mounted_catalogs_[mountpoint] = loaded_catalogs_[mountpoint];
74
44
  loaded_catalogs_.erase(mountpoint);
75
44
  return new Catalog(mountpoint, catalog_hash, parent_catalog);
76
}
77
78
79
4
shash::Any ClientCatalogManager::GetRootHash() {
80
4
  ReadLock();
81
4
  shash::Any result = mounted_catalogs_[PathString("", 0)];
82
4
  Unlock();
83
4
  return result;
84
}
85
86
87
/**
88
 * Specialized initialization that uses a fixed root hash.
89
 */
90
27
bool ClientCatalogManager::InitFixed(
91
  const shash::Any &root_hash,
92
  bool alternative_path)
93
{
94
  LogCvmfs(kLogCatalog, kLogDebug, "Initialize catalog with root hash %s",
95
27
           root_hash.ToString().c_str());
96
27
  WriteLock();
97
27
  fixed_alt_root_catalog_ = alternative_path;
98
27
  bool attached = MountCatalog(PathString("", 0), root_hash, NULL);
99
27
  Unlock();
100
101
27
  if (!attached) {
102
    LogCvmfs(kLogCatalog, kLogDebug, "failed to initialize root catalog");
103
  }
104
105
27
  return attached;
106
}
107
108
109
55
LoadError ClientCatalogManager::LoadCatalog(
110
  const PathString  &mountpoint,
111
  const shash::Any  &hash,
112
  std::string *catalog_path,
113
  shash::Any *catalog_hash)
114
{
115
  string cvmfs_path = "file catalog at " + repo_name_ + ":" +
116
    (mountpoint.IsEmpty() ?
117


55
      "/" : string(mountpoint.GetChars(), mountpoint.GetLength()));
118
  bool retval;
119
120
  // send the catalog hash to a blind memory position if it zero (save some ifs)
121
55
  shash::Any blind_hash;
122
55
  if (catalog_hash == NULL) {
123
    catalog_hash = &blind_hash;
124
  }
125
126
  // Load a particular catalog
127
55
  if (!hash.IsNull()) {
128
31
    cvmfs_path += " (" + hash.ToString() + ")";
129
31
    string alt_catalog_path = "";
130

31
    if (mountpoint.IsEmpty() && fixed_alt_root_catalog_)
131
      alt_catalog_path = hash.MakeAlternativePath();
132
    LoadError load_error =
133
31
      LoadCatalogCas(hash, cvmfs_path, alt_catalog_path, catalog_path);
134
31
    if (load_error == catalog::kLoadNew)
135
31
      loaded_catalogs_[mountpoint] = hash;
136
31
    *catalog_hash = hash;
137
31
    return load_error;
138
  }
139
140
  // Happens only on init/remount, i.e. quota won't delete a cached catalog
141
24
  string checksum_dir = fetcher_->cache_mgr()->GetBackingDirectory();
142
24
  if (checksum_dir.empty())
143
    checksum_dir = ".";
144
24
  shash::Any cache_hash(shash::kSha1, shash::kSuffixCatalog);
145
24
  uint64_t cache_last_modified = 0;
146
147
  retval = manifest::Manifest::ReadChecksum(
148
24
    repo_name_, checksum_dir, &cache_hash, &cache_last_modified);
149
24
  if (retval) {
150
    LogCvmfs(kLogCache, kLogDebug, "cached copy publish date %s (hash %s)",
151
             StringifyTime(cache_last_modified, true).c_str(),
152
2
             cache_hash.ToString().c_str());
153
  } else {
154
22
    LogCvmfs(kLogCache, kLogDebug, "unable to read local checksum");
155
  }
156
157
  // Load and verify remote checksum
158
  manifest::Failures manifest_failure;
159
24
  CachedManifestEnsemble ensemble(fetcher_->cache_mgr(), this);
160
  manifest_failure = manifest::Fetch("", repo_name_, cache_last_modified,
161
                                     &cache_hash, signature_mgr_,
162
                                     fetcher_->download_mgr(),
163
24
                                     &ensemble);
164
24
  if (manifest_failure != manifest::kFailOk) {
165
    LogCvmfs(kLogCache, kLogDebug, "failed to fetch manifest (%d - %s)",
166
11
             manifest_failure, manifest::Code2Ascii(manifest_failure));
167
168
11
    LoadError success_code = catalog::kLoadUp2Date;
169
170
    // Network unavailable but cached copy updated externally?
171
    std::map<PathString, shash::Any>::const_iterator iter =
172
11
      mounted_catalogs_.find(mountpoint);
173
11
    if (iter != mounted_catalogs_.end()) {
174
      if (iter->second != cache_hash) {
175
        success_code = catalog::kLoadNew;
176
      }
177
    }
178
179
11
    if (catalog_path) {
180
      LoadError success_code =
181
11
        LoadCatalogCas(cache_hash, cvmfs_path, "", catalog_path);
182
11
      if (success_code != catalog::kLoadNew)
183
11
        return success_code;
184
      loaded_catalogs_[mountpoint] = cache_hash;
185
    }
186
187
    *catalog_hash = cache_hash;
188
    offline_mode_ = true;
189
    return success_code;
190
  }
191
192
13
  offline_mode_ = false;
193
13
  cvmfs_path += " (" + ensemble.manifest->catalog_hash().ToString() + ")";
194
  LogCvmfs(kLogCache, kLogDebug, "remote checksum is %s",
195
13
           ensemble.manifest->catalog_hash().ToString().c_str());
196
197
  // Short way out, use cached copy
198
13
  if (ensemble.manifest->catalog_hash() == cache_hash) {
199
2
    if (catalog_path) {
200
      LoadError error =
201
2
        LoadCatalogCas(cache_hash, cvmfs_path, "", catalog_path);
202
2
      if (error == catalog::kLoadNew) {
203
2
        loaded_catalogs_[mountpoint] = cache_hash;
204
2
        *catalog_hash = cache_hash;
205
2
        return catalog::kLoadUp2Date;
206
      }
207
      LogCvmfs(kLogCache, kLogDebug,
208
               "unable to open catalog from local checksum, downloading");
209
    } else {
210
      loaded_catalogs_[mountpoint] = cache_hash;
211
      *catalog_hash = cache_hash;
212
      return catalog::kLoadUp2Date;
213
    }
214
  }
215
11
  if (!catalog_path)
216
    return catalog::kLoadNew;
217
218
  // Load new catalog
219
  catalog::LoadError load_retval =
220
    LoadCatalogCas(ensemble.manifest->catalog_hash(),
221
                   cvmfs_path,
222
                   ensemble.manifest->has_alt_catalog_path() ?
223
                     ensemble.manifest->MakeCatalogPath() : "",
224

11
                   catalog_path);
225
11
  if (load_retval != catalog::kLoadNew)
226
    return load_retval;
227
11
  loaded_catalogs_[mountpoint] = ensemble.manifest->catalog_hash();
228
11
  *catalog_hash = ensemble.manifest->catalog_hash();
229
230
  // Store new manifest and certificate
231
  fetcher_->cache_mgr()->CommitFromMem(ensemble.manifest->certificate(),
232
                                       ensemble.cert_buf, ensemble.cert_size,
233
11
                                       "certificate for " + repo_name_);
234
11
  ensemble.manifest->ExportChecksum(workspace_, 0600);
235
11
  return catalog::kLoadNew;
236
}
237
238
239
55
LoadError ClientCatalogManager::LoadCatalogCas(
240
  const shash::Any &hash,
241
  const string &name,
242
  const std::string &alt_catalog_path,
243
  string *catalog_path)
244
{
245
55
  assert(hash.suffix == shash::kSuffixCatalog);
246
  int fd = fetcher_->Fetch(hash, CacheManager::kSizeUnknown, name,
247
55
    zlib::kZlibDefault, CacheManager::kTypeCatalog, alt_catalog_path);
248
55
  if (fd >= 0) {
249
44
    *catalog_path = "@" + StringifyInt(fd);
250
44
    return kLoadNew;
251
  }
252
253
11
  if (fd == -ENOSPC)
254
    return kLoadNoSpace;
255
256
11
  return kLoadFail;
257
}
258
259
260
void ClientCatalogManager::UnloadCatalog(const Catalog *catalog) {
261
  LogCvmfs(kLogCache, kLogDebug, "unloading catalog %s",
262
           catalog->mountpoint().c_str());
263
264
  map<PathString, shash::Any>::iterator iter =
265
    mounted_catalogs_.find(catalog->mountpoint());
266
  assert(iter != mounted_catalogs_.end());
267
  fetcher_->cache_mgr()->quota_mgr()->Unpin(iter->second);
268
  mounted_catalogs_.erase(iter);
269
  const catalog::Counters &counters = catalog->GetCounters();
270
  loaded_inodes_ -= counters.GetSelfEntries();
271
}
272
273
274
/**
275
 * Checks if the current repository revision is blacklisted.  The format
276
 * of the blacklist lines is '<REPO N' where REPO is the repository name,
277
 * N is the revision number, and the two parts are separated by whitespace.
278
 * Any revision of REPO less than N is blacklisted.
279
 * Note: no extra characters are allowed after N, not even whitespace.
280
 * @return true if it is blacklisted, false otherwise
281
 */
282
40
bool ClientCatalogManager::IsRevisionBlacklisted() {
283
40
  uint64_t revision = GetRevision();
284
285
  LogCvmfs(kLogCache, kLogDebug, "checking if %s revision %u is blacklisted",
286
40
           repo_name_.c_str(), revision);
287
288
40
  vector<string> blacklist = signature_mgr_->GetBlacklist();
289

40
  for (unsigned i = 0; i < blacklist.size(); ++i) {
290
3
    std::string line = blacklist[i];
291
3
    if (line[0] != '<')
292
      continue;
293
3
    unsigned idx = repo_name_.length() + 1;
294
3
    if (line.length() <= idx)
295
      continue;
296

3
    if ((line[idx] != ' ') && (line[idx] != '\t'))
297
      continue;
298
3
    if (line.substr(1, idx - 1) != repo_name_)
299
      continue;
300
3
    ++idx;
301

6
    while ((line[idx] == ' ') || (line[idx] == '\t'))
302
      ++idx;
303
3
    if (idx >= line.length())
304
      continue;
305
    uint64_t rev;
306
3
    if (!String2Uint64Parse(line.substr(idx), &rev))
307
      continue;
308
3
    if (revision < rev)
309
3
      return true;
310
  }
311
312
37
  return false;
313
}
314
315
316
//------------------------------------------------------------------------------
317
318
319
17
void CachedManifestEnsemble::FetchCertificate(const shash::Any &hash) {
320
  uint64_t size;
321
  bool retval = cache_mgr_->Open2Mem(
322
17
    hash, "certificate for " + catalog_mgr_->repo_name(), &cert_buf, &size);
323
17
  cert_size = size;
324
17
  if (retval)
325
    perf::Inc(catalog_mgr_->n_certificate_hits_);
326
  else
327
17
    perf::Inc(catalog_mgr_->n_certificate_misses_);
328
17
}
329
330
}  // namespace catalog