CernVM-FS  2.10.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
magic_xattr.cc
Go to the documentation of this file.
1 
5 #include "magic_xattr.h"
6 
7 #include <cassert>
8 #include <string>
9 #include <vector>
10 
11 #include "catalog_mgr_client.h"
12 #include "fetch.h"
13 #include "logging.h"
14 #include "mountpoint.h"
15 #include "quota.h"
16 #include "signature.h"
17 #include "util/string.h"
18 
20  bool hide_magic_xattrs)
21  : mount_point_(mountpoint),
22  hide_magic_xattrs_(hide_magic_xattrs)
23 {
24  Register("user.catalog_counters", new CatalogCountersMagicXattr());
25  Register("user.external_host", new ExternalHostMagicXattr());
26  Register("user.external_timeout", new ExternalTimeoutMagicXattr());
27  Register("user.fqrn", new FqrnMagicXattr());
28  Register("user.host", new HostMagicXattr());
29  Register("user.host_list", new HostListMagicXattr());
30  Register("user.ncleanup24", new NCleanup24MagicXattr());
31  Register("user.nclg", new NClgMagicXattr());
32  Register("user.ndiropen", new NDirOpenMagicXattr());
33  Register("user.ndownload", new NDownloadMagicXattr());
34  Register("user.nioerr", new NIOErrMagicXattr());
35  Register("user.nopen", new NOpenMagicXattr());
36  Register("user.hitrate", new HitrateMagicXattr());
37  Register("user.logbuffer", new LogBufferXattr());
38  Register("user.proxy", new ProxyMagicXattr());
39  Register("user.pubkeys", new PubkeysMagicXattr());
40  Register("user.repo_counters", new RepoCountersMagicXattr());
41  Register("user.repo_metainfo", new RepoMetainfoMagicXattr());
42  Register("user.revision", new RevisionMagicXattr());
43  Register("user.root_hash", new RootHashMagicXattr());
44  Register("user.rx", new RxMagicXattr());
45  Register("user.speed", new SpeedMagicXattr());
46  Register("user.tag", new TagMagicXattr());
47  Register("user.timeout", new TimeoutMagicXattr());
48  Register("user.timeout_direct", new TimeoutDirectMagicXattr());
49  Register("user.timestamp_last_ioerr", new TimestampLastIOErrMagicXattr());
50  Register("user.usedfd", new UsedFdMagicXattr());
51  Register("user.useddirp", new UsedDirPMagicXattr());
52  Register("user.version", new VersionMagicXattr());
53 
54  Register("user.hash", new HashMagicXattr());
55  Register("user.lhash", new LHashMagicXattr());
56 
57  Register("user.chunk_list", new ChunkListMagicXattr());
58  Register("user.chunks", new ChunksMagicXattr());
59  Register("user.compression", new CompressionMagicXattr());
60  Register("user.direct_io", new DirectIoMagicXattr());
61  Register("user.external_file", new ExternalFileMagicXattr());
62 
63  Register("user.rawlink", new RawlinkMagicXattr());
64  Register("xfsroot.rawlink", new RawlinkMagicXattr());
65 
66  Register("user.authz", new AuthzMagicXattr());
67 }
68 
70  if (hide_magic_xattrs()) {
71  return "";
72  }
73 
74  std::string result;
75  std::map<std::string, BaseMagicXattr *>::iterator it = xattr_list_.begin();
76  for (; it != xattr_list_.end(); ++it) {
77  MagicXattrFlavor flavor = (*it).second->GetXattrFlavor();
78  // Skip those which should not be displayed
79  switch (flavor) {
80  case kXattrBase:
81  break;
82  case kXattrWithHash:
83  if (dirent->checksum().IsNull()) continue;
84  break;
85  case kXattrRegular:
86  if (!dirent->IsRegular()) continue;
87  break;
88  case kXattrSymlink:
89  if (!dirent->IsLink()) continue;
90  break;
91  case kXattrAuthz:
92  if (!mount_point_->has_membership_req()) continue;
93  break;
94  default:
95  PANIC("unknown magic xattr flavor");
96  }
97  result += (*it).first;
98  result.push_back('\0');
99  }
100 
101  return result;
102 }
103 
105  PathString path,
107 {
108  BaseMagicXattr *result;
109  if (xattr_list_.count(name) > 0) {
110  result = xattr_list_[name];
111  } else {
112  return NULL;
113  }
114 
115  result->Lock(path, d);
116 
117  return result;
118 }
119 
120 void MagicXattrManager::Register(const std::string &name,
121  BaseMagicXattr *magic_xattr)
122 {
123  if (xattr_list_.count(name) > 0) {
125  "Magic extended attribute with name %s already registered",
126  name.c_str());
127  }
128  magic_xattr->mount_point_ = mount_point_;
129  xattr_list_[name] = magic_xattr;
130 }
131 
134 }
135 
137  return mount_point_->membership_req();
138 }
139 
141  return kXattrAuthz;
142 }
143 
145  counters_ =
147  return true;
148 }
149 
151  std::string res;
152  res = "catalog_mountpoint: " + subcatalog_path_ + "\n";
153  res += counters_.GetCsvMap();
154  return res;
155 }
156 
158  chunk_list_ = "hash,offset,size\n";
159  if (!dirent_->IsRegular()) {
160  return false;
161  }
162  if (dirent_->IsChunkedFile()) {
163  FileChunkList chunks;
164  if (!mount_point_->catalog_mgr()
166  || chunks.IsEmpty())
167  {
168  LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr, "file %s is marked as "
169  "'chunked', but no chunks found.", path_.c_str());
170  return false;
171  } else {
172  for (size_t i = 0; i < chunks.size(); ++i) {
173  chunk_list_ += chunks.At(i).content_hash().ToString() + ",";
174  chunk_list_ += StringifyInt(chunks.At(i).offset()) + ",";
175  chunk_list_ += StringifyUint(chunks.At(i).size()) + "\n";
176  }
177  }
178  } else {
179  chunk_list_ += dirent_->checksum().ToString() + ",";
180  chunk_list_ += "0,";
181  chunk_list_ += StringifyUint(dirent_->size()) + "\n";
182  }
183  return true;
184 }
185 
187  return chunk_list_;
188 }
189 
191  if (!dirent_->IsRegular()) {
192  return false;
193  }
194  if (dirent_->IsChunkedFile()) {
195  FileChunkList chunks;
196  if (!mount_point_->catalog_mgr()
198  || chunks.IsEmpty())
199  {
200  LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr, "file %s is marked as "
201  "'chunked', but no chunks found.", path_.c_str());
202  return false;
203  } else {
204  n_chunks_ = chunks.size();
205  }
206  } else {
207  n_chunks_ = 1;
208  }
209 
210  return true;
211 }
212 
214  return StringifyUint(n_chunks_);
215 }
216 
218  return dirent_->IsRegular();
219 }
220 
223 }
224 
226  return dirent_->IsRegular();
227 }
228 
230  return dirent_->IsDirectIo() ? "1" : "0";
231 }
232 
234  return dirent_->IsRegular();
235 }
236 
238  return dirent_->IsExternalFile() ? "1" : "0";
239 }
240 
242  std::vector<string> host_chain;
243  std::vector<int> rtt;
244  unsigned current_host;
246  &host_chain, &rtt, &current_host);
247  if (host_chain.size()) {
248  return std::string(host_chain[current_host]);
249  } else {
250  return "internal error: no hosts defined";
251  }
252 }
253 
255  unsigned seconds, seconds_direct;
256  mount_point_->external_download_mgr()->GetTimeout(&seconds, &seconds_direct);
257  return StringifyUint(seconds_direct);
258 }
259 
261  return mount_point_->fqrn();
262 }
263 
265  return !dirent_->checksum().IsNull();
266 }
267 
269  return dirent_->checksum().ToString();
270 }
271 
273  std::vector<std::string> host_chain;
274  std::vector<int> rtt;
275  unsigned current_host;
276  mount_point_->download_mgr()->GetHostInfo(&host_chain, &rtt, &current_host);
277  if (host_chain.size()) {
278  return std::string(host_chain[current_host]);
279  } else {
280  return "internal error: no hosts defined";
281  }
282 }
283 
285  std::string result;
286  std::vector<std::string> host_chain;
287  std::vector<int> rtt;
288  unsigned current_host;
289  mount_point_->download_mgr()->GetHostInfo(&host_chain, &rtt, &current_host);
290  if (host_chain.size()) {
291  result = host_chain[current_host];
292  for (unsigned i = 1; i < host_chain.size(); ++i) {
293  result +=
294  ";" + host_chain[(i+current_host) % host_chain.size()];
295  }
296  } else {
297  result = "internal error: no hosts defined";
298  }
299  return result;
300 }
301 
303  return !dirent_->checksum().IsNull();
304 }
305 
307  string result;
308  CacheManager::ObjectInfo object_info;
309  object_info.description = path_.ToString();
311  object_info.type = CacheManager::kTypeVolatile;
312  int fd = mount_point_->file_system()->cache_mgr()->Open(
313  CacheManager::Bless(dirent_->checksum(), object_info));
314  if (fd < 0) {
315  result = "Not in cache";
316  } else {
318  int retval_i =
319  mount_point_->file_system()->cache_mgr()->ChecksumFd(fd, &hash);
320  if (retval_i != 0)
321  result = "I/O error (" + StringifyInt(retval_i) + ")";
322  else
323  result = hash.ToString();
325  }
326  return result;
327 }
328 
329 LogBufferXattr::LogBufferXattr() : BaseMagicXattr(), throttle_(1, 500, 2000) { }
330 
333  std::vector<LogBufferEntry> buffer = GetLogBuffer();
334  std::string result;
335  for (unsigned i = 0; i < buffer.size(); ++i) {
336  result += "[" + StringifyTime(buffer[i].timestamp, true /* UTC */) +
337  " UTC] " + buffer[i].message + "\n";
338  }
339  return result;
340 }
341 
343  QuotaManager *quota_mgr =
346  return StringifyInt(-1);
347  } else {
348  const uint64_t period_s = 24 * 60 * 60;
349  const uint64_t rate = quota_mgr->GetCleanupRate(period_s);
350  return StringifyInt(rate);
351  }
352 }
353 
356  return true;
357 }
358 
360  return StringifyInt(n_catalogs_);
361 }
362 
365 }
366 
368  return mount_point_->statistics()->Lookup("fetch.n_downloads")->Print();
369 }
370 
373 }
374 
376  return mount_point_->file_system()->n_fs_open()->ToString();
377 }
378 
380  int64_t n_invocations =
381  mount_point_->statistics()->Lookup("fetch.n_invocations")->Get();
382  if (n_invocations == 0)
383  return "n/a";
384 
385  int64_t n_downloads =
386  mount_point_->statistics()->Lookup("fetch.n_downloads")->Get();
387  float hitrate = 100. * (1. -
388  (static_cast<float>(n_downloads) / static_cast<float>(n_invocations)));
389  return StringifyDouble(hitrate);
390 }
391 
393  vector< vector<download::DownloadManager::ProxyInfo> > proxy_chain;
394  unsigned current_group;
396  &proxy_chain, &current_group, NULL);
397  if (proxy_chain.size()) {
398  return proxy_chain[current_group][0].url;
399  } else {
400  return "DIRECT";
401  }
402 }
403 
406  return true;
407 }
408 
410  return pubkeys_;
411 }
412 
414  return dirent_->IsLink();
415 }
416 
418  return dirent_->symlink().ToString();
419 }
420 
423  return true;
424 }
425 
427  return counters_.GetCsvMap();
428 }
429 
431 
433  if (!mount_point_->catalog_mgr()->manifest()) {
434  error_reason_ = "manifest not available";
435  return true;
436  }
437 
439  if (metainfo_hash_.IsNull()) {
440  error_reason_ = "metainfo not available";
441  return true;
442  }
443  return true;
444 }
445 
447  if (metainfo_hash_.IsNull()) {
448  return error_reason_;
449  }
450 
451  int fd = mount_point_->fetcher()->
453  "metainfo (" + metainfo_hash_.ToString() + ")",
455  if (fd < 0) {
456  return "Failed to open metadata file";
457  }
458  uint64_t actual_size = mount_point_->file_system()->cache_mgr()->GetSize(fd);
459  if (actual_size > kMaxMetainfoLength) {
461  return "Failed to open: metadata file is too big";
462  }
463  char buffer[kMaxMetainfoLength];
464  int bytes_read =
465  mount_point_->file_system()->cache_mgr()->Pread(fd, buffer, actual_size, 0);
467  if (bytes_read < 0) {
468  return "Failed to read metadata file";
469  }
470  return string(buffer, buffer + bytes_read);
471 }
472 
475  return true;
476 }
477 
479  return StringifyUint(revision_);
480 }
481 
484  return true;
485 }
486 
488  return root_hash_.ToString();
489 }
490 
491 std::string RxMagicXattr::GetValue() {
492  perf::Statistics *statistics = mount_point_->statistics();
493  int64_t rx = statistics->Lookup("download.sz_transferred_bytes")->Get();
494  return StringifyInt(rx/1024);
495 }
496 
498  perf::Statistics *statistics = mount_point_->statistics();
499  int64_t rx = statistics->Lookup("download.sz_transferred_bytes")->Get();
500  int64_t time = statistics->Lookup("download.sz_transfer_time")->Get();
501  if (time == 0)
502  return "n/a";
503  else
504  return StringifyInt((1000 * (rx/1024))/time);
505 }
506 
509  return true;
510 }
511 
512 std::string TagMagicXattr::GetValue() {
513  return tag_;
514 }
515 
517  unsigned seconds, seconds_direct;
518  mount_point_->download_mgr()->GetTimeout(&seconds, &seconds_direct);
519  return StringifyUint(seconds);
520 }
521 
523  unsigned seconds, seconds_direct;
524  mount_point_->download_mgr()->GetTimeout(&seconds, &seconds_direct);
525  return StringifyUint(seconds_direct);
526 }
527 
529  return StringifyInt(
531 }
532 
535 }
536 
539 }
540 
542  return std::string(VERSION) + "." + std::string(CVMFS_PATCH_LEVEL);
543 }
std::vector< LogBufferEntry > GetLogBuffer()
Definition: logging.cc:524
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
std::string ToString()
Definition: statistics.cc:24
int ChecksumFd(int fd, shash::Any *id)
Definition: cache.cc:39
virtual std::string GetValue()
Definition: magic_xattr.cc:342
bool IsExternalFile() const
const Counters & GetCounters() const
Definition: catalog.h:175
bool IsNull() const
Definition: hash.h:382
MagicXattrManager(MountPoint *mountpoint, bool hide_magic_xattrs)
Definition: magic_xattr.cc:19
virtual int64_t GetSize(int fd)=0
FileSystem * file_system()
Definition: mountpoint.h:442
MagicXattrFlavor
Definition: magic_xattr.h:20
Item At(const size_t index) const
Definition: bigvector.h:50
virtual bool PrepareValueFenced()
Definition: magic_xattr.cc:157
virtual std::string GetValue()
Definition: magic_xattr.cc:491
cvmfs::Fetcher * fetcher()
Definition: mountpoint.h:438
std::string subcatalog_path_
Definition: magic_xattr.h:167
virtual uint64_t GetCleanupRate(uint64_t period_s)=0
bool IsChunkedFile() const
perf::Statistics * statistics()
Definition: mountpoint.h:458
void Register(const std::string &name, BaseMagicXattr *magic_xattr)
Definition: magic_xattr.cc:120
virtual bool PrepareValueFenced()
Definition: magic_xattr.cc:482
std::string Print()
Definition: statistics.cc:25
#define PANIC(...)
Definition: exception.h:26
virtual std::string GetValue()
Definition: magic_xattr.cc:150
uint64_t size() const
std::string ToString(const bool with_suffix=false) const
Definition: hash.h:248
virtual std::string GetValue()
Definition: magic_xattr.cc:497
virtual std::string GetValue()
Definition: magic_xattr.cc:241
std::string fqrn() const
Definition: mountpoint.h:440
catalog::Counters counters_
Definition: magic_xattr.h:168
BaseMagicXattr * GetLocked(const std::string &name, PathString path, catalog::DirectoryEntry *d)
Definition: magic_xattr.cc:104
virtual std::string GetValue()
Definition: magic_xattr.cc:516
std::string error_reason_
Definition: magic_xattr.h:304
virtual std::string GetValue()
Definition: magic_xattr.cc:426
const shash::Any & content_hash() const
Definition: file_chunk.h:41
shash::Any root_hash_
Definition: magic_xattr.h:318
virtual std::string GetValue()
Definition: magic_xattr.cc:254
virtual bool PrepareValueFenced()
Definition: magic_xattr.cc:507
std::string AlgorithmName(const zlib::Algorithms alg)
Definition: compression.cc:158
MountPoint * mount_point_
Definition: cvmfs.cc:121
virtual std::string GetValue()
Definition: magic_xattr.cc:537
bool IsDirectIo() const
catalog::DirectoryEntry * dirent_
Definition: magic_xattr.h:79
Algorithms algorithm
Definition: hash.h:124
virtual std::string GetValue()
Definition: magic_xattr.cc:213
virtual int Open(const BlessedObject &object)=0
string StringifyTime(const time_t seconds, const bool utc)
Definition: string.cc:105
string StringifyDouble(const double value)
Definition: string.cc:96
virtual std::string GetValue()
Definition: magic_xattr.cc:284
virtual bool PrepareValueFenced()
Definition: magic_xattr.cc:217
std::string StringifyUint(const uint64_t value)
Definition: string.cc:84
virtual std::string GetValue()
Definition: magic_xattr.cc:306
shash::Any checksum() const
MountPoint * mount_point_
Definition: magic_xattr.h:77
virtual bool PrepareValueFenced()
Definition: magic_xattr.cc:144
bool has_membership_req()
Definition: mountpoint.h:444
std::string description
Definition: cache.h:97
std::string chunk_list_
Definition: magic_xattr.h:175
virtual std::string GetValue()
Definition: magic_xattr.cc:136
static uint64_t kMaxMetainfoLength
Definition: magic_xattr.h:301
MountPoint * mount_point_
Definition: magic_xattr.h:156
virtual std::string GetValue()
Definition: magic_xattr.cc:379
perf::Counter * n_fs_open()
Definition: mountpoint.h:214
virtual std::string GetValue()
Definition: magic_xattr.cc:331
std::string membership_req()
Definition: mountpoint.h:453
virtual bool PrepareValueFenced()
Definition: magic_xattr.cc:354
void Throttle()
Definition: backoff.cc:50
std::string GetListString(catalog::DirectoryEntry *dirent)
Definition: magic_xattr.cc:69
virtual std::string GetValue()
Definition: magic_xattr.cc:237
std::string GetActivePubkeys() const
Definition: signature.cc:357
Counter * Lookup(const std::string &name) const
Definition: statistics.cc:62
virtual std::string GetValue()
Definition: magic_xattr.cc:478
bool IsLink() const
virtual bool PrepareValueFenced()
Definition: magic_xattr.cc:473
void GetProxyInfo(std::vector< std::vector< ProxyInfo > > *proxy_chain, unsigned *current_group, unsigned *fallback_group)
Definition: download.cc:2556
std::string repository_tag()
Definition: mountpoint.h:456
catalog::ClientCatalogManager * catalog_mgr()
Definition: mountpoint.h:429
bool IsRegular() const
virtual std::string GetValue()
Definition: magic_xattr.cc:512
virtual std::string GetValue()
Definition: magic_xattr.cc:260
ObjectType type
Definition: cache.h:93
virtual std::string GetValue()
Definition: magic_xattr.cc:375
virtual bool PrepareValueFenced()
Definition: magic_xattr.cc:421
perf::Counter * no_open_dirs()
Definition: mountpoint.h:221
virtual std::string GetValue()
Definition: magic_xattr.cc:533
Failures Fetch(const std::string &base_url, const std::string &repository_name, const uint64_t minimum_timestamp, const shash::Any *base_catalog, signature::SignatureManager *signature_manager, download::DownloadManager *download_manager, ManifestEnsemble *ensemble)
virtual bool PrepareValueFenced()
Definition: magic_xattr.cc:302
off_t offset() const
Definition: file_chunk.h:42
zlib::Algorithms compression_algorithm() const
CacheManager * cache_mgr()
Definition: mountpoint.h:194
perf::Counter * no_open_files()
Definition: mountpoint.h:222
virtual std::string GetValue()
Definition: magic_xattr.cc:268
LinkString symlink() const
IoErrorInfo * io_error_info()
Definition: mountpoint.h:218
download::DownloadManager * download_mgr()
Definition: mountpoint.h:431
bool hide_magic_xattrs()
Definition: magic_xattr.h:152
virtual std::string GetValue()
Definition: magic_xattr.cc:541
virtual std::string GetValue()
Definition: magic_xattr.cc:487
string StringifyInt(const int64_t value)
Definition: string.cc:78
virtual std::string GetValue()
Definition: magic_xattr.cc:363
virtual int Close(int fd)=0
virtual std::string GetValue()
Definition: magic_xattr.cc:367
shash::Algorithms hash_algorithm() const
perf::Counter * n_fs_dir_open()
Definition: mountpoint.h:210
virtual std::string GetValue()
Definition: magic_xattr.cc:392
int64_t Get()
Definition: statistics.h:32
virtual std::string GetValue()
Definition: magic_xattr.cc:371
CatalogT * GetRootCatalog() const
Definition: catalog_mgr.h:192
std::string tag_
Definition: magic_xattr.h:333
bool IsEmpty() const
Definition: bigvector.h:67
static BlessedObject Bless(const shash::Any &id)
Definition: cache.h:125
shash::Any metainfo_hash_
Definition: magic_xattr.h:303
signature::SignatureManager * signature_mgr()
Definition: mountpoint.h:459
BackoffThrottle throttle_
Definition: magic_xattr.h:241
uint64_t n_chunks_
Definition: magic_xattr.h:182
virtual std::string GetValue()
Definition: magic_xattr.cc:186
std::string ToString() const
Definition: shortstring.h:114
virtual std::string GetValue()
Definition: magic_xattr.cc:446
QuotaManager * quota_mgr()
Definition: cache.h:198
virtual std::string GetValue()
Definition: magic_xattr.cc:359
virtual std::string GetValue()
Definition: magic_xattr.cc:272
PathString path_
Definition: magic_xattr.h:78
virtual std::string GetValue()
Definition: magic_xattr.cc:528
virtual std::string GetValue()
Definition: magic_xattr.cc:221
bool ListFileChunks(const PathString &path, const shash::Algorithms interpret_hashes_as, FileChunkList *chunks)
size_t size() const
Definition: file_chunk.h:43
std::map< std::string, BaseMagicXattr * > xattr_list_
Definition: magic_xattr.h:155
catalog::Counters LookupCounters(const PathString &path, std::string *subcatalog_path)
virtual bool PrepareValueFenced()
Definition: magic_xattr.cc:264
void GetTimeout(unsigned *seconds_proxy, unsigned *seconds_direct)
Definition: download.cc:1896
virtual bool PrepareValueFenced()
Definition: magic_xattr.cc:432
virtual std::string GetValue()
Definition: magic_xattr.cc:229
catalog::Counters counters_
Definition: magic_xattr.h:294
virtual bool PrepareValueFenced()
Definition: magic_xattr.cc:404
virtual MagicXattrFlavor GetXattrFlavor()
Definition: magic_xattr.cc:140
std::string pubkeys_
Definition: magic_xattr.h:282
manifest::Manifest * manifest() const
virtual std::string GetValue()
Definition: magic_xattr.cc:522
void Lock(PathString path, catalog::DirectoryEntry *dirent)
Definition: magic_xattr.h:63
std::string GetCsvMap() const
const char * c_str() const
Definition: shortstring.h:118
virtual bool PrepareValueFenced()
Definition: magic_xattr.cc:190
void GetHostInfo(std::vector< std::string > *host_chain, std::vector< int > *rtt, unsigned *current_host)
Definition: download.cc:1940
virtual std::string GetValue()
Definition: magic_xattr.cc:409
virtual int64_t Pread(int fd, void *buf, uint64_t size, uint64_t offset)=0
virtual bool PrepareValueFenced()
Definition: magic_xattr.cc:225
virtual bool PrepareValueFenced()
Definition: magic_xattr.cc:233
shash::Any meta_info() const
Definition: manifest.h:131
virtual bool HasCapability(Capabilities capability)=0
static const uint64_t kSizeUnknown
Definition: cache.h:72
size_t size() const
Definition: bigvector.h:100
download::DownloadManager * external_download_mgr()
Definition: mountpoint.h:432
virtual bool PrepareValueFenced()
Definition: magic_xattr.cc:132