CernVM-FS  2.12.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
swissknife_check.cc
Go to the documentation of this file.
1 
7 #define __STDC_FORMAT_MACROS
8 
9 #include "cvmfs_config.h"
10 #include "swissknife_check.h"
11 
12 #include <inttypes.h>
13 #include <unistd.h>
14 
15 #include <cassert>
16 #include <map>
17 #include <queue>
18 #include <set>
19 #include <string>
20 #include <vector>
21 
22 #include "catalog_sql.h"
23 #include "compression.h"
24 #include "file_chunk.h"
25 #include "history_sqlite.h"
26 #include "manifest.h"
27 #include "network/download.h"
28 #include "reflog.h"
29 #include "sanitizer.h"
30 #include "shortstring.h"
31 #include "util/exception.h"
32 #include "util/logging.h"
33 #include "util/pointer.h"
34 #include "util/posix.h"
35 
36 using namespace std; // NOLINT
37 
38 // for map of duplicate entries; as in kvstore.cc
39 static inline uint32_t hasher_any(const shash::Any &key) {
40  // We'll just do the same thing as hasher_md5, since every hash is at
41  // least as large.
42  return *const_cast<uint32_t *>(
43  reinterpret_cast<const uint32_t *>(key.digest) + 1);
44 }
45 
46 
47 namespace swissknife {
48 
49 CommandCheck::CommandCheck()
50  : check_chunks_(false)
51  , no_duplicates_map_(false)
52  , is_remote_(false) {
53  const shash::Any hash_null;
54  duplicates_map_.Init(16, hash_null, hasher_any);
55 }
56 
58  const catalog::DirectoryEntry &b,
59  const bool compare_names,
60  const bool is_transition_point)
61 {
62  typedef catalog::DirectoryEntry::Difference Difference;
63 
65  if (diffs == Difference::kIdentical) {
66  return true;
67  }
68 
69  // in case of a nested catalog transition point the controlling flags are
70  // supposed to differ. If this is the only difference we are done...
71  if (is_transition_point &&
72  (diffs ^ Difference::kNestedCatalogTransitionFlags) == 0) {
73  return true;
74  }
75 
76  bool retval = true;
77  if (compare_names) {
78  if (diffs & Difference::kName) {
79  LogCvmfs(kLogCvmfs, kLogStderr, "names differ: %s / %s",
80  a.name().c_str(), b.name().c_str());
81  retval = false;
82  }
83  }
84  if (diffs & Difference::kLinkcount) {
85  LogCvmfs(kLogCvmfs, kLogStderr, "linkcounts differ: %u / %u",
86  a.linkcount(), b.linkcount());
87  retval = false;
88  }
89  if (diffs & Difference::kHardlinkGroup) {
90  LogCvmfs(kLogCvmfs, kLogStderr, "hardlink groups differ: %u / %u",
92  retval = false;
93  }
94  if (diffs & Difference::kSize) {
95  LogCvmfs(kLogCvmfs, kLogStderr, "sizes differ: %" PRIu64 " / %" PRIu64,
96  a.size(), b.size());
97  retval = false;
98  }
99  if (diffs & Difference::kMode) {
100  LogCvmfs(kLogCvmfs, kLogStderr, "modes differ: %u / %u",
101  a.mode(), b.mode());
102  retval = false;
103  }
104  if (diffs & Difference::kMtime) {
105  LogCvmfs(kLogCvmfs, kLogStderr, "timestamps differ: %lu / %lu",
106  a.mtime(), b.mtime());
107  retval = false;
108  }
109  if (diffs & Difference::kChecksum) {
110  LogCvmfs(kLogCvmfs, kLogStderr, "content hashes differ: %s / %s",
111  a.checksum().ToString().c_str(), b.checksum().ToString().c_str());
112  retval = false;
113  }
114  if (diffs & Difference::kSymlink) {
115  LogCvmfs(kLogCvmfs, kLogStderr, "symlinks differ: %s / %s",
116  a.symlink().c_str(), b.symlink().c_str());
117  retval = false;
118  }
119  if (diffs & Difference::kExternalFileFlag) {
120  LogCvmfs(kLogCvmfs, kLogStderr, "external file flag differs: %d / %d "
121  "(%s / %s)", a.IsExternalFile(), b.IsExternalFile(),
122  a.name().c_str(), b.name().c_str());
123  retval = false;
124  }
125  if (diffs & Difference::kHasXattrsFlag) {
126  LogCvmfs(kLogCvmfs, kLogStderr, "extended attributes differ: %d / %d "
127  "(%s / %s)", a.HasXattrs(), b.HasXattrs(),
128  a.name().c_str(), b.name().c_str());
129  retval = false;
130  }
131 
132  return retval;
133 }
134 
135 
137  const catalog::Counters &b)
138 {
139  const catalog::Counters::FieldsMap map_a = a.GetFieldsMap();
140  const catalog::Counters::FieldsMap map_b = b.GetFieldsMap();
141 
142  bool retval = true;
143  catalog::Counters::FieldsMap::const_iterator i = map_a.begin();
144  catalog::Counters::FieldsMap::const_iterator iend = map_a.end();
145  for (; i != iend; ++i) {
146  catalog::Counters::FieldsMap::const_iterator comp = map_b.find(i->first);
147  assert(comp != map_b.end());
148 
149  if (*(i->second) != *(comp->second)) {
151  "catalog statistics mismatch: %s (expected: %" PRIu64 " / "
152  "in catalog: %" PRIu64 ")",
153  comp->first.c_str(), *(i->second), *(comp->second));
154  retval = false;
155  }
156  }
157 
158  return retval;
159 }
160 
161 
165 bool CommandCheck::Exists(const string &file)
166 {
167  if (!is_remote_) {
168  return FileExists(file) || SymlinkExists(file);
169  } else {
170  const string url = repo_base_path_ + "/" + file;
171  LogCvmfs(kLogCvmfs, kLogVerboseMsg, "[Exists::url] %s", url.c_str());
172  download::JobInfo head(&url, false);
173  return download_manager()->Fetch(&head) == download::kFailOk;
174  }
175 }
176 
177 
181 string CommandCheck::FetchPath(const string &path) {
182  string tmp_path;
183  FILE *f = CreateTempFile(temp_directory_ + "/cvmfstmp", kDefaultFileMode,
184  "w+", &tmp_path);
185  assert(f != NULL);
186 
187  const string url = repo_base_path_ + "/" + path;
188  if (is_remote_) {
189  cvmfs::FileSink filesink(f);
190  download::JobInfo download_job(&url, false, false, NULL, &filesink);
191  download::Failures retval = download_manager()->Fetch(&download_job);
192  if (retval != download::kFailOk) {
193  PANIC(kLogStderr, "failed to read %s", url.c_str());
194  }
195  } else {
196  bool retval = CopyPath2File(url, f);
197  if (!retval) {
198  PANIC(kLogStderr, "failed to read %s", url.c_str());
199  }
200  }
201 
202  fclose(f);
203  return tmp_path;
204 }
205 
206 
212  const shash::Any &reflog_hash,
214 {
215  LogCvmfs(kLogCvmfs, kLogStdout, "Inspecting log of references");
216  string reflog_path = FetchPath(".cvmfsreflog");
217  shash::Any computed_hash(reflog_hash.algorithm);
218  manifest::Reflog::HashDatabase(reflog_path, &computed_hash);
219  if (computed_hash != reflog_hash) {
221  "The .cvmfsreflog has unexpected content hash %s (expected %s)",
222  computed_hash.ToString().c_str(), reflog_hash.ToString().c_str());
223  unlink(reflog_path.c_str());
224  return false;
225  }
226 
228  assert(reflog.IsValid());
229  reflog->TakeDatabaseFileOwnership();
230 
231  if (!reflog->ContainsCatalog(manifest->catalog_hash())) {
233  "failed to find catalog root hash %s in .cvmfsreflog",
234  manifest->catalog_hash().ToString().c_str());
235  return false;
236  }
237 
238  if (!reflog->ContainsCertificate(manifest->certificate())) {
240  "failed to find certificate hash %s in .cvmfsreflog",
241  manifest->certificate().ToString().c_str());
242  return false;
243  }
244 
245  if (!manifest->history().IsNull() &&
246  !reflog->ContainsHistory(manifest->history()))
247  {
249  "failed to find tag database's hash %s in .cvmfsreflog",
250  manifest->history().ToString().c_str());
251  return false;
252  }
253 
254  if (!manifest->meta_info().IsNull() &&
255  !reflog->ContainsMetainfo(manifest->meta_info()))
256  {
258  "failed to find meta info hash %s in .cvmfsreflog",
259  manifest->meta_info().ToString().c_str());
260  return false;
261  }
262 
263  return true;
264 }
265 
266 
271  LogCvmfs(kLogCvmfs, kLogStdout, "Inspecting tag database");
272  bool retval;
273  vector<history::History::Tag> tags;
274  retval = history->List(&tags);
275  if (!retval) {
276  LogCvmfs(kLogCvmfs, kLogStderr, "failed to enumerate tags");
277  return false;
278  }
279  vector<history::History::Branch> branches;
280  retval = history->ListBranches(&branches);
281  if (!retval) {
282  LogCvmfs(kLogCvmfs, kLogStderr, "failed to enumerate branches");
283  return false;
284  }
285 
286  bool result = true;
287 
288  map<string, uint64_t> initial_revisions;
289  sanitizer::BranchSanitizer sanitizer;
290  for (unsigned i = 0; i < branches.size(); ++i) {
291  if (!sanitizer.IsValid(branches[i].branch)) {
292  LogCvmfs(kLogCvmfs, kLogStderr, "invalid branch name: %s",
293  branches[i].branch.c_str());
294  result = false;
295  }
296  initial_revisions[branches[i].branch] = branches[i].initial_revision;
297  }
298 
299  set<string> used_branches; // all branches referenced in tag db
300  // TODO(jblomer): same root hash implies same size and revision
301  for (unsigned i = 0; i < tags.size(); ++i) {
302  used_branches.insert(tags[i].branch);
303  const map<string, uint64_t>::const_iterator iter =
304  initial_revisions.find(tags[i].branch);
305  if (iter == initial_revisions.end()) {
306  LogCvmfs(kLogCvmfs, kLogStderr, "invalid branch %s in tag %s",
307  tags[i].branch.c_str(), tags[i].name.c_str());
308  result = false;
309  } else {
310  if (tags[i].revision < iter->second) {
311  LogCvmfs(kLogCvmfs, kLogStderr, "invalid revision %" PRIu64
312  " of tag %s", tags[i].revision, tags[i].name.c_str());
313  result = false;
314  }
315  }
316  }
317 
318  if (used_branches.size() != branches.size()) {
319  LogCvmfs(kLogCvmfs, kLogStderr, "unused, dangling branches stored");
320  result = false;
321  }
322 
323  return result;
324 }
325 
326 
335  const PathString &path,
336  catalog::DeltaCounters *computed_counters,
337  set<PathString> *bind_mountpoints)
338 {
340  catalog::DirectoryEntry this_directory;
341 
342  if (!catalog->LookupPath(path, &this_directory)) {
343  LogCvmfs(kLogCvmfs, kLogStderr, "failed to lookup %s",
344  path.c_str());
345  return false;
346  }
347  if (!catalog->ListingPath(path, &entries)) {
348  LogCvmfs(kLogCvmfs, kLogStderr, "failed to list %s",
349  path.c_str());
350  return false;
351  }
352 
353  uint32_t num_subdirs = 0;
354  bool retval = true;
355  typedef map< uint32_t, vector<catalog::DirectoryEntry> > HardlinkMap;
356  HardlinkMap hardlinks;
357  bool found_nested_marker = false;
358 
359  for (unsigned i = 0; i < entries.size(); ++i) {
360  // for performance reasons, keep track of files already checked
361  // and only run requests once per hash
362  const bool entry_needs_check =
363  !entries[i].checksum().IsNull() && !entries[i].IsExternalFile() &&
364  // fallback cli option can force the entry to be checked
366  !duplicates_map_.Contains(entries[i].checksum()));
367  if (entry_needs_check && !no_duplicates_map_)
368  duplicates_map_.Insert(entries[i].checksum(), 1);
369 
370  PathString full_path(path);
371  full_path.Append("/", 1);
372  full_path.Append(entries[i].name().GetChars(),
373  entries[i].name().GetLength());
374  LogCvmfs(kLogCvmfs, kLogVerboseMsg, "[path] %s [needs check] %i",
375  full_path.c_str(), entry_needs_check);
376 
377 
378  // Name must not be empty
379  if (entries[i].name().IsEmpty()) {
380  LogCvmfs(kLogCvmfs, kLogStderr, "empty path at %s",
381  full_path.c_str());
382  retval = false;
383  }
384 
385  // Catalog markers should indicate nested catalogs
386  if (entries[i].name() == NameString(string(".cvmfscatalog"))) {
387  if (catalog->mountpoint() != path) {
389  "found abandoned nested catalog marker at %s",
390  full_path.c_str());
391  retval = false;
392  }
393  found_nested_marker = true;
394  }
395 
396  // Check if checksum is not null
397  if (entries[i].IsRegular() && !entries[i].IsChunkedFile() &&
398  entries[i].checksum().IsNull())
399  {
401  "regular file pointing to zero-hash: '%s'", full_path.c_str());
402  retval = false;
403  }
404 
405  // Check if the chunk is there
406  if (check_chunks_ && entry_needs_check)
407  {
408  string chunk_path = "data/" + entries[i].checksum().MakePath();
409  if (entries[i].IsDirectory())
410  chunk_path += shash::kSuffixMicroCatalog;
411  if (!Exists(chunk_path)) {
412  LogCvmfs(kLogCvmfs, kLogStderr, "data chunk %s (%s) missing",
413  entries[i].checksum().ToString().c_str(), full_path.c_str());
414  retval = false;
415  }
416  }
417 
418  // Add hardlinks to counting map
419  if ((entries[i].linkcount() > 1) && !entries[i].IsDirectory()) {
420  if (entries[i].hardlink_group() == 0) {
421  LogCvmfs(kLogCvmfs, kLogStderr, "invalid hardlink group for %s",
422  full_path.c_str());
423  retval = false;
424  } else {
425  HardlinkMap::iterator hardlink_group =
426  hardlinks.find(entries[i].hardlink_group());
427  if (hardlink_group == hardlinks.end()) {
428  hardlinks[entries[i].hardlink_group()];
429  hardlinks[entries[i].hardlink_group()].push_back(entries[i]);
430  } else {
431  if (!CompareEntries(entries[i], (hardlink_group->second)[0], false)) {
432  LogCvmfs(kLogCvmfs, kLogStderr, "hardlink %s doesn't match",
433  full_path.c_str());
434  retval = false;
435  }
436  hardlink_group->second.push_back(entries[i]);
437  } // Hardlink added to map
438  } // Hardlink group > 0
439  } // Hardlink found
440 
441  // For any kind of entry, the linkcount should be > 0
442  if (entries[i].linkcount() == 0) {
443  LogCvmfs(kLogCvmfs, kLogStderr, "Entry %s has linkcount 0.",
444  entries[i].name().c_str());
445  retval = false;
446  }
447 
448  // Checks depending of entry type
449  if (!entries[i].IsRegular()) {
450  if (entries[i].IsDirectIo()) {
451  LogCvmfs(kLogCvmfs, kLogStderr, "invalid direct i/o flag found: %s",
452  full_path.c_str());
453  retval = false;
454  }
455  }
456  if (entries[i].IsDirectory()) {
457  computed_counters->self.directories++;
458  num_subdirs++;
459  // Directory size
460  // if (entries[i].size() < 4096) {
461  // LogCvmfs(kLogCvmfs, kLogStderr, "invalid file size for %s",
462  // full_path.c_str());
463  // retval = false;
464  // }
465  // No directory hardlinks
466  if (entries[i].hardlink_group() != 0) {
467  LogCvmfs(kLogCvmfs, kLogStderr, "directory hardlink found at %s",
468  full_path.c_str());
469  retval = false;
470  }
471  if (entries[i].IsNestedCatalogMountpoint() ||
472  entries[i].IsBindMountpoint())
473  {
474  // Find transition point
475  if (entries[i].IsNestedCatalogMountpoint())
476  computed_counters->self.nested_catalogs++;
477  shash::Any tmp;
478  uint64_t tmp2;
479  PathString mountpoint(full_path);
480  if (!catalog->FindNested(mountpoint, &tmp, &tmp2)) {
481  LogCvmfs(kLogCvmfs, kLogStderr, "nested catalog at %s not registered",
482  full_path.c_str());
483  retval = false;
484  }
485 
486  // check that the nested mountpoint is empty in the current catalog
487  catalog::DirectoryEntryList nested_entries;
488  if (catalog->ListingPath(full_path, &nested_entries) &&
489  !nested_entries.empty()) {
490  LogCvmfs(kLogCvmfs, kLogStderr, "non-empty nested catalog mountpoint "
491  "at %s.",
492  full_path.c_str());
493  retval = false;
494  }
495 
496  if (entries[i].IsBindMountpoint()) {
497  bind_mountpoints->insert(full_path);
498  if (entries[i].IsNestedCatalogMountpoint()) {
500  "bind mountpoint and nested mountpoint mutually exclusive"
501  " at %s.", full_path.c_str());
502  retval = false;
503  }
504  }
505  } else {
506  // Recurse
507  if (!Find(catalog, full_path, computed_counters, bind_mountpoints))
508  retval = false;
509  }
510  } else if (entries[i].IsLink()) {
511  computed_counters->self.symlinks++;
512  // No hash for symbolics links
513  if (!entries[i].checksum().IsNull()) {
514  LogCvmfs(kLogCvmfs, kLogStderr, "symbolic links with hash at %s",
515  full_path.c_str());
516  retval = false;
517  }
518  // Right size of symbolic link?
519  if (entries[i].size() != entries[i].symlink().GetLength()) {
520  LogCvmfs(kLogCvmfs, kLogStderr, "wrong symbolic link size for %s; "
521  "expected %u, got %lu", full_path.c_str(),
522  entries[i].symlink().GetLength(), entries[i].size());
523  retval = false;
524  }
525  } else if (entries[i].IsRegular()) {
526  computed_counters->self.regular_files++;
527  computed_counters->self.file_size += entries[i].size();
528  } else if (entries[i].IsSpecial()) {
529  computed_counters->self.specials++;
530  // Size zero for special files
531  if (entries[i].size() != 0) {
533  "unexpected non-zero special file size %s",
534  full_path.c_str());
535  retval = false;
536  }
537  // No hash for special files
538  if (!entries[i].checksum().IsNull()) {
539  LogCvmfs(kLogCvmfs, kLogStderr, "special file with hash at %s",
540  full_path.c_str());
541  retval = false;
542  }
543  // No symlink
544  if (entries[i].symlink().GetLength() > 0) {
546  "special file with non-zero symlink at %s", full_path.c_str());
547  retval = false;
548  }
549  } else {
550  LogCvmfs(kLogCvmfs, kLogStderr, "unknown file type %s",
551  full_path.c_str());
552  retval = false;
553  }
554 
555  if (entries[i].HasXattrs()) {
556  computed_counters->self.xattrs++;
557  }
558 
559  if (entries[i].IsExternalFile()) {
560  computed_counters->self.externals++;
561  computed_counters->self.external_file_size += entries[i].size();
562  if (!entries[i].IsRegular()) {
564  "only regular files can be external: %s", full_path.c_str());
565  retval = false;
566  }
567  }
568 
569  // checking file chunk integrity
570  if (entries[i].IsChunkedFile()) {
571  FileChunkList chunks;
572  catalog->ListPathChunks(full_path, entries[i].hash_algorithm(), &chunks);
573 
574  computed_counters->self.chunked_files++;
575  computed_counters->self.chunked_file_size += entries[i].size();
576  computed_counters->self.file_chunks += chunks.size();
577 
578  // do we find file chunks for the chunked file in this catalog?
579  if (chunks.size() == 0) {
580  LogCvmfs(kLogCvmfs, kLogStderr, "no file chunks found for big file %s",
581  full_path.c_str());
582  retval = false;
583  }
584 
585  size_t aggregated_file_size = 0;
586  off_t next_offset = 0;
587 
588  for (unsigned j = 0; j < chunks.size(); ++j) {
589  FileChunk this_chunk = chunks.At(j);
590  // check if the chunk boundaries fit together...
591  if (next_offset != this_chunk.offset()) {
592  LogCvmfs(kLogCvmfs, kLogStderr, "misaligned chunk offsets for %s",
593  full_path.c_str());
594  retval = false;
595  }
596  next_offset = this_chunk.offset() + this_chunk.size();
597  aggregated_file_size += this_chunk.size();
598 
599  // are all data chunks in the data store?
600  if (check_chunks_ && !entries[i].IsExternalFile()) {
601  const shash::Any &chunk_hash = this_chunk.content_hash();
602  // for performance reasons, only perform the check once
603  // and skip if the hash has been checked before
604  bool chunk_needs_check = true;
605  if (!no_duplicates_map_ && !duplicates_map_.Contains(chunk_hash)) {
606  duplicates_map_.Insert(chunk_hash, 1);
607  } else if (!no_duplicates_map_) {
608  chunk_needs_check = false;
609  }
610  if (chunk_needs_check) {
611  const string chunk_path = "data/" + chunk_hash.MakePath();
612  if (!Exists(chunk_path)) {
613  LogCvmfs(kLogCvmfs, kLogStderr, "partial data chunk %s (%s -> "
614  "offset: %ld | size: %lu) missing",
615  this_chunk.content_hash().ToStringWithSuffix().c_str(),
616  full_path.c_str(),
617  this_chunk.offset(),
618  this_chunk.size());
619  retval = false;
620  }
621  }
622  }
623  }
624 
625  // is the aggregated chunk size equal to the actual file size?
626  if (aggregated_file_size != entries[i].size()) {
627  LogCvmfs(kLogCvmfs, kLogStderr, "chunks of file %s produce a size "
628  "mismatch. Calculated %zu bytes | %lu "
629  "bytes expected",
630  full_path.c_str(),
631  aggregated_file_size,
632  entries[i].size());
633  retval = false;
634  }
635  }
636  } // Loop through entries
637 
638  // Check if nested catalog marker has been found
639  if (!path.IsEmpty() && (path == catalog->mountpoint()) &&
640  !found_nested_marker)
641  {
642  LogCvmfs(kLogCvmfs, kLogStderr, "nested catalog without marker at %s",
643  path.c_str());
644  retval = false;
645  }
646 
647  // Check directory linkcount
648  if (this_directory.linkcount() != num_subdirs + 2) {
649  LogCvmfs(kLogCvmfs, kLogStderr, "wrong linkcount for %s; "
650  "expected %u, got %u",
651  path.c_str(), num_subdirs + 2, this_directory.linkcount());
652  retval = false;
653  }
654 
655  // Check hardlink linkcounts
656  for (HardlinkMap::const_iterator i = hardlinks.begin(),
657  iEnd = hardlinks.end(); i != iEnd; ++i)
658  {
659  if (i->second[0].linkcount() != i->second.size()) {
660  LogCvmfs(kLogCvmfs, kLogStderr, "hardlink linkcount wrong for %s, "
661  "expected %lu, got %u",
662  (path.ToString() + "/" + i->second[0].name().ToString()).c_str(),
663  i->second.size(), i->second[0].linkcount());
664  retval = false;
665  }
666  }
667 
668  return retval;
669 }
670 
671 
672 string CommandCheck::DownloadPiece(const shash::Any catalog_hash) {
673  string source = "data/" + catalog_hash.MakePath();
674  const string dest = temp_directory_ + "/" + catalog_hash.ToString();
675  const string url = repo_base_path_ + "/" + source;
676 
677  cvmfs::PathSink pathsink(dest);
678  download::JobInfo download_catalog(&url, true, false, &catalog_hash,
679  &pathsink);
680  download::Failures retval = download_manager()->Fetch(&download_catalog);
681  if (retval != download::kFailOk) {
682  LogCvmfs(kLogCvmfs, kLogStderr, "failed to download object %s (%d)",
683  catalog_hash.ToString().c_str(), retval);
684  return "";
685  }
686 
687  return dest;
688 }
689 
690 
691 string CommandCheck::DecompressPiece(const shash::Any catalog_hash) {
692  string source = "data/" + catalog_hash.MakePath();
693  const string dest = temp_directory_ + "/" + catalog_hash.ToString();
694  if (!zlib::DecompressPath2Path(source, dest))
695  return "";
696 
697  return dest;
698 }
699 
700 
702  const shash::Any &catalog_hash,
703  const uint64_t catalog_size) {
704  string tmp_file;
705  if (!is_remote_)
706  tmp_file = DecompressPiece(catalog_hash);
707  else
708  tmp_file = DownloadPiece(catalog_hash);
709 
710  if (tmp_file == "") {
711  LogCvmfs(kLogCvmfs, kLogStderr, "failed to load catalog %s",
712  catalog_hash.ToString().c_str());
713  return NULL;
714  }
715 
716  catalog::Catalog *catalog =
717  catalog::Catalog::AttachFreely(path, tmp_file, catalog_hash);
718  int64_t catalog_file_size = GetFileSize(tmp_file);
719  assert(catalog_file_size > 0);
720  unlink(tmp_file.c_str());
721 
722  if ((catalog_size > 0) && (uint64_t(catalog_file_size) != catalog_size)) {
723  LogCvmfs(kLogCvmfs, kLogStderr, "catalog file size mismatch, "
724  "expected %" PRIu64 ", got %" PRIu64,
725  catalog_size, catalog_file_size);
726  delete catalog;
727  return NULL;
728  }
729 
730  return catalog;
731 }
732 
733 
734 bool CommandCheck::FindSubtreeRootCatalog(const string &subtree_path,
735  shash::Any *root_hash,
736  uint64_t *root_size) {
737  catalog::Catalog *current_catalog = FetchCatalog("", *root_hash);
738  if (current_catalog == NULL) {
739  return false;
740  }
741 
742  typedef vector<string> Tokens;
743  const Tokens path_tokens = SplitString(subtree_path, '/');
744 
745  string current_path = "";
746 
747  Tokens::const_iterator i = path_tokens.begin();
748  Tokens::const_iterator iend = path_tokens.end();
749  for (; i != iend; ++i) {
750  if (i->empty()) {
751  continue;
752  }
753 
754  current_path += "/" + *i;
755  if (current_catalog->FindNested(PathString(current_path),
756  root_hash,
757  root_size)) {
758  delete current_catalog;
759 
760  if (current_path.length() < subtree_path.length()) {
761  current_catalog = FetchCatalog(current_path, *root_hash);
762  if (current_catalog == NULL) {
763  break;
764  }
765  } else {
766  return true;
767  }
768  }
769  }
770  return false;
771 }
772 
773 
777 bool CommandCheck::InspectTree(const string &path,
778  const shash::Any &catalog_hash,
779  const uint64_t catalog_size,
780  const bool is_nested_catalog,
781  const catalog::DirectoryEntry *transition_point,
782  catalog::DeltaCounters *computed_counters)
783 {
784  LogCvmfs(kLogCvmfs, kLogStdout | kLogInform, "[inspecting catalog] %s at %s",
785  catalog_hash.ToString().c_str(), path == "" ? "/" : path.c_str());
786 
787  const catalog::Catalog *catalog = FetchCatalog(path,
788  catalog_hash,
789  catalog_size);
790  if (catalog == NULL) {
791  LogCvmfs(kLogCvmfs, kLogStderr, "failed to open catalog %s",
792  catalog_hash.ToString().c_str());
793  return false;
794  }
795 
796  int retval = true;
797 
798  if (catalog->root_prefix() != PathString(path.data(), path.length())) {
799  LogCvmfs(kLogCvmfs, kLogStderr, "root prefix mismatch; "
800  "expected %s, got %s",
801  path.c_str(), catalog->root_prefix().c_str());
802  retval = false;
803  }
804 
805  // Check transition point
806  catalog::DirectoryEntry root_entry;
807  if (!catalog->LookupPath(catalog->root_prefix(), &root_entry)) {
808  LogCvmfs(kLogCvmfs, kLogStderr, "failed to lookup root entry (%s)",
809  path.c_str());
810  retval = false;
811  }
812  if (!root_entry.IsDirectory()) {
813  LogCvmfs(kLogCvmfs, kLogStderr, "root entry not a directory (%s)",
814  path.c_str());
815  retval = false;
816  }
817  if (is_nested_catalog) {
818  if (transition_point != NULL &&
819  !CompareEntries(*transition_point, root_entry, true, true)) {
821  "transition point and root entry differ (%s)", path.c_str());
822  retval = false;
823  }
824  if (!root_entry.IsNestedCatalogRoot()) {
826  "nested catalog root expected but not found (%s)", path.c_str());
827  retval = false;
828  }
829  } else {
830  if (root_entry.IsNestedCatalogRoot()) {
832  "nested catalog root found but not expected (%s)", path.c_str());
833  retval = false;
834  }
835  }
836 
837  // Traverse the catalog
838  set<PathString> bind_mountpoints;
839  if (!Find(catalog, PathString(path.data(), path.length()),
840  computed_counters, &bind_mountpoints))
841  {
842  retval = false;
843  }
844 
845  // Check number of entries
846  if (root_entry.HasXattrs())
847  computed_counters->self.xattrs++;
848  const uint64_t num_found_entries =
849  1 +
850  computed_counters->self.regular_files +
851  computed_counters->self.symlinks +
852  computed_counters->self.specials +
853  computed_counters->self.directories;
854  if (num_found_entries != catalog->GetNumEntries()) {
855  LogCvmfs(kLogCvmfs, kLogStderr, "dangling entries in catalog, "
856  "expected %" PRIu64 ", got %" PRIu64,
857  catalog->GetNumEntries(), num_found_entries);
858  retval = false;
859  }
860 
861  // Recurse into nested catalogs
862  const catalog::Catalog::NestedCatalogList &nested_catalogs =
863  catalog->ListNestedCatalogs();
864  const catalog::Catalog::NestedCatalogList own_nested_catalogs =
865  catalog->ListOwnNestedCatalogs();
866  if (own_nested_catalogs.size() !=
867  static_cast<uint64_t>(computed_counters->self.nested_catalogs))
868  {
869  LogCvmfs(kLogCvmfs, kLogStderr, "number of nested catalogs does not match;"
870  " expected %lu, got %lu", computed_counters->self.nested_catalogs,
871  own_nested_catalogs.size());
872  retval = false;
873  }
874  set<PathString> nested_catalog_paths;
875  for (catalog::Catalog::NestedCatalogList::const_iterator i =
876  nested_catalogs.begin(), iEnd = nested_catalogs.end(); i != iEnd; ++i)
877  {
878  nested_catalog_paths.insert(i->mountpoint);
879  }
880  if (nested_catalog_paths.size() != nested_catalogs.size()) {
882  "duplicates among nested catalogs and bind mountpoints");
883  retval = false;
884  }
885 
886  for (catalog::Catalog::NestedCatalogList::const_iterator i =
887  nested_catalogs.begin(), iEnd = nested_catalogs.end(); i != iEnd; ++i)
888  {
889  if (bind_mountpoints.find(i->mountpoint) != bind_mountpoints.end()) {
890  catalog::DirectoryEntry bind_mountpoint;
891  PathString mountpoint("/" + i->mountpoint.ToString().substr(1));
892  if (!catalog->LookupPath(mountpoint, &bind_mountpoint)) {
893  LogCvmfs(kLogCvmfs, kLogStderr, "failed to lookup bind mountpoint %s",
894  mountpoint.c_str());
895  retval = false;
896  }
897  LogCvmfs(kLogCvmfs, kLogDebug, "skipping bind mountpoint %s",
898  i->mountpoint.c_str());
899  continue;
900  }
901  catalog::DirectoryEntry nested_transition_point;
902  if (!catalog->LookupPath(i->mountpoint, &nested_transition_point)) {
903  LogCvmfs(kLogCvmfs, kLogStderr, "failed to lookup transition point %s",
904  i->mountpoint.c_str());
905  retval = false;
906  } else {
907  catalog::DeltaCounters nested_counters;
908  const bool is_nested = true;
909  if (!InspectTree(i->mountpoint.ToString(), i->hash, i->size, is_nested,
910  &nested_transition_point, &nested_counters))
911  retval = false;
912  nested_counters.PopulateToParent(computed_counters);
913  }
914  }
915 
916  // Check statistics counters
917  // Additionally account for root directory
918  computed_counters->self.directories++;
919  catalog::Counters compare_counters;
920  compare_counters.ApplyDelta(*computed_counters);
921  const catalog::Counters stored_counters = catalog->GetCounters();
922  if (!CompareCounters(compare_counters, stored_counters)) {
923  LogCvmfs(kLogCvmfs, kLogStderr, "statistics counter mismatch [%s]",
924  catalog_hash.ToString().c_str());
925  retval = false;
926  }
927 
928  delete catalog;
929  return retval;
930 }
931 
932 
934  string tag_name;
935  string subtree_path = "";
936  string pubkey_path = "";
937  string trusted_certs = "";
938  string repo_name = "";
939  string reflog_chksum_path = "";
940 
941  temp_directory_ = (args.find('t') != args.end()) ? *args.find('t')->second
942  : "/tmp";
943  if (args.find('n') != args.end())
944  tag_name = *args.find('n')->second;
945  if (args.find('c') != args.end())
946  check_chunks_ = true;
947  if (args.find('d') != args.end())
948  no_duplicates_map_ = true;
949  if (args.find('l') != args.end()) {
950  unsigned log_level =
951  kLogLevel0 << String2Uint64(*args.find('l')->second);
952  if (log_level > kLogNone) {
953  LogCvmfs(kLogCvmfs, kLogStderr, "invalid log level");
954  return 1;
955  }
956  SetLogVerbosity(static_cast<LogLevels>(log_level));
957  }
958  if (args.find('k') != args.end())
959  pubkey_path = *args.find('k')->second;
960  if (DirectoryExists(pubkey_path))
961  pubkey_path = JoinStrings(FindFilesBySuffix(pubkey_path, ".pub"), ":");
962  if (args.find('z') != args.end())
963  trusted_certs = *args.find('z')->second;
964  if (args.find('N') != args.end())
965  repo_name = *args.find('N')->second;
966 
967  repo_base_path_ = MakeCanonicalPath(*args.find('r')->second);
968  if (args.find('s') != args.end())
969  subtree_path = MakeCanonicalPath(*args.find('s')->second);
970  if (args.find('R') != args.end())
971  reflog_chksum_path = *args.find('R')->second;
972 
973  // Repository can be HTTP address or on local file system
975 
976  // initialize the (swissknife global) download and signature managers
977  if (is_remote_) {
978  const bool follow_redirects = (args.count('L') > 0);
979  const string proxy = (args.count('@') > 0) ? *args.find('@')->second : "";
980  if (!this->InitDownloadManager(follow_redirects, proxy)) {
981  return 1;
982  }
983 
984  if (pubkey_path.empty() || repo_name.empty()) {
985  LogCvmfs(kLogCvmfs, kLogStderr, "please provide pubkey and repo name for "
986  "remote repositories");
987  return 1;
988  }
989 
990  if (!this->InitVerifyingSignatureManager(pubkey_path, trusted_certs)) {
991  return 1;
992  }
993  }
994 
995  // Load Manifest
997  bool successful = true;
998 
999  if (is_remote_) {
1000  manifest = FetchRemoteManifest(repo_base_path_, repo_name);
1001  } else {
1002  if (chdir(repo_base_path_.c_str()) != 0) {
1003  LogCvmfs(kLogCvmfs, kLogStderr, "failed to switch to directory %s",
1004  repo_base_path_.c_str());
1005  return 1;
1006  }
1007  manifest = OpenLocalManifest(".cvmfspublished");
1008  }
1009 
1010  if (!manifest.IsValid()) {
1011  LogCvmfs(kLogCvmfs, kLogStderr, "failed to load repository manifest");
1012  return 1;
1013  }
1014 
1015  // Check meta-info object
1016  if (!manifest->meta_info().IsNull()) {
1017  string tmp_file;
1018  if (!is_remote_)
1019  tmp_file = DecompressPiece(manifest->meta_info());
1020  else
1021  tmp_file = DownloadPiece(manifest->meta_info());
1022  if (tmp_file == "") {
1023  LogCvmfs(kLogCvmfs, kLogStderr, "failed to load repository metainfo %s",
1024  manifest->meta_info().ToString().c_str());
1025  return 1;
1026  }
1027  unlink(tmp_file.c_str());
1028  }
1029 
1030  shash::Any reflog_hash;
1031  if (!reflog_chksum_path.empty()) {
1032  if (!manifest::Reflog::ReadChecksum(reflog_chksum_path, &reflog_hash)) {
1033  LogCvmfs(kLogCvmfs, kLogStderr, "failed to read reflog checksum file");
1034  return 1;
1035  }
1036  } else {
1037  reflog_hash = manifest->reflog_hash();
1038  }
1039 
1040  if (Exists(".cvmfsreflog")) {
1041  if (reflog_hash.IsNull()) {
1042  // If there is a reflog, we want to check it
1044  ".cvmfsreflog present but no checksum provided, aborting");
1045  return 1;
1046  }
1047  bool retval = InspectReflog(reflog_hash, manifest.weak_ref());
1048  if (!retval) {
1049  LogCvmfs(kLogCvmfs, kLogStderr, "failed to verify reflog");
1050  return 1;
1051  }
1052  } else {
1053  if (!reflog_hash.IsNull()) {
1054  // There is a checksum but no reflog; possibly the checksum is for the
1055  // from the manifest for the stratum 0 reflog
1056  if (!reflog_chksum_path.empty()) {
1058  "local reflog checksum set but reflog itself is missing, "
1059  "aborting");
1060  return 1;
1061  }
1062  }
1063  }
1064 
1065  // Load history
1067  if (!manifest->history().IsNull()) {
1068  string tmp_file;
1069  if (!is_remote_)
1070  tmp_file = DecompressPiece(manifest->history());
1071  else
1072  tmp_file = DownloadPiece(manifest->history());
1073  if (tmp_file == "") {
1074  LogCvmfs(kLogCvmfs, kLogStderr, "failed to load history database %s",
1075  manifest->history().ToString().c_str());
1076  return 1;
1077  }
1078  tag_db = history::SqliteHistory::Open(tmp_file);
1079  if (!tag_db.IsValid()) {
1080  LogCvmfs(kLogCvmfs, kLogStderr, "failed to open history database %s",
1081  manifest->history().ToString().c_str());
1082  return 1;
1083  }
1084  tag_db->TakeDatabaseFileOwnership();
1085  successful = InspectHistory(tag_db.weak_ref()) && successful;
1086  }
1087 
1088  if (manifest->has_alt_catalog_path()) {
1089  if (!Exists(manifest->certificate().MakeAlternativePath())) {
1091  "failed to find alternative certificate link %s",
1092  manifest->certificate().MakeAlternativePath().c_str());
1093  return 1;
1094  }
1095  if (!Exists(manifest->catalog_hash().MakeAlternativePath())) {
1097  "failed to find alternative catalog link %s",
1098  manifest->catalog_hash().MakeAlternativePath().c_str());
1099  return 1;
1100  }
1101  }
1102 
1103  shash::Any root_hash = manifest->catalog_hash();
1104  uint64_t root_size = manifest->catalog_size();
1105  if (tag_name != "") {
1106  if (!tag_db.IsValid()) {
1107  LogCvmfs(kLogCvmfs, kLogStderr, "no history");
1108  return 1;
1109  }
1111  const bool retval = tag_db->GetByName(tag_name, &tag);
1112  if (!retval) {
1113  LogCvmfs(kLogCvmfs, kLogStderr, "no such tag: %s", tag_name.c_str());
1114  return 1;
1115  }
1116  root_hash = tag.root_hash;
1117  root_size = tag.size;
1118  LogCvmfs(kLogCvmfs, kLogStdout, "Inspecting repository tag %s",
1119  tag_name.c_str());
1120  }
1121 
1122  const bool is_nested_catalog = (!subtree_path.empty());
1123  if (is_nested_catalog && !FindSubtreeRootCatalog( subtree_path,
1124  &root_hash,
1125  &root_size)) {
1126  LogCvmfs(kLogCvmfs, kLogStderr, "cannot find nested catalog at %s",
1127  subtree_path.c_str());
1128  return 1;
1129  }
1130 
1131 
1132  catalog::DeltaCounters computed_counters;
1133  successful = InspectTree(subtree_path,
1134  root_hash,
1135  root_size,
1136  is_nested_catalog,
1137  NULL,
1138  &computed_counters) && successful;
1139 
1140  if (!successful) {
1141  LogCvmfs(kLogCvmfs, kLogStderr, "CATALOG PROBLEMS OR OTHER ERRORS FOUND");
1142  return 1;
1143  }
1144 
1145  LogCvmfs(kLogCvmfs, kLogStdout, "no problems found");
1146  return 0;
1147 }
1148 
1149 } // namespace swissknife
uint32_t linkcount() const
void SetLogVerbosity(const LogLevels max_level)
Definition: logging.cc:261
void TakeDatabaseFileOwnership()
Definition: reflog.cc:307
bool IsExternalFile() const
bool ContainsHistory(const shash::Any &history) const
Definition: reflog.cc:239
bool InspectHistory(history::History *history)
bool IsNull() const
Definition: hash.h:383
Differences CompareTo(const DirectoryEntry &other) const
const manifest::Manifest * manifest() const
Definition: repository.h:125
ShortString< kDefaultMaxName, 1 > NameString
Definition: shortstring.h:218
Item At(const size_t index) const
Definition: bigvector.h:50
time_t mtime() const
bool IsDirectory() const
static SqliteHistory * Open(const std::string &file_name)
bool ListPathChunks(const PathString &path, const shash::Algorithms interpret_hashes_as, FileChunkList *chunks) const
Definition: catalog.h:147
T * weak_ref() const
Definition: pointer.h:42
const int kDefaultFileMode
Definition: posix.h:32
static bool ReadChecksum(const std::string &path, shash::Any *checksum)
Definition: reflog.cc:47
#define PANIC(...)
Definition: exception.h:29
FILE * CreateTempFile(const std::string &path_prefix, const int mode, const char *open_flags, std::string *final_path)
Definition: posix.cc:1005
uint64_t size() const
string JoinStrings(const vector< string > &strings, const string &joint)
Definition: string.cc:325
std::string ToString(const bool with_suffix=false) const
Definition: hash.h:249
const history::History * history() const
std::string ToStringWithSuffix() const
Definition: hash.h:304
bool LookupPath(const PathString &path, DirectoryEntry *dirent) const
Definition: catalog.h:124
bool CompareEntries(const catalog::DirectoryEntry &a, const catalog::DirectoryEntry &b, const bool compare_names, const bool is_transition_point=false)
void ApplyDelta(const DeltaCounters &delta)
bool IsHttpUrl(const std::string &path)
Definition: posix.cc:168
bool ListingPath(const PathString &path, DirectoryEntryList *listing, const bool expand_symlink=true) const
Definition: catalog.h:132
const char kSuffixMicroCatalog
Definition: hash.h:56
manifest::Manifest * FetchRemoteManifest(const std::string &repository_url, const std::string &repository_name, const shash::Any &base_hash=shash::Any()) const
Definition: server_tool.cc:133
const shash::Any & content_hash() const
Definition: file_chunk.h:41
catalog::Catalog * FetchCatalog(const std::string &path, const shash::Any &catalog_hash, const uint64_t catalog_size=0)
std::map< std::string, const Counters_t * > FieldsMap
assert((mem||(size==0))&&"Out Of Memory")
bool InspectReflog(const shash::Any &reflog_hash, manifest::Manifest *manifest)
Algorithms algorithm
Definition: hash.h:125
shash::Any checksum() const
static uint32_t hasher_any(const shash::Any &key)
bool InitVerifyingSignatureManager(const std::string &pubkey_path, const std::string &trusted_certs="")
Definition: server_tool.cc:44
unsigned int mode() const
unsigned char digest[digest_size_]
Definition: hash.h:124
bool Find(const catalog::Catalog *catalog, const PathString &path, catalog::DeltaCounters *computed_counters, std::set< PathString > *bind_mountpoints)
bool ContainsMetainfo(const shash::Any &metainfo) const
Definition: reflog.cc:245
bool CopyPath2File(const std::string &src, FILE *fdest)
Definition: compression.cc:46
bool SymlinkExists(const std::string &path)
Definition: posix.cc:823
static Reflog * Open(const std::string &database_path)
Definition: reflog.cc:17
bool IsValid(const std::string &input) const
Definition: sanitizer.cc:114
bool IsNestedCatalogRoot() const
bool FileExists(const std::string &path)
Definition: posix.cc:791
std::string DownloadPiece(const shash::Any catalog_hash)
std::vector< DirectoryEntry > DirectoryEntryList
NameString name() const
int Main(const ArgumentList &args)
virtual bool ListBranches(std::vector< Branch > *branches) const =0
std::string DecompressPiece(const shash::Any catalog_hash)
bool HasXattrs() const
download::DownloadManager * download_manager() const
Definition: server_tool.cc:108
virtual bool List(std::vector< Tag > *tags) const =0
vector< string > SplitString(const string &str, char delim)
Definition: string.cc:290
bool Exists(const std::string &file)
off_t offset() const
Definition: file_chunk.h:42
shash::Any certificate() const
Definition: manifest.h:134
void PopulateToParent(DeltaCounters *parent) const
std::string FetchPath(const std::string &path)
LinkString symlink() const
void Insert(const Key &key, const Value &value)
Definition: smallhash.h:109
PathString mountpoint() const
Definition: catalog.h:179
static void HashDatabase(const std::string &database_path, shash::Any *hash_reflog)
Definition: reflog.cc:322
shash::Any catalog_hash() const
Definition: manifest.h:132
void Append(const char *chars, const unsigned length)
Definition: shortstring.h:80
SmallHashDynamic< shash::Any, char > duplicates_map_
bool FindSubtreeRootCatalog(const std::string &subtree_path, shash::Any *root_hash, uint64_t *root_size)
bool DirectoryExists(const std::string &path)
Definition: posix.cc:813
bool CompareCounters(const catalog::Counters &a, const catalog::Counters &b)
bool Contains(const Key &key) const
Definition: smallhash.h:102
manifest::Manifest * OpenLocalManifest(const std::string path) const
Definition: server_tool.cc:118
std::string ToString() const
Definition: shortstring.h:141
std::vector< NestedCatalog > NestedCatalogList
Definition: catalog.h:208
bool ContainsCatalog(const shash::Any &catalog) const
Definition: reflog.cc:222
bool IsEmpty() const
Definition: shortstring.h:137
uint64_t String2Uint64(const string &value)
Definition: string.cc:228
std::map< char, SharedPtr< std::string > > ArgumentList
Definition: swissknife.h:72
ShortString< kDefaultMaxPath, 0 > PathString
Definition: shortstring.h:217
Failures Fetch(JobInfo *info)
Definition: download.cc:1834
size_t size() const
Definition: file_chunk.h:43
shash::Any history() const
Definition: manifest.h:135
bool InspectTree(const std::string &path, const shash::Any &catalog_hash, const uint64_t catalog_size, const bool is_nested_catalog, const catalog::DirectoryEntry *transition_point, catalog::DeltaCounters *computed_counters)
shash::Any root_hash
Definition: history.h:89
static Catalog * AttachFreely(const std::string &imaginary_mountpoint, const std::string &file, const shash::Any &catalog_hash, Catalog *parent=NULL, const bool is_nested=false)
Definition: catalog.cc:29
bool FindNested(const PathString &mountpoint, shash::Any *hash, uint64_t *size) const
Definition: catalog.cc:690
bool DecompressPath2Path(const string &src, const string &dest)
Definition: compression.cc:381
int64_t GetFileSize(const std::string &path)
Definition: posix.cc:801
const int kLogVerboseMsg
std::string MakePath() const
Definition: hash.h:316
std::string MakeCanonicalPath(const std::string &path)
Definition: posix.cc:98
void Init(uint32_t expected_size, Key empty, uint32_t(*hasher)(const Key &key))
Definition: smallhash.h:60
bool InitDownloadManager(const bool follow_redirects, const std::string &proxy, const unsigned max_pool_handles=1)
Definition: server_tool.cc:17
const char * c_str() const
Definition: shortstring.h:145
uint32_t hardlink_group() const
bool ContainsCertificate(const shash::Any &certificate) const
Definition: reflog.cc:215
static void size_t size
Definition: smalloc.h:54
std::vector< std::string > FindFilesBySuffix(const std::string &dir, const std::string &suffix)
Definition: posix.cc:1124
unsigned int Differences
shash::Any meta_info() const
Definition: manifest.h:139
size_t size() const
Definition: bigvector.h:121
CVMFS_EXPORT void LogCvmfs(const LogSource source, const int mask, const char *format,...)
Definition: logging.cc:528