CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
swissknife_check.cc
Go to the documentation of this file.
1 
7 #define __STDC_FORMAT_MACROS
8 
9 #include "cvmfs_config.h"
10 #include "swissknife_check.h"
11 
12 #include <inttypes.h>
13 #include <unistd.h>
14 
15 #include <cassert>
16 #include <map>
17 #include <queue>
18 #include <set>
19 #include <string>
20 #include <vector>
21 
22 #include "catalog_sql.h"
23 #include "compression.h"
24 #include "download.h"
25 #include "file_chunk.h"
26 #include "history_sqlite.h"
27 #include "logging.h"
28 #include "manifest.h"
29 #include "reflog.h"
30 #include "sanitizer.h"
31 #include "shortstring.h"
32 #include "util/exception.h"
33 #include "util/pointer.h"
34 #include "util/posix.h"
35 
36 using namespace std; // NOLINT
37 
38 namespace swissknife {
39 
40 bool CommandCheck::CompareEntries(const catalog::DirectoryEntry &a,
41  const catalog::DirectoryEntry &b,
42  const bool compare_names,
43  const bool is_transition_point)
44 {
45  typedef catalog::DirectoryEntry::Difference Difference;
46 
48  if (diffs == Difference::kIdentical) {
49  return true;
50  }
51 
52  // in case of a nested catalog transition point the controlling flags are
53  // supposed to differ. If this is the only difference we are done...
54  if (is_transition_point &&
55  (diffs ^ Difference::kNestedCatalogTransitionFlags) == 0) {
56  return true;
57  }
58 
59  bool retval = true;
60  if (compare_names) {
61  if (diffs & Difference::kName) {
62  LogCvmfs(kLogCvmfs, kLogStderr, "names differ: %s / %s",
63  a.name().c_str(), b.name().c_str());
64  retval = false;
65  }
66  }
67  if (diffs & Difference::kLinkcount) {
68  LogCvmfs(kLogCvmfs, kLogStderr, "linkcounts differ: %lu / %lu",
69  a.linkcount(), b.linkcount());
70  retval = false;
71  }
72  if (diffs & Difference::kHardlinkGroup) {
73  LogCvmfs(kLogCvmfs, kLogStderr, "hardlink groups differ: %lu / %lu",
75  retval = false;
76  }
77  if (diffs & Difference::kSize) {
78  LogCvmfs(kLogCvmfs, kLogStderr, "sizes differ: %" PRIu64 " / %" PRIu64,
79  a.size(), b.size());
80  retval = false;
81  }
82  if (diffs & Difference::kMode) {
83  LogCvmfs(kLogCvmfs, kLogStderr, "modes differ: %lu / %lu",
84  a.mode(), b.mode());
85  retval = false;
86  }
87  if (diffs & Difference::kMtime) {
88  LogCvmfs(kLogCvmfs, kLogStderr, "timestamps differ: %lu / %lu",
89  a.mtime(), b.mtime());
90  retval = false;
91  }
92  if (diffs & Difference::kChecksum) {
93  LogCvmfs(kLogCvmfs, kLogStderr, "content hashes differ: %s / %s",
94  a.checksum().ToString().c_str(), b.checksum().ToString().c_str());
95  retval = false;
96  }
97  if (diffs & Difference::kSymlink) {
98  LogCvmfs(kLogCvmfs, kLogStderr, "symlinks differ: %s / %s",
99  a.symlink().c_str(), b.symlink().c_str());
100  retval = false;
101  }
102  if (diffs & Difference::kExternalFileFlag) {
103  LogCvmfs(kLogCvmfs, kLogStderr, "external file flag differs: %d / %d "
104  "(%s / %s)", a.IsExternalFile(), b.IsExternalFile(),
105  a.name().c_str(), b.name().c_str());
106  retval = false;
107  }
108  if (diffs & Difference::kHasXattrsFlag) {
109  LogCvmfs(kLogCvmfs, kLogStderr, "extended attributes differ: %d / %d "
110  "(%s / %s)", a.HasXattrs(), b.HasXattrs(),
111  a.name().c_str(), b.name().c_str());
112  retval = false;
113  }
114 
115  return retval;
116 }
117 
118 
119 bool CommandCheck::CompareCounters(const catalog::Counters &a,
120  const catalog::Counters &b)
121 {
122  const catalog::Counters::FieldsMap map_a = a.GetFieldsMap();
123  const catalog::Counters::FieldsMap map_b = b.GetFieldsMap();
124 
125  bool retval = true;
126  catalog::Counters::FieldsMap::const_iterator i = map_a.begin();
127  catalog::Counters::FieldsMap::const_iterator iend = map_a.end();
128  for (; i != iend; ++i) {
129  catalog::Counters::FieldsMap::const_iterator comp = map_b.find(i->first);
130  assert(comp != map_b.end());
131 
132  if (*(i->second) != *(comp->second)) {
134  "catalog statistics mismatch: %s (expected: %" PRIu64 " / "
135  "in catalog: %" PRIu64 ")",
136  comp->first.c_str(), *(i->second), *(comp->second));
137  retval = false;
138  }
139  }
140 
141  return retval;
142 }
143 
144 
148 bool CommandCheck::Exists(const string &file)
149 {
150  if (!is_remote_) {
151  return FileExists(file) || SymlinkExists(file);
152  } else {
153  const string url = repo_base_path_ + "/" + file;
154  download::JobInfo head(&url, false);
155  return download_manager()->Fetch(&head) == download::kFailOk;
156  }
157 }
158 
159 
163 string CommandCheck::FetchPath(const string &path) {
164  string tmp_path;
165  FILE *f = CreateTempFile(temp_directory_ + "/cvmfstmp", kDefaultFileMode,
166  "w+", &tmp_path);
167  assert(f != NULL);
168 
169  const string url = repo_base_path_ + "/" + path;
170  if (is_remote_) {
171  download::JobInfo download_job(&url, false, false, f, NULL);
172  download::Failures retval = download_manager()->Fetch(&download_job);
173  if (retval != download::kFailOk) {
174  PANIC(kLogStderr, "failed to read %s", url.c_str());
175  }
176  } else {
177  bool retval = CopyPath2File(url, f);
178  if (!retval) {
179  PANIC(kLogStderr, "failed to read %s", url.c_str());
180  }
181  }
182 
183  fclose(f);
184  return tmp_path;
185 }
186 
187 
192 bool CommandCheck::InspectReflog(
193  const shash::Any &reflog_hash,
195 {
196  LogCvmfs(kLogCvmfs, kLogStdout, "Inspecting log of references");
197  string reflog_path = FetchPath(".cvmfsreflog");
198  shash::Any computed_hash(reflog_hash.algorithm);
199  manifest::Reflog::HashDatabase(reflog_path, &computed_hash);
200  if (computed_hash != reflog_hash) {
202  "The .cvmfsreflog has unexpected content hash %s (expected %s)",
203  computed_hash.ToString().c_str(), reflog_hash.ToString().c_str());
204  unlink(reflog_path.c_str());
205  return false;
206  }
207 
209  assert(reflog.IsValid());
210  reflog->TakeDatabaseFileOwnership();
211 
212  if (!reflog->ContainsCatalog(manifest->catalog_hash())) {
214  "failed to find catalog root hash %s in .cvmfsreflog",
215  manifest->catalog_hash().ToString().c_str());
216  return false;
217  }
218 
219  if (!reflog->ContainsCertificate(manifest->certificate())) {
221  "failed to find certificate hash %s in .cvmfsreflog",
222  manifest->certificate().ToString().c_str());
223  return false;
224  }
225 
226  if (!manifest->history().IsNull() &&
227  !reflog->ContainsHistory(manifest->history()))
228  {
230  "failed to find tag database's hash %s in .cvmfsreflog",
231  manifest->history().ToString().c_str());
232  return false;
233  }
234 
235  if (!manifest->meta_info().IsNull() &&
236  !reflog->ContainsMetainfo(manifest->meta_info()))
237  {
239  "failed to find meta info hash %s in .cvmfsreflog",
240  manifest->meta_info().ToString().c_str());
241  return false;
242  }
243 
244  return true;
245 }
246 
247 
251 bool CommandCheck::InspectHistory(history::History *history) {
252  LogCvmfs(kLogCvmfs, kLogStdout, "Inspecting tag database");
253  bool retval;
254  vector<history::History::Tag> tags;
255  retval = history->List(&tags);
256  if (!retval) {
257  LogCvmfs(kLogCvmfs, kLogStderr, "failed to enumerate tags");
258  return false;
259  }
260  vector<history::History::Branch> branches;
261  retval = history->ListBranches(&branches);
262  if (!retval) {
263  LogCvmfs(kLogCvmfs, kLogStderr, "failed to enumerate branches");
264  return false;
265  }
266 
267  bool result = true;
268 
269  map<string, unsigned> initial_revisions;
270  sanitizer::BranchSanitizer sanitizer;
271  for (unsigned i = 0; i < branches.size(); ++i) {
272  if (!sanitizer.IsValid(branches[i].branch)) {
273  LogCvmfs(kLogCvmfs, kLogStderr, "invalid branch name: %s",
274  branches[i].branch.c_str());
275  result = false;
276  }
277  initial_revisions[branches[i].branch] = branches[i].initial_revision;
278  }
279 
280  set<string> used_branches; // all branches referenced in tag db
281  // TODO(jblomer): same root hash implies same size and revision
282  for (unsigned i = 0; i < tags.size(); ++i) {
283  used_branches.insert(tags[i].branch);
284  map<string, unsigned>::const_iterator iter =
285  initial_revisions.find(tags[i].branch);
286  if (iter == initial_revisions.end()) {
287  LogCvmfs(kLogCvmfs, kLogStderr, "invalid branch %s in tag %s",
288  tags[i].branch.c_str(), tags[i].name.c_str());
289  result = false;
290  } else {
291  if (tags[i].revision < iter->second) {
292  LogCvmfs(kLogCvmfs, kLogStderr, "invalid revision %u of tag %s",
293  tags[i].revision, tags[i].name.c_str());
294  result = false;
295  }
296  }
297  }
298 
299  if (used_branches.size() != branches.size()) {
300  LogCvmfs(kLogCvmfs, kLogStderr, "unused, dangling branches stored");
301  result = false;
302  }
303 
304  return result;
305 }
306 
307 
311 bool CommandCheck::Find(const catalog::Catalog *catalog,
312  const PathString &path,
313  catalog::DeltaCounters *computed_counters,
314  set<PathString> *bind_mountpoints)
315 {
317  catalog::DirectoryEntry this_directory;
318 
319  if (!catalog->LookupPath(path, &this_directory)) {
320  LogCvmfs(kLogCvmfs, kLogStderr, "failed to lookup %s",
321  path.c_str());
322  return false;
323  }
324  if (!catalog->ListingPath(path, &entries)) {
325  LogCvmfs(kLogCvmfs, kLogStderr, "failed to list %s",
326  path.c_str());
327  return false;
328  }
329 
330  uint32_t num_subdirs = 0;
331  bool retval = true;
332  typedef map< uint32_t, vector<catalog::DirectoryEntry> > HardlinkMap;
333  HardlinkMap hardlinks;
334  bool found_nested_marker = false;
335 
336  for (unsigned i = 0; i < entries.size(); ++i) {
337  PathString full_path(path);
338  full_path.Append("/", 1);
339  full_path.Append(entries[i].name().GetChars(),
340  entries[i].name().GetLength());
341  LogCvmfs(kLogCvmfs, kLogVerboseMsg, "[path] %s",
342  full_path.c_str());
343 
344  // Name must not be empty
345  if (entries[i].name().IsEmpty()) {
346  LogCvmfs(kLogCvmfs, kLogStderr, "empty path at %s",
347  full_path.c_str());
348  retval = false;
349  }
350 
351  // Catalog markers should indicate nested catalogs
352  if (entries[i].name() == NameString(string(".cvmfscatalog"))) {
353  if (catalog->mountpoint() != path) {
355  "found abandoned nested catalog marker at %s",
356  full_path.c_str());
357  retval = false;
358  }
359  found_nested_marker = true;
360  }
361 
362  // Check if checksum is not null
363  if (entries[i].IsRegular() && !entries[i].IsChunkedFile() &&
364  entries[i].checksum().IsNull())
365  {
367  "regular file pointing to zero-hash: '%s'", full_path.c_str());
368  retval = false;
369  }
370 
371  // Check if the chunk is there
372  if (check_chunks_ &&
373  !entries[i].checksum().IsNull() && !entries[i].IsExternalFile())
374  {
375  string chunk_path = "data/" + entries[i].checksum().MakePath();
376  if (entries[i].IsDirectory())
377  chunk_path += shash::kSuffixMicroCatalog;
378  if (!Exists(chunk_path)) {
379  LogCvmfs(kLogCvmfs, kLogStderr, "data chunk %s (%s) missing",
380  entries[i].checksum().ToString().c_str(), full_path.c_str());
381  retval = false;
382  }
383  }
384 
385  // Add hardlinks to counting map
386  if ((entries[i].linkcount() > 1) && !entries[i].IsDirectory()) {
387  if (entries[i].hardlink_group() == 0) {
388  LogCvmfs(kLogCvmfs, kLogStderr, "invalid hardlink group for %s",
389  full_path.c_str());
390  retval = false;
391  } else {
392  HardlinkMap::iterator hardlink_group =
393  hardlinks.find(entries[i].hardlink_group());
394  if (hardlink_group == hardlinks.end()) {
395  hardlinks[entries[i].hardlink_group()];
396  hardlinks[entries[i].hardlink_group()].push_back(entries[i]);
397  } else {
398  if (!CompareEntries(entries[i], (hardlink_group->second)[0], false)) {
399  LogCvmfs(kLogCvmfs, kLogStderr, "hardlink %s doesn't match",
400  full_path.c_str());
401  retval = false;
402  }
403  hardlink_group->second.push_back(entries[i]);
404  } // Hardlink added to map
405  } // Hardlink group > 0
406  } // Hardlink found
407 
408  // For any kind of entry, the linkcount should be > 0
409  if (entries[i].linkcount() == 0) {
410  LogCvmfs(kLogCvmfs, kLogStderr, "Entry %s has linkcount 0.",
411  entries[i].name().c_str());
412  retval = false;
413  }
414 
415  // Checks depending of entry type
416  if (!entries[i].IsRegular()) {
417  if (entries[i].IsDirectIo()) {
418  LogCvmfs(kLogCvmfs, kLogStderr, "invalid direct i/o flag found: %s",
419  full_path.c_str());
420  retval = false;
421  }
422  }
423  if (entries[i].IsDirectory()) {
424  computed_counters->self.directories++;
425  num_subdirs++;
426  // Directory size
427  // if (entries[i].size() < 4096) {
428  // LogCvmfs(kLogCvmfs, kLogStderr, "invalid file size for %s",
429  // full_path.c_str());
430  // retval = false;
431  // }
432  // No directory hardlinks
433  if (entries[i].hardlink_group() != 0) {
434  LogCvmfs(kLogCvmfs, kLogStderr, "directory hardlink found at %s",
435  full_path.c_str());
436  retval = false;
437  }
438  if (entries[i].IsNestedCatalogMountpoint() ||
439  entries[i].IsBindMountpoint())
440  {
441  // Find transition point
442  if (entries[i].IsNestedCatalogMountpoint())
443  computed_counters->self.nested_catalogs++;
444  shash::Any tmp;
445  uint64_t tmp2;
446  PathString mountpoint(full_path);
447  if (!catalog->FindNested(mountpoint, &tmp, &tmp2)) {
448  LogCvmfs(kLogCvmfs, kLogStderr, "nested catalog at %s not registered",
449  full_path.c_str());
450  retval = false;
451  }
452 
453  // check that the nested mountpoint is empty in the current catalog
454  catalog::DirectoryEntryList nested_entries;
455  if (catalog->ListingPath(full_path, &nested_entries) &&
456  !nested_entries.empty()) {
457  LogCvmfs(kLogCvmfs, kLogStderr, "non-empty nested catalog mountpoint "
458  "at %s.",
459  full_path.c_str());
460  retval = false;
461  }
462 
463  if (entries[i].IsBindMountpoint()) {
464  bind_mountpoints->insert(full_path);
465  if (entries[i].IsNestedCatalogMountpoint()) {
467  "bind mountpoint and nested mountpoint mutually exclusive"
468  " at %s.", full_path.c_str());
469  retval = false;
470  }
471  }
472  } else {
473  // Recurse
474  if (!Find(catalog, full_path, computed_counters, bind_mountpoints))
475  retval = false;
476  }
477  } else if (entries[i].IsLink()) {
478  computed_counters->self.symlinks++;
479  // No hash for symbolics links
480  if (!entries[i].checksum().IsNull()) {
481  LogCvmfs(kLogCvmfs, kLogStderr, "symbolic links with hash at %s",
482  full_path.c_str());
483  retval = false;
484  }
485  // Right size of symbolic link?
486  if (entries[i].size() != entries[i].symlink().GetLength()) {
487  LogCvmfs(kLogCvmfs, kLogStderr, "wrong symbolic link size for %s; ",
488  "expected %s, got %s", full_path.c_str(),
489  entries[i].symlink().GetLength(), entries[i].size());
490  retval = false;
491  }
492  } else if (entries[i].IsRegular()) {
493  computed_counters->self.regular_files++;
494  computed_counters->self.file_size += entries[i].size();
495  } else if (entries[i].IsSpecial()) {
496  computed_counters->self.specials++;
497  // Size zero for special files
498  if (entries[i].size() != 0) {
500  "unexpected non-zero special file size %s",
501  full_path.c_str());
502  retval = false;
503  }
504  // No hash for special files
505  if (!entries[i].checksum().IsNull()) {
506  LogCvmfs(kLogCvmfs, kLogStderr, "special file with hash at %s",
507  full_path.c_str());
508  retval = false;
509  }
510  // No symlink
511  if (entries[i].symlink().GetLength() > 0) {
513  "special file with non-zero symlink at %s", full_path.c_str());
514  retval = false;
515  }
516  } else {
517  LogCvmfs(kLogCvmfs, kLogStderr, "unknown file type %s",
518  full_path.c_str());
519  retval = false;
520  }
521 
522  if (entries[i].HasXattrs()) {
523  computed_counters->self.xattrs++;
524  }
525 
526  if (entries[i].IsExternalFile()) {
527  computed_counters->self.externals++;
528  computed_counters->self.external_file_size += entries[i].size();
529  if (!entries[i].IsRegular()) {
531  "only regular files can be external: %s", full_path.c_str());
532  retval = false;
533  }
534  }
535 
536  // checking file chunk integrity
537  if (entries[i].IsChunkedFile()) {
538  FileChunkList chunks;
539  catalog->ListPathChunks(full_path, entries[i].hash_algorithm(), &chunks);
540 
541  computed_counters->self.chunked_files++;
542  computed_counters->self.chunked_file_size += entries[i].size();
543  computed_counters->self.file_chunks += chunks.size();
544 
545  // do we find file chunks for the chunked file in this catalog?
546  if (chunks.size() == 0) {
547  LogCvmfs(kLogCvmfs, kLogStderr, "no file chunks found for big file %s",
548  full_path.c_str());
549  retval = false;
550  }
551 
552  size_t aggregated_file_size = 0;
553  off_t next_offset = 0;
554 
555  for (unsigned j = 0; j < chunks.size(); ++j) {
556  FileChunk this_chunk = chunks.At(j);
557  // check if the chunk boundaries fit together...
558  if (next_offset != this_chunk.offset()) {
559  LogCvmfs(kLogCvmfs, kLogStderr, "misaligned chunk offsets for %s",
560  full_path.c_str());
561  retval = false;
562  }
563  next_offset = this_chunk.offset() + this_chunk.size();
564  aggregated_file_size += this_chunk.size();
565 
566  // are all data chunks in the data store?
567  if (check_chunks_) {
568  const shash::Any &chunk_hash = this_chunk.content_hash();
569  const string chunk_path = "data/" + chunk_hash.MakePath();
570  if (!Exists(chunk_path)) {
571  LogCvmfs(kLogCvmfs, kLogStderr, "partial data chunk %s (%s -> "
572  "offset: %d | size: %d) missing",
573  this_chunk.content_hash().ToStringWithSuffix().c_str(),
574  full_path.c_str(),
575  this_chunk.offset(),
576  this_chunk.size());
577  retval = false;
578  }
579  }
580  }
581 
582  // is the aggregated chunk size equal to the actual file size?
583  if (aggregated_file_size != entries[i].size()) {
584  LogCvmfs(kLogCvmfs, kLogStderr, "chunks of file %s produce a size "
585  "mismatch. Calculated %d bytes | %d "
586  "bytes expected",
587  full_path.c_str(),
588  aggregated_file_size,
589  entries[i].size());
590  retval = false;
591  }
592  }
593  } // Loop through entries
594 
595  // Check if nested catalog marker has been found
596  if (!path.IsEmpty() && (path == catalog->mountpoint()) &&
597  !found_nested_marker)
598  {
599  LogCvmfs(kLogCvmfs, kLogStderr, "nested catalog without marker at %s",
600  path.c_str());
601  retval = false;
602  }
603 
604  // Check directory linkcount
605  if (this_directory.linkcount() != num_subdirs + 2) {
606  LogCvmfs(kLogCvmfs, kLogStderr, "wrong linkcount for %s; "
607  "expected %lu, got %lu",
608  path.c_str(), num_subdirs + 2, this_directory.linkcount());
609  retval = false;
610  }
611 
612  // Check hardlink linkcounts
613  for (HardlinkMap::const_iterator i = hardlinks.begin(),
614  iEnd = hardlinks.end(); i != iEnd; ++i)
615  {
616  if (i->second[0].linkcount() != i->second.size()) {
617  LogCvmfs(kLogCvmfs, kLogStderr, "hardlink linkcount wrong for %s, "
618  "expected %lu, got %lu",
619  (path.ToString() + "/" + i->second[0].name().ToString()).c_str(),
620  i->second.size(), i->second[0].linkcount());
621  retval = false;
622  }
623  }
624 
625  return retval;
626 }
627 
628 
629 string CommandCheck::DownloadPiece(const shash::Any catalog_hash) {
630  string source = "data/" + catalog_hash.MakePath();
631  const string dest = temp_directory_ + "/" + catalog_hash.ToString();
632  const string url = repo_base_path_ + "/" + source;
633  download::JobInfo download_catalog(&url, true, false, &dest, &catalog_hash);
634  download::Failures retval = download_manager()->Fetch(&download_catalog);
635  if (retval != download::kFailOk) {
636  LogCvmfs(kLogCvmfs, kLogStderr, "failed to download catalog %s (%d)",
637  catalog_hash.ToString().c_str(), retval);
638  return "";
639  }
640 
641  return dest;
642 }
643 
644 
645 string CommandCheck::DecompressPiece(const shash::Any catalog_hash) {
646  string source = "data/" + catalog_hash.MakePath();
647  const string dest = temp_directory_ + "/" + catalog_hash.ToString();
648  if (!zlib::DecompressPath2Path(source, dest))
649  return "";
650 
651  return dest;
652 }
653 
654 
655 catalog::Catalog* CommandCheck::FetchCatalog(const string &path,
656  const shash::Any &catalog_hash,
657  const uint64_t catalog_size) {
658  string tmp_file;
659  if (!is_remote_)
660  tmp_file = DecompressPiece(catalog_hash);
661  else
662  tmp_file = DownloadPiece(catalog_hash);
663 
664  if (tmp_file == "") {
665  LogCvmfs(kLogCvmfs, kLogStderr, "failed to load catalog %s",
666  catalog_hash.ToString().c_str());
667  return NULL;
668  }
669 
670  catalog::Catalog *catalog =
671  catalog::Catalog::AttachFreely(path, tmp_file, catalog_hash);
672  int64_t catalog_file_size = GetFileSize(tmp_file);
673  assert(catalog_file_size > 0);
674  unlink(tmp_file.c_str());
675 
676  if ((catalog_size > 0) && (uint64_t(catalog_file_size) != catalog_size)) {
677  LogCvmfs(kLogCvmfs, kLogStderr, "catalog file size mismatch, "
678  "expected %" PRIu64 ", got %" PRIu64,
679  catalog_size, catalog_file_size);
680  delete catalog;
681  return NULL;
682  }
683 
684  return catalog;
685 }
686 
687 
688 bool CommandCheck::FindSubtreeRootCatalog(const string &subtree_path,
689  shash::Any *root_hash,
690  uint64_t *root_size) {
691  catalog::Catalog *current_catalog = FetchCatalog("", *root_hash);
692  if (current_catalog == NULL) {
693  return false;
694  }
695 
696  typedef vector<string> Tokens;
697  const Tokens path_tokens = SplitString(subtree_path, '/');
698 
699  string current_path = "";
700  bool found = false;
701 
702  Tokens::const_iterator i = path_tokens.begin();
703  Tokens::const_iterator iend = path_tokens.end();
704  for (; i != iend; ++i) {
705  if (i->empty()) {
706  continue;
707  }
708 
709  current_path += "/" + *i;
710  if (current_catalog->FindNested(PathString(current_path),
711  root_hash,
712  root_size)) {
713  delete current_catalog;
714 
715  if (current_path.length() < subtree_path.length()) {
716  current_catalog = FetchCatalog(current_path, *root_hash);
717  if (current_catalog == NULL) {
718  break;
719  }
720  } else {
721  found = true;
722  }
723  }
724  }
725 
726  return found;
727 }
728 
729 
733 bool CommandCheck::InspectTree(const string &path,
734  const shash::Any &catalog_hash,
735  const uint64_t catalog_size,
736  const bool is_nested_catalog,
737  const catalog::DirectoryEntry *transition_point,
738  catalog::DeltaCounters *computed_counters)
739 {
740  LogCvmfs(kLogCvmfs, kLogStdout, "[inspecting catalog] %s at %s",
741  catalog_hash.ToString().c_str(), path == "" ? "/" : path.c_str());
742 
743  const catalog::Catalog *catalog = FetchCatalog(path,
744  catalog_hash,
745  catalog_size);
746  if (catalog == NULL) {
747  LogCvmfs(kLogCvmfs, kLogStderr, "failed to open catalog %s",
748  catalog_hash.ToString().c_str());
749  return false;
750  }
751 
752  int retval = true;
753 
754  if (catalog->root_prefix() != PathString(path.data(), path.length())) {
755  LogCvmfs(kLogCvmfs, kLogStderr, "root prefix mismatch; "
756  "expected %s, got %s",
757  path.c_str(), catalog->root_prefix().c_str());
758  retval = false;
759  }
760 
761  // Check transition point
762  catalog::DirectoryEntry root_entry;
763  if (!catalog->LookupPath(catalog->root_prefix(), &root_entry)) {
764  LogCvmfs(kLogCvmfs, kLogStderr, "failed to lookup root entry (%s)",
765  path.c_str());
766  retval = false;
767  }
768  if (!root_entry.IsDirectory()) {
769  LogCvmfs(kLogCvmfs, kLogStderr, "root entry not a directory (%s)",
770  path.c_str());
771  retval = false;
772  }
773  if (is_nested_catalog) {
774  if (transition_point != NULL &&
775  !CompareEntries(*transition_point, root_entry, true, true)) {
777  "transition point and root entry differ (%s)", path.c_str());
778  retval = false;
779  }
780  if (!root_entry.IsNestedCatalogRoot()) {
782  "nested catalog root expected but not found (%s)", path.c_str());
783  retval = false;
784  }
785  } else {
786  if (root_entry.IsNestedCatalogRoot()) {
788  "nested catalog root found but not expected (%s)", path.c_str());
789  retval = false;
790  }
791  }
792 
793  // Traverse the catalog
794  set<PathString> bind_mountpoints;
795  if (!Find(catalog, PathString(path.data(), path.length()),
796  computed_counters, &bind_mountpoints))
797  {
798  retval = false;
799  }
800 
801  // Check number of entries
802  if (root_entry.HasXattrs())
803  computed_counters->self.xattrs++;
804  const uint64_t num_found_entries =
805  1 +
806  computed_counters->self.regular_files +
807  computed_counters->self.symlinks +
808  computed_counters->self.specials +
809  computed_counters->self.directories;
810  if (num_found_entries != catalog->GetNumEntries()) {
811  LogCvmfs(kLogCvmfs, kLogStderr, "dangling entries in catalog, "
812  "expected %" PRIu64 ", got %" PRIu64,
813  catalog->GetNumEntries(), num_found_entries);
814  retval = false;
815  }
816 
817  // Recurse into nested catalogs
818  const catalog::Catalog::NestedCatalogList &nested_catalogs =
819  catalog->ListNestedCatalogs();
820  const catalog::Catalog::NestedCatalogList own_nested_catalogs =
821  catalog->ListOwnNestedCatalogs();
822  if (own_nested_catalogs.size() !=
823  static_cast<uint64_t>(computed_counters->self.nested_catalogs))
824  {
825  LogCvmfs(kLogCvmfs, kLogStderr, "number of nested catalogs does not match;"
826  " expected %lu, got %lu", computed_counters->self.nested_catalogs,
827  own_nested_catalogs.size());
828  retval = false;
829  }
830  set<PathString> nested_catalog_paths;
831  for (catalog::Catalog::NestedCatalogList::const_iterator i =
832  nested_catalogs.begin(), iEnd = nested_catalogs.end(); i != iEnd; ++i)
833  {
834  nested_catalog_paths.insert(i->mountpoint);
835  }
836  if (nested_catalog_paths.size() != nested_catalogs.size()) {
838  "duplicates among nested catalogs and bind mountpoints");
839  retval = false;
840  }
841 
842  for (catalog::Catalog::NestedCatalogList::const_iterator i =
843  nested_catalogs.begin(), iEnd = nested_catalogs.end(); i != iEnd; ++i)
844  {
845  if (bind_mountpoints.find(i->mountpoint) != bind_mountpoints.end()) {
846  catalog::DirectoryEntry bind_mountpoint;
847  PathString mountpoint("/" + i->mountpoint.ToString().substr(1));
848  if (!catalog->LookupPath(mountpoint, &bind_mountpoint)) {
849  LogCvmfs(kLogCvmfs, kLogStderr, "failed to lookup bind mountpoint %s",
850  mountpoint.c_str());
851  retval = false;
852  }
853  LogCvmfs(kLogCvmfs, kLogDebug, "skipping bind mountpoint %s",
854  i->mountpoint.c_str());
855  continue;
856  }
857  catalog::DirectoryEntry nested_transition_point;
858  if (!catalog->LookupPath(i->mountpoint, &nested_transition_point)) {
859  LogCvmfs(kLogCvmfs, kLogStderr, "failed to lookup transition point %s",
860  i->mountpoint.c_str());
861  retval = false;
862  } else {
863  catalog::DeltaCounters nested_counters;
864  const bool is_nested = true;
865  if (!InspectTree(i->mountpoint.ToString(), i->hash, i->size, is_nested,
866  &nested_transition_point, &nested_counters))
867  retval = false;
868  nested_counters.PopulateToParent(computed_counters);
869  }
870  }
871 
872  // Check statistics counters
873  // Additionally account for root directory
874  computed_counters->self.directories++;
875  catalog::Counters compare_counters;
876  compare_counters.ApplyDelta(*computed_counters);
877  const catalog::Counters stored_counters = catalog->GetCounters();
878  if (!CompareCounters(compare_counters, stored_counters)) {
879  LogCvmfs(kLogCvmfs, kLogStderr, "statistics counter mismatch [%s]",
880  catalog_hash.ToString().c_str());
881  retval = false;
882  }
883 
884  delete catalog;
885  return retval;
886 }
887 
888 
889 int CommandCheck::Main(const swissknife::ArgumentList &args) {
890  string tag_name;
891  string subtree_path = "";
892  string pubkey_path = "";
893  string trusted_certs = "";
894  string repo_name = "";
895  string reflog_chksum_path = "";
896 
897  temp_directory_ = (args.find('t') != args.end()) ? *args.find('t')->second
898  : "/tmp";
899  if (args.find('n') != args.end())
900  tag_name = *args.find('n')->second;
901  if (args.find('c') != args.end())
902  check_chunks_ = true;
903  if (args.find('l') != args.end()) {
904  unsigned log_level =
905  1 << (kLogLevel0 + String2Uint64(*args.find('l')->second));
906  if (log_level > kLogNone) {
907  LogCvmfs(kLogCvmfs, kLogStderr, "invalid log level");
908  return 1;
909  }
910  SetLogVerbosity(static_cast<LogLevels>(log_level));
911  }
912  if (args.find('k') != args.end())
913  pubkey_path = *args.find('k')->second;
914  if (DirectoryExists(pubkey_path))
915  pubkey_path = JoinStrings(FindFilesBySuffix(pubkey_path, ".pub"), ":");
916  if (args.find('z') != args.end())
917  trusted_certs = *args.find('z')->second;
918  if (args.find('N') != args.end())
919  repo_name = *args.find('N')->second;
920 
921  repo_base_path_ = MakeCanonicalPath(*args.find('r')->second);
922  if (args.find('s') != args.end())
923  subtree_path = MakeCanonicalPath(*args.find('s')->second);
924  if (args.find('R') != args.end())
925  reflog_chksum_path = *args.find('R')->second;
926 
927  // Repository can be HTTP address or on local file system
928  is_remote_ = IsHttpUrl(repo_base_path_);
929 
930  // initialize the (swissknife global) download and signature managers
931  if (is_remote_) {
932  const bool follow_redirects = (args.count('L') > 0);
933  if (!this->InitDownloadManager(follow_redirects)) {
934  return 1;
935  }
936 
937  if (pubkey_path.empty() || repo_name.empty()) {
938  LogCvmfs(kLogCvmfs, kLogStderr, "please provide pubkey and repo name for "
939  "remote repositories");
940  return 1;
941  }
942 
943  if (!this->InitVerifyingSignatureManager(pubkey_path, trusted_certs)) {
944  return 1;
945  }
946  }
947 
948  // Load Manifest
950  bool successful = true;
951 
952  if (is_remote_) {
953  manifest = FetchRemoteManifest(repo_base_path_, repo_name);
954  } else {
955  if (chdir(repo_base_path_.c_str()) != 0) {
956  LogCvmfs(kLogCvmfs, kLogStderr, "failed to switch to directory %s",
957  repo_base_path_.c_str());
958  return 1;
959  }
960  manifest = OpenLocalManifest(".cvmfspublished");
961  }
962 
963  if (!manifest.IsValid()) {
964  LogCvmfs(kLogCvmfs, kLogStderr, "failed to load repository manifest");
965  return 1;
966  }
967 
968  shash::Any reflog_hash;
969  if (!reflog_chksum_path.empty()) {
970  if (!manifest::Reflog::ReadChecksum(reflog_chksum_path, &reflog_hash)) {
971  LogCvmfs(kLogCvmfs, kLogStderr, "failed to read reflog checksum file");
972  return 1;
973  }
974  } else {
975  reflog_hash = manifest->reflog_hash();
976  }
977 
978  if (Exists(".cvmfsreflog")) {
979  if (reflog_hash.IsNull()) {
980  // If there is a reflog, we want to check it
982  ".cvmfsreflog present but no checksum provided, aborting");
983  return 1;
984  }
985  bool retval = InspectReflog(reflog_hash, manifest.weak_ref());
986  if (!retval) {
987  LogCvmfs(kLogCvmfs, kLogStderr, "failed to verify reflog");
988  return 1;
989  }
990  } else {
991  if (!reflog_hash.IsNull()) {
992  // There is a checksum but no reflog; possibly the checksum is for the
993  // from the manifest for the stratum 0 reflog
994  if (!reflog_chksum_path.empty()) {
996  "local reflog checksum set but reflog itself is missing, "
997  "aborting");
998  return 1;
999  }
1000  }
1001  }
1002 
1003  // Load history
1005  if (!manifest->history().IsNull()) {
1006  string tmp_file;
1007  if (!is_remote_)
1008  tmp_file = DecompressPiece(manifest->history());
1009  else
1010  tmp_file = DownloadPiece(manifest->history());
1011  if (tmp_file == "") {
1012  LogCvmfs(kLogCvmfs, kLogStderr, "failed to load history database %s",
1013  manifest->history().ToString().c_str());
1014  return 1;
1015  }
1016  tag_db = history::SqliteHistory::Open(tmp_file);
1017  if (!tag_db.IsValid()) {
1018  LogCvmfs(kLogCvmfs, kLogStderr, "failed to open history database %s",
1019  manifest->history().ToString().c_str());
1020  return 1;
1021  }
1022  tag_db->TakeDatabaseFileOwnership();
1023  successful = InspectHistory(tag_db.weak_ref()) && successful;
1024  }
1025 
1026  if (manifest->has_alt_catalog_path()) {
1027  if (!Exists(manifest->certificate().MakeAlternativePath())) {
1029  "failed to find alternative certificate link %s",
1030  manifest->certificate().MakeAlternativePath().c_str());
1031  return 1;
1032  }
1033  if (!Exists(manifest->catalog_hash().MakeAlternativePath())) {
1035  "failed to find alternative catalog link %s",
1036  manifest->catalog_hash().MakeAlternativePath().c_str());
1037  return 1;
1038  }
1039  }
1040 
1041  shash::Any root_hash = manifest->catalog_hash();
1042  uint64_t root_size = manifest->catalog_size();
1043  if (tag_name != "") {
1044  if (!tag_db.IsValid()) {
1045  LogCvmfs(kLogCvmfs, kLogStderr, "no history");
1046  return 1;
1047  }
1049  const bool retval = tag_db->GetByName(tag_name, &tag);
1050  if (!retval) {
1051  LogCvmfs(kLogCvmfs, kLogStderr, "no such tag: %s", tag_name.c_str());
1052  return 1;
1053  }
1054  root_hash = tag.root_hash;
1055  root_size = tag.size;
1056  LogCvmfs(kLogCvmfs, kLogStdout, "Inspecting repository tag %s",
1057  tag_name.c_str());
1058  }
1059 
1060  const bool is_nested_catalog = (!subtree_path.empty());
1061  if (is_nested_catalog && !FindSubtreeRootCatalog( subtree_path,
1062  &root_hash,
1063  &root_size)) {
1064  LogCvmfs(kLogCvmfs, kLogStderr, "cannot find nested catalog at %s",
1065  subtree_path.c_str());
1066  return 1;
1067  }
1068 
1069  catalog::DeltaCounters computed_counters;
1070  successful = InspectTree(subtree_path,
1071  root_hash,
1072  root_size,
1073  is_nested_catalog,
1074  NULL,
1075  &computed_counters) && successful;
1076 
1077  if (!successful) {
1078  LogCvmfs(kLogCvmfs, kLogStderr, "CATALOG PROBLEMS OR OTHER ERRORS FOUND");
1079  return 1;
1080  }
1081 
1082  LogCvmfs(kLogCvmfs, kLogStdout, "no problems found");
1083  return 0;
1084 }
1085 
1086 } // namespace swissknife
uint32_t linkcount() const
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
void TakeDatabaseFileOwnership()
Definition: reflog.cc:307
bool IsExternalFile() const
bool ContainsHistory(const shash::Any &history) const
Definition: reflog.cc:239
bool IsNull() const
Definition: hash.h:379
Differences CompareTo(const DirectoryEntry &other) const
const manifest::Manifest * manifest() const
Definition: repository.h:123
ShortString< kDefaultMaxName, 1 > NameString
Definition: shortstring.h:191
Item At(const size_t index) const
Definition: bigvector.h:50
time_t mtime() const
bool IsDirectory() const
static SqliteHistory * Open(const std::string &file_name)
bool ListPathChunks(const PathString &path, const shash::Algorithms interpret_hashes_as, FileChunkList *chunks) const
Definition: catalog.h:147
T * weak_ref() const
Definition: pointer.h:42
vector< string > SplitString(const string &str, const char delim, const unsigned max_chunks)
Definition: string.cc:288
const int kDefaultFileMode
Definition: posix.h:31
static bool ReadChecksum(const std::string &path, shash::Any *checksum)
Definition: reflog.cc:47
#define PANIC(...)
Definition: exception.h:26
FILE * CreateTempFile(const std::string &path_prefix, const int mode, const char *open_flags, std::string *final_path)
Definition: posix.cc:1030
uint64_t size() const
string JoinStrings(const vector< string > &strings, const string &joint)
Definition: string.cc:318
std::string ToString(const bool with_suffix=false) const
Definition: hash.h:245
const history::History * history() const
std::string ToStringWithSuffix() const
Definition: hash.h:300
bool LookupPath(const PathString &path, DirectoryEntry *dirent) const
Definition: catalog.h:124
void ApplyDelta(const DeltaCounters &delta)
bool IsHttpUrl(const std::string &path)
Definition: posix.cc:204
bool ListingPath(const PathString &path, DirectoryEntryList *listing, const bool expand_symlink=true) const
Definition: catalog.h:132
const char kSuffixMicroCatalog
Definition: hash.h:54
const shash::Any & content_hash() const
Definition: file_chunk.h:41
std::map< std::string, const Counters_t * > FieldsMap
assert((mem||(size==0))&&"Out Of Memory")
Algorithms algorithm
Definition: hash.h:123
shash::Any checksum() const
unsigned int mode() const
bool ContainsMetainfo(const shash::Any &metainfo) const
Definition: reflog.cc:245
bool CopyPath2File(const std::string &src, FILE *fdest)
Definition: compression.cc:46
bool SymlinkExists(const std::string &path)
Definition: posix.cc:848
static Reflog * Open(const std::string &database_path)
Definition: reflog.cc:17
bool IsValid(const std::string &input) const
Definition: sanitizer.cc:114
bool IsNestedCatalogRoot() const
bool FileExists(const std::string &path)
Definition: posix.cc:816
std::vector< DirectoryEntry > DirectoryEntryList
NameString name() const
virtual bool ListBranches(std::vector< Branch > *branches) const =0
bool HasXattrs() const
virtual bool List(std::vector< Tag > *tags) const =0
off_t offset() const
Definition: file_chunk.h:42
shash::Any certificate() const
Definition: manifest.h:126
void PopulateToParent(DeltaCounters *parent) const
LinkString symlink() const
void SetLogVerbosity(const LogLevels min_level)
Definition: logging.cc:207
PathString mountpoint() const
Definition: catalog.h:179
static void HashDatabase(const std::string &database_path, shash::Any *hash_reflog)
Definition: reflog.cc:322
shash::Any catalog_hash() const
Definition: manifest.h:124
void Append(const char *chars, const unsigned length)
Definition: shortstring.h:70
bool DirectoryExists(const std::string &path)
Definition: posix.cc:838
std::string ToString() const
Definition: shortstring.h:114
std::vector< NestedCatalog > NestedCatalogList
Definition: catalog.h:208
bool ContainsCatalog(const shash::Any &catalog) const
Definition: reflog.cc:222
bool IsEmpty() const
Definition: shortstring.h:110
uint64_t String2Uint64(const string &value)
Definition: string.cc:228
std::map< char, SharedPtr< std::string > > ArgumentList
Definition: swissknife.h:72
ShortString< kDefaultMaxPath, 0 > PathString
Definition: shortstring.h:190
size_t size() const
Definition: file_chunk.h:43
shash::Any history() const
Definition: manifest.h:127
shash::Any root_hash
Definition: history.h:114
static Catalog * AttachFreely(const std::string &imaginary_mountpoint, const std::string &file, const shash::Any &catalog_hash, Catalog *parent=NULL, const bool is_nested=false)
Definition: catalog.cc:29
bool FindNested(const PathString &mountpoint, shash::Any *hash, uint64_t *size) const
Definition: catalog.cc:690
bool DecompressPath2Path(const string &src, const string &dest)
Definition: compression.cc:381
int64_t GetFileSize(const std::string &path)
Definition: posix.cc:826
const int kLogVerboseMsg
std::string MakePath() const
Definition: hash.h:312
std::string MakeCanonicalPath(const std::string &path)
Definition: posix.cc:96
const char * c_str() const
Definition: shortstring.h:118
uint32_t hardlink_group() const
bool ContainsCertificate(const shash::Any &certificate) const
Definition: reflog.cc:215
static void size_t size
Definition: smalloc.h:47
std::vector< std::string > FindFilesBySuffix(const std::string &dir, const std::string &suffix)
Definition: posix.cc:1149
unsigned int Differences
shash::Any meta_info() const
Definition: manifest.h:131
size_t size() const
Definition: bigvector.h:100