GCC Code Coverage Report


Directory: cvmfs/
File: cvmfs/swissknife_check.cc
Date: 2026-05-19 11:45:12
Exec Total Coverage
Lines: 0 601 0.0%
Branches: 0 434 0.0%

Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * This tool checks a cvmfs repository for file catalog errors.
5 */
6
7 #include "swissknife_check.h"
8
9 #include <inttypes.h>
10 #include <unistd.h>
11
12 #include <cassert>
13 #include <map>
14 #include <set>
15 #include <string>
16 #include <vector>
17
18 #include "catalog_sql.h"
19 #include "compression/compression.h"
20 #include "file_chunk.h"
21 #include "history_sqlite.h"
22 #include "manifest.h"
23 #include "network/download.h"
24 #include "network/sink_path.h"
25 #include "reflog.h"
26 #include "sanitizer.h"
27 #include "shortstring.h"
28 #include "util/exception.h"
29 #include "util/logging.h"
30 #include "util/pointer.h"
31 #include "util/posix.h"
32
33 using namespace std; // NOLINT
34
35 // for map of duplicate entries; as in kvstore.cc
36 static inline uint32_t hasher_any(const shash::Any &key) {
37 // We'll just do the same thing as hasher_md5, since every hash is at
38 // least as large.
39 return *const_cast<uint32_t *>(reinterpret_cast<const uint32_t *>(key.digest)
40 + 1);
41 }
42
43
44 namespace swissknife {
45
46 CommandCheck::CommandCheck()
47 : check_chunks_(false), no_duplicates_map_(false), is_remote_(false) {
48 const shash::Any hash_null;
49 duplicates_map_.Init(16, hash_null, hasher_any);
50 }
51
52 bool CommandCheck::CompareEntries(const catalog::DirectoryEntry &a,
53 const catalog::DirectoryEntry &b,
54 const bool compare_names,
55 const bool is_transition_point) {
56 typedef catalog::DirectoryEntry::Difference Difference;
57
58 const catalog::DirectoryEntry::Differences diffs = a.CompareTo(b);
59 if (diffs == Difference::kIdentical) {
60 return true;
61 }
62
63 // in case of a nested catalog transition point the controlling flags are
64 // supposed to differ. If this is the only difference we are done...
65 if (is_transition_point
66 && (diffs ^ Difference::kNestedCatalogTransitionFlags) == 0) {
67 return true;
68 }
69
70 bool retval = true;
71 if (compare_names) {
72 if (diffs & Difference::kName) {
73 LogCvmfs(kLogCvmfs, kLogStderr, "names differ: %s / %s", a.name().c_str(),
74 b.name().c_str());
75 retval = false;
76 }
77 }
78 if (diffs & Difference::kLinkcount) {
79 LogCvmfs(kLogCvmfs, kLogStderr, "linkcounts differ: %u / %u", a.linkcount(),
80 b.linkcount());
81 retval = false;
82 }
83 if (diffs & Difference::kHardlinkGroup) {
84 LogCvmfs(kLogCvmfs, kLogStderr, "hardlink groups differ: %u / %u",
85 a.hardlink_group(), b.hardlink_group());
86 retval = false;
87 }
88 if (diffs & Difference::kSize) {
89 LogCvmfs(kLogCvmfs, kLogStderr, "sizes differ: %" PRIu64 " / %" PRIu64,
90 a.size(), b.size());
91 retval = false;
92 }
93 if (diffs & Difference::kMode) {
94 LogCvmfs(kLogCvmfs, kLogStderr, "modes differ: %u / %u", a.mode(),
95 b.mode());
96 retval = false;
97 }
98 if (diffs & Difference::kMtime) {
99 LogCvmfs(kLogCvmfs, kLogStderr, "timestamps differ: %lu / %lu", a.mtime(),
100 b.mtime());
101 retval = false;
102 }
103 if (diffs & Difference::kChecksum) {
104 LogCvmfs(kLogCvmfs, kLogStderr, "content hashes differ: %s / %s",
105 a.checksum().ToString().c_str(), b.checksum().ToString().c_str());
106 retval = false;
107 }
108 if (diffs & Difference::kSymlink) {
109 LogCvmfs(kLogCvmfs, kLogStderr, "symlinks differ: %s / %s",
110 a.symlink().c_str(), b.symlink().c_str());
111 retval = false;
112 }
113 if (diffs & Difference::kExternalFileFlag) {
114 LogCvmfs(kLogCvmfs, kLogStderr,
115 "external file flag differs: %d / %d "
116 "(%s / %s)",
117 a.IsExternalFile(), b.IsExternalFile(), a.name().c_str(),
118 b.name().c_str());
119 retval = false;
120 }
121 if (diffs & Difference::kHasXattrsFlag) {
122 LogCvmfs(kLogCvmfs, kLogStderr,
123 "extended attributes differ: %d / %d "
124 "(%s / %s)",
125 a.HasXattrs(), b.HasXattrs(), a.name().c_str(), b.name().c_str());
126 retval = false;
127 }
128 if (!is_transition_point) {
129 if (diffs & Difference::kUid) {
130 LogCvmfs(kLogCvmfs, kLogStderr, "uids differ: %d / %d (%s / %s)", a.uid(),
131 b.uid(), a.name().c_str(), b.name().c_str());
132 retval = false;
133 }
134 if (diffs & Difference::kGid) {
135 LogCvmfs(kLogCvmfs, kLogStderr, "gids differ: %d / %d (%s / %s)", a.gid(),
136 b.gid(), a.name().c_str(), b.name().c_str());
137 retval = false;
138 }
139 }
140
141 return retval;
142 }
143
144
145 bool CommandCheck::CompareCounters(const catalog::Counters &a,
146 const catalog::Counters &b) {
147 const catalog::Counters::FieldsMap map_a = a.GetFieldsMap();
148 const catalog::Counters::FieldsMap map_b = b.GetFieldsMap();
149
150 bool retval = true;
151 catalog::Counters::FieldsMap::const_iterator i = map_a.begin();
152 const catalog::Counters::FieldsMap::const_iterator iend = map_a.end();
153 for (; i != iend; ++i) {
154 const catalog::Counters::FieldsMap::const_iterator comp = map_b.find(
155 i->first);
156 assert(comp != map_b.end());
157
158 if (*(i->second) != *(comp->second)) {
159 LogCvmfs(kLogCvmfs, kLogStderr,
160 "catalog statistics mismatch: %s (expected: %" PRIu64 " / "
161 "in catalog: %" PRIu64 ")",
162 comp->first.c_str(), *(i->second), *(comp->second));
163 retval = false;
164 }
165 }
166
167 return retval;
168 }
169
170
171 /**
172 * Checks for existence of a file either locally or via HTTP head
173 */
174 bool CommandCheck::Exists(const string &file) {
175 if (!is_remote_) {
176 return FileExists(file) || SymlinkExists(file);
177 } else {
178 const string url = repo_base_path_ + "/" + file;
179 LogCvmfs(kLogCvmfs, kLogVerboseMsg, "[Exists::url] %s", url.c_str());
180 download::JobInfo head(&url, false);
181 return download_manager()->Fetch(&head) == download::kFailOk;
182 }
183 }
184
185
186 /**
187 * Copies a file from the repository into a temporary file.
188 */
189 string CommandCheck::FetchPath(const string &path) {
190 string tmp_path;
191 FILE *f = CreateTempFile(temp_directory_ + "/cvmfstmp", kDefaultFileMode,
192 "w+", &tmp_path);
193 assert(f != NULL);
194
195 const string url = repo_base_path_ + "/" + path;
196 if (is_remote_) {
197 cvmfs::FileSink filesink(f);
198 download::JobInfo download_job(&url, false, false, NULL, &filesink);
199 const download::Failures retval = download_manager()->Fetch(&download_job);
200 if (retval != download::kFailOk) {
201 PANIC(kLogStderr, "failed to read %s", url.c_str());
202 }
203 } else {
204 const bool retval = CopyPath2File(url, f);
205 if (!retval) {
206 PANIC(kLogStderr, "failed to read %s", url.c_str());
207 }
208 }
209
210 fclose(f);
211 return tmp_path;
212 }
213
214
215 /**
216 * Verifies reflog checksum and looks for presence of the entry points
217 * referenced in the manifest.
218 */
219 bool CommandCheck::InspectReflog(const shash::Any &reflog_hash,
220 manifest::Manifest *manifest) {
221 LogCvmfs(kLogCvmfs, kLogStdout, "Inspecting log of references");
222 const string reflog_path = FetchPath(".cvmfsreflog");
223 shash::Any computed_hash(reflog_hash.algorithm);
224 manifest::Reflog::HashDatabase(reflog_path, &computed_hash);
225 if (computed_hash != reflog_hash) {
226 LogCvmfs(kLogCvmfs, kLogStderr,
227 "The .cvmfsreflog has unexpected content hash %s (expected %s)",
228 computed_hash.ToString().c_str(), reflog_hash.ToString().c_str());
229 unlink(reflog_path.c_str());
230 return false;
231 }
232
233 const UniquePtr<manifest::Reflog> reflog(manifest::Reflog::Open(reflog_path));
234 assert(reflog.IsValid());
235 reflog->TakeDatabaseFileOwnership();
236
237 if (!reflog->ContainsCatalog(manifest->catalog_hash())) {
238 LogCvmfs(kLogCvmfs, kLogStderr,
239 "failed to find catalog root hash %s in .cvmfsreflog",
240 manifest->catalog_hash().ToString().c_str());
241 return false;
242 }
243
244 if (!reflog->ContainsCertificate(manifest->certificate())) {
245 LogCvmfs(kLogCvmfs, kLogStderr,
246 "failed to find certificate hash %s in .cvmfsreflog",
247 manifest->certificate().ToString().c_str());
248 return false;
249 }
250
251 if (!manifest->history().IsNull()
252 && !reflog->ContainsHistory(manifest->history())) {
253 LogCvmfs(kLogCvmfs, kLogStderr,
254 "failed to find tag database's hash %s in .cvmfsreflog",
255 manifest->history().ToString().c_str());
256 return false;
257 }
258
259 if (!manifest->meta_info().IsNull()
260 && !reflog->ContainsMetainfo(manifest->meta_info())) {
261 LogCvmfs(kLogCvmfs, kLogStderr,
262 "failed to find meta info hash %s in .cvmfsreflog",
263 manifest->meta_info().ToString().c_str());
264 return false;
265 }
266
267 return true;
268 }
269
270
271 /**
272 * Verifies the logical consistency of the tag database.
273 */
274 bool CommandCheck::InspectHistory(history::History *history) {
275 LogCvmfs(kLogCvmfs, kLogStdout, "Inspecting tag database");
276 bool retval;
277 vector<history::History::Tag> tags;
278 retval = history->List(&tags);
279 if (!retval) {
280 LogCvmfs(kLogCvmfs, kLogStderr, "failed to enumerate tags");
281 return false;
282 }
283 vector<history::History::Branch> branches;
284 retval = history->ListBranches(&branches);
285 if (!retval) {
286 LogCvmfs(kLogCvmfs, kLogStderr, "failed to enumerate branches");
287 return false;
288 }
289
290 bool result = true;
291
292 map<string, uint64_t> initial_revisions;
293 const sanitizer::BranchSanitizer sanitizer;
294 for (unsigned i = 0; i < branches.size(); ++i) {
295 if (!sanitizer.IsValid(branches[i].branch)) {
296 LogCvmfs(kLogCvmfs, kLogStderr, "invalid branch name: %s",
297 branches[i].branch.c_str());
298 result = false;
299 }
300 initial_revisions[branches[i].branch] = branches[i].initial_revision;
301 }
302
303 set<string> used_branches; // all branches referenced in tag db
304 // TODO(jblomer): same root hash implies same size and revision
305 for (unsigned i = 0; i < tags.size(); ++i) {
306 used_branches.insert(tags[i].branch);
307 const map<string, uint64_t>::const_iterator iter = initial_revisions.find(
308 tags[i].branch);
309 if (iter == initial_revisions.end()) {
310 LogCvmfs(kLogCvmfs, kLogStderr, "invalid branch %s in tag %s",
311 tags[i].branch.c_str(), tags[i].name.c_str());
312 result = false;
313 } else {
314 if (tags[i].revision < iter->second) {
315 LogCvmfs(kLogCvmfs, kLogStderr,
316 "invalid revision %" PRIu64 " of tag %s", tags[i].revision,
317 tags[i].name.c_str());
318 result = false;
319 }
320 }
321 }
322
323 if (used_branches.size() != branches.size()) {
324 LogCvmfs(kLogCvmfs, kLogStderr, "unused, dangling branches stored");
325 result = false;
326 }
327
328 return result;
329 }
330
331
332 /**
333 * Recursive catalog walk-through
334 *
335 * TODO(vavolkl): This method is large and does a lot of checks
336 * that could be split into smaller ones.
337 *
338 */
339 bool CommandCheck::Find(const catalog::Catalog *catalog,
340 const PathString &path,
341 catalog::DeltaCounters *computed_counters,
342 set<PathString> *bind_mountpoints) {
343 catalog::DirectoryEntryList entries;
344 catalog::DirectoryEntry this_directory;
345
346 if (!catalog->LookupPath(path, &this_directory)) {
347 LogCvmfs(kLogCvmfs, kLogStderr, "failed to lookup %s", path.c_str());
348 return false;
349 }
350 if (!catalog->ListingPath(path, &entries)) {
351 LogCvmfs(kLogCvmfs, kLogStderr, "failed to list %s", path.c_str());
352 return false;
353 }
354
355 uint32_t num_subdirs = 0;
356 bool retval = true;
357 typedef map<uint32_t, vector<catalog::DirectoryEntry> > HardlinkMap;
358 HardlinkMap hardlinks;
359 bool found_nested_marker = false;
360
361 for (unsigned i = 0; i < entries.size(); ++i) {
362 // for performance reasons, keep track of files already checked
363 // and only run requests once per hash
364 const bool entry_needs_check = !entries[i].checksum().IsNull() &&
365 !entries[i].IsExternalFile() &&
366 !(catalog::g_ignore_legacy_bulk_hashes &&
367 entries[i].IsChunkedFile()) &&
368 // fallback cli option can force the entry to
369 // be checked
370 (no_duplicates_map_
371 || !duplicates_map_.Contains(
372 entries[i].checksum()));
373 if (entry_needs_check && !no_duplicates_map_)
374 duplicates_map_.Insert(entries[i].checksum(), 1);
375
376 PathString full_path(path);
377 full_path.Append("/", 1);
378 full_path.Append(entries[i].name().GetChars(),
379 entries[i].name().GetLength());
380 LogCvmfs(kLogCvmfs, kLogVerboseMsg, "[path] %s [needs check] %i",
381 full_path.c_str(), entry_needs_check);
382
383
384 // Name must not be empty
385 if (entries[i].name().IsEmpty()) {
386 LogCvmfs(kLogCvmfs, kLogStderr, "empty path at %s", full_path.c_str());
387 retval = false;
388 }
389
390 // Catalog markers should indicate nested catalogs
391 if (entries[i].name() == NameString(string(".cvmfscatalog"))) {
392 if (catalog->mountpoint() != path) {
393 LogCvmfs(kLogCvmfs, kLogStderr,
394 "found abandoned nested catalog marker at %s",
395 full_path.c_str());
396 retval = false;
397 }
398 found_nested_marker = true;
399 }
400
401 // Check if checksum is not null
402 if (entries[i].IsRegular() && !entries[i].IsChunkedFile()
403 && entries[i].checksum().IsNull()) {
404 LogCvmfs(kLogCvmfs, kLogStderr,
405 "regular file pointing to zero-hash: '%s'", full_path.c_str());
406 retval = false;
407 }
408
409 // Check if the chunk is there
410 if (check_chunks_ && entry_needs_check) {
411 string chunk_path = "data/" + entries[i].checksum().MakePath();
412 if (entries[i].IsDirectory())
413 chunk_path += shash::kSuffixMicroCatalog;
414 if (!Exists(chunk_path)) {
415 LogCvmfs(kLogCvmfs, kLogStderr, "data chunk %s (%s) missing",
416 entries[i].checksum().ToString().c_str(), full_path.c_str());
417 retval = false;
418 }
419 }
420
421 // Add hardlinks to counting map
422 if ((entries[i].linkcount() > 1) && !entries[i].IsDirectory()) {
423 if (entries[i].hardlink_group() == 0) {
424 LogCvmfs(kLogCvmfs, kLogStderr, "invalid hardlink group for %s",
425 full_path.c_str());
426 retval = false;
427 } else {
428 const HardlinkMap::iterator hardlink_group = hardlinks.find(
429 entries[i].hardlink_group());
430 if (hardlink_group == hardlinks.end()) {
431 hardlinks[entries[i].hardlink_group()];
432 hardlinks[entries[i].hardlink_group()].push_back(entries[i]);
433 } else {
434 if (!CompareEntries(entries[i], (hardlink_group->second)[0], false)) {
435 LogCvmfs(kLogCvmfs, kLogStderr, "hardlink %s doesn't match",
436 full_path.c_str());
437 retval = false;
438 }
439 hardlink_group->second.push_back(entries[i]);
440 } // Hardlink added to map
441 } // Hardlink group > 0
442 } // Hardlink found
443
444 // For any kind of entry, the linkcount should be > 0
445 if (entries[i].linkcount() == 0) {
446 LogCvmfs(kLogCvmfs, kLogStderr, "Entry %s has linkcount 0.",
447 entries[i].name().c_str());
448 retval = false;
449 }
450
451 // Checks depending of entry type
452 if (!entries[i].IsRegular()) {
453 if (entries[i].IsDirectIo()) {
454 LogCvmfs(kLogCvmfs, kLogStderr, "invalid direct i/o flag found: %s",
455 full_path.c_str());
456 retval = false;
457 }
458 }
459 if (entries[i].IsDirectory()) {
460 computed_counters->self.directories++;
461 num_subdirs++;
462 // Directory size
463 // if (entries[i].size() < 4096) {
464 // LogCvmfs(kLogCvmfs, kLogStderr, "invalid file size for %s",
465 // full_path.c_str());
466 // retval = false;
467 // }
468 // No directory hardlinks
469 if (entries[i].hardlink_group() != 0) {
470 LogCvmfs(kLogCvmfs, kLogStderr, "directory hardlink found at %s",
471 full_path.c_str());
472 retval = false;
473 }
474 if (entries[i].IsNestedCatalogMountpoint()
475 || entries[i].IsBindMountpoint()) {
476 // Find transition point
477 if (entries[i].IsNestedCatalogMountpoint())
478 computed_counters->self.nested_catalogs++;
479 shash::Any tmp;
480 uint64_t tmp2;
481 const PathString mountpoint(full_path);
482 if (!catalog->FindNested(mountpoint, &tmp, &tmp2)) {
483 LogCvmfs(kLogCvmfs, kLogStderr, "nested catalog at %s not registered",
484 full_path.c_str());
485 retval = false;
486 }
487
488 // check that the nested mountpoint is empty in the current catalog
489 catalog::DirectoryEntryList nested_entries;
490 if (catalog->ListingPath(full_path, &nested_entries)
491 && !nested_entries.empty()) {
492 LogCvmfs(kLogCvmfs, kLogStderr,
493 "non-empty nested catalog mountpoint "
494 "at %s.",
495 full_path.c_str());
496 retval = false;
497 }
498
499 if (entries[i].IsBindMountpoint()) {
500 bind_mountpoints->insert(full_path);
501 if (entries[i].IsNestedCatalogMountpoint()) {
502 LogCvmfs(kLogCvmfs, kLogStderr,
503 "bind mountpoint and nested mountpoint mutually exclusive"
504 " at %s.",
505 full_path.c_str());
506 retval = false;
507 }
508 }
509 } else {
510 // Recurse
511 if (!Find(catalog, full_path, computed_counters, bind_mountpoints))
512 retval = false;
513 }
514 } else if (entries[i].IsLink()) {
515 computed_counters->self.symlinks++;
516 // No hash for symbolics links
517 if (!entries[i].checksum().IsNull()) {
518 LogCvmfs(kLogCvmfs, kLogStderr, "symbolic links with hash at %s",
519 full_path.c_str());
520 retval = false;
521 }
522 // Right size of symbolic link?
523 if (entries[i].size() != entries[i].symlink().GetLength()) {
524 LogCvmfs(kLogCvmfs, kLogStderr,
525 "wrong symbolic link size for %s; "
526 "expected %u, got %lu",
527 full_path.c_str(), entries[i].symlink().GetLength(),
528 entries[i].size());
529 retval = false;
530 }
531 } else if (entries[i].IsRegular()) {
532 computed_counters->self.regular_files++;
533 computed_counters->self.file_size += entries[i].size();
534 } else if (entries[i].IsSpecial()) {
535 computed_counters->self.specials++;
536 // Size zero for special files
537 if (entries[i].size() != 0) {
538 LogCvmfs(kLogCvmfs, kLogStderr,
539 "unexpected non-zero special file size %s", full_path.c_str());
540 retval = false;
541 }
542 // No hash for special files
543 if (!entries[i].checksum().IsNull()) {
544 LogCvmfs(kLogCvmfs, kLogStderr, "special file with hash at %s",
545 full_path.c_str());
546 retval = false;
547 }
548 // No symlink
549 if (entries[i].symlink().GetLength() > 0) {
550 LogCvmfs(kLogCvmfs, kLogStderr,
551 "special file with non-zero symlink at %s", full_path.c_str());
552 retval = false;
553 }
554 } else {
555 LogCvmfs(kLogCvmfs, kLogStderr, "unknown file type %s",
556 full_path.c_str());
557 retval = false;
558 }
559
560 if (entries[i].HasXattrs()) {
561 computed_counters->self.xattrs++;
562 }
563
564 if (entries[i].IsExternalFile()) {
565 computed_counters->self.externals++;
566 computed_counters->self.external_file_size += entries[i].size();
567 if (!entries[i].IsRegular()) {
568 LogCvmfs(kLogCvmfs, kLogStderr,
569 "only regular files can be external: %s", full_path.c_str());
570 retval = false;
571 }
572 }
573
574 // checking file chunk integrity
575 if (entries[i].IsChunkedFile()) {
576 FileChunkList chunks;
577 catalog->ListPathChunks(full_path, entries[i].hash_algorithm(), &chunks);
578
579 computed_counters->self.chunked_files++;
580 computed_counters->self.chunked_file_size += entries[i].size();
581 computed_counters->self.file_chunks += chunks.size();
582
583 // do we find file chunks for the chunked file in this catalog?
584 if (chunks.size() == 0) {
585 LogCvmfs(kLogCvmfs, kLogStderr, "no file chunks found for big file %s",
586 full_path.c_str());
587 retval = false;
588 }
589
590 size_t aggregated_file_size = 0;
591 off_t next_offset = 0;
592
593 for (unsigned j = 0; j < chunks.size(); ++j) {
594 const FileChunk this_chunk = chunks.At(j);
595 // check if the chunk boundaries fit together...
596 if (next_offset != this_chunk.offset()) {
597 LogCvmfs(kLogCvmfs, kLogStderr, "misaligned chunk offsets for %s",
598 full_path.c_str());
599 retval = false;
600 }
601 next_offset = this_chunk.offset() + this_chunk.size();
602 aggregated_file_size += this_chunk.size();
603
604 // are all data chunks in the data store?
605 if (check_chunks_ && !entries[i].IsExternalFile()) {
606 const shash::Any &chunk_hash = this_chunk.content_hash();
607 // for performance reasons, only perform the check once
608 // and skip if the hash has been checked before
609 bool chunk_needs_check = true;
610 if (!no_duplicates_map_ && !duplicates_map_.Contains(chunk_hash)) {
611 duplicates_map_.Insert(chunk_hash, 1);
612 } else if (!no_duplicates_map_) {
613 chunk_needs_check = false;
614 }
615 if (chunk_needs_check) {
616 const string chunk_path = "data/" + chunk_hash.MakePath();
617 if (!Exists(chunk_path)) {
618 LogCvmfs(kLogCvmfs, kLogStderr,
619 "partial data chunk %s (%s -> "
620 "offset: %ld | size: %lu) missing",
621 this_chunk.content_hash().ToStringWithSuffix().c_str(),
622 full_path.c_str(), this_chunk.offset(),
623 this_chunk.size());
624 retval = false;
625 }
626 }
627 }
628 }
629
630 // is the aggregated chunk size equal to the actual file size?
631 if (aggregated_file_size != entries[i].size()) {
632 LogCvmfs(kLogCvmfs, kLogStderr,
633 "chunks of file %s produce a size "
634 "mismatch. Calculated %zu bytes | %lu "
635 "bytes expected",
636 full_path.c_str(), aggregated_file_size, entries[i].size());
637 retval = false;
638 }
639 }
640 } // Loop through entries
641
642 // Check if nested catalog marker has been found
643 if (!path.IsEmpty() && (path == catalog->mountpoint())
644 && !found_nested_marker) {
645 LogCvmfs(kLogCvmfs, kLogStderr, "nested catalog without marker at %s",
646 path.c_str());
647 retval = false;
648 }
649
650 // Check directory linkcount
651 if (this_directory.linkcount() != num_subdirs + 2) {
652 LogCvmfs(kLogCvmfs, kLogStderr,
653 "wrong linkcount for %s; "
654 "expected %u, got %u",
655 path.c_str(), num_subdirs + 2, this_directory.linkcount());
656 retval = false;
657 }
658
659 // Check hardlink linkcounts
660 for (HardlinkMap::const_iterator i = hardlinks.begin(),
661 iEnd = hardlinks.end();
662 i != iEnd;
663 ++i) {
664 if (i->second[0].linkcount() != i->second.size()) {
665 LogCvmfs(kLogCvmfs, kLogStderr,
666 "hardlink linkcount wrong for %s, "
667 "expected %lu, got %u",
668 (path.ToString() + "/" + i->second[0].name().ToString()).c_str(),
669 i->second.size(), i->second[0].linkcount());
670 retval = false;
671 }
672 }
673
674 return retval;
675 }
676
677
678 string CommandCheck::DownloadPiece(const shash::Any catalog_hash) {
679 const string source = "data/" + catalog_hash.MakePath();
680 const string dest = temp_directory_ + "/" + catalog_hash.ToString();
681 const string url = repo_base_path_ + "/" + source;
682
683 cvmfs::PathSink pathsink(dest);
684 download::JobInfo download_catalog(&url, true, false, &catalog_hash,
685 &pathsink);
686 const download::Failures retval = download_manager()->Fetch(
687 &download_catalog);
688 if (retval != download::kFailOk) {
689 LogCvmfs(kLogCvmfs, kLogStderr, "failed to download object %s (%d)",
690 catalog_hash.ToString().c_str(), retval);
691 return "";
692 }
693
694 return dest;
695 }
696
697
698 string CommandCheck::DecompressPiece(const shash::Any catalog_hash) {
699 const string source = "data/" + catalog_hash.MakePath();
700 const string dest = temp_directory_ + "/" + catalog_hash.ToString();
701 if (!zlib::DecompressPath2Path(source, dest))
702 return "";
703
704 return dest;
705 }
706
707
708 catalog::Catalog *CommandCheck::FetchCatalog(const string &path,
709 const shash::Any &catalog_hash,
710 const uint64_t catalog_size) {
711 string tmp_file;
712 if (!is_remote_)
713 tmp_file = DecompressPiece(catalog_hash);
714 else
715 tmp_file = DownloadPiece(catalog_hash);
716
717 if (tmp_file == "") {
718 LogCvmfs(kLogCvmfs, kLogStderr, "failed to load catalog %s",
719 catalog_hash.ToString().c_str());
720 return NULL;
721 }
722
723 catalog::Catalog *catalog = catalog::Catalog::AttachFreely(path, tmp_file,
724 catalog_hash);
725 const int64_t catalog_file_size = GetFileSize(tmp_file);
726 if (catalog_file_size <= 0) {
727 LogCvmfs(kLogCvmfs, kLogStderr, "Error downloading catalog %s at %s %s",
728 catalog_hash.ToString().c_str(), path.c_str(), tmp_file.c_str());
729 assert(catalog_file_size > 0);
730 }
731 unlink(tmp_file.c_str());
732
733 if ((catalog_size > 0) && (uint64_t(catalog_file_size) != catalog_size)) {
734 LogCvmfs(kLogCvmfs, kLogStderr,
735 "catalog file size mismatch, "
736 "expected %" PRIu64 ", got %" PRIu64,
737 catalog_size, catalog_file_size);
738 delete catalog;
739 return NULL;
740 }
741
742 return catalog;
743 }
744
745
746 bool CommandCheck::FindSubtreeRootCatalog(const string &subtree_path,
747 shash::Any *root_hash,
748 uint64_t *root_size) {
749 catalog::Catalog *current_catalog = FetchCatalog("", *root_hash);
750 if (current_catalog == NULL) {
751 return false;
752 }
753
754 typedef vector<string> Tokens;
755 const Tokens path_tokens = SplitString(subtree_path, '/');
756
757 string current_path = "";
758
759 Tokens::const_iterator i = path_tokens.begin();
760 const Tokens::const_iterator iend = path_tokens.end();
761 for (; i != iend; ++i) {
762 if (i->empty()) {
763 continue;
764 }
765
766 current_path += "/" + *i;
767 if (current_catalog->FindNested(PathString(current_path), root_hash,
768 root_size)) {
769 delete current_catalog;
770
771 if (current_path.length() < subtree_path.length()) {
772 current_catalog = FetchCatalog(current_path, *root_hash);
773 if (current_catalog == NULL) {
774 break;
775 }
776 } else {
777 return true;
778 }
779 }
780 }
781 return false;
782 }
783
784
785 /**
786 * Recursion on nested catalog level. No ownership of computed_counters.
787 */
788 bool CommandCheck::InspectTree(const string &path,
789 const shash::Any &catalog_hash,
790 const uint64_t catalog_size,
791 const bool is_nested_catalog,
792 const catalog::DirectoryEntry *transition_point,
793 catalog::DeltaCounters *computed_counters) {
794 LogCvmfs(kLogCvmfs, kLogStdout | kLogInform, "[inspecting catalog] %s at %s",
795 catalog_hash.ToString().c_str(), path == "" ? "/" : path.c_str());
796
797 const catalog::Catalog *catalog = FetchCatalog(path, catalog_hash,
798 catalog_size);
799 if (catalog == NULL) {
800 LogCvmfs(kLogCvmfs, kLogStderr, "failed to open catalog %s",
801 catalog_hash.ToString().c_str());
802 return false;
803 }
804
805 int retval = true;
806
807 if (catalog->root_prefix() != PathString(path.data(), path.length())) {
808 LogCvmfs(kLogCvmfs, kLogStderr,
809 "root prefix mismatch; "
810 "expected %s, got %s",
811 path.c_str(), catalog->root_prefix().c_str());
812 retval = false;
813 }
814
815 // Check transition point
816 catalog::DirectoryEntry root_entry;
817 if (!catalog->LookupPath(catalog->root_prefix(), &root_entry)) {
818 LogCvmfs(kLogCvmfs, kLogStderr, "failed to lookup root entry (%s)",
819 path.c_str());
820 retval = false;
821 }
822 if (!root_entry.IsDirectory()) {
823 LogCvmfs(kLogCvmfs, kLogStderr, "root entry not a directory (%s)",
824 path.c_str());
825 retval = false;
826 }
827 if (is_nested_catalog) {
828 if (transition_point != NULL
829 && !CompareEntries(*transition_point, root_entry, true, true)) {
830 LogCvmfs(kLogCvmfs, kLogStderr,
831 "transition point and root entry differ (%s)", path.c_str());
832 retval = false;
833 }
834 if (!root_entry.IsNestedCatalogRoot()) {
835 LogCvmfs(kLogCvmfs, kLogStderr,
836 "nested catalog root expected but not found (%s)", path.c_str());
837 retval = false;
838 }
839 } else {
840 if (root_entry.IsNestedCatalogRoot()) {
841 LogCvmfs(kLogCvmfs, kLogStderr,
842 "nested catalog root found but not expected (%s)", path.c_str());
843 retval = false;
844 }
845 }
846
847 // Traverse the catalog
848 set<PathString> bind_mountpoints;
849 if (!Find(catalog, PathString(path.data(), path.length()), computed_counters,
850 &bind_mountpoints)) {
851 retval = false;
852 }
853
854 // Check number of entries
855 if (root_entry.HasXattrs())
856 computed_counters->self.xattrs++;
857 const uint64_t num_found_entries = 1 + computed_counters->self.regular_files
858 + computed_counters->self.symlinks
859 + computed_counters->self.specials
860 + computed_counters->self.directories;
861 if (num_found_entries != catalog->GetNumEntries()) {
862 LogCvmfs(kLogCvmfs, kLogStderr,
863 "dangling entries in catalog, "
864 "expected %" PRIu64 ", got %" PRIu64,
865 catalog->GetNumEntries(), num_found_entries);
866 retval = false;
867 }
868
869 // Recurse into nested catalogs
870 const catalog::Catalog::NestedCatalogList
871 &nested_catalogs = catalog->ListNestedCatalogs();
872 const catalog::Catalog::NestedCatalogList
873 own_nested_catalogs = catalog->ListOwnNestedCatalogs();
874 if (own_nested_catalogs.size()
875 != static_cast<uint64_t>(computed_counters->self.nested_catalogs)) {
876 LogCvmfs(kLogCvmfs, kLogStderr,
877 "number of nested catalogs does not match;"
878 " expected %lu, got %lu",
879 computed_counters->self.nested_catalogs,
880 own_nested_catalogs.size());
881 retval = false;
882 }
883 set<PathString> nested_catalog_paths;
884 for (catalog::Catalog::NestedCatalogList::const_iterator
885 i = nested_catalogs.begin(),
886 iEnd = nested_catalogs.end();
887 i != iEnd;
888 ++i) {
889 nested_catalog_paths.insert(i->mountpoint);
890 }
891 if (nested_catalog_paths.size() != nested_catalogs.size()) {
892 LogCvmfs(kLogCvmfs, kLogStderr,
893 "duplicates among nested catalogs and bind mountpoints");
894 retval = false;
895 }
896
897 for (catalog::Catalog::NestedCatalogList::const_iterator
898 i = nested_catalogs.begin(),
899 iEnd = nested_catalogs.end();
900 i != iEnd;
901 ++i) {
902 if (bind_mountpoints.find(i->mountpoint) != bind_mountpoints.end()) {
903 catalog::DirectoryEntry bind_mountpoint;
904 const PathString mountpoint("/" + i->mountpoint.ToString().substr(1));
905 if (!catalog->LookupPath(mountpoint, &bind_mountpoint)) {
906 LogCvmfs(kLogCvmfs, kLogStderr, "failed to lookup bind mountpoint %s",
907 mountpoint.c_str());
908 retval = false;
909 }
910 LogCvmfs(kLogCvmfs, kLogDebug, "skipping bind mountpoint %s",
911 i->mountpoint.c_str());
912 continue;
913 }
914 catalog::DirectoryEntry nested_transition_point;
915 if (!catalog->LookupPath(i->mountpoint, &nested_transition_point)) {
916 LogCvmfs(kLogCvmfs, kLogStderr, "failed to lookup transition point %s",
917 i->mountpoint.c_str());
918 retval = false;
919 } else {
920 catalog::DeltaCounters nested_counters;
921 const bool is_nested = true;
922 if (!InspectTree(i->mountpoint.ToString(), i->hash, i->size, is_nested,
923 &nested_transition_point, &nested_counters))
924 retval = false;
925 nested_counters.PopulateToParent(computed_counters);
926 }
927 }
928
929 // Check statistics counters
930 // Additionally account for root directory
931 computed_counters->self.directories++;
932 catalog::Counters compare_counters;
933 compare_counters.ApplyDelta(*computed_counters);
934 const catalog::Counters stored_counters = catalog->GetCounters();
935 if (!CompareCounters(compare_counters, stored_counters)) {
936 LogCvmfs(kLogCvmfs, kLogStderr, "statistics counter mismatch [%s]",
937 catalog_hash.ToString().c_str());
938 retval = false;
939 }
940
941 delete catalog;
942 return retval;
943 }
944
945
946 int CommandCheck::Main(const swissknife::ArgumentList &args) {
947 string tag_name;
948 string subtree_path = "";
949 string pubkey_path = "";
950 string repo_name = "";
951 string reflog_chksum_path = "";
952
953 temp_directory_ = (args.find('t') != args.end()) ? *args.find('t')->second
954 : "/tmp";
955 if (args.find('n') != args.end())
956 tag_name = *args.find('n')->second;
957 if (args.find('c') != args.end())
958 check_chunks_ = true;
959 if (args.find('d') != args.end())
960 no_duplicates_map_ = true;
961 if (args.find('l') != args.end()) {
962 const unsigned log_level = kLogLevel0
963 << String2Uint64(*args.find('l')->second);
964 if (log_level > kLogNone) {
965 LogCvmfs(kLogCvmfs, kLogStderr, "invalid log level");
966 return 1;
967 }
968 SetLogVerbosity(static_cast<LogLevels>(log_level));
969 }
970 if (args.find('k') != args.end())
971 pubkey_path = *args.find('k')->second;
972 if (DirectoryExists(pubkey_path))
973 pubkey_path = JoinStrings(FindFilesBySuffix(pubkey_path, ".pub"), ":");
974 if (args.find('N') != args.end())
975 repo_name = *args.find('N')->second;
976
977 repo_base_path_ = MakeCanonicalPath(*args.find('r')->second);
978 if (args.find('s') != args.end())
979 subtree_path = MakeCanonicalPath(*args.find('s')->second);
980 if (args.find('R') != args.end())
981 reflog_chksum_path = *args.find('R')->second;
982
983 // Repository can be HTTP address or on local file system
984 is_remote_ = IsHttpUrl(repo_base_path_);
985
986 // initialize the (swissknife global) download and signature managers
987 if (is_remote_) {
988 const bool follow_redirects = (args.count('L') > 0);
989 const string proxy = (args.count('@') > 0) ? *args.find('@')->second : "";
990 if (!this->InitDownloadManager(follow_redirects, proxy)) {
991 return 1;
992 }
993
994 if (pubkey_path.empty() || repo_name.empty()) {
995 LogCvmfs(kLogCvmfs, kLogStderr,
996 "please provide pubkey and repo name for "
997 "remote repositories");
998 return 1;
999 }
1000
1001 if (!this->InitSignatureManager(pubkey_path)) {
1002 return 1;
1003 }
1004 }
1005
1006 // Load Manifest
1007 UniquePtr<manifest::Manifest> manifest;
1008 bool successful = true;
1009
1010 if (is_remote_) {
1011 manifest = FetchRemoteManifest(repo_base_path_, repo_name);
1012 } else {
1013 if (chdir(repo_base_path_.c_str()) != 0) {
1014 LogCvmfs(kLogCvmfs, kLogStderr, "failed to switch to directory %s",
1015 repo_base_path_.c_str());
1016 return 1;
1017 }
1018 manifest = OpenLocalManifest(".cvmfspublished");
1019 }
1020
1021 if (!manifest.IsValid()) {
1022 LogCvmfs(kLogCvmfs, kLogStderr, "failed to load repository manifest");
1023 return 1;
1024 }
1025
1026 // Check meta-info object
1027 if (!manifest->meta_info().IsNull()) {
1028 string tmp_file;
1029 if (!is_remote_)
1030 tmp_file = DecompressPiece(manifest->meta_info());
1031 else
1032 tmp_file = DownloadPiece(manifest->meta_info());
1033 if (tmp_file == "") {
1034 LogCvmfs(kLogCvmfs, kLogStderr, "failed to load repository metainfo %s",
1035 manifest->meta_info().ToString().c_str());
1036 return 1;
1037 }
1038 unlink(tmp_file.c_str());
1039 }
1040
1041 shash::Any reflog_hash;
1042 if (!reflog_chksum_path.empty()) {
1043 if (!manifest::Reflog::ReadChecksum(reflog_chksum_path, &reflog_hash)) {
1044 LogCvmfs(kLogCvmfs, kLogStderr, "failed to read reflog checksum file");
1045 return 1;
1046 }
1047 } else {
1048 reflog_hash = manifest->reflog_hash();
1049 }
1050
1051 if (Exists(".cvmfsreflog")) {
1052 if (reflog_hash.IsNull()) {
1053 // If there is a reflog, we want to check it
1054 LogCvmfs(kLogCvmfs, kLogStderr,
1055 ".cvmfsreflog present but no checksum provided, aborting");
1056 return 1;
1057 }
1058 const bool retval = InspectReflog(reflog_hash, manifest.weak_ref());
1059 if (!retval) {
1060 LogCvmfs(kLogCvmfs, kLogStderr, "failed to verify reflog");
1061 return 1;
1062 }
1063 } else {
1064 if (!reflog_hash.IsNull()) {
1065 // There is a checksum but no reflog; possibly the checksum is for the
1066 // from the manifest for the stratum 0 reflog
1067 if (!reflog_chksum_path.empty()) {
1068 LogCvmfs(kLogCvmfs, kLogStderr,
1069 "local reflog checksum set but reflog itself is missing, "
1070 "aborting");
1071 return 1;
1072 }
1073 }
1074 }
1075
1076 // Load history
1077 UniquePtr<history::History> tag_db;
1078 if (!manifest->history().IsNull()) {
1079 string tmp_file;
1080 if (!is_remote_)
1081 tmp_file = DecompressPiece(manifest->history());
1082 else
1083 tmp_file = DownloadPiece(manifest->history());
1084 if (tmp_file == "") {
1085 LogCvmfs(kLogCvmfs, kLogStderr, "failed to load history database %s",
1086 manifest->history().ToString().c_str());
1087 return 1;
1088 }
1089 tag_db = history::SqliteHistory::Open(tmp_file);
1090 if (!tag_db.IsValid()) {
1091 LogCvmfs(kLogCvmfs, kLogStderr, "failed to open history database %s",
1092 manifest->history().ToString().c_str());
1093 return 1;
1094 }
1095 tag_db->TakeDatabaseFileOwnership();
1096 successful = InspectHistory(tag_db.weak_ref()) && successful;
1097 }
1098
1099 if (manifest->has_alt_catalog_path()) {
1100 if (!Exists(manifest->certificate().MakeAlternativePath())) {
1101 LogCvmfs(kLogCvmfs, kLogStderr,
1102 "failed to find alternative certificate link %s",
1103 manifest->certificate().MakeAlternativePath().c_str());
1104 return 1;
1105 }
1106 if (!Exists(manifest->catalog_hash().MakeAlternativePath())) {
1107 LogCvmfs(kLogCvmfs, kLogStderr,
1108 "failed to find alternative catalog link %s",
1109 manifest->catalog_hash().MakeAlternativePath().c_str());
1110 return 1;
1111 }
1112 }
1113
1114 shash::Any root_hash = manifest->catalog_hash();
1115 uint64_t root_size = manifest->catalog_size();
1116 if (tag_name != "") {
1117 if (!tag_db.IsValid()) {
1118 LogCvmfs(kLogCvmfs, kLogStderr, "no history");
1119 return 1;
1120 }
1121 history::History::Tag tag;
1122 const bool retval = tag_db->GetByName(tag_name, &tag);
1123 if (!retval) {
1124 LogCvmfs(kLogCvmfs, kLogStderr, "no such tag: %s", tag_name.c_str());
1125 return 1;
1126 }
1127 root_hash = tag.root_hash;
1128 root_size = tag.size;
1129 LogCvmfs(kLogCvmfs, kLogStdout, "Inspecting repository tag %s",
1130 tag_name.c_str());
1131 }
1132
1133 const bool is_nested_catalog = (!subtree_path.empty());
1134 if (is_nested_catalog
1135 && !FindSubtreeRootCatalog(subtree_path, &root_hash, &root_size)) {
1136 LogCvmfs(kLogCvmfs, kLogStderr, "cannot find nested catalog at %s",
1137 subtree_path.c_str());
1138 return 1;
1139 }
1140
1141
1142 catalog::DeltaCounters computed_counters;
1143 successful = InspectTree(subtree_path,
1144 root_hash,
1145 root_size,
1146 is_nested_catalog,
1147 NULL,
1148 &computed_counters)
1149 && successful;
1150
1151 if (!successful) {
1152 LogCvmfs(kLogCvmfs, kLogStderr, "CATALOG PROBLEMS OR OTHER ERRORS FOUND");
1153 return 1;
1154 }
1155
1156 LogCvmfs(kLogCvmfs, kLogStdout, "no problems found");
1157 return 0;
1158 }
1159
1160 } // namespace swissknife
1161