GCC Code Coverage Report


Directory: cvmfs/
File: cvmfs/swissknife_check.cc
Date: 2026-04-05 02:35:23
Exec Total Coverage
Lines: 0 601 0.0%
Branches: 0 434 0.0%

Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * This tool checks a cvmfs repository for file catalog errors.
5 */
6
7 #define __STDC_FORMAT_MACROS
8
9
10 #include "swissknife_check.h"
11
12 #include <inttypes.h>
13 #include <unistd.h>
14
15 #include <cassert>
16 #include <map>
17 #include <queue>
18 #include <set>
19 #include <string>
20 #include <vector>
21
22 #include "catalog_sql.h"
23 #include "compression/compression.h"
24 #include "file_chunk.h"
25 #include "history_sqlite.h"
26 #include "manifest.h"
27 #include "network/download.h"
28 #include "reflog.h"
29 #include "sanitizer.h"
30 #include "shortstring.h"
31 #include "util/exception.h"
32 #include "util/logging.h"
33 #include "util/pointer.h"
34 #include "util/posix.h"
35
36 using namespace std; // NOLINT
37
38 // for map of duplicate entries; as in kvstore.cc
39 static inline uint32_t hasher_any(const shash::Any &key) {
40 // We'll just do the same thing as hasher_md5, since every hash is at
41 // least as large.
42 return *const_cast<uint32_t *>(reinterpret_cast<const uint32_t *>(key.digest)
43 + 1);
44 }
45
46
47 namespace swissknife {
48
49 CommandCheck::CommandCheck()
50 : check_chunks_(false), no_duplicates_map_(false), is_remote_(false) {
51 const shash::Any hash_null;
52 duplicates_map_.Init(16, hash_null, hasher_any);
53 }
54
55 bool CommandCheck::CompareEntries(const catalog::DirectoryEntry &a,
56 const catalog::DirectoryEntry &b,
57 const bool compare_names,
58 const bool is_transition_point) {
59 typedef catalog::DirectoryEntry::Difference Difference;
60
61 const catalog::DirectoryEntry::Differences diffs = a.CompareTo(b);
62 if (diffs == Difference::kIdentical) {
63 return true;
64 }
65
66 // in case of a nested catalog transition point the controlling flags are
67 // supposed to differ. If this is the only difference we are done...
68 if (is_transition_point
69 && (diffs ^ Difference::kNestedCatalogTransitionFlags) == 0) {
70 return true;
71 }
72
73 bool retval = true;
74 if (compare_names) {
75 if (diffs & Difference::kName) {
76 LogCvmfs(kLogCvmfs, kLogStderr, "names differ: %s / %s", a.name().c_str(),
77 b.name().c_str());
78 retval = false;
79 }
80 }
81 if (diffs & Difference::kLinkcount) {
82 LogCvmfs(kLogCvmfs, kLogStderr, "linkcounts differ: %u / %u", a.linkcount(),
83 b.linkcount());
84 retval = false;
85 }
86 if (diffs & Difference::kHardlinkGroup) {
87 LogCvmfs(kLogCvmfs, kLogStderr, "hardlink groups differ: %u / %u",
88 a.hardlink_group(), b.hardlink_group());
89 retval = false;
90 }
91 if (diffs & Difference::kSize) {
92 LogCvmfs(kLogCvmfs, kLogStderr, "sizes differ: %" PRIu64 " / %" PRIu64,
93 a.size(), b.size());
94 retval = false;
95 }
96 if (diffs & Difference::kMode) {
97 LogCvmfs(kLogCvmfs, kLogStderr, "modes differ: %u / %u", a.mode(),
98 b.mode());
99 retval = false;
100 }
101 if (diffs & Difference::kMtime) {
102 LogCvmfs(kLogCvmfs, kLogStderr, "timestamps differ: %lu / %lu", a.mtime(),
103 b.mtime());
104 retval = false;
105 }
106 if (diffs & Difference::kChecksum) {
107 LogCvmfs(kLogCvmfs, kLogStderr, "content hashes differ: %s / %s",
108 a.checksum().ToString().c_str(), b.checksum().ToString().c_str());
109 retval = false;
110 }
111 if (diffs & Difference::kSymlink) {
112 LogCvmfs(kLogCvmfs, kLogStderr, "symlinks differ: %s / %s",
113 a.symlink().c_str(), b.symlink().c_str());
114 retval = false;
115 }
116 if (diffs & Difference::kExternalFileFlag) {
117 LogCvmfs(kLogCvmfs, kLogStderr,
118 "external file flag differs: %d / %d "
119 "(%s / %s)",
120 a.IsExternalFile(), b.IsExternalFile(), a.name().c_str(),
121 b.name().c_str());
122 retval = false;
123 }
124 if (diffs & Difference::kHasXattrsFlag) {
125 LogCvmfs(kLogCvmfs, kLogStderr,
126 "extended attributes differ: %d / %d "
127 "(%s / %s)",
128 a.HasXattrs(), b.HasXattrs(), a.name().c_str(), b.name().c_str());
129 retval = false;
130 }
131 if (!is_transition_point) {
132 if (diffs & Difference::kUid) {
133 LogCvmfs(kLogCvmfs, kLogStderr, "uids differ: %d / %d (%s / %s)", a.uid(),
134 b.uid(), a.name().c_str(), b.name().c_str());
135 retval = false;
136 }
137 if (diffs & Difference::kGid) {
138 LogCvmfs(kLogCvmfs, kLogStderr, "gids differ: %d / %d (%s / %s)", a.gid(),
139 b.gid(), a.name().c_str(), b.name().c_str());
140 retval = false;
141 }
142 }
143
144 return retval;
145 }
146
147
148 bool CommandCheck::CompareCounters(const catalog::Counters &a,
149 const catalog::Counters &b) {
150 const catalog::Counters::FieldsMap map_a = a.GetFieldsMap();
151 const catalog::Counters::FieldsMap map_b = b.GetFieldsMap();
152
153 bool retval = true;
154 catalog::Counters::FieldsMap::const_iterator i = map_a.begin();
155 const catalog::Counters::FieldsMap::const_iterator iend = map_a.end();
156 for (; i != iend; ++i) {
157 const catalog::Counters::FieldsMap::const_iterator comp = map_b.find(
158 i->first);
159 assert(comp != map_b.end());
160
161 if (*(i->second) != *(comp->second)) {
162 LogCvmfs(kLogCvmfs, kLogStderr,
163 "catalog statistics mismatch: %s (expected: %" PRIu64 " / "
164 "in catalog: %" PRIu64 ")",
165 comp->first.c_str(), *(i->second), *(comp->second));
166 retval = false;
167 }
168 }
169
170 return retval;
171 }
172
173
174 /**
175 * Checks for existence of a file either locally or via HTTP head
176 */
177 bool CommandCheck::Exists(const string &file) {
178 if (!is_remote_) {
179 return FileExists(file) || SymlinkExists(file);
180 } else {
181 const string url = repo_base_path_ + "/" + file;
182 LogCvmfs(kLogCvmfs, kLogVerboseMsg, "[Exists::url] %s", url.c_str());
183 download::JobInfo head(&url, false);
184 return download_manager()->Fetch(&head) == download::kFailOk;
185 }
186 }
187
188
189 /**
190 * Copies a file from the repository into a temporary file.
191 */
192 string CommandCheck::FetchPath(const string &path) {
193 string tmp_path;
194 FILE *f = CreateTempFile(temp_directory_ + "/cvmfstmp", kDefaultFileMode,
195 "w+", &tmp_path);
196 assert(f != NULL);
197
198 const string url = repo_base_path_ + "/" + path;
199 if (is_remote_) {
200 cvmfs::FileSink filesink(f);
201 download::JobInfo download_job(&url, false, false, NULL, &filesink);
202 const download::Failures retval = download_manager()->Fetch(&download_job);
203 if (retval != download::kFailOk) {
204 PANIC(kLogStderr, "failed to read %s", url.c_str());
205 }
206 } else {
207 const bool retval = CopyPath2File(url, f);
208 if (!retval) {
209 PANIC(kLogStderr, "failed to read %s", url.c_str());
210 }
211 }
212
213 fclose(f);
214 return tmp_path;
215 }
216
217
218 /**
219 * Verifies reflog checksum and looks for presence of the entry points
220 * referenced in the manifest.
221 */
222 bool CommandCheck::InspectReflog(const shash::Any &reflog_hash,
223 manifest::Manifest *manifest) {
224 LogCvmfs(kLogCvmfs, kLogStdout, "Inspecting log of references");
225 const string reflog_path = FetchPath(".cvmfsreflog");
226 shash::Any computed_hash(reflog_hash.algorithm);
227 manifest::Reflog::HashDatabase(reflog_path, &computed_hash);
228 if (computed_hash != reflog_hash) {
229 LogCvmfs(kLogCvmfs, kLogStderr,
230 "The .cvmfsreflog has unexpected content hash %s (expected %s)",
231 computed_hash.ToString().c_str(), reflog_hash.ToString().c_str());
232 unlink(reflog_path.c_str());
233 return false;
234 }
235
236 const UniquePtr<manifest::Reflog> reflog(manifest::Reflog::Open(reflog_path));
237 assert(reflog.IsValid());
238 reflog->TakeDatabaseFileOwnership();
239
240 if (!reflog->ContainsCatalog(manifest->catalog_hash())) {
241 LogCvmfs(kLogCvmfs, kLogStderr,
242 "failed to find catalog root hash %s in .cvmfsreflog",
243 manifest->catalog_hash().ToString().c_str());
244 return false;
245 }
246
247 if (!reflog->ContainsCertificate(manifest->certificate())) {
248 LogCvmfs(kLogCvmfs, kLogStderr,
249 "failed to find certificate hash %s in .cvmfsreflog",
250 manifest->certificate().ToString().c_str());
251 return false;
252 }
253
254 if (!manifest->history().IsNull()
255 && !reflog->ContainsHistory(manifest->history())) {
256 LogCvmfs(kLogCvmfs, kLogStderr,
257 "failed to find tag database's hash %s in .cvmfsreflog",
258 manifest->history().ToString().c_str());
259 return false;
260 }
261
262 if (!manifest->meta_info().IsNull()
263 && !reflog->ContainsMetainfo(manifest->meta_info())) {
264 LogCvmfs(kLogCvmfs, kLogStderr,
265 "failed to find meta info hash %s in .cvmfsreflog",
266 manifest->meta_info().ToString().c_str());
267 return false;
268 }
269
270 return true;
271 }
272
273
274 /**
275 * Verifies the logical consistency of the tag database.
276 */
277 bool CommandCheck::InspectHistory(history::History *history) {
278 LogCvmfs(kLogCvmfs, kLogStdout, "Inspecting tag database");
279 bool retval;
280 vector<history::History::Tag> tags;
281 retval = history->List(&tags);
282 if (!retval) {
283 LogCvmfs(kLogCvmfs, kLogStderr, "failed to enumerate tags");
284 return false;
285 }
286 vector<history::History::Branch> branches;
287 retval = history->ListBranches(&branches);
288 if (!retval) {
289 LogCvmfs(kLogCvmfs, kLogStderr, "failed to enumerate branches");
290 return false;
291 }
292
293 bool result = true;
294
295 map<string, uint64_t> initial_revisions;
296 const sanitizer::BranchSanitizer sanitizer;
297 for (unsigned i = 0; i < branches.size(); ++i) {
298 if (!sanitizer.IsValid(branches[i].branch)) {
299 LogCvmfs(kLogCvmfs, kLogStderr, "invalid branch name: %s",
300 branches[i].branch.c_str());
301 result = false;
302 }
303 initial_revisions[branches[i].branch] = branches[i].initial_revision;
304 }
305
306 set<string> used_branches; // all branches referenced in tag db
307 // TODO(jblomer): same root hash implies same size and revision
308 for (unsigned i = 0; i < tags.size(); ++i) {
309 used_branches.insert(tags[i].branch);
310 const map<string, uint64_t>::const_iterator iter = initial_revisions.find(
311 tags[i].branch);
312 if (iter == initial_revisions.end()) {
313 LogCvmfs(kLogCvmfs, kLogStderr, "invalid branch %s in tag %s",
314 tags[i].branch.c_str(), tags[i].name.c_str());
315 result = false;
316 } else {
317 if (tags[i].revision < iter->second) {
318 LogCvmfs(kLogCvmfs, kLogStderr,
319 "invalid revision %" PRIu64 " of tag %s", tags[i].revision,
320 tags[i].name.c_str());
321 result = false;
322 }
323 }
324 }
325
326 if (used_branches.size() != branches.size()) {
327 LogCvmfs(kLogCvmfs, kLogStderr, "unused, dangling branches stored");
328 result = false;
329 }
330
331 return result;
332 }
333
334
335 /**
336 * Recursive catalog walk-through
337 *
338 * TODO(vavolkl): This method is large and does a lot of checks
339 * that could be split into smaller ones.
340 *
341 */
342 bool CommandCheck::Find(const catalog::Catalog *catalog,
343 const PathString &path,
344 catalog::DeltaCounters *computed_counters,
345 set<PathString> *bind_mountpoints) {
346 catalog::DirectoryEntryList entries;
347 catalog::DirectoryEntry this_directory;
348
349 if (!catalog->LookupPath(path, &this_directory)) {
350 LogCvmfs(kLogCvmfs, kLogStderr, "failed to lookup %s", path.c_str());
351 return false;
352 }
353 if (!catalog->ListingPath(path, &entries)) {
354 LogCvmfs(kLogCvmfs, kLogStderr, "failed to list %s", path.c_str());
355 return false;
356 }
357
358 uint32_t num_subdirs = 0;
359 bool retval = true;
360 typedef map<uint32_t, vector<catalog::DirectoryEntry> > HardlinkMap;
361 HardlinkMap hardlinks;
362 bool found_nested_marker = false;
363
364 for (unsigned i = 0; i < entries.size(); ++i) {
365 // for performance reasons, keep track of files already checked
366 // and only run requests once per hash
367 const bool entry_needs_check = !entries[i].checksum().IsNull() &&
368 !entries[i].IsExternalFile() &&
369 !(catalog::g_ignore_legacy_bulk_hashes &&
370 entries[i].IsChunkedFile()) &&
371 // fallback cli option can force the entry to
372 // be checked
373 (no_duplicates_map_
374 || !duplicates_map_.Contains(
375 entries[i].checksum()));
376 if (entry_needs_check && !no_duplicates_map_)
377 duplicates_map_.Insert(entries[i].checksum(), 1);
378
379 PathString full_path(path);
380 full_path.Append("/", 1);
381 full_path.Append(entries[i].name().GetChars(),
382 entries[i].name().GetLength());
383 LogCvmfs(kLogCvmfs, kLogVerboseMsg, "[path] %s [needs check] %i",
384 full_path.c_str(), entry_needs_check);
385
386
387 // Name must not be empty
388 if (entries[i].name().IsEmpty()) {
389 LogCvmfs(kLogCvmfs, kLogStderr, "empty path at %s", full_path.c_str());
390 retval = false;
391 }
392
393 // Catalog markers should indicate nested catalogs
394 if (entries[i].name() == NameString(string(".cvmfscatalog"))) {
395 if (catalog->mountpoint() != path) {
396 LogCvmfs(kLogCvmfs, kLogStderr,
397 "found abandoned nested catalog marker at %s",
398 full_path.c_str());
399 retval = false;
400 }
401 found_nested_marker = true;
402 }
403
404 // Check if checksum is not null
405 if (entries[i].IsRegular() && !entries[i].IsChunkedFile()
406 && entries[i].checksum().IsNull()) {
407 LogCvmfs(kLogCvmfs, kLogStderr,
408 "regular file pointing to zero-hash: '%s'", full_path.c_str());
409 retval = false;
410 }
411
412 // Check if the chunk is there
413 if (check_chunks_ && entry_needs_check) {
414 string chunk_path = "data/" + entries[i].checksum().MakePath();
415 if (entries[i].IsDirectory())
416 chunk_path += shash::kSuffixMicroCatalog;
417 if (!Exists(chunk_path)) {
418 LogCvmfs(kLogCvmfs, kLogStderr, "data chunk %s (%s) missing",
419 entries[i].checksum().ToString().c_str(), full_path.c_str());
420 retval = false;
421 }
422 }
423
424 // Add hardlinks to counting map
425 if ((entries[i].linkcount() > 1) && !entries[i].IsDirectory()) {
426 if (entries[i].hardlink_group() == 0) {
427 LogCvmfs(kLogCvmfs, kLogStderr, "invalid hardlink group for %s",
428 full_path.c_str());
429 retval = false;
430 } else {
431 const HardlinkMap::iterator hardlink_group = hardlinks.find(
432 entries[i].hardlink_group());
433 if (hardlink_group == hardlinks.end()) {
434 hardlinks[entries[i].hardlink_group()];
435 hardlinks[entries[i].hardlink_group()].push_back(entries[i]);
436 } else {
437 if (!CompareEntries(entries[i], (hardlink_group->second)[0], false)) {
438 LogCvmfs(kLogCvmfs, kLogStderr, "hardlink %s doesn't match",
439 full_path.c_str());
440 retval = false;
441 }
442 hardlink_group->second.push_back(entries[i]);
443 } // Hardlink added to map
444 } // Hardlink group > 0
445 } // Hardlink found
446
447 // For any kind of entry, the linkcount should be > 0
448 if (entries[i].linkcount() == 0) {
449 LogCvmfs(kLogCvmfs, kLogStderr, "Entry %s has linkcount 0.",
450 entries[i].name().c_str());
451 retval = false;
452 }
453
454 // Checks depending of entry type
455 if (!entries[i].IsRegular()) {
456 if (entries[i].IsDirectIo()) {
457 LogCvmfs(kLogCvmfs, kLogStderr, "invalid direct i/o flag found: %s",
458 full_path.c_str());
459 retval = false;
460 }
461 }
462 if (entries[i].IsDirectory()) {
463 computed_counters->self.directories++;
464 num_subdirs++;
465 // Directory size
466 // if (entries[i].size() < 4096) {
467 // LogCvmfs(kLogCvmfs, kLogStderr, "invalid file size for %s",
468 // full_path.c_str());
469 // retval = false;
470 // }
471 // No directory hardlinks
472 if (entries[i].hardlink_group() != 0) {
473 LogCvmfs(kLogCvmfs, kLogStderr, "directory hardlink found at %s",
474 full_path.c_str());
475 retval = false;
476 }
477 if (entries[i].IsNestedCatalogMountpoint()
478 || entries[i].IsBindMountpoint()) {
479 // Find transition point
480 if (entries[i].IsNestedCatalogMountpoint())
481 computed_counters->self.nested_catalogs++;
482 shash::Any tmp;
483 uint64_t tmp2;
484 const PathString mountpoint(full_path);
485 if (!catalog->FindNested(mountpoint, &tmp, &tmp2)) {
486 LogCvmfs(kLogCvmfs, kLogStderr, "nested catalog at %s not registered",
487 full_path.c_str());
488 retval = false;
489 }
490
491 // check that the nested mountpoint is empty in the current catalog
492 catalog::DirectoryEntryList nested_entries;
493 if (catalog->ListingPath(full_path, &nested_entries)
494 && !nested_entries.empty()) {
495 LogCvmfs(kLogCvmfs, kLogStderr,
496 "non-empty nested catalog mountpoint "
497 "at %s.",
498 full_path.c_str());
499 retval = false;
500 }
501
502 if (entries[i].IsBindMountpoint()) {
503 bind_mountpoints->insert(full_path);
504 if (entries[i].IsNestedCatalogMountpoint()) {
505 LogCvmfs(kLogCvmfs, kLogStderr,
506 "bind mountpoint and nested mountpoint mutually exclusive"
507 " at %s.",
508 full_path.c_str());
509 retval = false;
510 }
511 }
512 } else {
513 // Recurse
514 if (!Find(catalog, full_path, computed_counters, bind_mountpoints))
515 retval = false;
516 }
517 } else if (entries[i].IsLink()) {
518 computed_counters->self.symlinks++;
519 // No hash for symbolics links
520 if (!entries[i].checksum().IsNull()) {
521 LogCvmfs(kLogCvmfs, kLogStderr, "symbolic links with hash at %s",
522 full_path.c_str());
523 retval = false;
524 }
525 // Right size of symbolic link?
526 if (entries[i].size() != entries[i].symlink().GetLength()) {
527 LogCvmfs(kLogCvmfs, kLogStderr,
528 "wrong symbolic link size for %s; "
529 "expected %u, got %lu",
530 full_path.c_str(), entries[i].symlink().GetLength(),
531 entries[i].size());
532 retval = false;
533 }
534 } else if (entries[i].IsRegular()) {
535 computed_counters->self.regular_files++;
536 computed_counters->self.file_size += entries[i].size();
537 } else if (entries[i].IsSpecial()) {
538 computed_counters->self.specials++;
539 // Size zero for special files
540 if (entries[i].size() != 0) {
541 LogCvmfs(kLogCvmfs, kLogStderr,
542 "unexpected non-zero special file size %s", full_path.c_str());
543 retval = false;
544 }
545 // No hash for special files
546 if (!entries[i].checksum().IsNull()) {
547 LogCvmfs(kLogCvmfs, kLogStderr, "special file with hash at %s",
548 full_path.c_str());
549 retval = false;
550 }
551 // No symlink
552 if (entries[i].symlink().GetLength() > 0) {
553 LogCvmfs(kLogCvmfs, kLogStderr,
554 "special file with non-zero symlink at %s", full_path.c_str());
555 retval = false;
556 }
557 } else {
558 LogCvmfs(kLogCvmfs, kLogStderr, "unknown file type %s",
559 full_path.c_str());
560 retval = false;
561 }
562
563 if (entries[i].HasXattrs()) {
564 computed_counters->self.xattrs++;
565 }
566
567 if (entries[i].IsExternalFile()) {
568 computed_counters->self.externals++;
569 computed_counters->self.external_file_size += entries[i].size();
570 if (!entries[i].IsRegular()) {
571 LogCvmfs(kLogCvmfs, kLogStderr,
572 "only regular files can be external: %s", full_path.c_str());
573 retval = false;
574 }
575 }
576
577 // checking file chunk integrity
578 if (entries[i].IsChunkedFile()) {
579 FileChunkList chunks;
580 catalog->ListPathChunks(full_path, entries[i].hash_algorithm(), &chunks);
581
582 computed_counters->self.chunked_files++;
583 computed_counters->self.chunked_file_size += entries[i].size();
584 computed_counters->self.file_chunks += chunks.size();
585
586 // do we find file chunks for the chunked file in this catalog?
587 if (chunks.size() == 0) {
588 LogCvmfs(kLogCvmfs, kLogStderr, "no file chunks found for big file %s",
589 full_path.c_str());
590 retval = false;
591 }
592
593 size_t aggregated_file_size = 0;
594 off_t next_offset = 0;
595
596 for (unsigned j = 0; j < chunks.size(); ++j) {
597 const FileChunk this_chunk = chunks.At(j);
598 // check if the chunk boundaries fit together...
599 if (next_offset != this_chunk.offset()) {
600 LogCvmfs(kLogCvmfs, kLogStderr, "misaligned chunk offsets for %s",
601 full_path.c_str());
602 retval = false;
603 }
604 next_offset = this_chunk.offset() + this_chunk.size();
605 aggregated_file_size += this_chunk.size();
606
607 // are all data chunks in the data store?
608 if (check_chunks_ && !entries[i].IsExternalFile()) {
609 const shash::Any &chunk_hash = this_chunk.content_hash();
610 // for performance reasons, only perform the check once
611 // and skip if the hash has been checked before
612 bool chunk_needs_check = true;
613 if (!no_duplicates_map_ && !duplicates_map_.Contains(chunk_hash)) {
614 duplicates_map_.Insert(chunk_hash, 1);
615 } else if (!no_duplicates_map_) {
616 chunk_needs_check = false;
617 }
618 if (chunk_needs_check) {
619 const string chunk_path = "data/" + chunk_hash.MakePath();
620 if (!Exists(chunk_path)) {
621 LogCvmfs(kLogCvmfs, kLogStderr,
622 "partial data chunk %s (%s -> "
623 "offset: %ld | size: %lu) missing",
624 this_chunk.content_hash().ToStringWithSuffix().c_str(),
625 full_path.c_str(), this_chunk.offset(),
626 this_chunk.size());
627 retval = false;
628 }
629 }
630 }
631 }
632
633 // is the aggregated chunk size equal to the actual file size?
634 if (aggregated_file_size != entries[i].size()) {
635 LogCvmfs(kLogCvmfs, kLogStderr,
636 "chunks of file %s produce a size "
637 "mismatch. Calculated %zu bytes | %lu "
638 "bytes expected",
639 full_path.c_str(), aggregated_file_size, entries[i].size());
640 retval = false;
641 }
642 }
643 } // Loop through entries
644
645 // Check if nested catalog marker has been found
646 if (!path.IsEmpty() && (path == catalog->mountpoint())
647 && !found_nested_marker) {
648 LogCvmfs(kLogCvmfs, kLogStderr, "nested catalog without marker at %s",
649 path.c_str());
650 retval = false;
651 }
652
653 // Check directory linkcount
654 if (this_directory.linkcount() != num_subdirs + 2) {
655 LogCvmfs(kLogCvmfs, kLogStderr,
656 "wrong linkcount for %s; "
657 "expected %u, got %u",
658 path.c_str(), num_subdirs + 2, this_directory.linkcount());
659 retval = false;
660 }
661
662 // Check hardlink linkcounts
663 for (HardlinkMap::const_iterator i = hardlinks.begin(),
664 iEnd = hardlinks.end();
665 i != iEnd;
666 ++i) {
667 if (i->second[0].linkcount() != i->second.size()) {
668 LogCvmfs(kLogCvmfs, kLogStderr,
669 "hardlink linkcount wrong for %s, "
670 "expected %lu, got %u",
671 (path.ToString() + "/" + i->second[0].name().ToString()).c_str(),
672 i->second.size(), i->second[0].linkcount());
673 retval = false;
674 }
675 }
676
677 return retval;
678 }
679
680
681 string CommandCheck::DownloadPiece(const shash::Any catalog_hash) {
682 const string source = "data/" + catalog_hash.MakePath();
683 const string dest = temp_directory_ + "/" + catalog_hash.ToString();
684 const string url = repo_base_path_ + "/" + source;
685
686 cvmfs::PathSink pathsink(dest);
687 download::JobInfo download_catalog(&url, true, false, &catalog_hash,
688 &pathsink);
689 const download::Failures retval = download_manager()->Fetch(
690 &download_catalog);
691 if (retval != download::kFailOk) {
692 LogCvmfs(kLogCvmfs, kLogStderr, "failed to download object %s (%d)",
693 catalog_hash.ToString().c_str(), retval);
694 return "";
695 }
696
697 return dest;
698 }
699
700
701 string CommandCheck::DecompressPiece(const shash::Any catalog_hash) {
702 const string source = "data/" + catalog_hash.MakePath();
703 const string dest = temp_directory_ + "/" + catalog_hash.ToString();
704 if (!zlib::DecompressPath2Path(source, dest))
705 return "";
706
707 return dest;
708 }
709
710
711 catalog::Catalog *CommandCheck::FetchCatalog(const string &path,
712 const shash::Any &catalog_hash,
713 const uint64_t catalog_size) {
714 string tmp_file;
715 if (!is_remote_)
716 tmp_file = DecompressPiece(catalog_hash);
717 else
718 tmp_file = DownloadPiece(catalog_hash);
719
720 if (tmp_file == "") {
721 LogCvmfs(kLogCvmfs, kLogStderr, "failed to load catalog %s",
722 catalog_hash.ToString().c_str());
723 return NULL;
724 }
725
726 catalog::Catalog *catalog = catalog::Catalog::AttachFreely(path, tmp_file,
727 catalog_hash);
728 const int64_t catalog_file_size = GetFileSize(tmp_file);
729 if (catalog_file_size <= 0) {
730 LogCvmfs(kLogCvmfs, kLogStderr, "Error downloading catalog %s at %s %s",
731 catalog_hash.ToString().c_str(), path.c_str(), tmp_file.c_str());
732 assert(catalog_file_size > 0);
733 }
734 unlink(tmp_file.c_str());
735
736 if ((catalog_size > 0) && (uint64_t(catalog_file_size) != catalog_size)) {
737 LogCvmfs(kLogCvmfs, kLogStderr,
738 "catalog file size mismatch, "
739 "expected %" PRIu64 ", got %" PRIu64,
740 catalog_size, catalog_file_size);
741 delete catalog;
742 return NULL;
743 }
744
745 return catalog;
746 }
747
748
749 bool CommandCheck::FindSubtreeRootCatalog(const string &subtree_path,
750 shash::Any *root_hash,
751 uint64_t *root_size) {
752 catalog::Catalog *current_catalog = FetchCatalog("", *root_hash);
753 if (current_catalog == NULL) {
754 return false;
755 }
756
757 typedef vector<string> Tokens;
758 const Tokens path_tokens = SplitString(subtree_path, '/');
759
760 string current_path = "";
761
762 Tokens::const_iterator i = path_tokens.begin();
763 const Tokens::const_iterator iend = path_tokens.end();
764 for (; i != iend; ++i) {
765 if (i->empty()) {
766 continue;
767 }
768
769 current_path += "/" + *i;
770 if (current_catalog->FindNested(PathString(current_path), root_hash,
771 root_size)) {
772 delete current_catalog;
773
774 if (current_path.length() < subtree_path.length()) {
775 current_catalog = FetchCatalog(current_path, *root_hash);
776 if (current_catalog == NULL) {
777 break;
778 }
779 } else {
780 return true;
781 }
782 }
783 }
784 return false;
785 }
786
787
788 /**
789 * Recursion on nested catalog level. No ownership of computed_counters.
790 */
791 bool CommandCheck::InspectTree(const string &path,
792 const shash::Any &catalog_hash,
793 const uint64_t catalog_size,
794 const bool is_nested_catalog,
795 const catalog::DirectoryEntry *transition_point,
796 catalog::DeltaCounters *computed_counters) {
797 LogCvmfs(kLogCvmfs, kLogStdout | kLogInform, "[inspecting catalog] %s at %s",
798 catalog_hash.ToString().c_str(), path == "" ? "/" : path.c_str());
799
800 const catalog::Catalog *catalog = FetchCatalog(path, catalog_hash,
801 catalog_size);
802 if (catalog == NULL) {
803 LogCvmfs(kLogCvmfs, kLogStderr, "failed to open catalog %s",
804 catalog_hash.ToString().c_str());
805 return false;
806 }
807
808 int retval = true;
809
810 if (catalog->root_prefix() != PathString(path.data(), path.length())) {
811 LogCvmfs(kLogCvmfs, kLogStderr,
812 "root prefix mismatch; "
813 "expected %s, got %s",
814 path.c_str(), catalog->root_prefix().c_str());
815 retval = false;
816 }
817
818 // Check transition point
819 catalog::DirectoryEntry root_entry;
820 if (!catalog->LookupPath(catalog->root_prefix(), &root_entry)) {
821 LogCvmfs(kLogCvmfs, kLogStderr, "failed to lookup root entry (%s)",
822 path.c_str());
823 retval = false;
824 }
825 if (!root_entry.IsDirectory()) {
826 LogCvmfs(kLogCvmfs, kLogStderr, "root entry not a directory (%s)",
827 path.c_str());
828 retval = false;
829 }
830 if (is_nested_catalog) {
831 if (transition_point != NULL
832 && !CompareEntries(*transition_point, root_entry, true, true)) {
833 LogCvmfs(kLogCvmfs, kLogStderr,
834 "transition point and root entry differ (%s)", path.c_str());
835 retval = false;
836 }
837 if (!root_entry.IsNestedCatalogRoot()) {
838 LogCvmfs(kLogCvmfs, kLogStderr,
839 "nested catalog root expected but not found (%s)", path.c_str());
840 retval = false;
841 }
842 } else {
843 if (root_entry.IsNestedCatalogRoot()) {
844 LogCvmfs(kLogCvmfs, kLogStderr,
845 "nested catalog root found but not expected (%s)", path.c_str());
846 retval = false;
847 }
848 }
849
850 // Traverse the catalog
851 set<PathString> bind_mountpoints;
852 if (!Find(catalog, PathString(path.data(), path.length()), computed_counters,
853 &bind_mountpoints)) {
854 retval = false;
855 }
856
857 // Check number of entries
858 if (root_entry.HasXattrs())
859 computed_counters->self.xattrs++;
860 const uint64_t num_found_entries = 1 + computed_counters->self.regular_files
861 + computed_counters->self.symlinks
862 + computed_counters->self.specials
863 + computed_counters->self.directories;
864 if (num_found_entries != catalog->GetNumEntries()) {
865 LogCvmfs(kLogCvmfs, kLogStderr,
866 "dangling entries in catalog, "
867 "expected %" PRIu64 ", got %" PRIu64,
868 catalog->GetNumEntries(), num_found_entries);
869 retval = false;
870 }
871
872 // Recurse into nested catalogs
873 const catalog::Catalog::NestedCatalogList
874 &nested_catalogs = catalog->ListNestedCatalogs();
875 const catalog::Catalog::NestedCatalogList
876 own_nested_catalogs = catalog->ListOwnNestedCatalogs();
877 if (own_nested_catalogs.size()
878 != static_cast<uint64_t>(computed_counters->self.nested_catalogs)) {
879 LogCvmfs(kLogCvmfs, kLogStderr,
880 "number of nested catalogs does not match;"
881 " expected %lu, got %lu",
882 computed_counters->self.nested_catalogs,
883 own_nested_catalogs.size());
884 retval = false;
885 }
886 set<PathString> nested_catalog_paths;
887 for (catalog::Catalog::NestedCatalogList::const_iterator
888 i = nested_catalogs.begin(),
889 iEnd = nested_catalogs.end();
890 i != iEnd;
891 ++i) {
892 nested_catalog_paths.insert(i->mountpoint);
893 }
894 if (nested_catalog_paths.size() != nested_catalogs.size()) {
895 LogCvmfs(kLogCvmfs, kLogStderr,
896 "duplicates among nested catalogs and bind mountpoints");
897 retval = false;
898 }
899
900 for (catalog::Catalog::NestedCatalogList::const_iterator
901 i = nested_catalogs.begin(),
902 iEnd = nested_catalogs.end();
903 i != iEnd;
904 ++i) {
905 if (bind_mountpoints.find(i->mountpoint) != bind_mountpoints.end()) {
906 catalog::DirectoryEntry bind_mountpoint;
907 const PathString mountpoint("/" + i->mountpoint.ToString().substr(1));
908 if (!catalog->LookupPath(mountpoint, &bind_mountpoint)) {
909 LogCvmfs(kLogCvmfs, kLogStderr, "failed to lookup bind mountpoint %s",
910 mountpoint.c_str());
911 retval = false;
912 }
913 LogCvmfs(kLogCvmfs, kLogDebug, "skipping bind mountpoint %s",
914 i->mountpoint.c_str());
915 continue;
916 }
917 catalog::DirectoryEntry nested_transition_point;
918 if (!catalog->LookupPath(i->mountpoint, &nested_transition_point)) {
919 LogCvmfs(kLogCvmfs, kLogStderr, "failed to lookup transition point %s",
920 i->mountpoint.c_str());
921 retval = false;
922 } else {
923 catalog::DeltaCounters nested_counters;
924 const bool is_nested = true;
925 if (!InspectTree(i->mountpoint.ToString(), i->hash, i->size, is_nested,
926 &nested_transition_point, &nested_counters))
927 retval = false;
928 nested_counters.PopulateToParent(computed_counters);
929 }
930 }
931
932 // Check statistics counters
933 // Additionally account for root directory
934 computed_counters->self.directories++;
935 catalog::Counters compare_counters;
936 compare_counters.ApplyDelta(*computed_counters);
937 const catalog::Counters stored_counters = catalog->GetCounters();
938 if (!CompareCounters(compare_counters, stored_counters)) {
939 LogCvmfs(kLogCvmfs, kLogStderr, "statistics counter mismatch [%s]",
940 catalog_hash.ToString().c_str());
941 retval = false;
942 }
943
944 delete catalog;
945 return retval;
946 }
947
948
949 int CommandCheck::Main(const swissknife::ArgumentList &args) {
950 string tag_name;
951 string subtree_path = "";
952 string pubkey_path = "";
953 string repo_name = "";
954 string reflog_chksum_path = "";
955
956 temp_directory_ = (args.find('t') != args.end()) ? *args.find('t')->second
957 : "/tmp";
958 if (args.find('n') != args.end())
959 tag_name = *args.find('n')->second;
960 if (args.find('c') != args.end())
961 check_chunks_ = true;
962 if (args.find('d') != args.end())
963 no_duplicates_map_ = true;
964 if (args.find('l') != args.end()) {
965 const unsigned log_level = kLogLevel0
966 << String2Uint64(*args.find('l')->second);
967 if (log_level > kLogNone) {
968 LogCvmfs(kLogCvmfs, kLogStderr, "invalid log level");
969 return 1;
970 }
971 SetLogVerbosity(static_cast<LogLevels>(log_level));
972 }
973 if (args.find('k') != args.end())
974 pubkey_path = *args.find('k')->second;
975 if (DirectoryExists(pubkey_path))
976 pubkey_path = JoinStrings(FindFilesBySuffix(pubkey_path, ".pub"), ":");
977 if (args.find('N') != args.end())
978 repo_name = *args.find('N')->second;
979
980 repo_base_path_ = MakeCanonicalPath(*args.find('r')->second);
981 if (args.find('s') != args.end())
982 subtree_path = MakeCanonicalPath(*args.find('s')->second);
983 if (args.find('R') != args.end())
984 reflog_chksum_path = *args.find('R')->second;
985
986 // Repository can be HTTP address or on local file system
987 is_remote_ = IsHttpUrl(repo_base_path_);
988
989 // initialize the (swissknife global) download and signature managers
990 if (is_remote_) {
991 const bool follow_redirects = (args.count('L') > 0);
992 const string proxy = (args.count('@') > 0) ? *args.find('@')->second : "";
993 if (!this->InitDownloadManager(follow_redirects, proxy)) {
994 return 1;
995 }
996
997 if (pubkey_path.empty() || repo_name.empty()) {
998 LogCvmfs(kLogCvmfs, kLogStderr,
999 "please provide pubkey and repo name for "
1000 "remote repositories");
1001 return 1;
1002 }
1003
1004 if (!this->InitSignatureManager(pubkey_path)) {
1005 return 1;
1006 }
1007 }
1008
1009 // Load Manifest
1010 UniquePtr<manifest::Manifest> manifest;
1011 bool successful = true;
1012
1013 if (is_remote_) {
1014 manifest = FetchRemoteManifest(repo_base_path_, repo_name);
1015 } else {
1016 if (chdir(repo_base_path_.c_str()) != 0) {
1017 LogCvmfs(kLogCvmfs, kLogStderr, "failed to switch to directory %s",
1018 repo_base_path_.c_str());
1019 return 1;
1020 }
1021 manifest = OpenLocalManifest(".cvmfspublished");
1022 }
1023
1024 if (!manifest.IsValid()) {
1025 LogCvmfs(kLogCvmfs, kLogStderr, "failed to load repository manifest");
1026 return 1;
1027 }
1028
1029 // Check meta-info object
1030 if (!manifest->meta_info().IsNull()) {
1031 string tmp_file;
1032 if (!is_remote_)
1033 tmp_file = DecompressPiece(manifest->meta_info());
1034 else
1035 tmp_file = DownloadPiece(manifest->meta_info());
1036 if (tmp_file == "") {
1037 LogCvmfs(kLogCvmfs, kLogStderr, "failed to load repository metainfo %s",
1038 manifest->meta_info().ToString().c_str());
1039 return 1;
1040 }
1041 unlink(tmp_file.c_str());
1042 }
1043
1044 shash::Any reflog_hash;
1045 if (!reflog_chksum_path.empty()) {
1046 if (!manifest::Reflog::ReadChecksum(reflog_chksum_path, &reflog_hash)) {
1047 LogCvmfs(kLogCvmfs, kLogStderr, "failed to read reflog checksum file");
1048 return 1;
1049 }
1050 } else {
1051 reflog_hash = manifest->reflog_hash();
1052 }
1053
1054 if (Exists(".cvmfsreflog")) {
1055 if (reflog_hash.IsNull()) {
1056 // If there is a reflog, we want to check it
1057 LogCvmfs(kLogCvmfs, kLogStderr,
1058 ".cvmfsreflog present but no checksum provided, aborting");
1059 return 1;
1060 }
1061 const bool retval = InspectReflog(reflog_hash, manifest.weak_ref());
1062 if (!retval) {
1063 LogCvmfs(kLogCvmfs, kLogStderr, "failed to verify reflog");
1064 return 1;
1065 }
1066 } else {
1067 if (!reflog_hash.IsNull()) {
1068 // There is a checksum but no reflog; possibly the checksum is for the
1069 // from the manifest for the stratum 0 reflog
1070 if (!reflog_chksum_path.empty()) {
1071 LogCvmfs(kLogCvmfs, kLogStderr,
1072 "local reflog checksum set but reflog itself is missing, "
1073 "aborting");
1074 return 1;
1075 }
1076 }
1077 }
1078
1079 // Load history
1080 UniquePtr<history::History> tag_db;
1081 if (!manifest->history().IsNull()) {
1082 string tmp_file;
1083 if (!is_remote_)
1084 tmp_file = DecompressPiece(manifest->history());
1085 else
1086 tmp_file = DownloadPiece(manifest->history());
1087 if (tmp_file == "") {
1088 LogCvmfs(kLogCvmfs, kLogStderr, "failed to load history database %s",
1089 manifest->history().ToString().c_str());
1090 return 1;
1091 }
1092 tag_db = history::SqliteHistory::Open(tmp_file);
1093 if (!tag_db.IsValid()) {
1094 LogCvmfs(kLogCvmfs, kLogStderr, "failed to open history database %s",
1095 manifest->history().ToString().c_str());
1096 return 1;
1097 }
1098 tag_db->TakeDatabaseFileOwnership();
1099 successful = InspectHistory(tag_db.weak_ref()) && successful;
1100 }
1101
1102 if (manifest->has_alt_catalog_path()) {
1103 if (!Exists(manifest->certificate().MakeAlternativePath())) {
1104 LogCvmfs(kLogCvmfs, kLogStderr,
1105 "failed to find alternative certificate link %s",
1106 manifest->certificate().MakeAlternativePath().c_str());
1107 return 1;
1108 }
1109 if (!Exists(manifest->catalog_hash().MakeAlternativePath())) {
1110 LogCvmfs(kLogCvmfs, kLogStderr,
1111 "failed to find alternative catalog link %s",
1112 manifest->catalog_hash().MakeAlternativePath().c_str());
1113 return 1;
1114 }
1115 }
1116
1117 shash::Any root_hash = manifest->catalog_hash();
1118 uint64_t root_size = manifest->catalog_size();
1119 if (tag_name != "") {
1120 if (!tag_db.IsValid()) {
1121 LogCvmfs(kLogCvmfs, kLogStderr, "no history");
1122 return 1;
1123 }
1124 history::History::Tag tag;
1125 const bool retval = tag_db->GetByName(tag_name, &tag);
1126 if (!retval) {
1127 LogCvmfs(kLogCvmfs, kLogStderr, "no such tag: %s", tag_name.c_str());
1128 return 1;
1129 }
1130 root_hash = tag.root_hash;
1131 root_size = tag.size;
1132 LogCvmfs(kLogCvmfs, kLogStdout, "Inspecting repository tag %s",
1133 tag_name.c_str());
1134 }
1135
1136 const bool is_nested_catalog = (!subtree_path.empty());
1137 if (is_nested_catalog
1138 && !FindSubtreeRootCatalog(subtree_path, &root_hash, &root_size)) {
1139 LogCvmfs(kLogCvmfs, kLogStderr, "cannot find nested catalog at %s",
1140 subtree_path.c_str());
1141 return 1;
1142 }
1143
1144
1145 catalog::DeltaCounters computed_counters;
1146 successful = InspectTree(subtree_path,
1147 root_hash,
1148 root_size,
1149 is_nested_catalog,
1150 NULL,
1151 &computed_counters)
1152 && successful;
1153
1154 if (!successful) {
1155 LogCvmfs(kLogCvmfs, kLogStderr, "CATALOG PROBLEMS OR OTHER ERRORS FOUND");
1156 return 1;
1157 }
1158
1159 LogCvmfs(kLogCvmfs, kLogStdout, "no problems found");
1160 return 0;
1161 }
1162
1163 } // namespace swissknife
1164