7 #define __STDC_FORMAT_MACROS
9 #include "cvmfs_config.h"
42 return *
const_cast<uint32_t *
>(
43 reinterpret_cast<const uint32_t *
>(key.
digest) + 1);
47 namespace swissknife {
49 CommandCheck::CommandCheck()
50 : check_chunks_(false)
51 , no_duplicates_map_(false)
59 const bool compare_names,
60 const bool is_transition_point)
65 if (diffs == Difference::kIdentical) {
71 if (is_transition_point &&
72 (diffs ^ Difference::kNestedCatalogTransitionFlags) == 0) {
78 if (diffs & Difference::kName) {
84 if (diffs & Difference::kLinkcount) {
89 if (diffs & Difference::kHardlinkGroup) {
94 if (diffs & Difference::kSize) {
99 if (diffs & Difference::kMode) {
104 if (diffs & Difference::kMtime) {
109 if (diffs & Difference::kChecksum) {
114 if (diffs & Difference::kSymlink) {
119 if (diffs & Difference::kExternalFileFlag) {
125 if (diffs & Difference::kHasXattrsFlag) {
143 catalog::Counters::FieldsMap::const_iterator i = map_a.begin();
144 catalog::Counters::FieldsMap::const_iterator iend = map_a.end();
145 for (; i != iend; ++i) {
146 catalog::Counters::FieldsMap::const_iterator comp = map_b.find(i->first);
147 assert(comp != map_b.end());
149 if (*(i->second) != *(comp->second)) {
151 "catalog statistics mismatch: %s (expected: %" PRIu64
" / "
152 "in catalog: %" PRIu64
")",
153 comp->first.c_str(), *(i->second), *(comp->second));
216 string reflog_path =
FetchPath(
".cvmfsreflog");
219 if (computed_hash != reflog_hash) {
221 "The .cvmfsreflog has unexpected content hash %s (expected %s)",
223 unlink(reflog_path.c_str());
233 "failed to find catalog root hash %s in .cvmfsreflog",
240 "failed to find certificate hash %s in .cvmfsreflog",
249 "failed to find tag database's hash %s in .cvmfsreflog",
258 "failed to find meta info hash %s in .cvmfsreflog",
273 vector<history::History::Tag> tags;
274 retval = history->
List(&tags);
279 vector<history::History::Branch> branches;
288 map<string, uint64_t> initial_revisions;
290 for (
unsigned i = 0; i < branches.size(); ++i) {
291 if (!sanitizer.
IsValid(branches[i].branch)) {
293 branches[i].branch.c_str());
296 initial_revisions[branches[i].branch] = branches[i].initial_revision;
299 set<string> used_branches;
301 for (
unsigned i = 0; i < tags.size(); ++i) {
302 used_branches.insert(tags[i].branch);
303 const map<string, uint64_t>::const_iterator iter =
304 initial_revisions.find(tags[i].branch);
305 if (iter == initial_revisions.end()) {
307 tags[i].branch.c_str(), tags[i].name.c_str());
310 if (tags[i].revision < iter->second) {
312 " of tag %s", tags[i].revision, tags[i].name.c_str());
318 if (used_branches.size() != branches.size()) {
337 set<PathString> *bind_mountpoints)
342 if (!catalog->
LookupPath(path, &this_directory)) {
353 uint32_t num_subdirs = 0;
355 typedef map< uint32_t, vector<catalog::DirectoryEntry> > HardlinkMap;
356 HardlinkMap hardlinks;
357 bool found_nested_marker =
false;
359 for (
unsigned i = 0; i < entries.size(); ++i) {
362 const bool entry_needs_check =
363 !entries[i].checksum().IsNull() && !entries[i].IsExternalFile() &&
372 full_path.
Append(entries[i].name().GetChars(),
373 entries[i].name().GetLength());
375 full_path.
c_str(), entry_needs_check);
379 if (entries[i].name().IsEmpty()) {
386 if (entries[i].name() ==
NameString(
string(
".cvmfscatalog"))) {
389 "found abandoned nested catalog marker at %s",
393 found_nested_marker =
true;
397 if (entries[i].IsRegular() && !entries[i].IsChunkedFile() &&
398 entries[i].checksum().IsNull())
401 "regular file pointing to zero-hash: '%s'", full_path.
c_str());
408 string chunk_path =
"data/" + entries[i].checksum().MakePath();
409 if (entries[i].IsDirectory())
411 if (!
Exists(chunk_path)) {
413 entries[i].checksum().ToString().c_str(), full_path.
c_str());
419 if ((entries[i].linkcount() > 1) && !entries[i].IsDirectory()) {
420 if (entries[i].hardlink_group() == 0) {
425 HardlinkMap::iterator hardlink_group =
426 hardlinks.find(entries[i].hardlink_group());
427 if (hardlink_group == hardlinks.end()) {
428 hardlinks[entries[i].hardlink_group()];
429 hardlinks[entries[i].hardlink_group()].push_back(entries[i]);
431 if (!
CompareEntries(entries[i], (hardlink_group->second)[0],
false)) {
436 hardlink_group->second.push_back(entries[i]);
442 if (entries[i].linkcount() == 0) {
444 entries[i].name().c_str());
449 if (!entries[i].IsRegular()) {
450 if (entries[i].IsDirectIo()) {
456 if (entries[i].IsDirectory()) {
457 computed_counters->
self.directories++;
466 if (entries[i].hardlink_group() != 0) {
471 if (entries[i].IsNestedCatalogMountpoint() ||
472 entries[i].IsBindMountpoint())
475 if (entries[i].IsNestedCatalogMountpoint())
476 computed_counters->
self.nested_catalogs++;
480 if (!catalog->
FindNested(mountpoint, &tmp, &tmp2)) {
488 if (catalog->
ListingPath(full_path, &nested_entries) &&
489 !nested_entries.empty()) {
496 if (entries[i].IsBindMountpoint()) {
497 bind_mountpoints->insert(full_path);
498 if (entries[i].IsNestedCatalogMountpoint()) {
500 "bind mountpoint and nested mountpoint mutually exclusive"
501 " at %s.", full_path.
c_str());
507 if (!
Find(catalog, full_path, computed_counters, bind_mountpoints))
510 }
else if (entries[i].IsLink()) {
511 computed_counters->
self.symlinks++;
513 if (!entries[i].checksum().IsNull()) {
519 if (entries[i].
size() != entries[i].symlink().GetLength()) {
521 "expected %u, got %lu", full_path.
c_str(),
522 entries[i].symlink().GetLength(), entries[i].size());
525 }
else if (entries[i].IsRegular()) {
526 computed_counters->
self.regular_files++;
527 computed_counters->
self.file_size += entries[i].size();
528 }
else if (entries[i].IsSpecial()) {
529 computed_counters->
self.specials++;
531 if (entries[i].
size() != 0) {
533 "unexpected non-zero special file size %s",
538 if (!entries[i].checksum().IsNull()) {
544 if (entries[i].symlink().GetLength() > 0) {
546 "special file with non-zero symlink at %s", full_path.
c_str());
555 if (entries[i].HasXattrs()) {
556 computed_counters->
self.xattrs++;
559 if (entries[i].IsExternalFile()) {
560 computed_counters->
self.externals++;
561 computed_counters->
self.external_file_size += entries[i].size();
562 if (!entries[i].IsRegular()) {
564 "only regular files can be external: %s", full_path.
c_str());
570 if (entries[i].IsChunkedFile()) {
572 catalog->
ListPathChunks(full_path, entries[i].hash_algorithm(), &chunks);
574 computed_counters->
self.chunked_files++;
575 computed_counters->
self.chunked_file_size += entries[i].size();
576 computed_counters->
self.file_chunks += chunks.
size();
579 if (chunks.
size() == 0) {
585 size_t aggregated_file_size = 0;
586 off_t next_offset = 0;
588 for (
unsigned j = 0; j < chunks.
size(); ++j) {
591 if (next_offset != this_chunk.
offset()) {
596 next_offset = this_chunk.
offset() + this_chunk.
size();
597 aggregated_file_size += this_chunk.
size();
604 bool chunk_needs_check =
true;
608 chunk_needs_check =
false;
610 if (chunk_needs_check) {
611 const string chunk_path =
"data/" + chunk_hash.
MakePath();
612 if (!
Exists(chunk_path)) {
614 "offset: %ld | size: %lu) missing",
626 if (aggregated_file_size != entries[i].
size()) {
628 "mismatch. Calculated %zu bytes | %lu "
631 aggregated_file_size,
640 !found_nested_marker)
648 if (this_directory.
linkcount() != num_subdirs + 2) {
650 "expected %u, got %u",
656 for (HardlinkMap::const_iterator i = hardlinks.begin(),
657 iEnd = hardlinks.end(); i != iEnd; ++i)
659 if (i->second[0].linkcount() != i->second.size()) {
661 "expected %lu, got %u",
662 (path.
ToString() +
"/" + i->second[0].name().ToString()).c_str(),
663 i->second.size(), i->second[0].linkcount());
673 string source =
"data/" + catalog_hash.
MakePath();
683 catalog_hash.
ToString().c_str(), retval);
692 string source =
"data/" + catalog_hash.
MakePath();
703 const uint64_t catalog_size) {
710 if (tmp_file ==
"") {
719 assert(catalog_file_size > 0);
720 unlink(tmp_file.c_str());
722 if ((catalog_size > 0) && (uint64_t(catalog_file_size) != catalog_size)) {
724 "expected %" PRIu64
", got %" PRIu64,
725 catalog_size, catalog_file_size);
736 uint64_t *root_size) {
738 if (current_catalog == NULL) {
742 typedef vector<string> Tokens;
743 const Tokens path_tokens =
SplitString(subtree_path,
'/');
745 string current_path =
"";
747 Tokens::const_iterator i = path_tokens.begin();
748 Tokens::const_iterator iend = path_tokens.end();
749 for (; i != iend; ++i) {
754 current_path +=
"/" + *i;
758 delete current_catalog;
760 if (current_path.length() < subtree_path.length()) {
761 current_catalog =
FetchCatalog(current_path, *root_hash);
762 if (current_catalog == NULL) {
779 const uint64_t catalog_size,
780 const bool is_nested_catalog,
785 catalog_hash.
ToString().c_str(), path ==
"" ?
"/" : path.c_str());
790 if (catalog == NULL) {
798 if (catalog->root_prefix() !=
PathString(path.data(), path.length())) {
800 "expected %s, got %s",
801 path.c_str(), catalog->root_prefix().c_str());
807 if (!catalog->LookupPath(catalog->root_prefix(), &root_entry)) {
817 if (is_nested_catalog) {
818 if (transition_point != NULL &&
821 "transition point and root entry differ (%s)", path.c_str());
826 "nested catalog root expected but not found (%s)", path.c_str());
832 "nested catalog root found but not expected (%s)", path.c_str());
838 set<PathString> bind_mountpoints;
840 computed_counters, &bind_mountpoints))
847 computed_counters->
self.xattrs++;
848 const uint64_t num_found_entries =
850 computed_counters->
self.regular_files +
851 computed_counters->
self.symlinks +
852 computed_counters->
self.specials +
853 computed_counters->
self.directories;
854 if (num_found_entries != catalog->GetNumEntries()) {
856 "expected %" PRIu64
", got %" PRIu64,
857 catalog->GetNumEntries(), num_found_entries);
863 catalog->ListNestedCatalogs();
865 catalog->ListOwnNestedCatalogs();
866 if (own_nested_catalogs.size() !=
867 static_cast<uint64_t
>(computed_counters->
self.nested_catalogs))
870 " expected %lu, got %lu", computed_counters->
self.nested_catalogs,
871 own_nested_catalogs.size());
874 set<PathString> nested_catalog_paths;
875 for (catalog::Catalog::NestedCatalogList::const_iterator i =
876 nested_catalogs.begin(), iEnd = nested_catalogs.end(); i != iEnd; ++i)
878 nested_catalog_paths.insert(i->mountpoint);
880 if (nested_catalog_paths.size() != nested_catalogs.size()) {
882 "duplicates among nested catalogs and bind mountpoints");
886 for (catalog::Catalog::NestedCatalogList::const_iterator i =
887 nested_catalogs.begin(), iEnd = nested_catalogs.end(); i != iEnd; ++i)
889 if (bind_mountpoints.find(i->mountpoint) != bind_mountpoints.end()) {
891 PathString mountpoint(
"/" + i->mountpoint.ToString().substr(1));
892 if (!catalog->LookupPath(mountpoint, &bind_mountpoint)) {
898 i->mountpoint.c_str());
902 if (!catalog->LookupPath(i->mountpoint, &nested_transition_point)) {
904 i->mountpoint.c_str());
908 const bool is_nested =
true;
909 if (!
InspectTree(i->mountpoint.ToString(), i->hash, i->size, is_nested,
910 &nested_transition_point, &nested_counters))
918 computed_counters->
self.directories++;
920 compare_counters.
ApplyDelta(*computed_counters);
935 string subtree_path =
"";
936 string pubkey_path =
"";
937 string trusted_certs =
"";
938 string repo_name =
"";
939 string reflog_chksum_path =
"";
941 temp_directory_ = (args.find(
't') != args.end()) ? *args.find(
't')->second
943 if (args.find(
'n') != args.end())
944 tag_name = *args.find(
'n')->second;
945 if (args.find(
'c') != args.end())
947 if (args.find(
'd') != args.end())
949 if (args.find(
'l') != args.end()) {
958 if (args.find(
'k') != args.end())
959 pubkey_path = *args.find(
'k')->second;
962 if (args.find(
'z') != args.end())
963 trusted_certs = *args.find(
'z')->second;
964 if (args.find(
'N') != args.end())
965 repo_name = *args.find(
'N')->second;
968 if (args.find(
's') != args.end())
970 if (args.find(
'R') != args.end())
971 reflog_chksum_path = *args.find(
'R')->second;
978 const bool follow_redirects = (args.count(
'L') > 0);
979 const string proxy = (args.count(
'@') > 0) ? *args.find(
'@')->second :
"";
984 if (pubkey_path.empty() || repo_name.empty()) {
986 "remote repositories");
997 bool successful =
true;
1016 if (!manifest->meta_info().IsNull()) {
1022 if (tmp_file ==
"") {
1024 manifest->meta_info().ToString().c_str());
1027 unlink(tmp_file.c_str());
1031 if (!reflog_chksum_path.empty()) {
1037 reflog_hash = manifest->reflog_hash();
1040 if (
Exists(
".cvmfsreflog")) {
1041 if (reflog_hash.
IsNull()) {
1044 ".cvmfsreflog present but no checksum provided, aborting");
1053 if (!reflog_hash.
IsNull()) {
1056 if (!reflog_chksum_path.empty()) {
1058 "local reflog checksum set but reflog itself is missing, "
1067 if (!manifest->history().IsNull()) {
1073 if (tmp_file ==
"") {
1075 manifest->history().ToString().c_str());
1081 manifest->history().ToString().c_str());
1084 tag_db->TakeDatabaseFileOwnership();
1088 if (manifest->has_alt_catalog_path()) {
1089 if (!
Exists(manifest->certificate().MakeAlternativePath())) {
1091 "failed to find alternative certificate link %s",
1092 manifest->certificate().MakeAlternativePath().c_str());
1095 if (!
Exists(manifest->catalog_hash().MakeAlternativePath())) {
1097 "failed to find alternative catalog link %s",
1098 manifest->catalog_hash().MakeAlternativePath().c_str());
1103 shash::Any root_hash = manifest->catalog_hash();
1104 uint64_t root_size = manifest->catalog_size();
1105 if (tag_name !=
"") {
1111 const bool retval = tag_db->GetByName(tag_name, &tag);
1117 root_size = tag.
size;
1122 const bool is_nested_catalog = (!subtree_path.empty());
1127 subtree_path.c_str());
1138 &computed_counters) && successful;
uint32_t linkcount() const
void SetLogVerbosity(const LogLevels max_level)
void TakeDatabaseFileOwnership()
bool IsExternalFile() const
bool ContainsHistory(const shash::Any &history) const
bool InspectHistory(history::History *history)
Differences CompareTo(const DirectoryEntry &other) const
const manifest::Manifest * manifest() const
ShortString< kDefaultMaxName, 1 > NameString
Item At(const size_t index) const
static SqliteHistory * Open(const std::string &file_name)
bool ListPathChunks(const PathString &path, const shash::Algorithms interpret_hashes_as, FileChunkList *chunks) const
const int kDefaultFileMode
static bool ReadChecksum(const std::string &path, shash::Any *checksum)
FILE * CreateTempFile(const std::string &path_prefix, const int mode, const char *open_flags, std::string *final_path)
string JoinStrings(const vector< string > &strings, const string &joint)
std::string ToString(const bool with_suffix=false) const
const history::History * history() const
std::string ToStringWithSuffix() const
bool LookupPath(const PathString &path, DirectoryEntry *dirent) const
bool CompareEntries(const catalog::DirectoryEntry &a, const catalog::DirectoryEntry &b, const bool compare_names, const bool is_transition_point=false)
void ApplyDelta(const DeltaCounters &delta)
bool IsHttpUrl(const std::string &path)
bool ListingPath(const PathString &path, DirectoryEntryList *listing, const bool expand_symlink=true) const
const char kSuffixMicroCatalog
const shash::Any & content_hash() const
catalog::Catalog * FetchCatalog(const std::string &path, const shash::Any &catalog_hash, const uint64_t catalog_size=0)
std::map< std::string, const Counters_t * > FieldsMap
assert((mem||(size==0))&&"Out Of Memory")
bool InspectReflog(const shash::Any &reflog_hash, manifest::Manifest *manifest)
std::string temp_directory_
shash::Any checksum() const
static uint32_t hasher_any(const shash::Any &key)
unsigned int mode() const
unsigned char digest[digest_size_]
bool Find(const catalog::Catalog *catalog, const PathString &path, catalog::DeltaCounters *computed_counters, std::set< PathString > *bind_mountpoints)
std::string repo_base_path_
bool ContainsMetainfo(const shash::Any &metainfo) const
bool CopyPath2File(const std::string &src, FILE *fdest)
FieldsMap GetFieldsMap() const
bool SymlinkExists(const std::string &path)
static Reflog * Open(const std::string &database_path)
bool IsNestedCatalogRoot() const
bool FileExists(const std::string &path)
std::string DownloadPiece(const shash::Any catalog_hash)
std::vector< DirectoryEntry > DirectoryEntryList
int Main(const ArgumentList &args)
virtual bool ListBranches(std::vector< Branch > *branches) const =0
std::string DecompressPiece(const shash::Any catalog_hash)
virtual bool List(std::vector< Tag > *tags) const =0
vector< string > SplitString(const string &str, char delim)
bool Exists(const std::string &file)
shash::Any certificate() const
void PopulateToParent(DeltaCounters *parent) const
std::string FetchPath(const std::string &path)
LinkString symlink() const
void Insert(const Key &key, const Value &value)
PathString mountpoint() const
static void HashDatabase(const std::string &database_path, shash::Any *hash_reflog)
shash::Any catalog_hash() const
void Append(const char *chars, const unsigned length)
manifest::Reflog * reflog
SmallHashDynamic< shash::Any, char > duplicates_map_
bool FindSubtreeRootCatalog(const std::string &subtree_path, shash::Any *root_hash, uint64_t *root_size)
bool DirectoryExists(const std::string &path)
bool CompareCounters(const catalog::Counters &a, const catalog::Counters &b)
bool Contains(const Key &key) const
std::string ToString() const
std::vector< NestedCatalog > NestedCatalogList
bool ContainsCatalog(const shash::Any &catalog) const
uint64_t String2Uint64(const string &value)
std::map< char, SharedPtr< std::string > > ArgumentList
ShortString< kDefaultMaxPath, 0 > PathString
Failures Fetch(JobInfo *info)
shash::Any history() const
bool InspectTree(const std::string &path, const shash::Any &catalog_hash, const uint64_t catalog_size, const bool is_nested_catalog, const catalog::DirectoryEntry *transition_point, catalog::DeltaCounters *computed_counters)
static Catalog * AttachFreely(const std::string &imaginary_mountpoint, const std::string &file, const shash::Any &catalog_hash, Catalog *parent=NULL, const bool is_nested=false)
bool FindNested(const PathString &mountpoint, shash::Any *hash, uint64_t *size) const
bool DecompressPath2Path(const string &src, const string &dest)
int64_t GetFileSize(const std::string &path)
std::string MakePath() const
std::string MakeCanonicalPath(const std::string &path)
void Init(uint32_t expected_size, Key empty, uint32_t(*hasher)(const Key &key))
const char * c_str() const
uint32_t hardlink_group() const
bool ContainsCertificate(const shash::Any &certificate) const
std::vector< std::string > FindFilesBySuffix(const std::string &dir, const std::string &suffix)
shash::Any meta_info() const
CVMFS_EXPORT void LogCvmfs(const LogSource source, const int mask, const char *format,...)