Head

GCC Code Coverage Report

Directory:	cvmfs/		Exec	Total	Coverage
File:	cvmfs/cvmfs_fsck.cc	Lines:	0	169	0.0 %
Date:	2019-02-03 02:48:13	Branches:	0	120	0.0 %


/**
 * This file is part of the CernVM File System.
 *
 * This tool checks a cvmfs cache directory for consistency.
 * If necessary, the managed cache db is removed so that
 * it will be rebuilt on next mount.
 */

#define _FILE_OFFSET_BITS 64

#include "cvmfs_config.h"

#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <stdint.h>
#include <sys/stat.h>
#include <unistd.h>

#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <string>

#include "atomic.h"
#include "compression.h"
#include "hash.h"
#include "logging.h"
#include "platform.h"
#include "smalloc.h"
#include "util/posix.h"

using namespace std;  // NOLINT

enum Errors {
  kErrorOk = 0,
  kErrorFixed = 1,
  kErrorReboot = 2,
  kErrorUnfixed = 4,
  kErrorOperational = 8,
  kErrorUsage = 16,
};

string *g_cache_dir;
atomic_int32 g_num_files;
atomic_int32 g_num_err_fixed;
atomic_int32 g_num_err_unfixed;
atomic_int32 g_num_err_operational;
atomic_int32 g_num_tmp_catalog;
/**
 * Traversal of the file system tree is serialized.
 */
pthread_mutex_t g_lock_traverse = PTHREAD_MUTEX_INITIALIZER;
DIR *g_DIRP_current = NULL;
int g_num_dirs = -1;  /**< Number of cache directories already examined. */
string *g_current_dir;  /**< Current cache sub directory */

int g_num_threads = 1;
bool g_fix_errors = false;
bool g_verbose = false;
atomic_int32 g_force_rebuild;
atomic_int32 g_modified_cache;


static void Usage() {
  LogCvmfs(kLogCvmfs, kLogStdout,
           "CernVM File System consistency checker, version %s\n\n"
           "This tool checks a cvmfs cache directory for consistency.\n"
           "If necessary, the managed cache db is removed so that\n"
           "it will be rebuilt on next mount.\n\n"
           "Usage: cvmfs_fsck [-v] [-p] [-f] [-j #threads] <cache directory>\n"
           "Options:\n"
           "  -v verbose output\n"
           "  -p try to fix automatically\n"
           "  -f force rebuild of managed cache db on next mount\n"
           "  -j number of concurrent integrity check worker threads\n",
           VERSION);
}


static bool GetNextFile(string *relative_path, string *hash_name) {
  platform_dirent64 *d = NULL;

  pthread_mutex_lock(&g_lock_traverse);
 get_next_file_again:
  while (g_DIRP_current && ((d = platform_readdir(g_DIRP_current)) != NULL)) {
    const string name = d->d_name;
    if ((name == ".") || (name == "..")) continue;

    platform_stat64 info;
    *relative_path = *g_current_dir + "/" + name;
    *hash_name = *g_current_dir + name;
    const string path = *g_cache_dir + "/" + *relative_path;
    if (platform_lstat(relative_path->c_str(), &info) != 0) {
      LogCvmfs(kLogCvmfs, kLogStdout, "Warning: failed to stat() %s (%d)",
               path.c_str(), errno);
      continue;
    }

    if (!S_ISREG(info.st_mode)) {
      LogCvmfs(kLogCvmfs, kLogStdout, "Warning: %s is not a regular file",
               path.c_str());
      continue;
    }

    break;
  }

  if (!d) {
    if (g_DIRP_current) {
      closedir(g_DIRP_current);
      g_DIRP_current = NULL;
    }
    g_num_dirs++;
    if (g_num_dirs < 256) {
      char hex[3];
      snprintf(hex, sizeof(hex), "%02x", g_num_dirs);
      *g_current_dir = string(hex, 2);

      if (g_verbose)
        LogCvmfs(kLogCvmfs, kLogStdout, "Entering %s", g_current_dir->c_str());
      if ((g_DIRP_current = opendir(hex)) == NULL) {
        LogCvmfs(kLogCvmfs, kLogStderr,
                 "Invalid cache directory, %s/%s does not exist",
                 g_cache_dir->c_str(), g_current_dir->c_str());
        pthread_mutex_unlock(&g_lock_traverse);
        exit(kErrorUnfixed);
      }
      goto get_next_file_again;
    }
  }
  pthread_mutex_unlock(&g_lock_traverse);

  if (d)
    return true;

  return false;
}


static void *MainCheck(void *data __attribute__((unused))) {
  string relative_path;
  string hash_name;

  while (GetNextFile(&relative_path, &hash_name)) {
    const string path = *g_cache_dir + "/" + relative_path;

    int n = atomic_xadd32(&g_num_files, 1);
    if (g_verbose)
      LogCvmfs(kLogCvmfs, kLogStdout, "Checking file %s", path.c_str());
    if (!g_verbose && ((n % 1000) == 0))
      LogCvmfs(kLogCvmfs, kLogStdout | kLogNoLinebreak, ".");

    if (relative_path[relative_path.length()-1] == 'T') {
      LogCvmfs(kLogCvmfs, kLogStdout,
               "Warning: temporary file catalog %s found", path.c_str());
      atomic_inc32(&g_num_tmp_catalog);
      if (g_fix_errors) {
        if (unlink(relative_path.c_str()) == 0) {
          LogCvmfs(kLogCvmfs, kLogStdout, "Fix: %s unlinked", path.c_str());
          atomic_inc32(&g_num_err_fixed);
        } else {
          LogCvmfs(kLogCvmfs, kLogStdout, "Error: failed to unlink %s",
                   path.c_str());
          atomic_inc32(&g_num_err_unfixed);
        }
      }
      continue;
    }

    int fd_src = open(relative_path.c_str() , O_RDONLY);
    if (fd_src < 0) {
      LogCvmfs(kLogCvmfs, kLogStdout, "Error: cannot open %s", path.c_str());
      atomic_inc32(&g_num_err_operational);
      continue;
    }
    // Don't thrash kernel buffers
    platform_disable_kcache(fd_src);

    // Compress every file and calculate SHA-1 of stream
    shash::Any expected_hash = shash::MkFromHexPtr(shash::HexPtr(hash_name));
    shash::Any hash(expected_hash.algorithm);
    if (!zlib::CompressFd2Null(fd_src, &hash)) {
      LogCvmfs(kLogCvmfs, kLogStdout, "Error: could not compress %s",
               path.c_str());
      atomic_inc32(&g_num_err_operational);
    } else {
      if (hash != expected_hash) {
        // If the hashes don't match, try hashing the uncompressed file
        if (!shash::HashFile(relative_path, &hash)) {
          LogCvmfs(kLogCvmfs, kLogStdout, "Error: could not hash %s",
                   path.c_str());
          atomic_inc32(&g_num_err_operational);
        }
        if (hash != expected_hash) {
          if (g_fix_errors) {
            const string quarantaine_path = "./quarantaine/" + hash_name;
            bool fixed = false;
            if (rename(relative_path.c_str(), quarantaine_path.c_str()) == 0) {
              LogCvmfs(kLogCvmfs, kLogStdout,
                       "Fix: %s is corrupted, moved to quarantaine folder",
                       path.c_str());
              fixed = true;
            } else {
              LogCvmfs(kLogCvmfs, kLogStdout,
                       "Warning: failed to move %s into quarantaine folder",
                       path.c_str());
              if (unlink(relative_path.c_str()) == 0) {
                LogCvmfs(kLogCvmfs, kLogStdout,
                         "Fix: %s is corrupted, file unlinked", path.c_str());
                fixed = true;
              } else {
                LogCvmfs(kLogCvmfs, kLogStdout,
                         "Error: %s is corrupted, could not unlink",
                         path.c_str());
              }
            }

            if (fixed) {
              atomic_inc32(&g_num_err_fixed);

              // Changes made, we have to rebuild the managed cache db
              atomic_cas32(&g_force_rebuild, 0, 1);
              atomic_cas32(&g_modified_cache, 0, 1);
            } else {
              atomic_inc32(&g_num_err_unfixed);
            }
          } else {
            LogCvmfs(kLogCvmfs, kLogStdout,
                     "Error: %s has compressed checksum %s, "
                     "delete this file from cache directory!",
                     path.c_str(), hash.ToString().c_str());
            atomic_inc32(&g_num_err_unfixed);
          }
        }
      }
    }
    close(fd_src);
  }

  return NULL;
}


int main(int argc, char **argv) {
  atomic_init32(&g_force_rebuild);
  atomic_init32(&g_modified_cache);
  g_current_dir = new string();

  int c;
  while ((c = getopt(argc, argv, "hvpfj:")) != -1) {
    switch (c) {
      case 'h':
        Usage();
        return kErrorOk;
      case 'v':
        g_verbose = true;
        break;
      case 'p':
        g_fix_errors = true;
        break;
      case 'f':
        atomic_cas32(&g_force_rebuild, 0, 1);
        break;
      case 'j':
        g_num_threads = atoi(optarg);
        if (g_num_threads < 1) {
          LogCvmfs(kLogCvmfs, kLogStdout,
                   "There is at least one worker thread required");
          return kErrorUsage;
        }
        break;
      case '?':
      default:
        Usage();
        return kErrorUsage;
    }
  }

  // Switch to cache directory
  if (optind >= argc) {
    Usage();
    return kErrorUsage;
  }
  g_cache_dir = new string(MakeCanonicalPath(argv[optind]));
  if (chdir(g_cache_dir->c_str()) != 0) {
    LogCvmfs(kLogCvmfs, kLogStderr, "Could not chdir to %s",
             g_cache_dir->c_str());
    return kErrorOperational;
  }

  // Check if txn directory is empty
  DIR *dirp_txn;
  if ((dirp_txn = opendir("txn")) == NULL) {
    LogCvmfs(kLogCvmfs, kLogStderr,
             "Invalid cache directory, %s/txn does not exist",
             g_cache_dir->c_str());
    return kErrorOperational;
  }
  platform_dirent64 *d;
  while ((d = platform_readdir(dirp_txn)) != NULL) {
    const string name = d->d_name;
    if ((name == ".") || (name == "..")) continue;

    LogCvmfs(kLogCvmfs, kLogStdout,
             "Warning: temporary directory %s/txn is not empty\n"
             "If this repository is currently _not_ mounted, "
             "you can remove its contents", g_cache_dir->c_str());
    break;
  }
  closedir(dirp_txn);

  // Run workers to recalculate checksums
  atomic_init32(&g_num_files);
  atomic_init32(&g_num_err_fixed);
  atomic_init32(&g_num_err_unfixed);
  atomic_init32(&g_num_err_operational);
  atomic_init32(&g_num_tmp_catalog);
  pthread_t *workers = reinterpret_cast<pthread_t *>(
    smalloc(g_num_threads * sizeof(pthread_t)));
  if (!g_verbose)
    LogCvmfs(kLogCvmfs, kLogStdout | kLogNoLinebreak, "Verifying: ");
  for (int i = 0; i < g_num_threads; ++i) {
    if (g_verbose)
      LogCvmfs(kLogCvmfs, kLogStdout, "Starting worker %d", i+1);
    if (pthread_create(&workers[i], NULL, MainCheck, NULL) != 0) {
      LogCvmfs(kLogCvmfs, kLogStdout, "Fatal: could not create worker thread");
      return kErrorOperational;
    }
  }
  for (int i = g_num_threads-1; i >= 0; --i) {
    pthread_join(workers[i], NULL);
    if (g_verbose)
      LogCvmfs(kLogCvmfs, kLogStdout, "Stopping worker %d", i+1);
  }
  free(workers);
  if (!g_verbose)
    LogCvmfs(kLogCvmfs, kLogStdout, "");
  LogCvmfs(kLogCvmfs, kLogStdout, "Verified %d files",
           atomic_read32(&g_num_files));

  if (atomic_read32(&g_num_tmp_catalog) > 0)
    LogCvmfs(kLogCvmfs, kLogStdout, "Temporary file catalogs were found.");

  if (atomic_read32(&g_force_rebuild)) {
    if (unlink("cachedb") == 0) {
      LogCvmfs(kLogCvmfs, kLogStdout,
               "Fix: managed cache db unlinked, will be rebuilt on next mount");
      atomic_inc32(&g_num_err_fixed);
    } else {
      if (errno != ENOENT) {
        LogCvmfs(kLogCvmfs, kLogStdout,
                 "Error: could not unlink managed cache database (%d)", errno);
        atomic_inc32(&g_num_err_unfixed);
      }
    }
  }

  if (atomic_read32(&g_modified_cache)) {
    LogCvmfs(kLogCvmfs, kLogStdout, "\n"
             "WARNING: There might by corrupted files in the kernel buffers.\n"
             "Remount CernVM-FS or run 'echo 3 > /proc/sys/vm/drop_caches'"
             "\n\n");
  }

  int retval = 0;
  if (atomic_read32(&g_num_err_fixed) > 0)
    retval |= kErrorFixed;
  if (atomic_read32(&g_num_err_unfixed) > 0)
    retval |= kErrorUnfixed;
  if (atomic_read32(&g_num_err_operational) > 0)
    retval |= kErrorOperational;

  return retval;
}


Generated by: GCOVR (Version 4.1)

Line	Branch	Exec	Source
1			/**
2			* This file is part of the CernVM File System.
3			*
4			* This tool checks a cvmfs cache directory for consistency.
5			* If necessary, the managed cache db is removed so that
6			* it will be rebuilt on next mount.
7			*/
8
9			#define _FILE_OFFSET_BITS 64
10
11			#include "cvmfs_config.h"
12
13			#include <dirent.h>
14			#include <errno.h>
15			#include <fcntl.h>
16			#include <pthread.h>
17			#include <stdint.h>
18			#include <sys/stat.h>
19			#include <unistd.h>
20
21			#include <cstdio>
22			#include <cstdlib>
23			#include <cstring>
24			#include <string>
25
26			#include "atomic.h"
27			#include "compression.h"
28			#include "hash.h"
29			#include "logging.h"
30			#include "platform.h"
31			#include "smalloc.h"
32			#include "util/posix.h"
33
34			using namespace std; // NOLINT
35
36			enum Errors {
37			kErrorOk = 0,
38			kErrorFixed = 1,
39			kErrorReboot = 2,
40			kErrorUnfixed = 4,
41			kErrorOperational = 8,
42			kErrorUsage = 16,
43			};
44
45			string *g_cache_dir;
46			atomic_int32 g_num_files;
47			atomic_int32 g_num_err_fixed;
48			atomic_int32 g_num_err_unfixed;
49			atomic_int32 g_num_err_operational;
50			atomic_int32 g_num_tmp_catalog;
51			/**
52			* Traversal of the file system tree is serialized.
53			*/
54			pthread_mutex_t g_lock_traverse = PTHREAD_MUTEX_INITIALIZER;
55			DIR *g_DIRP_current = NULL;
56			int g_num_dirs = -1; /*< Number of cache directories already examined. /
57			string g_current_dir; /< Current cache sub directory /
58
59			int g_num_threads = 1;
60			bool g_fix_errors = false;
61			bool g_verbose = false;
62			atomic_int32 g_force_rebuild;
63			atomic_int32 g_modified_cache;
64
65
66			static void Usage() {
67			LogCvmfs(kLogCvmfs, kLogStdout,
68			"CernVM File System consistency checker, version %s\n\n"
69			"This tool checks a cvmfs cache directory for consistency.\n"
70			"If necessary, the managed cache db is removed so that\n"
71			"it will be rebuilt on next mount.\n\n"
72			"Usage: cvmfs_fsck [-v] [-p] [-f] [-j #threads] <cache directory>\n"
73			"Options:\n"
74			" -v verbose output\n"
75			" -p try to fix automatically\n"
76			" -f force rebuild of managed cache db on next mount\n"
77			" -j number of concurrent integrity check worker threads\n",
78			VERSION);
79			}
80
81
82			static bool GetNextFile(string relative_path, string hash_name) {
83			platform_dirent64 *d = NULL;
84
85			pthread_mutex_lock(&g_lock_traverse);
86			get_next_file_again:
87			while (g_DIRP_current && ((d = platform_readdir(g_DIRP_current)) != NULL)) {
88			const string name = d->d_name;
89			if ((name == ".") \|\| (name == "..")) continue;
90
91			platform_stat64 info;
92			relative_path = g_current_dir + "/" + name;
93			hash_name = g_current_dir + name;
94			const string path = g_cache_dir + "/" + relative_path;
95			if (platform_lstat(relative_path->c_str(), &info) != 0) {
96			LogCvmfs(kLogCvmfs, kLogStdout, "Warning: failed to stat() %s (%d)",
97			path.c_str(), errno);
98			continue;
99			}
100
101			if (!S_ISREG(info.st_mode)) {
102			LogCvmfs(kLogCvmfs, kLogStdout, "Warning: %s is not a regular file",
103			path.c_str());
104			continue;
105			}
106
107			break;
108			}
109
110			if (!d) {
111			if (g_DIRP_current) {
112			closedir(g_DIRP_current);
113			g_DIRP_current = NULL;
114			}
115			g_num_dirs++;
116			if (g_num_dirs < 256) {
117			char hex[3];
118			snprintf(hex, sizeof(hex), "%02x", g_num_dirs);
119			*g_current_dir = string(hex, 2);
120
121			if (g_verbose)
122			LogCvmfs(kLogCvmfs, kLogStdout, "Entering %s", g_current_dir->c_str());
123			if ((g_DIRP_current = opendir(hex)) == NULL) {
124			LogCvmfs(kLogCvmfs, kLogStderr,
125			"Invalid cache directory, %s/%s does not exist",
126			g_cache_dir->c_str(), g_current_dir->c_str());
127			pthread_mutex_unlock(&g_lock_traverse);
128			exit(kErrorUnfixed);
129			}
130			goto get_next_file_again;
131			}
132			}
133			pthread_mutex_unlock(&g_lock_traverse);
134
135			if (d)
136			return true;
137
138			return false;
139			}
140
141
142			static void MainCheck(void data __attribute__((unused))) {
143			string relative_path;
144			string hash_name;
145
146			while (GetNextFile(&relative_path, &hash_name)) {
147			const string path = *g_cache_dir + "/" + relative_path;
148
149			int n = atomic_xadd32(&g_num_files, 1);
150			if (g_verbose)
151			LogCvmfs(kLogCvmfs, kLogStdout, "Checking file %s", path.c_str());
152			if (!g_verbose && ((n % 1000) == 0))
153			LogCvmfs(kLogCvmfs, kLogStdout \| kLogNoLinebreak, ".");
154
155			if (relative_path[relative_path.length()-1] == 'T') {
156			LogCvmfs(kLogCvmfs, kLogStdout,
157			"Warning: temporary file catalog %s found", path.c_str());
158			atomic_inc32(&g_num_tmp_catalog);
159			if (g_fix_errors) {
160			if (unlink(relative_path.c_str()) == 0) {
161			LogCvmfs(kLogCvmfs, kLogStdout, "Fix: %s unlinked", path.c_str());
162			atomic_inc32(&g_num_err_fixed);
163			} else {
164			LogCvmfs(kLogCvmfs, kLogStdout, "Error: failed to unlink %s",
165			path.c_str());
166			atomic_inc32(&g_num_err_unfixed);
167			}
168			}
169			continue;
170			}
171
172			int fd_src = open(relative_path.c_str() , O_RDONLY);
173			if (fd_src < 0) {
174			LogCvmfs(kLogCvmfs, kLogStdout, "Error: cannot open %s", path.c_str());
175			atomic_inc32(&g_num_err_operational);
176			continue;
177			}
178			// Don't thrash kernel buffers
179			platform_disable_kcache(fd_src);
180
181			// Compress every file and calculate SHA-1 of stream
182			shash::Any expected_hash = shash::MkFromHexPtr(shash::HexPtr(hash_name));
183			shash::Any hash(expected_hash.algorithm);
184			if (!zlib::CompressFd2Null(fd_src, &hash)) {
185			LogCvmfs(kLogCvmfs, kLogStdout, "Error: could not compress %s",
186			path.c_str());
187			atomic_inc32(&g_num_err_operational);
188			} else {
189			if (hash != expected_hash) {
190			// If the hashes don't match, try hashing the uncompressed file
191			if (!shash::HashFile(relative_path, &hash)) {
192			LogCvmfs(kLogCvmfs, kLogStdout, "Error: could not hash %s",
193			path.c_str());
194			atomic_inc32(&g_num_err_operational);
195			}
196			if (hash != expected_hash) {
197			if (g_fix_errors) {
198			const string quarantaine_path = "./quarantaine/" + hash_name;
199			bool fixed = false;
200			if (rename(relative_path.c_str(), quarantaine_path.c_str()) == 0) {
201			LogCvmfs(kLogCvmfs, kLogStdout,
202			"Fix: %s is corrupted, moved to quarantaine folder",
203			path.c_str());
204			fixed = true;
205			} else {
206			LogCvmfs(kLogCvmfs, kLogStdout,
207			"Warning: failed to move %s into quarantaine folder",
208			path.c_str());
209			if (unlink(relative_path.c_str()) == 0) {
210			LogCvmfs(kLogCvmfs, kLogStdout,
211			"Fix: %s is corrupted, file unlinked", path.c_str());
212			fixed = true;
213			} else {
214			LogCvmfs(kLogCvmfs, kLogStdout,
215			"Error: %s is corrupted, could not unlink",
216			path.c_str());
217			}
218			}
219
220			if (fixed) {
221			atomic_inc32(&g_num_err_fixed);
222
223			// Changes made, we have to rebuild the managed cache db
224			atomic_cas32(&g_force_rebuild, 0, 1);
225			atomic_cas32(&g_modified_cache, 0, 1);
226			} else {
227			atomic_inc32(&g_num_err_unfixed);
228			}
229			} else {
230			LogCvmfs(kLogCvmfs, kLogStdout,
231			"Error: %s has compressed checksum %s, "
232			"delete this file from cache directory!",
233			path.c_str(), hash.ToString().c_str());
234			atomic_inc32(&g_num_err_unfixed);
235			}
236			}
237			}
238			}
239			close(fd_src);
240			}
241
242			return NULL;
243			}
244
245
246			int main(int argc, char **argv) {
247			atomic_init32(&g_force_rebuild);
248			atomic_init32(&g_modified_cache);
249			g_current_dir = new string();
250
251			int c;
252			while ((c = getopt(argc, argv, "hvpfj:")) != -1) {
253			switch (c) {
254			case 'h':
255			Usage();
256			return kErrorOk;
257			case 'v':
258			g_verbose = true;
259			break;
260			case 'p':
261			g_fix_errors = true;
262			break;
263			case 'f':
264			atomic_cas32(&g_force_rebuild, 0, 1);
265			break;
266			case 'j':
267			g_num_threads = atoi(optarg);
268			if (g_num_threads < 1) {
269			LogCvmfs(kLogCvmfs, kLogStdout,
270			"There is at least one worker thread required");
271			return kErrorUsage;
272			}
273			break;
274			case '?':
275			default:
276			Usage();
277			return kErrorUsage;
278			}
279			}
280
281			// Switch to cache directory
282			if (optind >= argc) {
283			Usage();
284			return kErrorUsage;
285			}
286			g_cache_dir = new string(MakeCanonicalPath(argv[optind]));
287			if (chdir(g_cache_dir->c_str()) != 0) {
288			LogCvmfs(kLogCvmfs, kLogStderr, "Could not chdir to %s",
289			g_cache_dir->c_str());
290			return kErrorOperational;
291			}
292
293			// Check if txn directory is empty
294			DIR *dirp_txn;
295			if ((dirp_txn = opendir("txn")) == NULL) {
296			LogCvmfs(kLogCvmfs, kLogStderr,
297			"Invalid cache directory, %s/txn does not exist",
298			g_cache_dir->c_str());
299			return kErrorOperational;
300			}
301			platform_dirent64 *d;
302			while ((d = platform_readdir(dirp_txn)) != NULL) {
303			const string name = d->d_name;
304			if ((name == ".") \|\| (name == "..")) continue;
305
306			LogCvmfs(kLogCvmfs, kLogStdout,
307			"Warning: temporary directory %s/txn is not empty\n"
308			"If this repository is currently _not_ mounted, "
309			"you can remove its contents", g_cache_dir->c_str());
310			break;
311			}
312			closedir(dirp_txn);
313
314			// Run workers to recalculate checksums
315			atomic_init32(&g_num_files);
316			atomic_init32(&g_num_err_fixed);
317			atomic_init32(&g_num_err_unfixed);
318			atomic_init32(&g_num_err_operational);
319			atomic_init32(&g_num_tmp_catalog);
320			pthread_t workers = reinterpret_cast<pthread_t >(
321			smalloc(g_num_threads * sizeof(pthread_t)));
322			if (!g_verbose)
323			LogCvmfs(kLogCvmfs, kLogStdout \| kLogNoLinebreak, "Verifying: ");
324			for (int i = 0; i < g_num_threads; ++i) {
325			if (g_verbose)
326			LogCvmfs(kLogCvmfs, kLogStdout, "Starting worker %d", i+1);
327			if (pthread_create(&workers[i], NULL, MainCheck, NULL) != 0) {
328			LogCvmfs(kLogCvmfs, kLogStdout, "Fatal: could not create worker thread");
329			return kErrorOperational;
330			}
331			}
332			for (int i = g_num_threads-1; i >= 0; --i) {
333			pthread_join(workers[i], NULL);
334			if (g_verbose)
335			LogCvmfs(kLogCvmfs, kLogStdout, "Stopping worker %d", i+1);
336			}
337			free(workers);
338			if (!g_verbose)
339			LogCvmfs(kLogCvmfs, kLogStdout, "");
340			LogCvmfs(kLogCvmfs, kLogStdout, "Verified %d files",
341			atomic_read32(&g_num_files));
342
343			if (atomic_read32(&g_num_tmp_catalog) > 0)
344			LogCvmfs(kLogCvmfs, kLogStdout, "Temporary file catalogs were found.");
345
346			if (atomic_read32(&g_force_rebuild)) {
347			if (unlink("cachedb") == 0) {
348			LogCvmfs(kLogCvmfs, kLogStdout,
349			"Fix: managed cache db unlinked, will be rebuilt on next mount");
350			atomic_inc32(&g_num_err_fixed);
351			} else {
352			if (errno != ENOENT) {
353			LogCvmfs(kLogCvmfs, kLogStdout,
354			"Error: could not unlink managed cache database (%d)", errno);
355			atomic_inc32(&g_num_err_unfixed);
356			}
357			}
358			}
359
360			if (atomic_read32(&g_modified_cache)) {
361			LogCvmfs(kLogCvmfs, kLogStdout, "\n"
362			"WARNING: There might by corrupted files in the kernel buffers.\n"
363			"Remount CernVM-FS or run 'echo 3 > /proc/sys/vm/drop_caches'"
364			"\n\n");
365			}
366
367			int retval = 0;
368			if (atomic_read32(&g_num_err_fixed) > 0)
369			retval \|= kErrorFixed;
370			if (atomic_read32(&g_num_err_unfixed) > 0)
371			retval \|= kErrorUnfixed;
372			if (atomic_read32(&g_num_err_operational) > 0)
373			retval \|= kErrorOperational;
374
375			return retval;
376			}