CernVM-FS  2.12.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
fs_traversal.cc
Go to the documentation of this file.
1 
5 #include <errno.h>
6 #include <pthread.h>
7 #include <stdio.h>
8 
9 #include <fstream>
10 #include <map>
11 #include <vector>
12 
13 #include "cvmfs_config.h"
14 #include "libcvmfs.h"
19 #include "shrinkwrap/spec_tree.h"
20 #include "statistics.h"
21 #include "util/atomic.h"
22 #include "util/concurrency.h"
23 #include "util/logging.h"
24 #include "util/platform.h"
25 #include "util/posix.h"
26 #include "util/smalloc.h"
27 #include "util/string.h"
28 
29 using namespace std; //NOLINT
30 
31 // Taken from fsck
32 enum Errors {
33  kErrorOk = 0,
39 };
40 
41 int strcmp_list(const void* a, const void* b)
42 {
43  char const **char_a = (char const **)a;
44  char const **char_b = (char const **)b;
45 
46  return strcmp(*char_a, *char_b);
47 }
48 
49 namespace shrinkwrap {
50 
51 namespace {
52 
53 // No destructor is written to prevent double free and
54 // corruption of string pointers. After the copy written
55 // to the pipe, the new copy falls out of scope and
56 // is destroyed. If there is a destructor that frees the
57 // strings they get deleted after writing to the pipe.
58 class FileCopy {
59  public:
61  : src(NULL)
62  , dest(NULL) {}
63 
64  FileCopy(char *src, char *dest)
65  : src(strdup(src))
66  , dest(strdup(dest)) {}
67 
68  bool IsTerminateJob() const {
69  return ((src == NULL) && (dest == NULL));
70  }
71 
72  char *src;
73  char *dest;
74 };
75 
76 class RecDir {
77  public:
79  : dir(NULL)
80  , recursive(false) {}
81 
82  RecDir(const char *dir, bool recursive)
83  : dir(strdup(dir))
84  , recursive(recursive) {}
85 
86  ~RecDir() {
87  free(dir);
88  }
89 
90  char *dir;
91  bool recursive;
92 };
93 
94 unsigned num_parallel_ = 0;
95 bool recursive = true;
96 uint64_t stat_update_period_ = 0; // Off for testing
97 int pipe_chunks[2];
98 // required for concurrent reading
99 pthread_mutex_t lock_pipe = PTHREAD_MUTEX_INITIALIZER;
101 
102 vector<RecDir*> dirs_;
103 
105 
106 } // namespace
107 
108 struct fs_traversal* FindInterface(const char * type)
109 {
110  if (!strcmp(type, "posix")) {
111  return posix_get_interface();
112  } else if (!strcmp(type, "cvmfs")) {
113  return libcvmfs_get_interface();
114  }
116  "Unknown File System Interface : %s", type);
117  return NULL;
118 }
119 
120 bool cvmfs_attr_cmp(struct cvmfs_attr *src, struct cvmfs_attr *dest,
121  struct fs_traversal *dest_fs) {
122  if (!src) { return false; }
123  if (!dest) { return false; }
124  if (src->version != dest->version) { return false; }
125  if (src->size != dest->size) { return false; }
126 
127  // Actual contents of stat, mapped from DirectoryEntry
128  if ( (!S_ISLNK(src->st_mode) && src->st_mode != dest->st_mode)
129  || ((S_IFMT & src->st_mode) != (S_IFMT & dest->st_mode)) )
130  { return false; }
131 
132  if (!S_ISLNK(src->st_mode) && src->st_uid != dest->st_uid)
133  { return false; }
134  if (!S_ISLNK(src->st_mode) && src->st_gid != dest->st_gid)
135  { return false; }
136 
137 
138  // CVMFS related content
139  if (S_ISREG(src->st_mode) && src->cvm_checksum) {
140  if (dest->cvm_checksum != NULL) {
141  if (strcmp(src->cvm_checksum, dest->cvm_checksum)) {
142  return false;
143  }
144  } else if (!dest_fs->is_hash_consistent(dest_fs->context_, src)) {
145  return false;
146  }
147  }
148 
149  if (S_ISLNK(src->st_mode)) {
150  if (strcmp(src->cvm_symlink, dest->cvm_symlink)) { return false; }
151  }
152 
153  if (strcmp(src->cvm_name, dest->cvm_name)) { return false; }
154 
155  // TODO(nhazekam): Not comparing Xattrs yet
156  // void * cvm_xattrs;
157  return true;
158 }
159 
160 bool copyFile(
161  struct fs_traversal *src_fs,
162  const char *src_name,
163  struct fs_traversal *dest_fs,
164  const char *dest_name,
165  perf::Statistics *pstats
166 ) {
167  int retval = 0;
168  bool status = true;
169  size_t bytes_transferred = 0;
170 
171  // Get file handles from each respective filesystem
172  void *src = src_fs->get_handle(src_fs->context_, src_name);
173  void *dest = dest_fs->get_handle(dest_fs->context_, dest_name);
174 
175  retval = src_fs->do_fopen(src, fs_open_read);
176  if (retval != 0) {
178  "Failed open src : %s : %d : %s\n",
179  src_name, errno, strerror(errno));
180  status = false;
181  goto copy_fail;
182  }
183 
184  retval = dest_fs->do_fopen(dest, fs_open_write);
185  if (retval != 0) {
187  "Failed open dest : %s : %d : %s\n",
188  dest_name, errno, strerror(errno));
189  status = false;
190  goto copy_fail;
191  }
192 
193  while (status) {
194  char buffer[COPY_BUFFER_SIZE];
195 
196  size_t actual_read = 0;
197  retval = src_fs->do_fread(src, buffer, sizeof(buffer), &actual_read);
198  if (retval != 0) {
200  "Read failed : %d %s\n", errno, strerror(errno));
201  status = false;
202  goto copy_fail;
203  }
204  bytes_transferred+=actual_read;
205 
206  retval = dest_fs->do_fwrite(dest, buffer, actual_read);
207  if (retval != 0) {
209  "Write failed : %d %s\n", errno, strerror(errno));
210  status = false;
211  goto copy_fail;
212  }
213 
214  if (actual_read < COPY_BUFFER_SIZE) {
215  break;
216  }
217  }
218 
219  pstats->Lookup(SHRINKWRAP_STAT_DATA_BYTES)->Xadd(bytes_transferred);
220 
221  retval = src_fs->do_fclose(src);
222  if (retval != 0) {
224  "Failed close src : %s : %d : %s\n",
225  src_name, errno, strerror(errno));
226  status = false;
227  goto copy_fail;
228  }
229 
230  retval = dest_fs->do_fclose(dest);
231  if (retval != 0) {
233  "Failed close dest : %s : %d : %s\n",
234  dest_name, errno, strerror(errno));
235  status = false;
236  goto copy_fail;
237  }
238 
239 copy_fail:
240  // This function will close the file if still open
241  // and free the handle.
242  src_fs->do_ffree(src);
243  dest_fs->do_ffree(dest);
244 
245  return status;
246 }
247 
248 char *get_full_path(const char *dir, const char *entry) {
249  size_t len = 2 + strlen(dir)+ strlen(entry);
250  char * path = reinterpret_cast<char *>(smalloc(len));
251  snprintf(path, len, "%s/%s", dir, entry);
252  return path;
253 }
254 
256  struct fs_traversal *fs,
257  const char *entry,
258  struct cvmfs_attr **st,
259  bool get_hash
260 ) {
261  cvmfs_attr_free(*st);
262  *st = NULL;
263  *st = cvmfs_attr_init();
264  int retval = fs->get_stat(fs->context_, entry, *st, get_hash);
265  if (retval != 0) {
266  return false;
267  }
268  return true;
269 }
270 
271 bool getNext(
272  struct fs_traversal *fs,
273  const char *dir,
274  char **dir_list,
275  char **entry,
276  ssize_t *iter,
277  struct cvmfs_attr **st,
278  bool get_hash,
279  bool is_src,
280  perf::Statistics *pstats
281 ) {
282  size_t location = 0;
283  if (iter) {
284  location = (*iter)+1;
285  *iter = location;
286  }
287 
288  free(*entry);
289  *entry = NULL;
290 
291  if (dir_list && dir_list[location]) {
292  *entry = get_full_path(dir, dir_list[location]);
293  if (entry && is_src && !spec_tree_->IsMatching(string(*entry))) {
294  return getNext(fs, dir, dir_list, entry, iter,
295  st, get_hash, is_src, pstats);
296  }
297  if (entry && !updateStat(fs, *entry, st, get_hash)) {
298  return getNext(fs, dir, dir_list, entry, iter,
299  st, get_hash, is_src, pstats);
300  }
301  } else {
302  return false;
303  }
304  if (is_src) {
306  switch ((*st)->st_mode & S_IFMT) {
307  case S_IFREG:
309  pstats->Lookup(SHRINKWRAP_STAT_COUNT_BYTE)->Xadd((*st)->st_size);
310  break;
311  case S_IFDIR:
313  break;
314  case S_IFLNK:
316  break;
317  }
318  } else {
320  }
321  return true;
322 }
323 
325  struct fs_traversal *src,
326  const char *dir,
327  char ***buf,
328  size_t *len
329 ) {
330  int retval = spec_tree_->ListDir(dir, buf, len);
331 
332  if (retval == SPEC_READ_FS) {
333  src->list_dir(src->context_, dir, buf, len);
334  qsort(*buf, *len, sizeof(char *), strcmp_list);
335  }
336 }
337 
339  struct fs_traversal *src,
340  struct cvmfs_attr *src_st,
341  struct fs_traversal *dest,
342  struct cvmfs_attr *dest_st,
343  const char *entry,
344  perf::Statistics *pstats
345 ) {
346  bool result = true;
347  // They don't point to the same data, link new data
348  char *dest_data = dest->get_identifier(dest->context_, src_st);
349 
350  // Touch is atomic, if it fails something else will write file
351  if (!dest->touch(dest->context_, src_st)) {
352  char *src_ident = src->get_identifier(src->context_, src_st);
353  if (num_parallel_) {
354  FileCopy next_copy(src_ident, dest_data);
355 
356  WritePipe(pipe_chunks[1], &next_copy, sizeof(next_copy));
357  atomic_inc64(&copy_queue);
358  } else {
359  if (!copyFile(src, src_ident, dest, dest_data, pstats)) {
361  "Failed to copy %s->%s : %d : %s",
362  entry, dest_data, errno, strerror(errno));
363  errno = 0;
364  result = false;
365  }
366  }
368  free(src_ident);
369  } else {
372  }
373 
374  // The target always exists (either previously existed or just created).
375  // This is not done in parallel like copyFile
376  if (result && dest->do_link(dest->context_, entry, dest_data)) {
378  "Failed to link %s->%s : %d : %s",
379  entry, dest_data, errno, strerror(errno));
380  errno = 0;
381  result = false;
382  }
383  free(dest_data);
384  return result;
385 }
386 
388  struct fs_traversal *src,
389  struct cvmfs_attr *src_st,
390  struct fs_traversal *dest,
391  struct cvmfs_attr *dest_st,
392  const char *entry
393 ) {
394  if (dest->do_mkdir(dest->context_, entry, src_st)) {
395  if (errno == EEXIST) {
396  errno = 0;
397  if (dest->set_meta(dest->context_, entry, src_st)) {
399  "Traversal failed to set_meta %s", entry);
400  return false;
401  }
402  } else {
404  "Traversal failed to mkdir %s : %d : %s",
405  entry, errno, strerror(errno));
406  return false;
407  }
408  }
409  return true;
410 }
411 
412 void add_dir_for_sync(const char *dir, bool recursive) {
413  dirs_.push_back(new RecDir(dir, recursive));
414 }
415 
416 // Compares possibly null strings.
417 // If 1->handle destination
418 // If -1->handle source
419 // If 0->handle both
420 int robust_strcmp(const char *src, const char *dest) {
421  if (!src) {
422  return 1;
423  } else if (!dest) {
424  return -1;
425  } else {
426  return strcmp(src, dest);
427  }
428 }
429 
430 bool Sync(
431  const char *dir,
432  struct fs_traversal *src,
433  struct fs_traversal *dest,
434  bool recursive,
435  perf::Statistics *pstats) {
436  bool result = true;
437  int cmp = 0;
438 
439  char **src_dir = NULL;
440  size_t src_len = 0;
441  ssize_t src_iter = -1;
442  char * src_entry = NULL;
443  struct cvmfs_attr *src_st = cvmfs_attr_init();
444  list_src_dir(src, dir, &src_dir, &src_len);
445 
446  char **dest_dir = NULL;
447  size_t dest_len = 0;
448  ssize_t dest_iter = -1;
449  char * dest_entry = NULL;
450  struct cvmfs_attr *dest_st = cvmfs_attr_init();
451  dest->list_dir(dest->context_, dir, &dest_dir, &dest_len);
452  qsort(dest_dir, dest_len, sizeof(char *), strcmp_list);
453 
454  // While either have defined members and there hasn't been a failure
455  while (result) {
456  if (cmp <= 0) {
457  getNext(src, dir, src_dir, &src_entry, &src_iter,
458  &src_st, true, true, pstats);
459  }
460  if (cmp >= 0) {
461  getNext(dest, dir, dest_dir, &dest_entry, &dest_iter,
462  &dest_st, false, false, pstats);
463  } else {
464  // A destination entry was added
466  }
467 
468  // All entries in both have been visited
469  if (!src_entry && !dest_entry)
470  break;
471 
472  cmp = robust_strcmp(src_entry, dest_entry);
473 
474  if (cmp <= 0) {
475  // Compares stats to see if they are equivalent
476  if (cmp == 0 && cvmfs_attr_cmp(src_st, dest_st, dest)
477  // Also check internal hardlink consistency in destination file system
478  // where applicable:
479  && (dest_st->cvm_checksum == NULL
480  || dest->is_hash_consistent(dest->context_, dest_st))) {
481  if (S_ISDIR(src_st->st_mode) && recursive) {
482  add_dir_for_sync(src_entry, recursive);
483  }
484  continue;
485  }
486  // If not equal, bring dest up-to-date
487  switch (src_st->st_mode & S_IFMT) {
488  case S_IFREG:
489  if (!handle_file(src, src_st, dest, dest_st, src_entry, pstats))
490  result = false;
491  break;
492  case S_IFDIR:
493  if (!handle_dir(src, src_st, dest, dest_st, src_entry))
494  result = false;
495  if (result && recursive)
496  add_dir_for_sync(src_entry, recursive);
497  break;
498  case S_IFLNK:
499  // Should likely copy the source of the symlink target
500  if (dest->do_symlink(dest->context_, src_entry,
501  src_st->cvm_symlink, src_st) != 0) {
503  "Traversal failed to symlink %s->%s : %d : %s",
504  src_entry, src_st->cvm_symlink, errno, strerror(errno));
505  result = false;
506  }
507  break;
508  case S_IFIFO:
509  // Ignore pipe, but continue
511  "Skipping pipe: %s",
512  src_entry);
513  break;
514  default:
516  "Encountered unknown file type '%d' for source file %s",
517  src_st->st_mode & S_IFMT,
518  src_entry);
519  result = false;
520  }
521  /* Dest contains something missing from Src */
522  } else {
523  switch (dest_st->st_mode & S_IFMT) {
524  case S_IFREG:
525  case S_IFLNK:
526  if (dest->do_unlink(dest->context_, dest_entry) != 0) {
528  "Failed to unlink file %s", dest_entry);
529  result = false;
530  }
531  break;
532  case S_IFDIR:
533  // This is recursive so that the contents of the directory
534  // are removed before trying to remove the directory. If
535  // tail recursed, do_rmdir will fail as there are still
536  // contents.
537  if (!Sync(dest_entry, src, dest, true, pstats)) {
538  result = false;
539  break;
540  }
541  if (dest->do_rmdir(dest->context_, dest_entry) != 0) {
543  "Failed to remove directory %s", dest_entry);
544  result = false;
545  }
546  break;
547  default:
549  "Encountered unknown file type '%d' for destination file %s",
550  dest_st->st_mode & S_IFMT,
551  dest_entry);
552  result = false;
553  }
554  }
555  }
556 
557  cvmfs_list_free(src_dir);
558  free(src_entry);
559  if (src_st)
560  cvmfs_attr_free(src_st);
561 
562  cvmfs_list_free(dest_dir);
563  free(dest_entry);
564  if (dest_st)
565  cvmfs_attr_free(dest_st);
566 
567  return result;
568 }
569 
570 bool SyncFull(
571  struct fs_traversal *src,
572  struct fs_traversal *dest,
573  perf::Statistics *pstats,
574  uint64_t last_print_time) {
575  if (dirs_.empty()) {
576  dirs_.push_back(new RecDir("", true));
577  }
578  while (!dirs_.empty()) {
579  RecDir *next_dir = dirs_.back();
580  dirs_.pop_back();
581 
582  if (!Sync(next_dir->dir, src, dest, next_dir->recursive, pstats)) {
584  "File %s failed to copy\n", next_dir->dir);
585  return false;
586  }
587 
588  if (stat_update_period_ > 0 &&
589  platform_monotonic_time()-last_print_time > stat_update_period_) {
591  "%s",
592  pstats->PrintList(perf::Statistics::kPrintSimple).c_str());
593  last_print_time = platform_monotonic_time();
594  }
595 
596  delete next_dir;
597  }
598  return true;
599 }
600 
605  int parallel;
606 };
607 
611 };
612 
613 static void *MainWorker(void *data) {
614  MainWorkerSpecificContext *sc = static_cast<MainWorkerSpecificContext*>(data);
615  MainWorkerContext *mwc = sc->mwc;
616  perf::Counter *files_transferred
618 
619  while (1) {
620  FileCopy next_copy;
621  {
623  ReadPipe(pipe_chunks[0], &next_copy, sizeof(next_copy));
624  }
625  if (next_copy.IsTerminateJob())
626  break;
627 
628  if (!next_copy.src || !next_copy.dest) {
629  continue;
630  }
631  if (!copyFile(mwc->src_fs, next_copy.src, mwc->dest_fs,
632  next_copy.dest, mwc->pstats)) {
634  "File %s failed to copy\n", next_copy.src);
635  }
636  files_transferred->Inc();
637 
638  // Noted in FileCopy: This is freed here to prevent the strings from being
639  // double freed when a FileCopy goes out of scope here and at WritePipe
640  free(next_copy.src);
641  free(next_copy.dest);
642 
643  atomic_dec64(&copy_queue);
644  }
645  return NULL;
646 }
647 
649  perf::Statistics *result = new perf::Statistics();
651  "Number of files from source repository");
653  "Number of directories from source repository");
655  "Number of symlinks from source repository");
657  "Byte count of projected repository");
659  "Number of file system entries processed in the source");
661  "Number of file system entries processed in the destination");
663  "Number of data files transferred from source to destination");
665  "Bytes transferred from source to destination");
667  "Number of files not copied due to deduplication");
669  "Number of bytes not copied due to deduplication");
670  return result;
671 }
672 
674  struct fs_traversal *src,
675  struct fs_traversal *dest,
676  const char *base,
677  const char *spec,
678  uint64_t parallel,
679  uint64_t stat_period) {
680  num_parallel_ = parallel;
681  stat_update_period_ = stat_period;
682 
684 
685  // Initialization
686  atomic_init64(&copy_queue);
687 
688  pthread_t *workers = NULL;
689 
690  struct MainWorkerSpecificContext *specificWorkerContexts = NULL;
691 
692  MainWorkerContext *mwc = NULL;
693 
694  if (num_parallel_ > 0) {
695  workers =
696  reinterpret_cast<pthread_t *>(smalloc(sizeof(pthread_t) * num_parallel_));
697 
698  specificWorkerContexts =
699  reinterpret_cast<struct MainWorkerSpecificContext *>(
700  smalloc(sizeof(struct MainWorkerSpecificContext) * num_parallel_));
701 
702  mwc = new struct MainWorkerContext;
703  // Start Workers
705  LogCvmfs(kLogCvmfs, kLogStdout, "Starting %u workers", num_parallel_);
706  mwc->src_fs = src;
707  mwc->dest_fs = dest;
708  mwc->pstats = pstats;
709  mwc->parallel = num_parallel_;
710 
711  for (unsigned i = 0; i < num_parallel_; ++i) {
712  specificWorkerContexts[i].mwc = mwc;
713  specificWorkerContexts[i].num_thread = i;
714  int retval = pthread_create(&workers[i], NULL, MainWorker,
715  static_cast<void*>(&(specificWorkerContexts[i])));
716  assert(retval == 0);
717  }
718  }
719 
720  if (spec) {
721  delete spec_tree_;
723  }
724 
725  uint64_t last_print_time = 0;
727  int result = !SyncFull(src, dest, pstats, last_print_time);
728 
729  while (atomic_read64(&copy_queue) != 0) {
730  if (platform_monotonic_time() -last_print_time > stat_update_period_) {
732  "%s",
733  pstats->PrintList(perf::Statistics::kPrintSimple).c_str());
734  last_print_time = platform_monotonic_time();
735  }
736 
737  SafeSleepMs(100);
738  }
739 
740 
741  if (num_parallel_ > 0) {
742  LogCvmfs(kLogCvmfs, kLogStdout, "Stopping %u workers", num_parallel_);
743  for (unsigned i = 0; i < num_parallel_; ++i) {
744  FileCopy terminate_workers;
745  WritePipe(pipe_chunks[1], &terminate_workers, sizeof(terminate_workers));
746  }
747  for (unsigned i = 0; i < num_parallel_; ++i) {
748  int retval = pthread_join(workers[i], NULL);
749  assert(retval == 0);
750  }
752  delete workers;
753  delete specificWorkerContexts;
754  delete mwc;
755  }
757  "%s",
758  pstats->PrintList(perf::Statistics::kPrintHeader).c_str());
759  delete pstats;
760 
761  delete spec_tree_;
762 
763  return result;
764 }
765 
766 int GarbageCollect(struct fs_traversal *fs) {
768  "Performing garbage collection...");
769  uint64_t start_time = platform_monotonic_time();
770  int retval = fs->garbage_collector(fs->context_);
771  uint64_t end_time = platform_monotonic_time();
772  LogCvmfs(kLogCvmfs, kLogStdout, "Garbage collection took %lu seconds.",
773  (end_time-start_time));
774  return retval;
775 }
776 
777 } // namespace shrinkwrap
char * cvm_symlink
Definition: libcvmfs.h:172
Counter * Register(const std::string &name, const std::string &desc)
Definition: statistics.cc:160
int64_t atomic_int64
Definition: atomic.h:18
bool SyncFull(struct fs_traversal *src, struct fs_traversal *dest, perf::Statistics *pstats, uint64_t last_print_time)
struct fs_traversal * FindInterface(const char *type)
char * cvm_name
Definition: libcvmfs.h:174
int(* do_fopen)(void *file_ctx, fs_open_type op_mode)
#define SHRINKWRAP_STAT_DATA_BYTES_DEDUPED
Definition: fs_traversal.h:24
bool cvmfs_attr_cmp(struct cvmfs_attr *src, struct cvmfs_attr *dest, struct fs_traversal *dest_fs)
void list_src_dir(struct fs_traversal *src, const char *dir, char ***buf, size_t *len)
#define SHRINKWRAP_STAT_COUNT_FILE
Definition: fs_traversal.h:15
void cvmfs_attr_free(struct cvmfs_attr *attr)
Definition: libcvmfs.cc:53
bool getNext(struct fs_traversal *fs, const char *dir, char **dir_list, char **entry, ssize_t *iter, struct cvmfs_attr **st, bool get_hash, bool is_src, perf::Statistics *pstats)
#define SHRINKWRAP_STAT_DATA_FILES
Definition: fs_traversal.h:21
static void * MainWorker(void *data)
int(* set_meta)(struct fs_traversal_context *ctx, const char *path, const struct cvmfs_attr *stat)
bool(* is_hash_consistent)(struct fs_traversal_context *ctx, const struct cvmfs_attr *stat)
static SpecTree * Create(const std::string &path)
Definition: spec_tree.cc:21
#define SHRINKWRAP_STAT_COUNT_SYMLINK
Definition: fs_traversal.h:17
struct fs_traversal * posix_get_interface()
Definition: interface.cc:131
struct fs_traversal * libcvmfs_get_interface()
#define SHRINKWRAP_STAT_COUNT_DIR
Definition: fs_traversal.h:16
assert((mem||(size==0))&&"Out Of Memory")
int(* do_unlink)(struct fs_traversal_context *ctx, const char *path)
bool handle_file(struct fs_traversal *src, struct cvmfs_attr *src_st, struct fs_traversal *dest, struct cvmfs_attr *dest_st, const char *entry, perf::Statistics *pstats)
int(* garbage_collector)(struct fs_traversal_context *ctx)
int strcmp_list(const void *a, const void *b)
Definition: fs_traversal.cc:41
struct fs_traversal * dest_fs
int(* do_rmdir)(struct fs_traversal_context *ctx, const char *path)
void MakePipe(int pipe_fd[2])
Definition: posix.cc:492
void *(* get_handle)(struct fs_traversal_context *ctx, const char *identifier)
#define SHRINKWRAP_STAT_DATA_BYTES
Definition: fs_traversal.h:22
perf::Statistics * GetSyncStatTemplate()
char * cvm_checksum
Definition: libcvmfs.h:171
void(* do_ffree)(void *file_ctx)
int SyncInit(struct fs_traversal *src, struct fs_traversal *dest, const char *base, const char *spec, uint64_t parallel, uint64_t stat_period)
void cvmfs_list_free(char **buf)
Definition: libcvmfs.cc:469
Counter * Lookup(const std::string &name) const
Definition: statistics.cc:62
int(* touch)(struct fs_traversal_context *ctx, const struct cvmfs_attr *stat)
int(* do_symlink)(struct fs_traversal_context *ctx, const char *src, const char *dest, const struct cvmfs_attr *stat_info)
struct fs_traversal * src_fs
int(* do_link)(struct fs_traversal_context *ctx, const char *path, const char *identifier)
gid_t st_gid
Definition: libcvmfs.h:161
void(* list_dir)(struct fs_traversal_context *ctx, const char *dir, char ***buf, size_t *len)
#define SHRINKWRAP_STAT_ENTRIES_SRC
Definition: fs_traversal.h:19
off_t st_size
Definition: libcvmfs.h:163
#define COPY_BUFFER_SIZE
void Sync()
Definition: repository.cc:773
bool updateStat(struct fs_traversal *fs, const char *entry, struct cvmfs_attr **st, bool get_hash)
void add_dir_for_sync(const char *dir, bool recursive)
uint64_t platform_monotonic_time()
int robust_strcmp(const char *src, const char *dest)
void Inc()
Definition: statistics.h:30
int(* do_fclose)(void *file_ctx)
int(* get_stat)(struct fs_traversal_context *ctx, const char *path, struct cvmfs_attr *stat, bool get_hash)
int64_t Xadd(const int64_t delta)
Definition: statistics.h:34
#define SHRINKWRAP_STAT_ENTRIES_DEST
Definition: fs_traversal.h:20
int(* do_fwrite)(void *file_ctx, const char *buff, size_t len)
Definition: mutex.h:42
bool copyFile(struct fs_traversal *src_fs, const char *src_name, struct fs_traversal *dest_fs, const char *dest_name, perf::Statistics *pstats)
std::string PrintList(const PrintOptions print_options)
Definition: statistics.cc:79
void GarbageCollect()
unsigned version
Definition: libcvmfs.h:152
uint64_t size
Definition: libcvmfs.h:153
int(* do_fread)(void *file_ctx, char *buff, size_t len, size_t *read_len)
uid_t st_uid
Definition: libcvmfs.h:160
void SafeSleepMs(const unsigned ms)
Definition: posix.cc:1975
int(* do_mkdir)(struct fs_traversal_context *ctx, const char *path, const struct cvmfs_attr *stat)
bool IsMatching(std::string path)
Definition: spec_tree.cc:47
#define SHRINKWRAP_STAT_DATA_FILES_DEDUPED
Definition: fs_traversal.h:23
int ListDir(const char *dir, char ***buf, size_t *len)
Definition: spec_tree.cc:195
char *(* get_identifier)(struct fs_traversal_context *ctx, const struct cvmfs_attr *stat)
char * get_full_path(const char *dir, const char *entry)
struct MainWorkerContext * mwc
void WritePipe(int fd, const void *buf, size_t nbyte)
Definition: posix.cc:501
void ReadPipe(int fd, void *buf, size_t nbyte)
Definition: posix.cc:513
void ClosePipe(int pipe_fd[2])
Definition: posix.cc:551
struct cvmfs_attr * cvmfs_attr_init()
Definition: libcvmfs.cc:39
bool handle_dir(struct fs_traversal *src, struct cvmfs_attr *src_st, struct fs_traversal *dest, struct cvmfs_attr *dest_st, const char *entry)
#define SHRINKWRAP_STAT_COUNT_BYTE
Definition: fs_traversal.h:18
struct fs_traversal_context * context_
Errors
Definition: cvmfs_fsck.cc:37
mode_t st_mode
Definition: libcvmfs.h:158
CVMFS_EXPORT void LogCvmfs(const LogSource source, const int mask, const char *format,...)
Definition: logging.cc:528