20 #define _FILE_OFFSET_BITS 64
22 #define __STDC_FORMAT_MACROS
31 #include <sys/capability.h>
62 PrintError(
"overlay (copy on write) directory does not exist");
70 PrintError(
"cvmfs read/only repository does not exist");
83 PrintError(
"data store directory does not exist");
89 PrintError(
"file chunk size values are not sane");
95 PrintError(
"Session token file has to be provided "
96 "when upstream type is gw.");
105 const string manifest_path = *args.find(
'o')->second;
106 const string dir_temp = *args.find(
't')->second;
107 const string spooler_definition = *args.find(
'r')->second;
108 const string repo_name = *args.find(
'n')->second;
109 const string reflog_chksum_path = *args.find(
'R')->second;
110 if (args.find(
'l') != args.end()) {
120 if (args.find(
'a') != args.end()) {
128 const bool volatile_content = (args.count(
'v') > 0);
129 const bool garbage_collectable = (args.count(
'z') > 0);
130 std::string voms_authz;
131 if (args.find(
'V') != args.end()) {
132 voms_authz = *args.find(
'V')->second;
142 dir_temp, volatile_content, voms_authz, spooler.
weak_ref()));
143 if (!manifest.IsValid()) {
144 PrintError(
"Swissknife Sync: Failed to create new repository");
150 PrintError(
"Swissknife Sync: Failed to create fresh Reflog");
159 spooler->UploadReflog(reflog_path);
160 spooler->WaitForUpload();
161 unlink(reflog_path.c_str());
162 if (spooler->GetNumberOfErrors()) {
166 assert(!reflog_chksum_path.empty());
170 const bool needs_bootstrap_shortcuts = !voms_authz.empty();
171 manifest->set_garbage_collectability(garbage_collectable);
172 manifest->set_has_alt_catalog_path(needs_bootstrap_shortcuts);
174 if (!manifest->Export(manifest_path)) {
175 PrintError(
"Swissknife Sync: Failed to create new repository");
183 const string source = *args.find(
'i')->second;
184 const string dest = *args.find(
'o')->second;
185 const string spooler_definition = *args.find(
'r')->second;
187 if (args.find(
'a') != args.end()) {
190 PrintError(
"Swissknife Sync: Unknown hash algorithm");
196 upload::Spooler *
spooler = upload::Spooler::Construct(sd);
198 spooler->Upload(source, dest);
199 spooler->WaitForUpload();
201 if (spooler->GetNumberOfErrors() > 0) {
213 const string file_to_peek = *args.find(
'd')->second;
214 const string spooler_definition = *args.find(
'r')->second;
218 upload::Spooler *
spooler = upload::Spooler::Construct(sd);
220 const bool success = spooler->Peek(file_to_peek);
222 if (spooler->GetNumberOfErrors() > 0) {
224 file_to_peek.c_str());
229 file_to_peek.c_str());
233 file_to_peek.c_str());
241 const string file_to_delete = *args.find(
'o')->second;
242 const string spooler_definition = *args.find(
'r')->second;
246 upload::Spooler *
spooler = upload::Spooler::Construct(sd);
248 spooler->RemoveAsync(file_to_delete);
249 spooler->WaitForUpload();
251 if (spooler->GetNumberOfErrors() > 0) {
253 file_to_delete.c_str());
263 const string dirtab_file = *args.find(
'd')->second;
268 const string stratum0 = *args.find(
'w')->second;
269 const string dir_temp = *args.find(
't')->second;
270 verbose_ = (args.find(
'x') != args.end());
275 "Swissknife Sync: Didn't find a dirtab at '%s'. Skipping...",
276 dirtab_file.c_str());
284 "Swissknife Sync: Invalid or not readable dirtab '%s'",
285 dirtab_file.c_str());
289 "Swissknife Sync: Found %lu rules in dirtab '%s'",
290 dirtab->
RuleCount(), dirtab_file.c_str());
293 const bool auto_manage_catalog_files =
true;
294 const bool follow_redirects = (args.count(
'L') > 0);
295 const string proxy = (args.count(
'@') > 0) ? *args.find(
'@')->second :
"";
296 if (!InitDownloadManager(follow_redirects, proxy)) {
300 base_hash, stratum0, dir_temp, download_manager(), statistics(),
301 auto_manage_catalog_files);
302 catalog_manager.
Init();
304 vector<string> new_nested_catalogs;
305 DetermineNestedCatalogCandidates(*dirtab, &catalog_manager,
306 &new_nested_catalogs);
307 const bool success = CreateCatalogMarkers(new_nested_catalogs);
310 return (success) ? 0 : 1;
322 char resolved_cstr[PATH_MAX];
323 char *retval = realpath(name, resolved_cstr);
327 const std::string resolved(resolved_cstr);
328 if (resolved == *g_glob_uniondir)
330 if (!
HasPrefix(resolved, (*g_glob_uniondir) +
"/",
false )) {
340 return opendir(name);
346 return readdir(static_cast<DIR *>(dirp));
352 return lstat(name, st);
358 return stat(name, st);
367 vector<string> *nested_catalog_candidates) {
370 catalog::Dirtab::Rules::const_iterator i = lookup_rules.begin();
371 const catalog::Dirtab::Rules::const_iterator iend = lookup_rules.end();
372 for (; i != iend; ++i) {
377 const std::string &glob_string = i->pathspec.GetGlobString();
378 const std::string &glob_string_abs = union_dir_ + glob_string;
379 const int glob_flags = GLOB_ONLYDIR | GLOB_NOSORT | GLOB_PERIOD
388 const int glob_retval = glob(glob_string_abs.c_str(), glob_flags, NULL,
391 g_glob_uniondir = NULL;
393 if (glob_retval == 0) {
396 "Swissknife Sync: Found %lu entries for pathspec (%s)",
397 glob_res.gl_pathc, glob_string.c_str());
398 FilterCandidatesFromGlobResult(dirtab, glob_res.gl_pathv,
399 glob_res.gl_pathc, catalog_manager,
400 nested_catalog_candidates);
401 }
else if (glob_retval == GLOB_NOMATCH) {
403 "Swissknife Sync: WARNING: cannot apply pathspec %s",
404 glob_string.c_str());
407 "Swissknife Sync: Failed to run glob matching (%s)",
408 glob_string.c_str());
418 std::vector<std::string> *nested_catalog_candidates) {
420 for (
size_t i = 0; i < npaths; ++i) {
422 const std::string candidate(paths[i]);
423 const std::string candidate_rel = candidate.substr(union_dir_.size());
427 const int lstat_retval =
platform_lstat(candidate.c_str(), &candidate_info);
428 if (lstat_retval != 0) {
431 "Error in processing .cvmfsdirtab: cannot access %s (%d)",
432 candidate.c_str(), errno);
435 assert(lstat_retval == 0);
436 if (!S_ISDIR(candidate_info.st_mode)) {
441 "The '%s' dirtab entry does not point to a directory "
442 "but to a file or a symbolic link",
443 candidate_rel.c_str());
448 assert(candidate_rel.size() >= 2);
449 if (candidate_rel.substr(candidate_rel.size() - 2) ==
"/."
450 || candidate_rel.substr(candidate_rel.size() - 3) ==
"/..") {
457 "Swissknife Sync: Candidate '%s' is excluded by dirtab",
458 candidate_rel.c_str());
466 const bool lookup_success = catalog_manager->
LookupPath(
468 if (!lookup_success) {
470 "Swissknife Sync: Didn't find '%s' in catalogs, could "
471 "be a new directory and nested catalog.",
472 candidate_rel.c_str());
473 nested_catalog_candidates->push_back(candidate);
477 "Swissknife Sync: Found '%s' in catalogs but is not a "
478 "nested catalog yet.",
479 candidate_rel.c_str());
480 nested_catalog_candidates->push_back(candidate);
490 && !
FileExists(union_dir_ + candidate_rel +
"/.cvmfscatalog")) {
492 "Swissknife Sync: WARNING: '%s' should be a nested "
493 "catalog according to the dirtab. "
495 candidate_rel.c_str());
496 nested_catalog_candidates->push_back(candidate);
500 "Found '%s' in catalogs and it already is a nested catalog.",
501 candidate_rel.c_str());
508 const std::vector<std::string> &new_nested_catalogs) {
512 std::vector<std::string>::const_iterator k = new_nested_catalogs.begin();
513 const std::vector<std::string>::const_iterator kend = new_nested_catalogs
515 for (; k != kend; ++k) {
516 assert(!k->empty() && k->size() > union_dir_.size());
519 const std::string marker_path = *k +
"/.cvmfscatalog";
526 const int fd = open(marker_path.c_str(), O_CREAT, mode);
529 "Swissknife Sync: Failed to create nested catalog marker "
530 "at '%s' (errno: %d)",
531 marker_path.c_str(), errno);
540 "Swissknife Sync: Auto-creating nested catalog in %s",
549 chunk_arg(
char param,
size_t *save_to) : param(param), save_to(save_to) { }
556 typedef std::vector<chunk_arg> ChunkArgs;
559 ChunkArgs chunk_args;
565 ChunkArgs::const_iterator i = chunk_args.begin();
566 const ChunkArgs::const_iterator iend = chunk_args.end();
567 for (; i != iend; ++i) {
568 const swissknife::ArgumentList::const_iterator arg = args.find(i->param);
570 if (arg != args.end()) {
571 const size_t arg_value =
static_cast<size_t>(
String2Uint64(*arg->second));
573 *i->save_to = arg_value;
588 const std::string watchdog_dir =
"/tmp";
589 char watchdog_path[PATH_MAX];
590 const std::string timestamp =
GetGMTimestamp(
"%Y.%m.%d-%H.%M.%S");
591 const int path_size =
592 snprintf(watchdog_path,
sizeof(watchdog_path),
593 "%s/cvmfs-swissknife-sync-stacktrace.%s.%d",
594 watchdog_dir.c_str(), timestamp.c_str(), getpid());
596 assert(path_size < PATH_MAX);
598 watchdog->
Spawn(std::string(watchdog_path));
609 params.
stratum0 = *args.find(
'w')->second;
614 params.
repo_name = *args.find(
'N')->second;
618 if (args.find(
'f') != args.end())
620 if (args.find(
'A') != args.end())
622 if (args.find(
'x') != args.end())
624 if (args.find(
'y') != args.end())
626 if (args.find(
'm') != args.end())
628 if (args.find(
'i') != args.end())
630 if (args.find(
'd') != args.end())
632 if (args.find(
'V') != args.end())
634 if (args.find(
'F') != args.end())
636 if (args.find(
'k') != args.end())
638 if (args.find(
'j') != args.end())
640 if (args.find(
'Y') != args.end())
642 if (args.find(
'W') != args.end())
644 if (args.find(
'S') != args.end()) {
649 "Swissknife Sync: Invalid virtual catalog options: %s",
650 args.find(
'S')->second->c_str());
654 if (args.find(
'z') != args.end()) {
655 const unsigned log_level =
664 if (args.find(
'X') != args.end())
666 if (args.find(
'M') != args.end())
669 if (args.find(
'p') != args.end()) {
671 if (!ReadFileChunkingArgs(args, ¶ms)) {
672 PrintError(
"Swissknife Sync: Failed to read file chunk size values");
676 if (args.find(
'O') != args.end()) {
680 if (args.find(
'e') != args.end()) {
683 PrintError(
"Swissknife Sync: Unknown hash algorithm");
687 if (args.find(
'Z') != args.end()) {
689 *args.find(
'Z')->second);
692 if (args.find(
'E') != args.end())
694 if (args.find(
'Q') != args.end()) {
699 if (args.find(
'R') != args.end()) {
704 if (args.find(
'U') != args.end()) {
710 if (args.find(
'v') != args.end()) {
712 if (!sanitizer.
IsValid(*args.find(
'v')->second)) {
713 PrintError(
"Swissknife Sync: Invalid revision number");
721 if (args.find(
'q') != args.end()) {
725 if (args.find(
'0') != args.end()) {
729 if (args.find(
'T') != args.end()) {
733 if (args.find(
'g') != args.end()) {
737 if (args.find(
'P') != args.end()) {
741 if (args.find(
'H') != args.end()) {
742 params.
key_file = *args.find(
'H')->second;
745 if (args.find(
'D') != args.end()) {
749 if (args.find(
'J') != args.end()) {
753 if (args.find(
'G') != args.end()) {
757 const bool upload_statsdb = (args.count(
'I') > 0);
781 params.
spooler = upload::Spooler::Construct(spooler_definition,
782 &publish_statistics);
786 spooler_definition_catalogs, &publish_statistics));
787 if (!spooler_catalogs.
IsValid())
790 const bool follow_redirects = (args.count(
'L') > 0);
791 const string proxy = (args.count(
'@') > 0) ? *args.find(
'@')->second :
"";
792 if (!InitDownloadManager(follow_redirects, proxy)) {
813 params.
base_hash = manifest->catalog_hash();
827 const std::string old_root_hash = manifest->catalog_hash().ToString(
true);
835 catalog_manager.
Init();
843 && ((params.
ttl_seconds != catalog_manager.GetTTL())
844 || !catalog_manager.HasExplicitTTL())) {
846 "Swissknife Sync: Setting repository TTL to %" PRIu64
"s",
862 "Swissknife Sync: unknown union file system: %s",
869 "Swissknife Sync: Initialization of the synchronisation "
876 assert(!manifest->history().IsNull());
878 manifest.
weak_ref(), download_manager(), &catalog_manager, ¶ms);
884 "Swissknife Sync: Adding contents of authz file %s to"
887 const int fd = open(params.
authz_file.c_str(), O_RDONLY);
890 "Swissknife Sync: Unable to open authz file (%s)"
891 "from the publication process: %s",
896 std::string new_authz;
900 if (!read_successful) {
902 "Swissknife Sync: Failed to read authz file (%s): %s",
907 catalog_manager.SetVOMSAuthz(new_authz);
911 PrintError(
"Swissknife Sync: Something went wrong during sync");
914 if (upload_statsdb) {
922 "publish.revision",
"Published revision number");
923 revision_counter->
Set(
924 static_cast<int64_t>(catalog_manager.GetRootCatalog()->revision()));
928 "Swissknife Sync: Wait for all uploads to finish");
929 params.
spooler->WaitForUpload();
930 spooler_catalogs->WaitForUpload();
931 params.
spooler->FinalizeSession(
false);
934 "Swissknife Sync: Exporting repository manifest");
940 const std::string new_root_hash = manifest->catalog_hash().ToString(
true);
942 if (!spooler_catalogs->FinalizeSession(
true, old_root_hash, new_root_hash,
944 PrintError(
"Swissknife Sync: Failed to commit transaction.");
947 if (upload_statsdb) {
956 if (upload_statsdb) {
964 PrintError(
"Swissknife Sync: Failed to create new repository");
void SetLogVerbosity(const LogLevels max_level)
std::string database_file() const
Algorithms ParseCompressionAlgorithm(const std::string &algorithm_option)
size_t avg_file_chunk_size
const manifest::Manifest * manifest() const
int Main(const ArgumentList &args)
std::string GetGMTimestamp(const std::string &format)
bool UploadStatistics(upload::Spooler *spooler, std::string local_path="")
SpoolerDefinition Dup2DefaultCompression() const
std::vector< Rule > Rules
bool generate_legacy_bulk_chunks
unsigned file_mbyte_limit
uint64_t max_concurrent_write_jobs
const int kDefaultFileMode
static const unsigned kDefaultFileMbyteLimit
CVMFS_EXPORT const LogSource source
virtual bool Initialize()
static const unsigned kDefaultNestedKcatalogLimit
static const unsigned kDefaultRootKcatalogLimit
int Main(const ArgumentList &args)
static bool ParseActions(const std::string &action_desc, int *actions)
size_t min_file_chunk_size
unsigned num_upload_tasks
std::string spooler_definition
std::string manifest_path
bool ReadFileChunkingArgs(const swissknife::ArgumentList &args, SyncParameters *params)
void SetName(const std::string &name)
const Rules & positive_rules() const
bool CheckParams(const SyncParameters &p)
zlib::Algorithms compression_alg
unsigned nested_kcatalog_limit
int GlobLstat(const char *name, struct stat *st)
assert((mem||(size==0))&&"Out Of Memory")
bool LookupPath(const PathString &path, const LookupOptions options, DirectoryEntry *entry)
unsigned int number_of_concurrent_uploads
upload::Spooler * spooler
void SetDescription(const std::string &description)
std::string union_fs_type
bool GlobCheckPath(const char *name)
const unsigned kLookupDefault
int Main(const ArgumentList &args)
bool IsNestedCatalogMountpoint() const
virtual bool IsOpposing(const std::string &path) const
chunk_arg(char param, size_t *save_to)
bool IsNestedCatalogRoot() const
bool FileExists(const std::string &path)
int Main(const ArgumentList &args)
static Watchdog * Create(FnOnCrash on_crash)
static manifest::Manifest * CreateRepository(const std::string &dir_temp, const bool volatile_content, const std::string &voms_authz, upload::Spooler *spooler)
void Set(const int64_t val)
void FilterCandidatesFromGlobResult(const catalog::Dirtab &dirtab, char **paths, const size_t npaths, catalog::SimpleCatalogManager *catalog_manager, std::vector< std::string > *nested_catalog_candidates)
bool CheckParams(const swissknife::CommandLease::Parameters &p)
const char kSuffixCatalog
upload::Spooler * spooler
unsigned int num_upload_tasks
static Dirtab * Create(const std::string &dirtab_path)
static const int kActionNone
int Main(const ArgumentList &args)
bool ignore_special_files
static void HashDatabase(const std::string &database_path, shash::Any *hash_reflog)
manifest::Reflog * reflog
int Main(const ArgumentList &args)
struct dirent * GlobReaddir(void *dirp)
bool HasPrefix(const string &str, const string &prefix, const bool ignore_case)
void DropDatabaseFileOwnership()
static StatisticsDatabase * OpenStandardDB(const std::string repo_name)
bool DirectoryExists(const std::string &path)
void * GlobOpendir(const char *name)
bool stop_for_catalog_tweaks
bool SafeReadToString(int fd, std::string *final_result)
unsigned root_kcatalog_limit
size_t max_file_chunk_size
bool StorePublishStatistics(const perf::Statistics *statistics, const std::string &start_time, const bool success)
int GlobStat(const char *name, struct stat *st)
std::string session_token_file
bool ObtainDacReadSearchCapability()
virtual void Traverse()=0
uint64_t String2Uint64(const string &value)
std::map< char, SharedPtr< std::string > > ArgumentList
Algorithms ParseHashAlgorithm(const string &algorithm_option)
void GlobClosedir(void *dirp)
Any MkFromHexPtr(const HexPtr hex, const char suffix)
void DetermineNestedCatalogCandidates(const catalog::Dirtab &dirtab, catalog::SimpleCatalogManager *catalog_manager, std::vector< std::string > *nested_catalog_candidates)
void Generate(int actions)
void Spawn(const std::string &crash_dump_path)
static bool WriteChecksum(const std::string &path, const shash::Any &value)
bool ignore_xdir_hardlinks
std::string MakeCanonicalPath(const std::string &path)
void PrintError(const string &message)
const upload::Spooler * spooler_catalogs() const
static const uint64_t kDefaultTTL
bool CreateCatalogMarkers(const std::vector< std::string > &new_nested_catalogs)
CVMFS_EXPORT void LogCvmfs(const LogSource source, const int mask, const char *format,...)
std::string * g_glob_uniondir