20 #define _FILE_OFFSET_BITS 64
22 #define __STDC_FORMAT_MACROS
31 #include <sys/capability.h>
62 PrintError(
"overlay (copy on write) directory does not exist");
70 PrintError(
"cvmfs read/only repository does not exist");
83 PrintError(
"data store directory does not exist");
89 PrintError(
"file chunk size values are not sane");
95 PrintError(
"Session token file has to be provided "
96 "when upstream type is gw.");
105 const string manifest_path = *args.find(
'o')->second;
106 const string dir_temp = *args.find(
't')->second;
107 const string spooler_definition = *args.find(
'r')->second;
108 const string repo_name = *args.find(
'n')->second;
109 const string reflog_chksum_path = *args.find(
'R')->second;
110 if (args.find(
'l') != args.end()) {
119 if (args.find(
'a') != args.end()) {
127 const bool volatile_content = (args.count(
'v') > 0);
128 const bool garbage_collectable = (args.count(
'z') > 0);
129 std::string voms_authz;
130 if (args.find(
'V') != args.end()) {
131 voms_authz = *args.find(
'V')->second;
141 dir_temp, volatile_content, voms_authz, spooler.
weak_ref()));
142 if (!manifest.IsValid()) {
143 PrintError(
"Swissknife Sync: Failed to create new repository");
149 PrintError(
"Swissknife Sync: Failed to create fresh Reflog");
158 spooler->UploadReflog(reflog_path);
159 spooler->WaitForUpload();
160 unlink(reflog_path.c_str());
161 if (spooler->GetNumberOfErrors()) {
165 assert(!reflog_chksum_path.empty());
169 const bool needs_bootstrap_shortcuts = !voms_authz.empty();
170 manifest->set_garbage_collectability(garbage_collectable);
171 manifest->set_has_alt_catalog_path(needs_bootstrap_shortcuts);
173 if (!manifest->Export(manifest_path)) {
174 PrintError(
"Swissknife Sync: Failed to create new repository");
182 const string source = *args.find(
'i')->second;
183 const string dest = *args.find(
'o')->second;
184 const string spooler_definition = *args.find(
'r')->second;
186 if (args.find(
'a') != args.end()) {
189 PrintError(
"Swissknife Sync: Unknown hash algorithm");
195 upload::Spooler *
spooler = upload::Spooler::Construct(sd);
197 spooler->Upload(source, dest);
198 spooler->WaitForUpload();
200 if (spooler->GetNumberOfErrors() > 0) {
212 const string file_to_peek = *args.find(
'd')->second;
213 const string spooler_definition = *args.find(
'r')->second;
217 upload::Spooler *
spooler = upload::Spooler::Construct(sd);
219 const bool success = spooler->Peek(file_to_peek);
221 if (spooler->GetNumberOfErrors() > 0) {
223 file_to_peek.c_str());
228 file_to_peek.c_str());
232 file_to_peek.c_str());
240 const string file_to_delete = *args.find(
'o')->second;
241 const string spooler_definition = *args.find(
'r')->second;
245 upload::Spooler *
spooler = upload::Spooler::Construct(sd);
247 spooler->RemoveAsync(file_to_delete);
248 spooler->WaitForUpload();
250 if (spooler->GetNumberOfErrors() > 0) {
252 file_to_delete.c_str());
262 const string dirtab_file = *args.find(
'd')->second;
267 const string stratum0 = *args.find(
'w')->second;
268 const string dir_temp = *args.find(
't')->second;
269 verbose_ = (args.find(
'x') != args.end());
274 "Swissknife Sync: Didn't find a dirtab at '%s'. Skipping...",
275 dirtab_file.c_str());
283 "Swissknife Sync: Invalid or not readable dirtab '%s'",
284 dirtab_file.c_str());
288 "Swissknife Sync: Found %lu rules in dirtab '%s'",
289 dirtab->
RuleCount(), dirtab_file.c_str());
292 const bool auto_manage_catalog_files =
true;
293 const bool follow_redirects = (args.count(
'L') > 0);
294 const string proxy = (args.count(
'@') > 0) ? *args.find(
'@')->second :
"";
295 if (!InitDownloadManager(follow_redirects, proxy)) {
299 base_hash, stratum0, dir_temp, download_manager(), statistics(),
300 auto_manage_catalog_files);
301 catalog_manager.
Init();
303 vector<string> new_nested_catalogs;
304 DetermineNestedCatalogCandidates(*dirtab, &catalog_manager,
305 &new_nested_catalogs);
306 const bool success = CreateCatalogMarkers(new_nested_catalogs);
309 return (success) ? 0 : 1;
321 char resolved_cstr[PATH_MAX];
322 char *retval = realpath(name, resolved_cstr);
326 std::string resolved(resolved_cstr);
327 if (resolved == *g_glob_uniondir)
329 if (!
HasPrefix(resolved, (*g_glob_uniondir) +
"/",
false )) {
339 return opendir(name);
345 return readdir(static_cast<DIR *>(dirp));
351 return lstat(name, st);
357 return stat(name, st);
366 vector<string> *nested_catalog_candidates) {
369 catalog::Dirtab::Rules::const_iterator i = lookup_rules.begin();
370 const catalog::Dirtab::Rules::const_iterator iend = lookup_rules.end();
371 for (; i != iend; ++i) {
376 const std::string &glob_string = i->pathspec.GetGlobString();
377 const std::string &glob_string_abs = union_dir_ + glob_string;
378 const int glob_flags = GLOB_ONLYDIR | GLOB_NOSORT | GLOB_PERIOD
387 const int glob_retval = glob(glob_string_abs.c_str(), glob_flags, NULL,
390 g_glob_uniondir = NULL;
392 if (glob_retval == 0) {
395 "Swissknife Sync: Found %lu entries for pathspec (%s)",
396 glob_res.gl_pathc, glob_string.c_str());
397 FilterCandidatesFromGlobResult(dirtab, glob_res.gl_pathv,
398 glob_res.gl_pathc, catalog_manager,
399 nested_catalog_candidates);
400 }
else if (glob_retval == GLOB_NOMATCH) {
402 "Swissknife Sync: WARNING: cannot apply pathspec %s",
403 glob_string.c_str());
406 "Swissknife Sync: Failed to run glob matching (%s)",
407 glob_string.c_str());
417 std::vector<std::string> *nested_catalog_candidates) {
419 for (
size_t i = 0; i < npaths; ++i) {
421 const std::string candidate(paths[i]);
422 const std::string candidate_rel = candidate.substr(union_dir_.size());
426 const int lstat_retval =
platform_lstat(candidate.c_str(), &candidate_info);
427 if (lstat_retval != 0) {
430 "Error in processing .cvmfsdirtab: cannot access %s (%d)",
431 candidate.c_str(), errno);
434 assert(lstat_retval == 0);
435 if (!S_ISDIR(candidate_info.st_mode)) {
440 "The '%s' dirtab entry does not point to a directory "
441 "but to a file or a symbolic link",
442 candidate_rel.c_str());
447 assert(candidate_rel.size() >= 2);
448 if (candidate_rel.substr(candidate_rel.size() - 2) ==
"/."
449 || candidate_rel.substr(candidate_rel.size() - 3) ==
"/..") {
456 "Swissknife Sync: Candidate '%s' is excluded by dirtab",
457 candidate_rel.c_str());
465 const bool lookup_success = catalog_manager->
LookupPath(
467 if (!lookup_success) {
469 "Swissknife Sync: Didn't find '%s' in catalogs, could "
470 "be a new directory and nested catalog.",
471 candidate_rel.c_str());
472 nested_catalog_candidates->push_back(candidate);
476 "Swissknife Sync: Found '%s' in catalogs but is not a "
477 "nested catalog yet.",
478 candidate_rel.c_str());
479 nested_catalog_candidates->push_back(candidate);
489 && !
FileExists(union_dir_ + candidate_rel +
"/.cvmfscatalog")) {
491 "Swissknife Sync: WARNING: '%s' should be a nested "
492 "catalog according to the dirtab. "
494 candidate_rel.c_str());
495 nested_catalog_candidates->push_back(candidate);
499 "Found '%s' in catalogs and it already is a nested catalog.",
500 candidate_rel.c_str());
507 const std::vector<std::string> &new_nested_catalogs) {
511 std::vector<std::string>::const_iterator k = new_nested_catalogs.begin();
512 const std::vector<std::string>::const_iterator kend = new_nested_catalogs
514 for (; k != kend; ++k) {
515 assert(!k->empty() && k->size() > union_dir_.size());
518 const std::string marker_path = *k +
"/.cvmfscatalog";
525 const int fd = open(marker_path.c_str(), O_CREAT, mode);
528 "Swissknife Sync: Failed to create nested catalog marker "
529 "at '%s' (errno: %d)",
530 marker_path.c_str(), errno);
539 "Swissknife Sync: Auto-creating nested catalog in %s",
548 chunk_arg(
char param,
size_t *save_to) : param(param), save_to(save_to) { }
555 typedef std::vector<chunk_arg> ChunkArgs;
558 ChunkArgs chunk_args;
564 ChunkArgs::const_iterator i = chunk_args.begin();
565 ChunkArgs::const_iterator iend = chunk_args.end();
566 for (; i != iend; ++i) {
567 swissknife::ArgumentList::const_iterator arg = args.find(i->param);
569 if (arg != args.end()) {
570 size_t arg_value =
static_cast<size_t>(
String2Uint64(*arg->second));
572 *i->save_to = arg_value;
587 std::string watchdog_dir =
"/tmp";
588 char watchdog_path[PATH_MAX];
590 int path_size = snprintf(watchdog_path,
sizeof(watchdog_path),
591 "%s/cvmfs-swissknife-sync-stacktrace.%s.%d",
592 watchdog_dir.c_str(), timestamp.c_str(), getpid());
594 assert(path_size < PATH_MAX);
596 watchdog->
Spawn(std::string(watchdog_path));
607 params.
stratum0 = *args.find(
'w')->second;
612 params.
repo_name = *args.find(
'N')->second;
616 if (args.find(
'f') != args.end())
618 if (args.find(
'A') != args.end())
620 if (args.find(
'x') != args.end())
622 if (args.find(
'y') != args.end())
624 if (args.find(
'm') != args.end())
626 if (args.find(
'i') != args.end())
628 if (args.find(
'd') != args.end())
630 if (args.find(
'V') != args.end())
632 if (args.find(
'F') != args.end())
634 if (args.find(
'k') != args.end())
636 if (args.find(
'j') != args.end())
638 if (args.find(
'Y') != args.end())
640 if (args.find(
'W') != args.end())
642 if (args.find(
'S') != args.end()) {
647 "Swissknife Sync: Invalid virtual catalog options: %s",
648 args.find(
'S')->second->c_str());
652 if (args.find(
'z') != args.end()) {
662 if (args.find(
'X') != args.end())
664 if (args.find(
'M') != args.end())
667 if (args.find(
'p') != args.end()) {
669 if (!ReadFileChunkingArgs(args, ¶ms)) {
670 PrintError(
"Swissknife Sync: Failed to read file chunk size values");
674 if (args.find(
'O') != args.end()) {
678 if (args.find(
'e') != args.end()) {
681 PrintError(
"Swissknife Sync: Unknown hash algorithm");
685 if (args.find(
'Z') != args.end()) {
687 *args.find(
'Z')->second);
690 if (args.find(
'E') != args.end())
692 if (args.find(
'Q') != args.end()) {
697 if (args.find(
'R') != args.end()) {
702 if (args.find(
'U') != args.end()) {
708 if (args.find(
'v') != args.end()) {
710 if (!sanitizer.
IsValid(*args.find(
'v')->second)) {
711 PrintError(
"Swissknife Sync: Invalid revision number");
719 if (args.find(
'q') != args.end()) {
723 if (args.find(
'0') != args.end()) {
727 if (args.find(
'T') != args.end()) {
731 if (args.find(
'g') != args.end()) {
735 if (args.find(
'P') != args.end()) {
739 if (args.find(
'H') != args.end()) {
740 params.
key_file = *args.find(
'H')->second;
743 if (args.find(
'D') != args.end()) {
747 if (args.find(
'J') != args.end()) {
751 if (args.find(
'G') != args.end()) {
755 const bool upload_statsdb = (args.count(
'I') > 0);
779 params.
spooler = upload::Spooler::Construct(spooler_definition,
780 &publish_statistics);
784 spooler_definition_catalogs, &publish_statistics));
785 if (!spooler_catalogs.
IsValid())
788 const bool follow_redirects = (args.count(
'L') > 0);
789 const string proxy = (args.count(
'@') > 0) ? *args.find(
'@')->second :
"";
790 if (!InitDownloadManager(follow_redirects, proxy)) {
811 params.
base_hash = manifest->catalog_hash();
825 const std::string old_root_hash = manifest->catalog_hash().ToString(
true);
833 catalog_manager.
Init();
841 && ((params.
ttl_seconds != catalog_manager.GetTTL())
842 || !catalog_manager.HasExplicitTTL())) {
844 "Swissknife Sync: Setting repository TTL to %" PRIu64
"s",
860 "Swissknife Sync: unknown union file system: %s",
867 "Swissknife Sync: Initialization of the synchronisation "
874 assert(!manifest->history().IsNull());
876 manifest.
weak_ref(), download_manager(), &catalog_manager, ¶ms);
882 "Swissknife Sync: Adding contents of authz file %s to"
885 int fd = open(params.
authz_file.c_str(), O_RDONLY);
888 "Swissknife Sync: Unable to open authz file (%s)"
889 "from the publication process: %s",
894 std::string new_authz;
898 if (!read_successful) {
900 "Swissknife Sync: Failed to read authz file (%s): %s",
905 catalog_manager.SetVOMSAuthz(new_authz);
909 PrintError(
"Swissknife Sync: Something went wrong during sync");
912 if (upload_statsdb) {
920 "publish.revision",
"Published revision number");
921 revision_counter->
Set(
922 static_cast<int64_t>(catalog_manager.GetRootCatalog()->revision()));
926 "Swissknife Sync: Wait for all uploads to finish");
927 params.
spooler->WaitForUpload();
928 spooler_catalogs->WaitForUpload();
929 params.
spooler->FinalizeSession(
false);
932 "Swissknife Sync: Exporting repository manifest");
938 const std::string new_root_hash = manifest->catalog_hash().ToString(
true);
940 if (!spooler_catalogs->FinalizeSession(
true, old_root_hash, new_root_hash,
942 PrintError(
"Swissknife Sync: Failed to commit transaction.");
945 if (upload_statsdb) {
954 if (upload_statsdb) {
962 PrintError(
"Swissknife Sync: Failed to create new repository");
void SetLogVerbosity(const LogLevels max_level)
std::string database_file() const
Algorithms ParseCompressionAlgorithm(const std::string &algorithm_option)
size_t avg_file_chunk_size
const manifest::Manifest * manifest() const
int Main(const ArgumentList &args)
std::string GetGMTimestamp(const std::string &format)
bool UploadStatistics(upload::Spooler *spooler, std::string local_path="")
SpoolerDefinition Dup2DefaultCompression() const
std::vector< Rule > Rules
bool generate_legacy_bulk_chunks
unsigned file_mbyte_limit
uint64_t max_concurrent_write_jobs
const int kDefaultFileMode
static const unsigned kDefaultFileMbyteLimit
CVMFS_EXPORT const LogSource source
virtual bool Initialize()
static const unsigned kDefaultNestedKcatalogLimit
static const unsigned kDefaultRootKcatalogLimit
int Main(const ArgumentList &args)
static bool ParseActions(const std::string &action_desc, int *actions)
size_t min_file_chunk_size
unsigned num_upload_tasks
std::string spooler_definition
std::string manifest_path
bool ReadFileChunkingArgs(const swissknife::ArgumentList &args, SyncParameters *params)
void SetName(const std::string &name)
const Rules & positive_rules() const
bool CheckParams(const SyncParameters &p)
zlib::Algorithms compression_alg
unsigned nested_kcatalog_limit
int GlobLstat(const char *name, struct stat *st)
assert((mem||(size==0))&&"Out Of Memory")
bool LookupPath(const PathString &path, const LookupOptions options, DirectoryEntry *entry)
unsigned int number_of_concurrent_uploads
upload::Spooler * spooler
void SetDescription(const std::string &description)
std::string union_fs_type
bool GlobCheckPath(const char *name)
const unsigned kLookupDefault
int Main(const ArgumentList &args)
bool IsNestedCatalogMountpoint() const
virtual bool IsOpposing(const std::string &path) const
chunk_arg(char param, size_t *save_to)
bool IsNestedCatalogRoot() const
bool FileExists(const std::string &path)
int Main(const ArgumentList &args)
static Watchdog * Create(FnOnCrash on_crash)
static manifest::Manifest * CreateRepository(const std::string &dir_temp, const bool volatile_content, const std::string &voms_authz, upload::Spooler *spooler)
void Set(const int64_t val)
void FilterCandidatesFromGlobResult(const catalog::Dirtab &dirtab, char **paths, const size_t npaths, catalog::SimpleCatalogManager *catalog_manager, std::vector< std::string > *nested_catalog_candidates)
bool CheckParams(const swissknife::CommandLease::Parameters &p)
const char kSuffixCatalog
upload::Spooler * spooler
unsigned int num_upload_tasks
static Dirtab * Create(const std::string &dirtab_path)
static const int kActionNone
int Main(const ArgumentList &args)
bool ignore_special_files
static void HashDatabase(const std::string &database_path, shash::Any *hash_reflog)
manifest::Reflog * reflog
int Main(const ArgumentList &args)
struct dirent * GlobReaddir(void *dirp)
bool HasPrefix(const string &str, const string &prefix, const bool ignore_case)
void DropDatabaseFileOwnership()
static StatisticsDatabase * OpenStandardDB(const std::string repo_name)
bool DirectoryExists(const std::string &path)
void * GlobOpendir(const char *name)
bool stop_for_catalog_tweaks
bool SafeReadToString(int fd, std::string *final_result)
unsigned root_kcatalog_limit
size_t max_file_chunk_size
bool StorePublishStatistics(const perf::Statistics *statistics, const std::string &start_time, const bool success)
int GlobStat(const char *name, struct stat *st)
std::string session_token_file
bool ObtainDacReadSearchCapability()
virtual void Traverse()=0
uint64_t String2Uint64(const string &value)
std::map< char, SharedPtr< std::string > > ArgumentList
Algorithms ParseHashAlgorithm(const string &algorithm_option)
void GlobClosedir(void *dirp)
Any MkFromHexPtr(const HexPtr hex, const char suffix)
void DetermineNestedCatalogCandidates(const catalog::Dirtab &dirtab, catalog::SimpleCatalogManager *catalog_manager, std::vector< std::string > *nested_catalog_candidates)
void Generate(int actions)
void Spawn(const std::string &crash_dump_path)
static bool WriteChecksum(const std::string &path, const shash::Any &value)
bool ignore_xdir_hardlinks
std::string MakeCanonicalPath(const std::string &path)
void PrintError(const string &message)
const upload::Spooler * spooler_catalogs() const
static const uint64_t kDefaultTTL
bool CreateCatalogMarkers(const std::vector< std::string > &new_nested_catalogs)
CVMFS_EXPORT void LogCvmfs(const LogSource source, const int mask, const char *format,...)
std::string * g_glob_uniondir