20 #define _FILE_OFFSET_BITS 64
22 #define __STDC_FORMAT_MACROS
25 #include "cvmfs_config.h"
32 #include <sys/capability.h>
63 PrintError(
"overlay (copy on write) directory does not exist");
71 PrintError(
"cvmfs read/only repository does not exist");
84 PrintError(
"data store directory does not exist");
90 PrintError(
"file chunk size values are not sane");
97 "Session token file has to be provided "
98 "when upstream type is gw.");
107 const string manifest_path = *args.find(
'o')->second;
108 const string dir_temp = *args.find(
't')->second;
109 const string spooler_definition = *args.find(
'r')->second;
110 const string repo_name = *args.find(
'n')->second;
111 const string reflog_chksum_path = *args.find(
'R')->second;
112 if (args.find(
'l') != args.end()) {
122 if (args.find(
'a') != args.end()) {
130 const bool volatile_content = (args.count(
'v') > 0);
131 const bool garbage_collectable = (args.count(
'z') > 0);
132 std::string voms_authz;
133 if (args.find(
'V') != args.end()) {
134 voms_authz = *args.find(
'V')->second;
144 dir_temp, volatile_content, voms_authz, spooler.
weak_ref()));
145 if (!manifest.IsValid()) {
146 PrintError(
"Failed to create new repository");
161 spooler->UploadReflog(reflog_path);
162 spooler->WaitForUpload();
163 unlink(reflog_path.c_str());
164 if (spooler->GetNumberOfErrors()) {
168 assert(!reflog_chksum_path.empty());
172 const bool needs_bootstrap_shortcuts = !voms_authz.empty();
173 manifest->set_garbage_collectability(garbage_collectable);
174 manifest->set_has_alt_catalog_path(needs_bootstrap_shortcuts);
176 if (!manifest->Export(manifest_path)) {
177 PrintError(
"Failed to create new repository");
185 const string source = *args.find(
'i')->second;
186 const string dest = *args.find(
'o')->second;
187 const string spooler_definition = *args.find(
'r')->second;
189 if (args.find(
'a') != args.end()) {
198 upload::Spooler *
spooler = upload::Spooler::Construct(sd);
200 spooler->Upload(source, dest);
201 spooler->WaitForUpload();
203 if (spooler->GetNumberOfErrors() > 0) {
214 const string file_to_peek = *args.find(
'd')->second;
215 const string spooler_definition = *args.find(
'r')->second;
219 upload::Spooler *
spooler = upload::Spooler::Construct(sd);
221 const bool success = spooler->Peek(file_to_peek);
223 if (spooler->GetNumberOfErrors() > 0) {
225 file_to_peek.c_str());
240 const string file_to_delete = *args.find(
'o')->second;
241 const string spooler_definition = *args.find(
'r')->second;
245 upload::Spooler *
spooler = upload::Spooler::Construct(sd);
247 spooler->RemoveAsync(file_to_delete);
248 spooler->WaitForUpload();
250 if (spooler->GetNumberOfErrors() > 0) {
252 file_to_delete.c_str());
262 const string dirtab_file = *args.find(
'd')->second;
267 const string stratum0 = *args.find(
'w')->second;
268 const string dir_temp = *args.find(
't')->second;
269 verbose_ = (args.find(
'x') != args.end());
274 "Didn't find a dirtab at '%s'. Skipping...", dirtab_file.c_str());
282 dirtab_file.c_str());
286 dirtab->
RuleCount(), dirtab_file.c_str());
289 const bool auto_manage_catalog_files =
true;
290 const bool follow_redirects = (args.count(
'L') > 0);
291 const string proxy = (args.count(
'@') > 0) ? *args.find(
'@')->second :
"";
292 if (!InitDownloadManager(follow_redirects, proxy)) {
296 base_hash, stratum0, dir_temp, download_manager(), statistics(),
297 auto_manage_catalog_files);
298 catalog_manager.
Init();
300 vector<string> new_nested_catalogs;
301 DetermineNestedCatalogCandidates(*dirtab, &catalog_manager,
302 &new_nested_catalogs);
303 const bool success = CreateCatalogMarkers(new_nested_catalogs);
306 return (success) ? 0 : 1;
318 char resolved_cstr[PATH_MAX];
319 char *retval = realpath(name, resolved_cstr);
320 if (retval == NULL)
return false;
322 std::string resolved(resolved_cstr);
323 if (resolved == *g_glob_uniondir)
return true;
324 if (!
HasPrefix(resolved, (*g_glob_uniondir) +
"/",
false )) {
333 return opendir(name);
337 closedir(static_cast<DIR *>(dirp));
341 return readdir(static_cast<DIR *>(dirp));
346 return lstat(name, st);
351 return stat(name, st);
360 vector<string> *nested_catalog_candidates) {
363 catalog::Dirtab::Rules::const_iterator i = lookup_rules.begin();
364 const catalog::Dirtab::Rules::const_iterator iend = lookup_rules.end();
365 for (; i != iend; ++i) {
370 const std::string &glob_string = i->pathspec.GetGlobString();
371 const std::string &glob_string_abs = union_dir_ + glob_string;
372 const int glob_flags = GLOB_ONLYDIR | GLOB_NOSORT | GLOB_PERIOD |
381 const int glob_retval =
382 glob(glob_string_abs.c_str(), glob_flags, NULL, &glob_res);
384 g_glob_uniondir = NULL;
386 if (glob_retval == 0) {
389 glob_res.gl_pathc, glob_string.c_str());
390 FilterCandidatesFromGlobResult(dirtab, glob_res.gl_pathv,
391 glob_res.gl_pathc, catalog_manager,
392 nested_catalog_candidates);
393 }
else if (glob_retval == GLOB_NOMATCH) {
395 glob_string.c_str());
398 glob_string.c_str());
408 std::vector<std::string> *nested_catalog_candidates) {
410 for (
size_t i = 0; i < npaths; ++i) {
412 const std::string candidate(paths[i]);
413 const std::string candidate_rel = candidate.substr(union_dir_.size());
417 const int lstat_retval =
platform_lstat(candidate.c_str(), &candidate_info);
418 if (lstat_retval != 0) {
420 "Error in processing .cvmfsdirtab: cannot access %s (%d)",
421 candidate.c_str(), errno);
424 assert(lstat_retval == 0);
425 if (!S_ISDIR(candidate_info.st_mode)) {
429 "The '%s' dirtab entry does not point to a directory "
430 "but to a file or a symbolic link",
431 candidate_rel.c_str());
436 assert(candidate_rel.size() >= 2);
437 if (candidate_rel.substr(candidate_rel.size() - 2) ==
"/." ||
438 candidate_rel.substr(candidate_rel.size() - 3) ==
"/..") {
445 candidate_rel.c_str());
453 const bool lookup_success = catalog_manager->
LookupPath(
455 if (!lookup_success) {
457 "Didn't find '%s' in catalogs, could "
458 "be a new directory and nested catalog.",
459 candidate_rel.c_str());
460 nested_catalog_candidates->push_back(candidate);
464 "Found '%s' in catalogs but is not a "
465 "nested catalog yet.",
466 candidate_rel.c_str());
467 nested_catalog_candidates->push_back(candidate);
477 !
FileExists(union_dir_ + candidate_rel +
"/.cvmfscatalog")) {
479 "WARNING: '%s' should be a nested "
480 "catalog according to the dirtab. "
482 candidate_rel.c_str());
483 nested_catalog_candidates->push_back(candidate);
486 "Found '%s' in catalogs and it already is a nested catalog.",
487 candidate_rel.c_str());
494 const std::vector<std::string> &new_nested_catalogs) {
498 std::vector<std::string>::const_iterator k = new_nested_catalogs.begin();
499 const std::vector<std::string>::const_iterator kend =
500 new_nested_catalogs.end();
501 for (; k != kend; ++k) {
502 assert(!k->empty() && k->size() > union_dir_.size());
505 const std::string marker_path = *k +
"/.cvmfscatalog";
512 const int fd = open(marker_path.c_str(), O_CREAT, mode);
515 "Failed to create nested catalog marker "
516 "at '%s' (errno: %d)",
517 marker_path.c_str(), errno);
534 chunk_arg(
char param,
size_t *save_to) : param(param), save_to(save_to) {}
541 typedef std::vector<chunk_arg> ChunkArgs;
544 ChunkArgs chunk_args;
550 ChunkArgs::const_iterator i = chunk_args.begin();
551 ChunkArgs::const_iterator iend = chunk_args.end();
552 for (; i != iend; ++i) {
553 swissknife::ArgumentList::const_iterator arg = args.find(i->param);
555 if (arg != args.end()) {
556 size_t arg_value =
static_cast<size_t>(
String2Uint64(*arg->second));
558 *i->save_to = arg_value;
573 std::string watchdog_dir =
"/tmp";
574 char watchdog_path[PATH_MAX];
576 int path_size = snprintf(watchdog_path,
sizeof(watchdog_path),
577 "%s/cvmfs-swissknife-sync-stacktrace.%s.%d",
578 watchdog_dir.c_str(), timestamp.c_str(), getpid());
580 assert(path_size < PATH_MAX);
582 watchdog->
Spawn(std::string(watchdog_path));
593 params.
stratum0 = *args.find(
'w')->second;
598 params.
repo_name = *args.find(
'N')->second;
602 if (args.find(
'f') != args.end())
604 if (args.find(
'A') != args.end()) params.
is_balanced =
true;
606 if (args.find(
'y') != args.end()) params.
dry_run =
true;
607 if (args.find(
'm') != args.end()) params.
mucatalogs =
true;
610 if (args.find(
'V') != args.end()) params.
voms_authz =
true;
611 if (args.find(
'F') != args.end()) params.
authz_file = *args.find(
'F')->second;
613 if (args.find(
'Y') != args.end()) params.
external_data =
true;
614 if (args.find(
'W') != args.end()) params.
direct_io =
true;
615 if (args.find(
'S') != args.end()) {
620 args.find(
'S')->second->c_str());
624 if (args.find(
'z') != args.end()) {
634 if (args.find(
'X') != args.end())
636 if (args.find(
'M') != args.end())
639 if (args.find(
'p') != args.end()) {
641 if (!ReadFileChunkingArgs(args, ¶ms)) {
642 PrintError(
"Failed to read file chunk size values");
646 if (args.find(
'O') != args.end()) {
650 if (args.find(
'e') != args.end()) {
657 if (args.find(
'Z') != args.end()) {
662 if (args.find(
'C') != args.end()) {
667 if (args.find(
'Q') != args.end()) {
672 if (args.find(
'R') != args.end()) {
677 if (args.find(
'U') != args.end()) {
683 if (args.find(
'v') != args.end()) {
685 if (!sanitizer.
IsValid(*args.find(
'v')->second)) {
694 if (args.find(
'q') != args.end()) {
698 if (args.find(
'0') != args.end()) {
702 if (args.find(
'T') != args.end()) {
706 if (args.find(
'g') != args.end()) {
710 if (args.find(
'P') != args.end()) {
714 if (args.find(
'H') != args.end()) {
715 params.
key_file = *args.find(
'H')->second;
718 if (args.find(
'D') != args.end()) {
722 if (args.find(
'J') != args.end()) {
726 const bool upload_statsdb = (args.count(
'I') > 0);
749 params.
spooler = upload::Spooler::Construct(spooler_definition,
750 &publish_statistics);
751 if (NULL == params.
spooler)
return 3;
753 upload::Spooler::Construct(spooler_definition_catalogs,
754 &publish_statistics));
755 if (!spooler_catalogs.
IsValid())
return 3;
757 const bool follow_redirects = (args.count(
'L') > 0);
758 const string proxy = (args.count(
'@') > 0) ? *args.find(
'@')->second :
"";
759 if (!InitDownloadManager(follow_redirects, proxy)) {
763 if (!InitVerifyingSignatureManager(params.
public_keys,
781 params.
base_hash = manifest->catalog_hash();
795 const std::string old_root_hash = manifest->catalog_hash().ToString(
true);
803 catalog_manager.
Init();
811 ((params.
ttl_seconds != catalog_manager.GetTTL()) ||
812 !catalog_manager.HasExplicitTTL())) {
835 "Initialization of the synchronisation "
842 assert(!manifest->history().IsNull());
844 manifest.
weak_ref(), download_manager(), &catalog_manager, ¶ms);
850 "Adding contents of authz file %s to"
853 int fd = open(params.
authz_file.c_str(), O_RDONLY);
856 "Unable to open authz file (%s)"
857 "from the publication process: %s",
862 std::string new_authz;
866 if (!read_successful) {
872 catalog_manager.SetVOMSAuthz(new_authz);
876 PrintError(
"something went wrong during sync");
879 if (upload_statsdb) {
886 perf::Counter *revision_counter = statistics()->Register(
"publish.revision",
887 "Published revision number");
888 revision_counter->
Set(static_cast<int64_t>(
889 catalog_manager.GetRootCatalog()->revision()));
893 params.
spooler->WaitForUpload();
894 spooler_catalogs->WaitForUpload();
895 params.
spooler->FinalizeSession(
false);
903 const std::string new_root_hash = manifest->catalog_hash().ToString(
true);
905 if (!spooler_catalogs->FinalizeSession(
true, old_root_hash, new_root_hash,
910 if (upload_statsdb) {
919 if (upload_statsdb) {
927 PrintError(
"Failed to create new repository");
#define LogCvmfs(source, mask,...)
void SetLogVerbosity(const LogLevels max_level)
std::string database_file() const
Algorithms ParseCompressionAlgorithm(const std::string &algorithm_option)
size_t avg_file_chunk_size
const manifest::Manifest * manifest() const
int Main(const ArgumentList &args)
std::string GetGMTimestamp(const std::string &format)
bool UploadStatistics(upload::Spooler *spooler, std::string local_path="")
SpoolerDefinition Dup2DefaultCompression() const
std::vector< Rule > Rules
bool generate_legacy_bulk_chunks
unsigned file_mbyte_limit
uint64_t max_concurrent_write_jobs
const int kDefaultFileMode
static const unsigned kDefaultFileMbyteLimit
virtual bool Initialize()
static const unsigned kDefaultNestedKcatalogLimit
static const unsigned kDefaultRootKcatalogLimit
int Main(const ArgumentList &args)
static bool ParseActions(const std::string &action_desc, int *actions)
size_t min_file_chunk_size
unsigned num_upload_tasks
std::string spooler_definition
std::string manifest_path
bool ReadFileChunkingArgs(const swissknife::ArgumentList &args, SyncParameters *params)
void SetName(const std::string &name)
const Rules & positive_rules() const
bool CheckParams(const SyncParameters &p)
zlib::Algorithms compression_alg
unsigned nested_kcatalog_limit
int GlobLstat(const char *name, struct stat *st)
assert((mem||(size==0))&&"Out Of Memory")
bool LookupPath(const PathString &path, const LookupOptions options, DirectoryEntry *entry)
unsigned int number_of_concurrent_uploads
upload::Spooler * spooler
void SetDescription(const std::string &description)
std::string union_fs_type
bool GlobCheckPath(const char *name)
const unsigned kLookupDefault
int Main(const ArgumentList &args)
bool IsNestedCatalogMountpoint() const
virtual bool IsOpposing(const std::string &path) const
chunk_arg(char param, size_t *save_to)
bool IsNestedCatalogRoot() const
bool FileExists(const std::string &path)
int Main(const ArgumentList &args)
static Watchdog * Create(FnOnCrash on_crash)
static manifest::Manifest * CreateRepository(const std::string &dir_temp, const bool volatile_content, const std::string &voms_authz, upload::Spooler *spooler)
void Set(const int64_t val)
void FilterCandidatesFromGlobResult(const catalog::Dirtab &dirtab, char **paths, const size_t npaths, catalog::SimpleCatalogManager *catalog_manager, std::vector< std::string > *nested_catalog_candidates)
bool CheckParams(const swissknife::CommandLease::Parameters &p)
const char kSuffixCatalog
upload::Spooler * spooler
unsigned int num_upload_tasks
static Dirtab * Create(const std::string &dirtab_path)
static const int kActionNone
int Main(const ArgumentList &args)
bool ignore_special_files
static void HashDatabase(const std::string &database_path, shash::Any *hash_reflog)
manifest::Reflog * reflog
int Main(const ArgumentList &args)
struct dirent * GlobReaddir(void *dirp)
bool HasPrefix(const string &str, const string &prefix, const bool ignore_case)
void DropDatabaseFileOwnership()
static StatisticsDatabase * OpenStandardDB(const std::string repo_name)
bool DirectoryExists(const std::string &path)
void * GlobOpendir(const char *name)
bool stop_for_catalog_tweaks
bool SafeReadToString(int fd, std::string *final_result)
unsigned root_kcatalog_limit
size_t max_file_chunk_size
bool StorePublishStatistics(const perf::Statistics *statistics, const std::string &start_time, const bool success)
int GlobStat(const char *name, struct stat *st)
std::string session_token_file
std::string trusted_certs
bool ObtainDacReadSearchCapability()
virtual void Traverse()=0
uint64_t String2Uint64(const string &value)
std::map< char, SharedPtr< std::string > > ArgumentList
Algorithms ParseHashAlgorithm(const string &algorithm_option)
void GlobClosedir(void *dirp)
Any MkFromHexPtr(const HexPtr hex, const char suffix)
void DetermineNestedCatalogCandidates(const catalog::Dirtab &dirtab, catalog::SimpleCatalogManager *catalog_manager, std::vector< std::string > *nested_catalog_candidates)
void Generate(int actions)
void Spawn(const std::string &crash_dump_path)
static bool WriteChecksum(const std::string &path, const shash::Any &value)
bool ignore_xdir_hardlinks
std::string MakeCanonicalPath(const std::string &path)
void PrintError(const string &message)
const upload::Spooler * spooler_catalogs() const
static const uint64_t kDefaultTTL
bool CreateCatalogMarkers(const std::vector< std::string > &new_nested_catalogs)
std::string * g_glob_uniondir