5 #define __STDC_FORMAT_MACROS
31 WritableCatalogManager::WritableCatalogManager(
33 const std::string &stratum0,
34 const string &dir_temp,
38 const unsigned nested_kcatalog_limit,
39 const unsigned root_kcatalog_limit,
40 const unsigned file_mbyte_limit,
48 , enforce_limits_(enforce_limits)
49 , nested_kcatalog_limit_(nested_kcatalog_limit)
50 , root_kcatalog_limit_(root_kcatalog_limit)
51 , file_mbyte_limit_(file_mbyte_limit)
52 , is_balanceable_(is_balanceable)
53 , max_weight_(max_weight)
54 , min_weight_(min_weight)
55 , balance_weight_(max_weight / 2)
58 reinterpret_cast<pthread_mutex_t *
>(smalloc(
sizeof(pthread_mutex_t)));
59 int retval = pthread_mutex_init(
sync_lock_, NULL);
62 reinterpret_cast<pthread_mutex_t *
>(smalloc(
sizeof(pthread_mutex_t)));
108 const string &dir_temp,
109 const bool volatile_content,
110 const std::string &voms_authz,
114 string file_path = dir_temp +
"/new_root_catalog";
122 root_entry.
mode_ = 16877;
123 root_entry.
size_ = 4096;
124 root_entry.
mtime_ = time(NULL);
125 root_entry.
uid_ = getuid();
126 root_entry.
gid_ = getgid();
129 string root_path =
"";
135 !new_clg_db->InsertInitialValues(root_path,
148 if (catalog_size < 0) {
149 unlink(file_path.c_str());
152 string file_path_compressed = file_path +
".compressed";
159 unlink(file_path.c_str());
162 unlink(file_path.c_str());
165 const string manifest_path = dir_temp +
"/manifest";
168 if (!voms_authz.empty()) {
173 spooler->Upload(file_path_compressed,
"data/" + hash_catalog.
MakePath());
174 spooler->WaitForUpload();
175 unlink(file_path_compressed.c_str());
176 if (spooler->GetNumberOfErrors() > 0) {
178 file_path_compressed.c_str());
213 if (NULL == dirent) {
216 bool found = catalog->
LookupPath(ps_path, dirent);
226 const std::string &path)
230 if (!retval)
return NULL;
268 if (!
FindCatalog(parent_path, &catalog, &parent_entry)) {
270 directory_path.c_str());
279 parent_path.c_str());
284 parent_catalog->UpdateEntry(parent_entry, parent_path);
298 const std::string source) {
313 bool destination_already_present =
315 if (destination_already_present) {
320 std::string destination_dirname;
321 std::string destination_filename;
322 SplitPath(destination, &destination_dirname, &destination_filename);
325 NameString(destination_filename.c_str(), destination_filename.length()));
340 const std::string &to_dir)
343 if (from_dir.empty() || to_dir.empty())
349 if (relative_source == relative_dest) {
350 PANIC(
kLogStderr,
"cannot clone tree into itself ('%s')", to_dir.c_str());
352 if (
HasPrefix(relative_dest, relative_source +
"/",
false )) {
354 "cannot clone tree into sub directory of source '%s' --> '%s'",
355 from_dir.c_str(), to_dir.c_str());
360 PANIC(
kLogStderr,
"path '%s' cannot be found, aborting", from_dir.c_str());
369 PANIC(
kLogStderr,
"destination '%s' exists, aborting", to_dir.c_str());
372 const std::string dest_parent =
GetParentPath(relative_dest);
391 const std::string &dest_parent_dir,
395 dest_parent_dir.c_str(), dest_name.
ToString().c_str());
404 dest_dirent.name_.Assign(dest_name);
406 dest_dirent.set_is_nested_catalog_mountpoint(
false);
407 dest_dirent.set_is_nested_catalog_root(
false);
416 std::string dest_dir = dest_parent_dir;
417 if (!dest_dir.empty())
418 dest_dir.push_back(
'/');
427 retval =
Listing(relative_source, &ls,
false );
429 for (
unsigned i = 0; i < ls.size(); ++i) {
433 sub_path.
Append(ls[i].name().GetChars(), ls[i].name().GetLength());
435 if (ls[i].IsDirectory()) {
441 ls[i].set_hardlink_group(0);
442 ls[i].set_linkcount(1);
445 if (ls[i].HasXattrs()) {
450 if (ls[i].IsChunkedFile()) {
454 PathString(relative_sub_path), ls[i].hash_algorithm(), &chunks);
458 AddFile(ls[i], xattrs, dest_dir);
473 const std::string &parent_directory)
476 string directory_path = parent_path +
"/";
482 if (!
FindCatalog(parent_path, &catalog, &parent_entry)) {
484 directory_path.c_str());
489 catalog->
AddEntry(fixed_hardlink_count, xattrs,
490 directory_path, parent_path);
496 parent_path.c_str());
501 parent_catalog->UpdateEntry(parent_entry, parent_path);
516 const std::string &parent_directory)
519 const string file_path = entry.
GetFullPath(parent_path);
533 unsigned mbytes = entry.
size() / (1024 * 1024);
536 "%s: file at %s is larger than %u megabytes (%u). "
537 "CernVM-FS works best with small files. "
538 "Please remove the file or increase the limit.",
546 catalog->
AddEntry(entry, xattrs, file_path, parent_path);
554 const std::string &parent_directory,
560 full_entry.set_is_chunked_file(
true);
562 AddFile(full_entry, xattrs, parent_directory);
565 const string file_path = entry.
GetFullPath(parent_path);
574 for (
unsigned i = 0; i < file_chunks.
size(); ++i) {
591 const std::string &parent_directory,
594 assert(entries.size() >= 1);
596 if (entries.size() == 1) {
600 return AddFile(fix_linkcount, xattrs, parent_directory);
601 return AddChunkedFile(fix_linkcount, xattrs, parent_directory, file_chunks);
605 parent_directory.c_str(), entries[0].name().c_str());
612 unsigned mbytes = entries[0].size() / (1024 * 1024);
615 "%s: hard link at %s is larger than %u megabytes (%u). "
616 "CernVM-FS works best with small files. "
617 "Please remove the file or increase the limit.",
619 (parent_path + entries[0].name().ToString()).c_str(),
624 (parent_path + entries[0].name().ToString()).c_str(),
632 "catalog for hardlink group containing '%s' cannot be found",
633 parent_path.c_str());
644 for (DirectoryEntryBaseList::const_iterator i = entries.begin(),
645 iEnd = entries.end(); i != iEnd; ++i)
647 string file_path = parent_path +
"/";
648 file_path.append(i->name().GetChars(), i->name().GetLength());
657 catalog->
AddEntry(hardlink, xattrs, file_path, parent_path);
659 for (
unsigned i = 0; i < file_chunks.
size(); ++i) {
675 "catalog for hardlink group containing '%s' cannot be found",
676 remove_path.c_str());
694 const std::string &directory_path)
709 catalog->
TouchEntry(entry, xattrs, entry_path);
719 PathString transition_path(entry_path.data(), entry_path.length());
720 bool retval = catalog->
LookupPath(transition_path,
721 &potential_transition_point);
725 "updating transition point at %s", entry_path.c_str());
729 uint64_t nested_size;
730 retval = catalog->
FindNested(transition_path, &nested_hash, &nested_size);
733 nested_catalog =
MountCatalog(transition_path, nested_hash, catalog);
734 assert(nested_catalog != NULL);
738 TouchEntry(entry, xattrs, entry_path);
754 const PathString ps_nested_root_path(nested_root_path);
762 if (!
FindCatalog(nested_root_path, &old_catalog, &new_root_entry)) {
764 "failed to create nested catalog '%s': "
765 "mountpoint was not found in current catalog structure",
766 nested_root_path.c_str());
773 const bool volatile_content =
false;
775 assert(NULL != new_catalog_db);
786 delete new_catalog_db;
787 new_catalog_db = NULL;
802 wr_new_catalog->TouchEntry(new_root_entry, xattrs, nested_root_path);
819 wr_new_catalog->ListOwnNestedCatalogs();
821 for (Catalog::NestedCatalogList::const_iterator i = grand_nested.begin(),
822 iEnd = grand_nested.end(); i != iEnd; ++i)
825 retval =
FindCatalog(i->mountpoint.ToString(), &grand_catalog);
830 DeltaCounters save_counters = wr_new_catalog->delta_counters_;
831 wr_new_catalog->delta_counters_ = fix_subtree_counters;
832 wr_new_catalog->UpdateCounters();
833 wr_new_catalog->delta_counters_ = save_counters;
857 if (!
FindCatalog(nested_root_path, &nested_catalog)) {
859 "failed to remove nested catalog '%s': "
860 "mountpoint was not found in current catalog structure",
861 nested_root_path.c_str());
878 "unable to delete the removed nested catalog database file '%s'",
901 const uint64_t new_size) {
913 "failed to swap nested catalog '%s': could not find parent '%s'",
914 nested_root_path.c_str(), parent_path.c_str());
920 if (old_attached_catalog) {
928 "failed to swap nested catalog '%s': already modified",
929 nested_root_path.c_str());
931 old_counters = old_attached_catalog->
GetCounters();
939 const bool old_found = parent->
FindNested(nested_root_ps, &old_hash,
944 "failed to swap nested catalog '%s': not found in parent",
945 nested_root_path.c_str());
949 if (!old_free_catalog.
IsValid()) {
952 "failed to swap nested catalog '%s': failed to load old catalog",
953 nested_root_path.c_str());
955 old_counters = old_free_catalog->GetCounters();
963 "failed to swap nested catalog '%s': failed to load new catalog",
964 nested_root_path.c_str());
970 const bool dirent_found = new_catalog->LookupPath(nested_root_ps, &dirent);
974 "failed to swap nested catalog '%s': missing dirent in new catalog",
975 nested_root_path.c_str());
978 const bool xattrs_found = new_catalog->LookupXattrsPath(nested_root_ps,
983 "failed to swap nested catalog '%s': missing xattrs in new catalog",
984 nested_root_path.c_str());
996 parent->
TouchEntry(dirent, xattrs, nested_root_path);
1000 new_catalog->GetCounters());
1049 const uint64_t manual_revision,
1056 if (manual_revision > 0) {
1057 const uint64_t revision = root_catalog->
GetRevision();
1058 if (revision >= manual_revision) {
1060 "smaller than the current root "
1061 "catalog's (%d). Skipped!",
1062 manual_revision, revision);
1071 if (getenv(
"_CVMFS_SERIALIZED_CATALOG_PROCESSING_") == NULL)
1075 if (
spooler_->GetNumberOfErrors() > 0) {
1121 const bool stop_for_tweaks) {
1137 WritableCatalogList::const_iterator i = leafs_to_snapshot.begin();
1138 const WritableCatalogList::const_iterator iend = leafs_to_snapshot.end();
1139 for (; i != iend; ++i) {
1149 return root_catalog_info;
1154 const bool stop_for_tweaks) {
1167 base_hash().ToStringWithSuffix().c_str());
1173 uint64_t size_previous;
1176 &hash_previous, &size_previous);
1181 "for nested catalog '%s'",
1189 uint64_t catalog_limit = uint64_t(1000) *
1190 uint64_t((catalog->
IsRoot()
1193 if ((catalog_limit > 0) &&
1196 "%s: catalog at %s has more than %u entries (%u). "
1197 "Large catalogs stress the CernVM-FS transport infrastructure. "
1198 "Please split it into nested catalogs or increase the limit.",
1209 if (stop_for_tweaks) {
1211 "(hit return to continue)",
1213 int read_char = getchar();
1214 assert(read_char != EOF);
1246 std::map<std::string, WritableCatalog*>::iterator c =
1249 catalog = c->second;
1253 assert(catalog_size > 0);
1268 const int remaining_dirty_children =
1275 if (remaining_dirty_children == 0) {
1280 }
else if (catalog->
IsRoot()) {
1284 root_catalog_info.
size = catalog_size;
1285 root_catalog_info.
ttl = catalog->
GetTTL();
1312 int dirty_children = 0;
1314 CatalogList::const_iterator i = children.begin();
1315 const CatalogList::const_iterator iend = children.end();
1316 for (; i != iend; ++i) {
1325 const bool is_dirty = wr_catalog->
IsDirty() || dirty_children > 0;
1326 const bool is_leaf = dirty_children == 0;
1327 if (is_dirty && is_leaf) {
1328 result->push_back(const_cast<WritableCatalog *>(wr_catalog));
1337 reverse(catalog_list.begin(), catalog_list.end());
1338 for (
unsigned i = 0; i < catalog_list.size(); ++i) {
1339 FixWeight(static_cast<WritableCatalog*>(catalog_list[i]));
1349 "Deleting an autogenerated catalog in '%s'",
1354 catalog->
RemoveEntry(path +
"/.cvmfsautocatalog");
1360 catalog_balancer.
Balance(catalog);
1379 int dirty_catalogs = (wr_catalog->
IsDirty()) ? 1 : 0;
1383 for (CatalogList::const_iterator i = children.begin(), iEnd = children.end();
1391 if (dirty_catalogs > 0)
1392 result->push_back(const_cast<WritableCatalog *>(wr_catalog));
1395 return dirty_catalogs;
1413 const bool stop_for_tweaks)
1426 WritableCatalogList::const_iterator i = catalogs_to_snapshot.begin();
1427 const WritableCatalogList::const_iterator iend = catalogs_to_snapshot.end();
1428 for (; i != iend; ++i) {
1438 (*i)->mountpoint().ToString().c_str());
1441 int64_t catalog_size =
GetFileSize((*i)->database_path());
1442 assert(catalog_size > 0);
1444 if ((*i)->HasParent()) {
1448 catalog_size, (*i)->delta_counters_);
1449 (*i)->delta_counters_.SetZero();
1450 }
else if ((*i)->IsRoot()) {
1451 root_catalog_info.
size = catalog_size;
1452 root_catalog_info.
ttl = (*i)->GetTTL();
1454 root_catalog_info.
revision = (*i)->GetRevision();
1459 spooler_->ProcessCatalog((*i)->database_path());
1464 return root_catalog_info;
bool CompressPath2Null(const string &src, shash::Any *compressed_hash)
uint32_t linkcount() const
int return_code
the return value of the spooler operation
void DetachSubtree(Catalog *catalog)
#define LogCvmfs(source, mask,...)
bool IsExternalFile() const
const Counters & GetCounters() const
void AddChunkedFile(const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &parent_directory, const FileChunkList &file_chunks)
const manifest::Manifest * manifest() const
void set_base_hash(const shash::Any &hash)
ShortString< kDefaultMaxName, 1 > NameString
void CloneTree(const std::string &from_dir, const std::string &to_dir)
void AddHardlinkGroup(const DirectoryEntryBaseList &entries, const XattrList &xattrs, const std::string &parent_directory, const FileChunkList &file_chunks)
const std::vector< Catalog * > & GetCatalogs() const
void set_dirty_children(const int count)
void set_is_chunked_file(const bool val)
void AddDirectory(const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &parent_directory)
std::string database_path() const
void set_catalog_hash(const shash::Any &catalog_hash)
NameString GetFileName(const PathString &path)
Catalog * LoadFreeCatalog(const PathString &mountpoint, const shash::Any &hash)
bool IsChunkedFile() const
unsigned nested_kcatalog_limit_
bool MountSubtree(const PathString &path, const Catalog *entry_point, bool can_listing, Catalog **leaf_catalog)
~WritableCatalogManager()
void RemoveDirectory(const std::string &directory_path)
uint32_t GetMaxLinkId() const
void UpdateNestedCatalog(const std::string &path, const shash::Any &hash, const uint64_t size, const DeltaCounters &child_counters)
WritableCatalog * GetHostingCatalog(const std::string &path)
void Assign(const char *chars, const unsigned length)
void set_linkcount(const uint32_t linkcount)
void set_is_nested_catalog_root(const bool val)
void FixWeight(WritableCatalog *catalog)
std::string ToStringWithSuffix() const
void Balance(catalog_t *catalog)
bool LookupPath(const PathString &path, DirectoryEntry *dirent) const
void UpdateLastModified()
void ScheduleCatalogProcessing(WritableCatalog *catalog)
std::map< std::string, WritableCatalog * > catalog_processing_map_
std::string CreateTempPath(const std::string &path_prefix, const int mode)
void DetachCatalog(Catalog *catalog)
void SetTTL(const uint64_t new_ttl)
static DeltaCounters Diff(const Counters &from, const Counters &to)
int GetModifiedCatalogsRecursively(const Catalog *catalog, WritableCatalogList *result) const
Counters_t GetSelfEntries() const
void set_revision(const uint64_t revision)
void InsertNestedCatalog(const std::string &mountpoint, Catalog *attached_reference, const shash::Any content_hash, const uint64_t size)
void TouchEntry(const DirectoryEntryBase &entry, const XattrList &xattrs, const shash::Md5 &path_hash)
assert((mem||(size==0))&&"Out Of Memory")
bool FindCatalog(const std::string &path, WritableCatalog **result, DirectoryEntry *dirent=NULL)
bool Commit(const bool stop_for_tweaks, const uint64_t manual_revision, manifest::Manifest *manifest)
CatalogList GetChildren() const
bool LookupPath(const PathString &path, const LookupOptions options, DirectoryEntry *entry)
upload::Spooler * spooler
void SetPreviousRevision(const shash::Any &hash)
pthread_mutex_t * catalog_processing_lock_
Catalog * FindChild(const PathString &mountpoint) const
std::vector< WritableCatalog * > WritableCatalogList
shash::Any checksum() const
void CloneTreeImpl(const PathString &source_dir, const std::string &dest_parent_dir, const NameString &dest_name)
const unsigned kLookupDefault
void AddAsSubtree(DeltaCounters *delta) const
bool IsNestedCatalogMountpoint() const
bool IsTransitionPoint(const std::string &mountpoint)
bool Listing(const PathString &path, DirectoryEntryList *listing, const bool expand_symlink)
void CatalogUploadCallback(const upload::SpoolerResult &result, const CatalogUploadContext clg_upload_context)
bool IsNestedCatalogRoot() const
void TouchDirectory(const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &directory_path)
uint64_t GetNumEntries() const
uint64_t GetRevision() const
void AddFileChunk(const std::string &entry_path, const FileChunk &chunk)
bool IsAutogenerated() const
std::vector< DirectoryEntry > DirectoryEntryList
static manifest::Manifest * CreateRepository(const std::string &dir_temp, const bool volatile_content, const std::string &voms_authz, upload::Spooler *spooler)
bool AttachCatalog(const std::string &db_path, Catalog *new_catalog)
void SplitPath(const std::string &path, std::string *dirname, std::string *filename)
const unsigned max_weight_
void ActivateCatalog(Catalog *catalog)
std::string local_path
the local_path previously given as input
bool LookupXattrsPath(const PathString &path, XattrList *xattrs) const
void ShrinkHardlinkGroup(const std::string &remove_path)
bool GetModifiedCatalogLeafsRecursively(Catalog *catalog, WritableCatalogList *result) const
const char kSuffixCatalog
pthread_mutex_t * sync_lock_
unsigned root_kcatalog_limit_
const std::string & dir_temp() const
void PopulateToParent(DeltaCounters *parent) const
bool InsertInitialValues(const std::string &root_path, const bool volatile_content, const std::string &voms_authz, const DirectoryEntry &root_entry=DirectoryEntry(kDirentNegative))
CatalogInfo SnapshotCatalogsSerialized(const bool stop_for_tweaks)
CatalogInfo SnapshotCatalogs(const bool stop_for_tweaks)
PathString mountpoint() const
static const inode_t kInvalidInode
void Append(const char *chars, const unsigned length)
void TakeDatabaseFileOwnership()
bool SetVOMSAuthz(const std::string &voms_authz)
bool HasPrefix(const string &str, const string &prefix, const bool ignore_case)
void Clone(const std::string from, const std::string to)
std::vector< Catalog * > CatalogList
void RemoveFile(const std::string &file_path)
bool IsBindMountpoint() const
std::string MakeRelativePath(const std::string &relative_path) const
void CatalogUploadSerializedCallback(const upload::SpoolerResult &result, const CatalogUploadContext unused)
void RemoveNestedCatalog(const std::string &mountpoint, Catalog **attached_reference)
Catalog * GetRootCatalog() const
upload::Spooler * spooler_
void AddEntry(const DirectoryEntry &entry, const XattrList &xattr, const std::string &entry_path, const std::string &parent_path)
void RemoveNestedCatalog(const std::string &mountpoint, const bool merge=true)
const unsigned min_weight_
void set_ttl(const uint32_t ttl)
shash::Algorithms GetHashAlgorithm() const
std::string ToString() const
std::vector< NestedCatalog > NestedCatalogList
bool ListFileChunks(const PathString &path, const shash::Algorithms interpret_hashes_as, FileChunkList *chunks)
void Partition(WritableCatalog *new_nested_catalog)
ShortString< kDefaultMaxPath, 0 > PathString
void SetRevision(const uint64_t new_revision)
void set_hardlink_group(const uint32_t group)
PathString GetParentPath(const PathString &path)
WritableCatalog * GetWritableParent() const
void set_catalog_size(const uint64_t catalog_size)
DeltaCounters delta_counters_
bool FindNested(const PathString &mountpoint, shash::Any *hash, uint64_t *size) const
void VacuumDatabaseIfNecessary()
std::string GetFullPath(const std::string &parent_directory) const
bool LookupXattrs(const PathString &path, XattrList *xattrs)
Catalog * MountCatalog(const PathString &mountpoint, const shash::Any &hash, Catalog *parent_catalog)
int64_t GetFileSize(const std::string &path)
std::vector< DirectoryEntryBase > DirectoryEntryBaseList
void set_has_alt_catalog_path(const bool &has_alt_path)
unsigned GetLength() const
CatalogT * FindCatalog(const PathString &path) const
std::string MakePath() const
void AddFile(const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &parent_directory)
Future< CatalogInfo > * root_catalog_info
const char * c_str() const
void SwapNestedCatalog(const string &mountpoint, const shash::Any &new_hash, const uint64_t new_size)
void RemoveEntry(const std::string &entry_path)
const char * GetChars() const
virtual bool IsWritable() const
void GetModifiedCatalogLeafs(WritableCatalogList *result) const
Catalog * CreateCatalog(const PathString &mountpoint, const shash::Any &catalog_hash, Catalog *parent_catalog)
void FinalizeCatalog(WritableCatalog *catalog, const bool stop_for_tweaks)
void IncLinkcount(const std::string &path_within_group, const int delta)
void PrecalculateListings()
int DecrementDirtyChildren()
const Item * AtPtr(const size_t index) const
void set_is_nested_catalog_mountpoint(const bool val)
bool CompressPath2Path(const string &src, const string &dest)
void CreateNestedCatalog(const std::string &mountpoint)
void UpdateEntry(const DirectoryEntry &entry, const shash::Md5 &path_hash)
static DerivedT * Create(const std::string &filename)
void GetModifiedCatalogs(WritableCatalogList *result) const
unsigned file_mbyte_limit_
void set_root_path(const std::string &root_path)
const shash::Any & base_hash() const