CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
catalog::WritableCatalogManager Class Reference

#include <catalog_mgr_rw.h>

Inheritance diagram for catalog::WritableCatalogManager:
Collaboration diagram for catalog::WritableCatalogManager:

Classes

struct  CatalogInfo
 
struct  CatalogUploadContext
 

Public Member Functions

 WritableCatalogManager (const shash::Any &base_hash, const std::string &stratum0, const std::string &dir_temp, upload::Spooler *spooler, download::DownloadManager *download_manager, bool enforce_limits, const unsigned nested_kcatalog_limit, const unsigned root_kcatalog_limit, const unsigned file_mbyte_limit, perf::Statistics *statistics, bool is_balanceable, unsigned max_weight, unsigned min_weight)
 
 ~WritableCatalogManager ()
 
void AddFile (const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &parent_directory)
 
void AddChunkedFile (const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &parent_directory, const FileChunkList &file_chunks)
 
void RemoveFile (const std::string &file_path)
 
void AddDirectory (const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &parent_directory)
 
void TouchDirectory (const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &directory_path)
 
void RemoveDirectory (const std::string &directory_path)
 
void Clone (const std::string from, const std::string to)
 
void CloneTree (const std::string &from_dir, const std::string &to_dir)
 
void AddHardlinkGroup (const DirectoryEntryBaseList &entries, const XattrList &xattrs, const std::string &parent_directory, const FileChunkList &file_chunks)
 
void ShrinkHardlinkGroup (const std::string &remove_path)
 
void CreateNestedCatalog (const std::string &mountpoint)
 
void RemoveNestedCatalog (const std::string &mountpoint, const bool merge=true)
 
void SwapNestedCatalog (const string &mountpoint, const shash::Any &new_hash, const uint64_t new_size)
 
bool IsTransitionPoint (const std::string &mountpoint)
 
WritableCatalogGetHostingCatalog (const std::string &path)
 
bool IsBalanceable () const
 
void PrecalculateListings ()
 
void SetTTL (const uint64_t new_ttl)
 
bool SetVOMSAuthz (const std::string &voms_authz)
 
bool Commit (const bool stop_for_tweaks, const uint64_t manual_revision, manifest::Manifest *manifest)
 
void Balance ()
 
- Public Member Functions inherited from catalog::SimpleCatalogManager
 SimpleCatalogManager (const shash::Any &base_hash, const std::string &stratum0, const std::string &dir_temp, download::DownloadManager *download_manager, perf::Statistics *statistics, const bool manage_catalog_files=false)
 
- Public Member Functions inherited from catalog::AbstractCatalogManager< Catalog >
 AbstractCatalogManager (perf::Statistics *statistics)
 
virtual ~AbstractCatalogManager ()
 
void SetInodeAnnotation (InodeAnnotation *new_annotation)
 
virtual bool Init ()
 
LoadError Remount (const bool dry_run)
 
LoadError ChangeRoot (const shash::Any &root_hash)
 
void DetachNested ()
 
bool LookupPath (const PathString &path, const LookupOptions options, DirectoryEntry *entry)
 
bool LookupPath (const std::string &path, const LookupOptions options, DirectoryEntry *entry)
 
bool LookupXattrs (const PathString &path, XattrList *xattrs)
 
bool LookupNested (const PathString &path, PathString *mountpoint, shash::Any *hash, uint64_t *size)
 
bool ListCatalogSkein (const PathString &path, std::vector< PathString > *result_list)
 
bool Listing (const PathString &path, DirectoryEntryList *listing, const bool expand_symlink)
 
bool Listing (const PathString &path, DirectoryEntryList *listing)
 
bool Listing (const std::string &path, DirectoryEntryList *listing)
 
bool ListingStat (const PathString &path, StatEntryList *listing)
 
bool ListFileChunks (const PathString &path, const shash::Algorithms interpret_hashes_as, FileChunkList *chunks)
 
void SetOwnerMaps (const OwnerMap &uid_map, const OwnerMap &gid_map)
 
void SetCatalogWatermark (unsigned limit)
 
shash::Any GetNestedCatalogHash (const PathString &mountpoint)
 
Statistics statistics () const
 
uint64_t inode_gauge ()
 
bool volatile_flag () const
 
uint64_t GetRevision () const
 
uint64_t GetTTL () const
 
bool HasExplicitTTL () const
 
bool GetVOMSAuthz (std::string *authz) const
 
int GetNumCatalogs () const
 
std::string PrintHierarchy () const
 
std::string PrintAllMemStatistics () const
 
inode_t GetRootInode () const
 
CatalogGetRootCatalog () const
 
inode_t MangleInode (const inode_t inode) const
 
catalog::Counters LookupCounters (const PathString &path, std::string *subcatalog_path)
 

Static Public Member Functions

static manifest::ManifestCreateRepository (const std::string &dir_temp, const bool volatile_content, const std::string &voms_authz, upload::Spooler *spooler)
 

Protected Member Functions

void EnforceSqliteMemLimit ()
 
CatalogCreateCatalog (const PathString &mountpoint, const shash::Any &catalog_hash, Catalog *parent_catalog)
 
void ActivateCatalog (Catalog *catalog)
 
void AddFile (const DirectoryEntry &entry, const XattrList &xattrs, const std::string &parent_directory)
 
- Protected Member Functions inherited from catalog::SimpleCatalogManager
virtual LoadError LoadCatalog (const PathString &mountpoint, const shash::Any &hash, std::string *catalog_path, shash::Any *catalog_hash)
 
const shash::Anybase_hash () const
 
void set_base_hash (const shash::Any &hash)
 
const std::string & dir_temp () const
 
std::string MakeRelativePath (const std::string &relative_path) const
 
- Protected Member Functions inherited from catalog::AbstractCatalogManager< Catalog >
virtual void UnloadCatalog (const Catalog *catalog)
 
const std::vector< Catalog * > & GetCatalogs () const
 
CatalogMountCatalog (const PathString &mountpoint, const shash::Any &hash, Catalog *parent_catalog)
 
bool MountSubtree (const PathString &path, const Catalog *entry_point, bool can_listing, Catalog **leaf_catalog)
 
CatalogLoadFreeCatalog (const PathString &mountpoint, const shash::Any &hash)
 
bool AttachCatalog (const std::string &db_path, Catalog *new_catalog)
 
void DetachCatalog (Catalog *catalog)
 
void DetachSubtree (Catalog *catalog)
 
void DetachSiblings (const PathString &current_tree)
 
void DetachAll ()
 
bool IsAttached (const PathString &root_path, Catalog **attached_catalog) const
 
CatalogFindCatalog (const PathString &path) const
 
void ReadLock () const
 
void WriteLock () const
 
void Unlock () const
 
- Protected Member Functions inherited from SingleCopy
 SingleCopy ()
 

Private Member Functions

bool FindCatalog (const std::string &path, WritableCatalog **result, DirectoryEntry *dirent=NULL)
 
void DoBalance ()
 
void FixWeight (WritableCatalog *catalog)
 
void CloneTreeImpl (const PathString &source_dir, const std::string &dest_parent_dir, const NameString &dest_name)
 
CatalogInfo SnapshotCatalogs (const bool stop_for_tweaks)
 
void FinalizeCatalog (WritableCatalog *catalog, const bool stop_for_tweaks)
 
void ScheduleCatalogProcessing (WritableCatalog *catalog)
 
void GetModifiedCatalogLeafs (WritableCatalogList *result) const
 
bool GetModifiedCatalogLeafsRecursively (Catalog *catalog, WritableCatalogList *result) const
 
void CatalogUploadCallback (const upload::SpoolerResult &result, const CatalogUploadContext clg_upload_context)
 
void SyncLock ()
 
void SyncUnlock ()
 
void GetModifiedCatalogs (WritableCatalogList *result) const
 
int GetModifiedCatalogsRecursively (const Catalog *catalog, WritableCatalogList *result) const
 
void CatalogUploadSerializedCallback (const upload::SpoolerResult &result, const CatalogUploadContext unused)
 
CatalogInfo SnapshotCatalogsSerialized (const bool stop_for_tweaks)
 

Private Attributes

pthread_mutex_t * sync_lock_
 
upload::Spooler * spooler_
 
pthread_mutex_t * catalog_processing_lock_
 
std::map< std::string,
WritableCatalog * > 
catalog_processing_map_
 
bool enforce_limits_
 
unsigned nested_kcatalog_limit_
 
unsigned root_kcatalog_limit_
 
unsigned file_mbyte_limit_
 
XattrList empty_xattrs
 
const bool is_balanceable_
 
const unsigned max_weight_
 
const unsigned min_weight_
 
const unsigned balance_weight_
 

Static Private Attributes

static const std::string kCatalogFilename
 

Friends

class CatalogBalancer< WritableCatalogManager >
 
class VirtualCatalog
 

Additional Inherited Members

- Public Types inherited from catalog::AbstractCatalogManager< Catalog >
typedef std::vector< Catalog * > CatalogList
 
typedef Catalog catalog_t
 
- Static Public Attributes inherited from catalog::AbstractCatalogManager< Catalog >
static const inode_t kInodeOffset
 

Detailed Description

Definition at line 70 of file catalog_mgr_rw.h.

Constructor & Destructor Documentation

catalog::WritableCatalogManager::WritableCatalogManager ( const shash::Any base_hash,
const std::string &  stratum0,
const std::string &  dir_temp,
upload::Spooler *  spooler,
download::DownloadManager download_manager,
bool  enforce_limits,
const unsigned  nested_kcatalog_limit,
const unsigned  root_kcatalog_limit,
const unsigned  file_mbyte_limit,
perf::Statistics statistics,
bool  is_balanceable,
unsigned  max_weight,
unsigned  min_weight 
)

Definition at line 31 of file catalog_mgr_rw.cc.

Here is the call graph for this function:

catalog::WritableCatalogManager::~WritableCatalogManager ( )

Definition at line 68 of file catalog_mgr_rw.cc.

Member Function Documentation

void catalog::WritableCatalogManager::ActivateCatalog ( Catalog catalog)
protectedvirtual

Reimplemented from catalog::AbstractCatalogManager< Catalog >.

Definition at line 96 of file catalog_mgr_rw.cc.

Here is the call graph for this function:

void catalog::WritableCatalogManager::AddChunkedFile ( const DirectoryEntryBase entry,
const XattrList xattrs,
const std::string &  parent_directory,
const FileChunkList file_chunks 
)

Definition at line 551 of file catalog_mgr_rw.cc.

Referenced by publish::SyncMediator::AddFile(), AddHardlinkGroup(), CloneTreeImpl(), and publish::SyncMediator::PublishFilesCallback().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::AddDirectory ( const DirectoryEntryBase entry,
const XattrList xattrs,
const std::string &  parent_directory 
)

Add a new directory to the catalogs.

Parameters
entrya DirectoryEntry structure describing the new directory
parent_directorythe absolute path of the directory containing the directory to be created
Returns
true on success, false otherwise

Definition at line 471 of file catalog_mgr_rw.cc.

Referenced by publish::SyncMediator::AddDirectory(), CloneTreeImpl(), catalog::VirtualCatalog::CreateBaseDirectory(), catalog::VirtualCatalog::CreateSnapshotDirectory(), and catalog::VirtualCatalog::InsertSnapshot().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::AddFile ( const DirectoryEntryBase entry,
const XattrList xattrs,
const std::string &  parent_directory 
)
inline
void catalog::WritableCatalogManager::AddFile ( const DirectoryEntry entry,
const XattrList xattrs,
const std::string &  parent_directory 
)
protected

Add a new file to the catalogs.

Parameters
entrya DirectoryEntry structure describing the new file
parent_directorythe absolute path of the directory containing the file to be created
Returns
true on success, false otherwise

Definition at line 513 of file catalog_mgr_rw.cc.

Here is the call graph for this function:

void catalog::WritableCatalogManager::AddHardlinkGroup ( const DirectoryEntryBaseList entries,
const XattrList xattrs,
const std::string &  parent_directory,
const FileChunkList file_chunks 
)

Add a hardlink group to the catalogs.

Parameters
entriesa list of DirectoryEntries describing the new files
parent_directorythe absolute path of the directory containing the files to be created
Returns
true on success, false otherwise

Definition at line 588 of file catalog_mgr_rw.cc.

Referenced by publish::SyncMediator::AddHardlinkGroup().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::Balance ( )
inline

Definition at line 148 of file catalog_mgr_rw.h.

Referenced by publish::SyncMediator::Commit().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::CatalogUploadCallback ( const upload::SpoolerResult result,
const CatalogUploadContext  clg_upload_context 
)
private

Definition at line 1227 of file catalog_mgr_rw.cc.

Referenced by SnapshotCatalogs().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::CatalogUploadSerializedCallback ( const upload::SpoolerResult result,
const CatalogUploadContext  unused 
)
private

Definition at line 1393 of file catalog_mgr_rw.cc.

Referenced by SnapshotCatalogsSerialized().

Here is the caller graph for this function:

void catalog::WritableCatalogManager::Clone ( const std::string  destination,
const std::string  source 
)

Clone the file called source changing its name into destination, the source file is keep intact. destination, the name of the new file, complete path source, the name of the file to clone, which must be already in the repository

Returns
void

Definition at line 297 of file catalog_mgr_rw.cc.

Referenced by publish::SyncMediator::Clone().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::CloneTree ( const std::string &  from_dir,
const std::string &  to_dir 
)

Copies an entire directory tree from the exisitng from_dir to the non-existing to_dir. The destination's parent directory must exist. On the catalog level, the new entries will be identical to the old ones except for their path hash fields.

Definition at line 339 of file catalog_mgr_rw.cc.

Referenced by publish::TransactionImpl().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::CloneTreeImpl ( const PathString source_dir,
const std::string &  dest_parent_dir,
const NameString dest_name 
)
private

Called from CloneTree(), assumes that from_dir and to_dir are sufficiently sanitized

Definition at line 389 of file catalog_mgr_rw.cc.

Referenced by CloneTree().

Here is the call graph for this function:

Here is the caller graph for this function:

bool catalog::WritableCatalogManager::Commit ( const bool  stop_for_tweaks,
const uint64_t  manual_revision,
manifest::Manifest manifest 
)

Definition at line 1042 of file catalog_mgr_rw.cc.

Referenced by publish::SyncMediator::Commit().

Here is the call graph for this function:

Here is the caller graph for this function:

Catalog * catalog::WritableCatalogManager::CreateCatalog ( const PathString mountpoint,
const shash::Any catalog_hash,
Catalog parent_catalog 
)
protectedvirtual

This method is virtual in AbstractCatalogManager. It returns a new catalog structure in the form the different CatalogManagers need it. In this case it returns a stub for a WritableCatalog.

Parameters
mountpointthe mount point of the catalog stub to create
catalog_hashthe content hash of the catalog to create
parent_catalogthe parent of the catalog stub to create
Returns
a pointer to the catalog stub structure created

Reimplemented from catalog::SimpleCatalogManager.

Definition at line 85 of file catalog_mgr_rw.cc.

Referenced by CreateNestedCatalog().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::CreateNestedCatalog ( const std::string &  mountpoint)

Create a new nested catalog. Includes moving all entries belonging there from it's parent catalog.

Parameters
mountpointthe path of the directory to become a nested root
Returns
true on success, false otherwise

Definition at line 751 of file catalog_mgr_rw.cc.

Referenced by catalog::CatalogBalancer< CatalogMgrT >::AddCatalog(), CloneTreeImpl(), catalog::VirtualCatalog::CreateBaseDirectory(), and publish::SyncMediator::CreateNestedCatalog().

Here is the call graph for this function:

Here is the caller graph for this function:

manifest::Manifest * catalog::WritableCatalogManager::CreateRepository ( const std::string &  dir_temp,
const bool  volatile_content,
const std::string &  voms_authz,
upload::Spooler *  spooler 
)
static

This method is invoked if we create a completely new repository. The new root catalog will already contain a root entry. It is uploaded by a Forklift to the upstream storage.

Returns
true on success, false otherwise

Definition at line 107 of file catalog_mgr_rw.cc.

Referenced by publish::CreateRootObjects(), and swissknife::CommandCreate::Main().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::DoBalance ( )
private

Definition at line 1329 of file catalog_mgr_rw.cc.

Referenced by Balance().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::EnforceSqliteMemLimit ( )
inlineprotectedvirtual

Reimplemented from catalog::AbstractCatalogManager< Catalog >.

Definition at line 158 of file catalog_mgr_rw.h.

void catalog::WritableCatalogManager::FinalizeCatalog ( WritableCatalog catalog,
const bool  stop_for_tweaks 
)
private

Definition at line 1147 of file catalog_mgr_rw.cc.

Referenced by CatalogUploadCallback(), SnapshotCatalogs(), and SnapshotCatalogsSerialized().

Here is the call graph for this function:

Here is the caller graph for this function:

bool catalog::WritableCatalogManager::FindCatalog ( const std::string &  path,
WritableCatalog **  result,
DirectoryEntry dirent = NULL 
)
private

Retrieve the catalog containing the given path. Other than AbstractCatalogManager::FindCatalog() this mounts nested catalogs if necessary and returns WritableCatalog objects. Furthermore it optionally returns the looked-up DirectoryEntry.

Parameters
paththe path to look for
resultthe retrieved catalog (as a pointer)
direntis set to looked up DirectoryEntry for 'path' if non-NULL
Returns
true if catalog was found

Definition at line 198 of file catalog_mgr_rw.cc.

Referenced by AddChunkedFile(), AddDirectory(), AddFile(), AddHardlinkGroup(), CreateNestedCatalog(), GetHostingCatalog(), IsTransitionPoint(), RemoveDirectory(), RemoveFile(), RemoveNestedCatalog(), ShrinkHardlinkGroup(), SwapNestedCatalog(), and TouchDirectory().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::FixWeight ( WritableCatalog catalog)
private

Definition at line 1337 of file catalog_mgr_rw.cc.

Referenced by DoBalance().

Here is the call graph for this function:

Here is the caller graph for this function:

WritableCatalog * catalog::WritableCatalogManager::GetHostingCatalog ( const std::string &  path)
void catalog::WritableCatalogManager::GetModifiedCatalogLeafs ( WritableCatalogList result) const
inlineprivate

Definition at line 197 of file catalog_mgr_rw.h.

Referenced by SnapshotCatalogs().

Here is the call graph for this function:

Here is the caller graph for this function:

bool catalog::WritableCatalogManager::GetModifiedCatalogLeafsRecursively ( Catalog catalog,
WritableCatalogList result 
) const
private

Finds dirty catalogs that can be snapshot right away and annotates all the other catalogs with their number of dirty decendants. Note that there is a convenience wrapper to start the recursion: WritableCatalogManager::GetModifiedCatalogLeafs()

Parameters
catalogthe catalog for this recursion step
resultthe result list to be appended to
Returns
true if 'catalog' is dirty

Definition at line 1300 of file catalog_mgr_rw.cc.

Referenced by GetModifiedCatalogLeafs(), and SwapNestedCatalog().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::GetModifiedCatalogs ( WritableCatalogList result) const
inlineprivate

Definition at line 214 of file catalog_mgr_rw.h.

Referenced by SnapshotCatalogsSerialized().

Here is the call graph for this function:

Here is the caller graph for this function:

int catalog::WritableCatalogManager::GetModifiedCatalogsRecursively ( const Catalog catalog,
WritableCatalogList result 
) const
private

Definition at line 1362 of file catalog_mgr_rw.cc.

Referenced by GetModifiedCatalogs().

Here is the call graph for this function:

Here is the caller graph for this function:

bool catalog::WritableCatalogManager::IsBalanceable ( ) const
inline

Definition at line 136 of file catalog_mgr_rw.h.

Referenced by Balance(), and publish::SyncMediator::Commit().

Here is the caller graph for this function:

bool catalog::WritableCatalogManager::IsTransitionPoint ( const std::string &  mountpoint)

Checks if a nested catalog starts at this path. The path must be valid.

Definition at line 1004 of file catalog_mgr_rw.cc.

Referenced by publish::SyncMediator::AddDirectory(), catalog::VirtualCatalog::EnsurePresence(), publish::SyncMediator::RemoveDirectory(), and publish::SyncMediator::TouchDirectory().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::PrecalculateListings ( )

TODO

Definition at line 1020 of file catalog_mgr_rw.cc.

Referenced by publish::SyncMediator::Commit().

Here is the caller graph for this function:

void catalog::WritableCatalogManager::RemoveDirectory ( const std::string &  path)

Remove the given directory from the catalogs.

Parameters
directory_paththe full path to the directory to be removed
Returns
true on success, false otherwise

Definition at line 261 of file catalog_mgr_rw.cc.

Referenced by catalog::VirtualCatalog::Remove(), publish::SyncMediator::RemoveDirectory(), catalog::VirtualCatalog::RemoveRecursively(), and catalog::VirtualCatalog::RemoveSnapshot().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::RemoveFile ( const std::string &  path)

Remove the given file from the catalogs.

Parameters
file_paththe full path to the file to be removed
Returns
true on success, false otherwise

Definition at line 240 of file catalog_mgr_rw.cc.

Referenced by Clone(), publish::SyncMediator::RemoveFile(), and catalog::VirtualCatalog::RemoveRecursively().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::RemoveNestedCatalog ( const std::string &  mountpoint,
const bool  merge = true 
)

Remove a nested catalog

If the merged parameter is true, when you remove a nested catalog all entries currently held by it will be merged into its parent catalog.

Parameters
mountpoint- the path of the nested catalog to be removed
merge- merge the subtree associated with the nested catalog into its parent catalog
Returns
- true on success, false otherwise

Definition at line 850 of file catalog_mgr_rw.cc.

Referenced by FixWeight(), catalog::VirtualCatalog::Remove(), and publish::SyncMediator::RemoveNestedCatalog().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::ScheduleCatalogProcessing ( WritableCatalog catalog)
private

Definition at line 1216 of file catalog_mgr_rw.cc.

Referenced by CatalogUploadCallback(), and SnapshotCatalogs().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::SetTTL ( const uint64_t  new_ttl)

Definition at line 1025 of file catalog_mgr_rw.cc.

Referenced by swissknife::CommandSync::Main().

Here is the call graph for this function:

Here is the caller graph for this function:

bool catalog::WritableCatalogManager::SetVOMSAuthz ( const std::string &  voms_authz)

Definition at line 1032 of file catalog_mgr_rw.cc.

Referenced by swissknife::Ingest::Main(), and swissknife::CommandSync::Main().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::ShrinkHardlinkGroup ( const std::string &  remove_path)

Definition at line 668 of file catalog_mgr_rw.cc.

Referenced by publish::SyncMediator::RemoveFile().

Here is the call graph for this function:

Here is the caller graph for this function:

WritableCatalogManager::CatalogInfo catalog::WritableCatalogManager::SnapshotCatalogs ( const bool  stop_for_tweaks)
private

Handles the snapshotting of dirty (i.e. modified) catalogs while trying to parallize the compression and upload as much as possible. We use a parallel depth first post order tree traversal based on 'continuations'.

The idea is as follows:

  1. find all leaf-catalogs (i.e. dirty catalogs with no dirty children) –> these can be processed and uploaded immedately and independently see WritableCatalogManager::GetModifiedCatalogLeafs()
  2. annotate non-leaf catalogs with their number of dirty children –> a finished child will notify it's parent and decrement this number see WritableCatalogManager::CatalogUploadCallback()
  3. if a non-leaf catalog's dirty children number reaches 0, it is scheduled for processing as well (continuation) –> the parallel processing walks bottom-up through the catalog tree see WritableCatalogManager::CatalogUploadCallback()
  4. when the root catalog is reached, we notify the main thread and return –> done through a Future<> in WritableCatalogManager::SnapshotCatalogs

Note: The catalog finalisation (see WritableCatalogManager::FinalizeCatalog) happens in a worker thread (i.e. the callback method) for non-leaf catalogs.

TODO(rmeusel): since all leaf catalogs are finalized in the main thread, we sacrafice some potential concurrency for simplicity.

Definition at line 1114 of file catalog_mgr_rw.cc.

Referenced by Commit().

Here is the call graph for this function:

Here is the caller graph for this function:

WritableCatalogManager::CatalogInfo catalog::WritableCatalogManager::SnapshotCatalogsSerialized ( const bool  stop_for_tweaks)
private

Definition at line 1406 of file catalog_mgr_rw.cc.

Referenced by Commit().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::SwapNestedCatalog ( const string &  mountpoint,
const shash::Any new_hash,
const uint64_t  new_size 
)

Swap in a new nested catalog

The old nested catalog must not have been already attached to the catalog tree. This method will not attach the new nested catalog to the catalog tree.

Parameters
mountpoint- the path of the nested catalog to be removed
new_hash- the hash of the new nested catalog
new_size- the size of the new nested catalog

Definition at line 899 of file catalog_mgr_rw.cc.

Here is the call graph for this function:

void catalog::WritableCatalogManager::SyncLock ( )
inlineprivate
void catalog::WritableCatalogManager::SyncUnlock ( )
inlineprivate
void catalog::WritableCatalogManager::TouchDirectory ( const DirectoryEntryBase entry,
const XattrList xattrs,
const std::string &  directory_path 
)

Update entry meta data (mode, owner, ...). CVMFS specific meta data (i.e. nested catalog transition points) are NOT changed by this method, although transition points intrinsics are taken into account, to keep nested catalogs consistent.

Parameters
entrythe directory entry to be touched
paththe path of the directory entry to be touched

Definition at line 692 of file catalog_mgr_rw.cc.

Referenced by publish::SyncMediator::TouchDirectory().

Here is the call graph for this function:

Here is the caller graph for this function:

Friends And Related Function Documentation

friend class CatalogBalancer< WritableCatalogManager >
friend

Definition at line 71 of file catalog_mgr_rw.h.

friend class VirtualCatalog
friend

Definition at line 74 of file catalog_mgr_rw.h.

Member Data Documentation

const unsigned catalog::WritableCatalogManager::balance_weight_
private

Defines the threshold that will be used to balance a catalog that has been overflowed. Its value should be lesser than max_weight_ and greater than min_weight. By default it is set to max_weight / 2.

Definition at line 275 of file catalog_mgr_rw.h.

Referenced by catalog::CatalogBalancer< CatalogMgrT >::PartitionOptimally().

pthread_mutex_t* catalog::WritableCatalogManager::catalog_processing_lock_
private
std::map<std::string, WritableCatalog*> catalog::WritableCatalogManager::catalog_processing_map_
private

Definition at line 235 of file catalog_mgr_rw.h.

Referenced by CatalogUploadCallback(), and ScheduleCatalogProcessing().

XattrList catalog::WritableCatalogManager::empty_xattrs
private

Directories don't have extended attributes at this point.

Definition at line 246 of file catalog_mgr_rw.h.

Referenced by Clone().

bool catalog::WritableCatalogManager::enforce_limits_
private

Definition at line 238 of file catalog_mgr_rw.h.

Referenced by AddFile(), AddHardlinkGroup(), and FinalizeCatalog().

unsigned catalog::WritableCatalogManager::file_mbyte_limit_
private

Definition at line 241 of file catalog_mgr_rw.h.

Referenced by AddFile(), and AddHardlinkGroup().

const bool catalog::WritableCatalogManager::is_balanceable_
private

It indicates whether this catalog manager supports balancing operations

Definition at line 251 of file catalog_mgr_rw.h.

Referenced by IsBalanceable().

const std::string catalog::WritableCatalogManager::kCatalogFilename
staticprivate

Definition at line 228 of file catalog_mgr_rw.h.

const unsigned catalog::WritableCatalogManager::max_weight_
private

Defines the maximum weight an autogenerated catalog can have. If after a publishing operation the catalog's weight is greater than this threshold it will be considered overflowed and will automatically be split in different sub-catalogs.

Definition at line 259 of file catalog_mgr_rw.h.

Referenced by FixWeight().

const unsigned catalog::WritableCatalogManager::min_weight_
private

Defines the minimum weight an autogenerated catalog can have. If after a publishing operation the catalog's weight is lesser than this threshold it will be considered underflowed and will automatically be merged with its parent. This last operation can provoke an overflow in the parent, though.

Definition at line 268 of file catalog_mgr_rw.h.

Referenced by FixWeight(), and catalog::CatalogBalancer< CatalogMgrT >::PartitionOptimally().

unsigned catalog::WritableCatalogManager::nested_kcatalog_limit_
private

Definition at line 239 of file catalog_mgr_rw.h.

Referenced by FinalizeCatalog().

unsigned catalog::WritableCatalogManager::root_kcatalog_limit_
private

Definition at line 240 of file catalog_mgr_rw.h.

Referenced by FinalizeCatalog().

pthread_mutex_t* catalog::WritableCatalogManager::sync_lock_
private

The documentation for this class was generated from the following files: