#include <catalog_mgr_rw.h>

Inheritance diagram for catalog::WritableCatalogManager:

Collaboration diagram for catalog::WritableCatalogManager:

Classes
struct	CatalogDownloadContext

struct	CatalogInfo

struct	CatalogUploadContext

Public Member Functions
	WritableCatalogManager (const shash::Any &base_hash, const std::string &stratum0, const std::string &dir_temp, upload::Spooler spooler, download::DownloadManager download_manager, bool enforce_limits, const unsigned nested_kcatalog_limit, const unsigned root_kcatalog_limit, const unsigned file_mbyte_limit, perf::Statistics *statistics, bool is_balanceable, unsigned max_weight, unsigned min_weight, const std::string &dir_cache="")

	~WritableCatalogManager ()

void	AddFile (const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &parent_directory)

void	AddChunkedFile (const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &parent_directory, const FileChunkList &file_chunks)

void	RemoveFile (const std::string &file_path)

void	AddDirectory (const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &parent_directory)

void	TouchDirectory (const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &directory_path)

void	RemoveDirectory (const std::string &directory_path)

void	Clone (const std::string from, const std::string to)

void	CloneTree (const std::string &from_dir, const std::string &to_dir)

void	AddHardlinkGroup (const DirectoryEntryBaseList &entries, const XattrList &xattrs, const std::string &parent_directory, const FileChunkList &file_chunks)

void	ShrinkHardlinkGroup (const std::string &remove_path)

void	CreateNestedCatalog (const std::string &mountpoint)

void	RemoveNestedCatalog (const std::string &mountpoint, const bool merge=true)

void	SwapNestedCatalog (const string &mountpoint, const shash::Any &new_hash, const uint64_t new_size)

void	GraftNestedCatalog (const string &mountpoint, const shash::Any &new_hash, const uint64_t new_size)

bool	IsTransitionPoint (const std::string &mountpoint)

WritableCatalog *	GetHostingCatalog (const std::string &path)

bool	IsBalanceable () const

void	PrecalculateListings ()

void	SetTTL (const uint64_t new_ttl)

bool	SetVOMSAuthz (const std::string &voms_authz)

bool	Commit (const bool stop_for_tweaks, const uint64_t manual_revision, manifest::Manifest *manifest)

void	Balance ()

void	LoadCatalogs (const std::string &base_path, const std::unordered_set< std::string > &dirs)

void	SetupSingleCatalogUploadCallback ()

void	RemoveSingleCatalogUploadCallback ()

void	AddCatalogToQueue (const std::string &path)

void	ScheduleReadyCatalogs ()

bool	LookupDirEntry (const std::string &path, const LookupOptions options, DirectoryEntry *dirent)

Public Member Functions inherited from catalog::SimpleCatalogManager
	SimpleCatalogManager (const shash::Any &base_hash, const std::string &stratum0, const std::string &dir_temp, download::DownloadManager download_manager, perf::Statistics statistics, const bool manage_catalog_files=false, const std::string &dir_cache="", const bool copy_to_tmp_dir=false)

virtual LoadReturn	LoadCatalogByHash (CatalogContext *ctlg_context)

Public Member Functions inherited from catalog::AbstractCatalogManager< Catalog >
	AbstractCatalogManager (perf::Statistics *statistics)

virtual	~AbstractCatalogManager ()

void	SetInodeAnnotation (InodeAnnotation *new_annotation)

virtual bool	Init ()

LoadReturn	RemountDryrun ()

LoadReturn	Remount ()

LoadReturn	ChangeRoot (const shash::Any &root_hash)

void	DetachNested ()

bool	LookupPath (const PathString &path, const LookupOptions options, DirectoryEntry *entry)

bool	LookupPath (const std::string &path, const LookupOptions options, DirectoryEntry *entry)

bool	LookupXattrs (const PathString &path, XattrList *xattrs)

bool	LookupNested (const PathString &path, PathString mountpoint, shash::Any hash, uint64_t *size)

bool	ListCatalogSkein (const PathString &path, std::vector< PathString > *result_list)

bool	Listing (const PathString &path, DirectoryEntryList *listing, const bool expand_symlink)

bool	Listing (const PathString &path, DirectoryEntryList *listing)

bool	Listing (const std::string &path, DirectoryEntryList *listing)

bool	ListingStat (const PathString &path, StatEntryList *listing)

bool	ListFileChunks (const PathString &path, const shash::Algorithms interpret_hashes_as, FileChunkList *chunks)

void	SetOwnerMaps (const OwnerMap &uid_map, const OwnerMap &gid_map)

void	SetCatalogWatermark (unsigned limit)

shash::Any	GetNestedCatalogHash (const PathString &mountpoint)

Statistics	statistics () const

uint64_t	inode_gauge ()

bool	volatile_flag () const

uint64_t	GetRevision () const

uint64_t	GetTimestamp () const

uint64_t	GetTTL () const

bool	HasExplicitTTL () const

bool	GetVOMSAuthz (std::string *authz) const

int	GetNumCatalogs () const

std::string	PrintHierarchy () const

std::string	PrintAllMemStatistics () const

inode_t	GetRootInode () const

Catalog *	GetRootCatalog () const

inode_t	MangleInode (const inode_t inode) const

catalog::Counters	LookupCounters (const PathString &path, std::string subcatalog_path, shash::Any hash)

Static Public Member Functions
static manifest::Manifest *	CreateRepository (const std::string &dir_temp, const bool volatile_content, const std::string &voms_authz, upload::Spooler *spooler)

Protected Member Functions
void	EnforceSqliteMemLimit ()

Catalog *	CreateCatalog (const PathString &mountpoint, const shash::Any &catalog_hash, Catalog *parent_catalog)

void	ActivateCatalog (Catalog *catalog)

void	AddFile (const DirectoryEntry &entry, const XattrList &xattrs, const std::string &parent_directory)

Protected Member Functions inherited from catalog::SimpleCatalogManager
virtual LoadReturn	GetNewRootCatalogContext (CatalogContext *result)

const shash::Any &	base_hash () const

void	set_base_hash (const shash::Any &hash)

const std::string &	dir_temp () const

std::string	MakeRelativePath (const std::string &relative_path) const

bool	UseLocalCache () const

Protected Member Functions inherited from catalog::AbstractCatalogManager< Catalog >
virtual void	UnloadCatalog (const Catalog *catalog)

const std::vector< Catalog * > &	GetCatalogs () const

virtual void	StageNestedCatalogByHash (const shash::Any &, const PathString &)

void	StageNestedCatalogAndUnlock (const PathString &path, const Catalog *parent, bool is_listable)

Catalog *	MountCatalog (const PathString &mountpoint, const shash::Any &hash, Catalog *parent_catalog)

bool	MountSubtree (const PathString &path, const Catalog entry_point, bool can_listing, Catalog *leaf_catalog)

Catalog *	LoadFreeCatalog (const PathString &mountpoint, const shash::Any &hash)

bool	AttachCatalog (const std::string &db_path, Catalog *new_catalog)

void	DetachCatalog (Catalog *catalog)

void	DetachSubtree (Catalog *catalog)

void	DetachSiblings (const PathString &current_tree)

void	DetachAll ()

bool	IsAttached (const PathString &root_path, Catalog **attached_catalog) const

Catalog *	FindCatalog (const PathString &path) const

uint64_t	GetRevisionNoLock () const

uint64_t	GetTimestampNoLock () const

void	ReadLock () const

void	WriteLock () const

void	Unlock () const

Protected Member Functions inherited from SingleCopy
	SingleCopy ()

Private Member Functions
bool	FindCatalog (const std::string &path, WritableCatalog *result, DirectoryEntry dirent=NULL)

void	DoBalance ()

void	FixWeight (WritableCatalog *catalog)

void	CloneTreeImpl (const PathString &source_dir, const std::string &dest_parent_dir, const NameString &dest_name)

void	CatalogDownloadCallback (const CatalogDownloadResult &result, const CatalogDownloadContext context)

void	SingleCatalogUploadCallback (const upload::SpoolerResult &result)

CatalogInfo	SnapshotCatalogs (const bool stop_for_tweaks)

void	FinalizeCatalog (WritableCatalog *catalog, const bool stop_for_tweaks)

void	ScheduleCatalogProcessing (WritableCatalog *catalog)

void	GetModifiedCatalogLeafs (WritableCatalogList *result) const

bool	GetModifiedCatalogLeafsRecursively (Catalog catalog, WritableCatalogList result) const

void	CatalogUploadCallback (const upload::SpoolerResult &result, const CatalogUploadContext clg_upload_context)

bool	CopyCatalogToLocalCache (const upload::SpoolerResult &result)

void	SyncLock ()

void	SyncUnlock ()

void	GetModifiedCatalogs (WritableCatalogList *result) const

int	GetModifiedCatalogsRecursively (const Catalog catalog, WritableCatalogList result) const

void	CatalogUploadSerializedCallback (const upload::SpoolerResult &result, const CatalogUploadContext unused)

CatalogInfo	SnapshotCatalogsSerialized (const bool stop_for_tweaks)

void	CatalogHashSerializedCallback (const CompressHashResult &result)

Private Attributes
pthread_mutex_t *	sync_lock_

upload::Spooler *	spooler_

pthread_mutex_t *	catalog_processing_lock_

std::map< std::string, WritableCatalog * >	catalog_processing_map_

std::list< WritableCatalog * >	pending_catalogs_

CatalogDownloadPipeline *	catalog_download_pipeline_

pthread_mutex_t *	catalog_download_lock_

std::unordered_map < std::string, Catalog * >	catalog_download_map_

pthread_mutex_t *	catalog_hash_lock_

std::map< std::string, shash::Any >	catalog_hash_map_

bool	enforce_limits_

unsigned	nested_kcatalog_limit_

unsigned	root_kcatalog_limit_

unsigned	file_mbyte_limit_

XattrList	empty_xattrs

const bool	is_balanceable_

const unsigned	max_weight_

const unsigned	min_weight_

const unsigned	balance_weight_

Static Private Attributes
static const std::string	kCatalogFilename

Friends
class	CatalogBalancer< WritableCatalogManager >

class	VirtualCatalog

Additional Inherited Members
Public Types inherited from catalog::AbstractCatalogManager< Catalog >
typedef std::vector< Catalog * >	CatalogList

typedef Catalog	catalog_t

Static Public Attributes inherited from catalog::AbstractCatalogManager< Catalog >
static const inode_t	kInodeOffset

Protected Attributes inherited from catalog::SimpleCatalogManager
std::string	dir_cache_

bool	copy_to_tmp_dir_

Detailed Description

Definition at line 75 of file catalog_mgr_rw.h.

Constructor & Destructor Documentation

catalog::WritableCatalogManager::WritableCatalogManager	(	const shash::Any &	base_hash,
		const std::string &	stratum0,
		const std::string &	dir_temp,
		upload::Spooler *	spooler,
		download::DownloadManager *	download_manager,
		bool	enforce_limits,
		const unsigned	nested_kcatalog_limit,
		const unsigned	root_kcatalog_limit,
		const unsigned	file_mbyte_limit,
		perf::Statistics *	statistics,
		bool	is_balanceable,
		unsigned	max_weight,
		unsigned	min_weight,
		const std::string &	dir_cache = `""`
	)

Definition at line 31 of file catalog_mgr_rw.cc.

Here is the call graph for this function:

catalog::WritableCatalogManager::~WritableCatalogManager ( )

Definition at line 69 of file catalog_mgr_rw.cc.

Member Function Documentation

void catalog::WritableCatalogManager::ActivateCatalog ( Catalog * catalog )

protectedvirtual

Reimplemented from catalog::AbstractCatalogManager< Catalog >.

Definition at line 94 of file catalog_mgr_rw.cc.

Here is the call graph for this function:

void catalog::WritableCatalogManager::AddCatalogToQueue ( const std::string & path )

Definition at line 1592 of file catalog_mgr_rw.cc.

Referenced by swissknife::IngestSQL::do_additions().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::AddChunkedFile	(	const DirectoryEntryBase &	entry,
		const XattrList &	xattrs,
		const std::string &	parent_directory,
		const FileChunkList &	file_chunks
	)

Definition at line 538 of file catalog_mgr_rw.cc.

Referenced by swissknife::IngestSQL::add_files(), publish::SyncMediator::AddFile(), AddHardlinkGroup(), CloneTreeImpl(), and publish::SyncMediator::PublishFilesCallback().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::AddDirectory	(	const DirectoryEntryBase &	entry,
		const XattrList &	xattrs,
		const std::string &	parent_directory
	)

Add a new directory to the catalogs.

Parameters

entry	a DirectoryEntry structure describing the new directory
parent_directory	the absolute path of the directory containing the directory to be created

Returns: true on success, false otherwise

Definition at line 462 of file catalog_mgr_rw.cc.

Referenced by publish::SyncMediator::AddDirectory(), CloneTreeImpl(), catalog::VirtualCatalog::CreateBaseDirectory(), catalog::VirtualCatalog::CreateSnapshotDirectory(), swissknife::IngestSQL::do_additions(), and catalog::VirtualCatalog::InsertSnapshot().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::AddFile	(	const DirectoryEntryBase &	entry,
		const XattrList &	xattrs,
		const std::string &	parent_directory
	)

inline

Definition at line 103 of file catalog_mgr_rw.h.

Referenced by swissknife::IngestSQL::add_symlinks(), catalog::CatalogBalancer< CatalogMgrT >::AddCatalogMarker(), AddChunkedFile(), publish::SyncMediator::AddFile(), AddHardlinkGroup(), Clone(), CloneTreeImpl(), catalog::VirtualCatalog::CreateNestedCatalogMarker(), swissknife::IngestSQL::do_additions(), and publish::SyncMediator::PublishFilesCallback().

Here is the caller graph for this function:

void catalog::WritableCatalogManager::AddFile	(	const DirectoryEntry &	entry,
		const XattrList &	xattrs,
		const std::string &	parent_directory
	)

protected

Add a new file to the catalogs.

Parameters

entry	a DirectoryEntry structure describing the new file
parent_directory	the absolute path of the directory containing the file to be created

Returns: true on success, false otherwise

Definition at line 502 of file catalog_mgr_rw.cc.

Here is the call graph for this function:

void catalog::WritableCatalogManager::AddHardlinkGroup	(	const DirectoryEntryBaseList &	entries,
		const XattrList &	xattrs,
		const std::string &	parent_directory,
		const FileChunkList &	file_chunks
	)

Add a hardlink group to the catalogs.

Parameters

entries	a list of DirectoryEntries describing the new files
parent_directory	the absolute path of the directory containing the files to be created

Returns: true on success, false otherwise

Definition at line 573 of file catalog_mgr_rw.cc.

Referenced by publish::SyncMediator::AddHardlinkGroup().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::Balance ( )

inline

Definition at line 155 of file catalog_mgr_rw.h.

Referenced by publish::SyncMediator::Commit().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::CatalogDownloadCallback	(	const CatalogDownloadResult &	result,
		const CatalogDownloadContext	context
	)

private

Definition at line 1707 of file catalog_mgr_rw.cc.

Referenced by LoadCatalogs().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::CatalogHashSerializedCallback ( const CompressHashResult & result )

private

Definition at line 1700 of file catalog_mgr_rw.cc.

void catalog::WritableCatalogManager::CatalogUploadCallback	(	const upload::SpoolerResult &	result,
		const CatalogUploadContext	clg_upload_context
	)

private

Definition at line 1348 of file catalog_mgr_rw.cc.

Referenced by SnapshotCatalogs().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::CatalogUploadSerializedCallback	(	const upload::SpoolerResult &	result,
		const CatalogUploadContext	unused
	)

private

Definition at line 1513 of file catalog_mgr_rw.cc.

Referenced by SnapshotCatalogsSerialized().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::Clone	(	const std::string	destination,
		const std::string	source
	)

Clone the file called source changing its name into destination, the source file is keep intact. destination, the name of the new file, complete path source, the name of the file to clone, which must be already in the repository

Returns: void

Definition at line 291 of file catalog_mgr_rw.cc.

Referenced by publish::SyncMediator::Clone().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::CloneTree	(	const std::string &	from_dir,
		const std::string &	to_dir
	)

Copies an entire directory tree from the existing from_dir to the non-existing to_dir. The destination's parent directory must exist. On the catalog level, the new entries will be identical to the old ones except for their path hash fields.

Definition at line 333 of file catalog_mgr_rw.cc.

Referenced by publish::TransactionImpl().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::CloneTreeImpl	(	const PathString &	source_dir,
		const std::string &	dest_parent_dir,
		const NameString &	dest_name
	)

private

Called from CloneTree(), assumes that from_dir and to_dir are sufficiently sanitized

Definition at line 382 of file catalog_mgr_rw.cc.

Referenced by CloneTree().

Here is the call graph for this function:

Here is the caller graph for this function:

bool catalog::WritableCatalogManager::Commit	(	const bool	stop_for_tweaks,
		const uint64_t	manual_revision,
		manifest::Manifest *	manifest
	)

Definition at line 1131 of file catalog_mgr_rw.cc.

Referenced by publish::SyncMediator::Commit().

Here is the call graph for this function:

Here is the caller graph for this function:

bool catalog::WritableCatalogManager::CopyCatalogToLocalCache ( const upload::SpoolerResult & result )

private

Copy catalog to local cache.server Must be an atomic write into the cache_dir As such: create a temporary copy in cache_dir/txn and then do a rename (which is atomic) to the actual cache path

Returns: true on success, otherwise false

Definition at line 1325 of file catalog_mgr_rw.cc.

Referenced by CatalogUploadCallback(), and CatalogUploadSerializedCallback().

Here is the call graph for this function:

Here is the caller graph for this function:

Catalog * catalog::WritableCatalogManager::CreateCatalog	(	const PathString &	mountpoint,
		const shash::Any &	catalog_hash,
		Catalog *	parent_catalog
	)

protectedvirtual

This method is virtual in AbstractCatalogManager. It returns a new catalog structure in the form the different CatalogManagers need it. In this case it returns a stub for a WritableCatalog.

Parameters

mountpoint	the mount point of the catalog stub to create
catalog_hash	the content hash of the catalog to create
parent_catalog	the parent of the catalog stub to create

Returns: a pointer to the catalog stub structure created

Reimplemented from catalog::SimpleCatalogManager.

Definition at line 86 of file catalog_mgr_rw.cc.

Referenced by CatalogDownloadCallback(), CreateNestedCatalog(), and LoadCatalogs().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::CreateNestedCatalog ( const std::string & mountpoint )

Create a new nested catalog. Includes moving all entries belonging there from it's parent catalog.

Parameters

mountpoint the path of the directory to become a nested root

Returns: true on success, false otherwise

Definition at line 734 of file catalog_mgr_rw.cc.

Referenced by catalog::CatalogBalancer< CatalogMgrT >::AddCatalog(), CloneTreeImpl(), catalog::VirtualCatalog::CreateBaseDirectory(), publish::SyncMediator::CreateNestedCatalog(), and swissknife::IngestSQL::do_additions().

Here is the call graph for this function:

Here is the caller graph for this function:

manifest::Manifest * catalog::WritableCatalogManager::CreateRepository	(	const std::string &	dir_temp,
		const bool	volatile_content,
		const std::string &	voms_authz,
		upload::Spooler *	spooler
	)

static

This method is invoked if we create a completely new repository. The new root catalog will already contain a root entry. It is uploaded by a Forklift to the upstream storage.

Returns: true on success, false otherwise

Definition at line 105 of file catalog_mgr_rw.cc.

Referenced by publish::CreateRootObjects(), and swissknife::CommandCreate::Main().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::DoBalance ( )

private

Definition at line 1453 of file catalog_mgr_rw.cc.

Referenced by Balance().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::EnforceSqliteMemLimit ( )

inlineprotectedvirtual

Reimplemented from catalog::AbstractCatalogManager< Catalog >.

Definition at line 177 of file catalog_mgr_rw.h.

void catalog::WritableCatalogManager::FinalizeCatalog	(	WritableCatalog *	catalog,
		const bool	stop_for_tweaks
	)

private

Definition at line 1237 of file catalog_mgr_rw.cc.

Referenced by CatalogUploadCallback(), ScheduleReadyCatalogs(), SnapshotCatalogs(), and SnapshotCatalogsSerialized().

Here is the call graph for this function:

Here is the caller graph for this function:

bool catalog::WritableCatalogManager::FindCatalog	(	const std::string &	path,
		WritableCatalog **	result,
		DirectoryEntry *	dirent = `NULL`
	)

private

Retrieve the catalog containing the given path. Other than AbstractCatalogManager::FindCatalog() this mounts nested catalogs if necessary and returns WritableCatalog objects. Furthermore it optionally returns the looked-up DirectoryEntry.

Parameters

path	the path to look for
result	the retrieved catalog (as a pointer)
dirent	is set to looked up DirectoryEntry for 'path' if non-NULL

Returns: true if catalog was found

Definition at line 193 of file catalog_mgr_rw.cc.

Referenced by AddCatalogToQueue(), AddChunkedFile(), AddDirectory(), AddFile(), AddHardlinkGroup(), CreateNestedCatalog(), GetHostingCatalog(), GraftNestedCatalog(), IsTransitionPoint(), RemoveDirectory(), RemoveFile(), RemoveNestedCatalog(), ShrinkHardlinkGroup(), SwapNestedCatalog(), and TouchDirectory().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::FixWeight ( WritableCatalog * catalog )

private

Definition at line 1461 of file catalog_mgr_rw.cc.

Referenced by DoBalance().

Here is the call graph for this function:

Here is the caller graph for this function:

WritableCatalog * catalog::WritableCatalogManager::GetHostingCatalog ( const std::string & path )

Definition at line 219 of file catalog_mgr_rw.cc.

Referenced by catalog::VirtualCatalog::CreateBaseDirectory(), catalog::VirtualCatalog::GetSortedTagsFromCatalog(), catalog::VirtualCatalog::InsertSnapshot(), catalog::VirtualCatalog::Remove(), and catalog::VirtualCatalog::RemoveSnapshot().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::GetModifiedCatalogLeafs ( WritableCatalogList * result ) const

inlineprivate

Definition at line 225 of file catalog_mgr_rw.h.

Referenced by SnapshotCatalogs().

Here is the call graph for this function:

Here is the caller graph for this function:

bool catalog::WritableCatalogManager::GetModifiedCatalogLeafsRecursively	(	Catalog *	catalog,
		WritableCatalogList *	result
	)		const

private

Finds dirty catalogs that can be snapshot right away and annotates all the other catalogs with their number of dirty descendants. Note that there is a convenience wrapper to start the recursion: WritableCatalogManager::GetModifiedCatalogLeafs()

Parameters

catalog	the catalog for this recursion step
result	the result list to be appended to

Returns: true if 'catalog' is dirty

Definition at line 1425 of file catalog_mgr_rw.cc.

Referenced by GetModifiedCatalogLeafs(), and SwapNestedCatalog().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::GetModifiedCatalogs ( WritableCatalogList * result ) const

inlineprivate

Definition at line 244 of file catalog_mgr_rw.h.

Referenced by SnapshotCatalogsSerialized().

Here is the call graph for this function:

Here is the caller graph for this function:

int catalog::WritableCatalogManager::GetModifiedCatalogsRecursively	(	const Catalog *	catalog,
		WritableCatalogList *	result
	)		const

private

Definition at line 1485 of file catalog_mgr_rw.cc.

Referenced by GetModifiedCatalogs().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::GraftNestedCatalog	(	const string &	mountpoint,
		const shash::Any &	new_hash,
		const uint64_t	new_size
	)

Install a nested catalog (catalog hierarchy) at a new, empty directory

The mountpoint directory must not yet exist. Its parent directory, however must exist. This method combines functionality from AddDirectory(), CreateNestedCatalog() and SwapNestedCatalog(). The new nested catalog won't get attached.

Parameters

mountpoint	- the path where the nested catalog should be installed
new_hash	- the hash of the new nested catalog
new_size	- the size of the new nested catalog

Definition at line 1003 of file catalog_mgr_rw.cc.

Here is the call graph for this function:

bool catalog::WritableCatalogManager::IsBalanceable ( ) const

inline

Definition at line 143 of file catalog_mgr_rw.h.

Referenced by Balance(), and publish::SyncMediator::Commit().

Here is the caller graph for this function:

bool catalog::WritableCatalogManager::IsTransitionPoint ( const std::string & mountpoint )

Checks if a nested catalog starts at this path. The path must be valid.

Definition at line 1093 of file catalog_mgr_rw.cc.

Referenced by publish::SyncMediator::AddDirectory(), catalog::VirtualCatalog::EnsurePresence(), recursively_delete_directory(), publish::SyncMediator::RemoveDirectory(), and publish::SyncMediator::TouchDirectory().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::LoadCatalogs	(	const std::string &	base_path,
		const std::unordered_set< std::string > &	dirs
	)

Definition at line 1660 of file catalog_mgr_rw.cc.

Referenced by swissknife::IngestSQL::process_sqlite().

Here is the call graph for this function:

Here is the caller graph for this function:

bool catalog::WritableCatalogManager::LookupDirEntry	(	const std::string &	path,
		const LookupOptions	options,
		DirectoryEntry *	dirent
	)

Definition at line 1691 of file catalog_mgr_rw.cc.

Referenced by swissknife::IngestSQL::add_files(), swissknife::IngestSQL::add_symlinks(), swissknife::IngestSQL::do_additions(), and swissknife::IngestSQL::do_deletions().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::PrecalculateListings ( )

TODO

Definition at line 1109 of file catalog_mgr_rw.cc.

Referenced by publish::SyncMediator::Commit().

Here is the caller graph for this function:

void catalog::WritableCatalogManager::RemoveDirectory ( const std::string & path )

Remove the given directory from the catalogs.

Parameters

directory_path the full path to the directory to be removed

Returns: true on success, false otherwise

Definition at line 255 of file catalog_mgr_rw.cc.

Referenced by recursively_delete_directory(), catalog::VirtualCatalog::Remove(), publish::SyncMediator::RemoveDirectory(), catalog::VirtualCatalog::RemoveRecursively(), and catalog::VirtualCatalog::RemoveSnapshot().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::RemoveFile ( const std::string & path )

Remove the given file from the catalogs.

Parameters

file_path the full path to the file to be removed

Returns: true on success, false otherwise

Definition at line 234 of file catalog_mgr_rw.cc.

Referenced by swissknife::IngestSQL::add_files(), swissknife::IngestSQL::add_symlinks(), Clone(), swissknife::IngestSQL::do_deletions(), recursively_delete_directory(), publish::SyncMediator::RemoveFile(), and catalog::VirtualCatalog::RemoveRecursively().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::RemoveNestedCatalog	(	const std::string &	mountpoint,
		const bool	merge = `true`
	)

Remove a nested catalog

If the merged parameter is true, when you remove a nested catalog all entries currently held by it will be merged into its parent catalog.

Parameters

mountpoint	- the path of the nested catalog to be removed
merge	- merge the subtree associated with the nested catalog into its parent catalog

Returns: - true on success, false otherwise

Definition at line 834 of file catalog_mgr_rw.cc.

Referenced by FixWeight(), recursively_delete_directory(), catalog::VirtualCatalog::Remove(), and publish::SyncMediator::RemoveNestedCatalog().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::RemoveSingleCatalogUploadCallback ( )

Definition at line 1585 of file catalog_mgr_rw.cc.

Referenced by swissknife::IngestSQL::do_additions().

Here is the caller graph for this function:

void catalog::WritableCatalogManager::ScheduleCatalogProcessing ( WritableCatalog * catalog )

private

Definition at line 1307 of file catalog_mgr_rw.cc.

Referenced by CatalogUploadCallback(), ScheduleReadyCatalogs(), and SnapshotCatalogs().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::ScheduleReadyCatalogs ( )

Definition at line 1604 of file catalog_mgr_rw.cc.

Referenced by swissknife::IngestSQL::do_additions().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::SetTTL ( const uint64_t new_ttl )

Definition at line 1114 of file catalog_mgr_rw.cc.

Here is the call graph for this function:

void catalog::WritableCatalogManager::SetupSingleCatalogUploadCallback ( )

Definition at line 1580 of file catalog_mgr_rw.cc.

Referenced by swissknife::IngestSQL::do_additions().

Here is the call graph for this function:

Here is the caller graph for this function:

bool catalog::WritableCatalogManager::SetVOMSAuthz ( const std::string & voms_authz )

Definition at line 1121 of file catalog_mgr_rw.cc.

Here is the call graph for this function:

void catalog::WritableCatalogManager::ShrinkHardlinkGroup ( const std::string & remove_path )

Definition at line 652 of file catalog_mgr_rw.cc.

Referenced by publish::SyncMediator::RemoveFile().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::SingleCatalogUploadCallback ( const upload::SpoolerResult & result )

private

Definition at line 1621 of file catalog_mgr_rw.cc.

Referenced by SetupSingleCatalogUploadCallback().

Here is the call graph for this function:

Here is the caller graph for this function:

WritableCatalogManager::CatalogInfo catalog::WritableCatalogManager::SnapshotCatalogs ( const bool stop_for_tweaks )

private

Handles the snapshotting of dirty (i.e. modified) catalogs while trying to parallelize the compression and upload as much as possible. We use a parallel depth first post order tree traversal based on 'continuations'.

The idea is as follows:

find all leaf-catalogs (i.e. dirty catalogs with no dirty children) –> these can be processed and uploaded immediately and independently see WritableCatalogManager::GetModifiedCatalogLeafs()
annotate non-leaf catalogs with their number of dirty children –> a finished child will notify it's parent and decrement this number see WritableCatalogManager::CatalogUploadCallback()
if a non-leaf catalog's dirty children number reaches 0, it is scheduled for processing as well (continuation) –> the parallel processing walks bottom-up through the catalog tree see WritableCatalogManager::CatalogUploadCallback()
when the root catalog is reached, we notify the main thread and return –> done through a Future<> in WritableCatalogManager::SnapshotCatalogs

Note: The catalog finalisation (see WritableCatalogManager::FinalizeCatalog) happens in a worker thread (i.e. the callback method) for non-leaf catalogs.

TODO(rmeusel): since all leaf catalogs are finalized in the main thread, we sacrifice some potential concurrency for simplicity.

Definition at line 1204 of file catalog_mgr_rw.cc.

Referenced by Commit().

Here is the call graph for this function:

Here is the caller graph for this function:

WritableCatalogManager::CatalogInfo catalog::WritableCatalogManager::SnapshotCatalogsSerialized ( const bool stop_for_tweaks )

private

Definition at line 1529 of file catalog_mgr_rw.cc.

Referenced by Commit().

Here is the call graph for this function:

Here is the caller graph for this function:

void catalog::WritableCatalogManager::SwapNestedCatalog	(	const string &	mountpoint,
		const shash::Any &	new_hash,
		const uint64_t	new_size
	)

Swap in a new nested catalog

The old nested catalog must not have been already attached to the catalog tree. This method will not attach the new nested catalog to the catalog tree.

Parameters

mountpoint	- the path of the nested catalog to be removed
new_hash	- the hash of the new nested catalog
new_size	- the size of the new nested catalog

Definition at line 883 of file catalog_mgr_rw.cc.

Here is the call graph for this function:

void catalog::WritableCatalogManager::SyncLock ( )

inlineprivate

Definition at line 239 of file catalog_mgr_rw.h.

Referenced by AddCatalogToQueue(), AddChunkedFile(), AddDirectory(), AddFile(), AddHardlinkGroup(), CatalogUploadCallback(), CreateNestedCatalog(), FinalizeCatalog(), GraftNestedCatalog(), IsTransitionPoint(), LookupDirEntry(), RemoveDirectory(), RemoveFile(), RemoveNestedCatalog(), SetTTL(), SetVOMSAuthz(), ShrinkHardlinkGroup(), SingleCatalogUploadCallback(), SwapNestedCatalog(), and TouchDirectory().

Here is the caller graph for this function:

void catalog::WritableCatalogManager::SyncUnlock ( )

inlineprivate

Definition at line 240 of file catalog_mgr_rw.h.

Referenced by AddCatalogToQueue(), AddChunkedFile(), AddDirectory(), AddFile(), AddHardlinkGroup(), CatalogUploadCallback(), CreateNestedCatalog(), FinalizeCatalog(), GraftNestedCatalog(), IsTransitionPoint(), LookupDirEntry(), RemoveDirectory(), RemoveFile(), RemoveNestedCatalog(), SetTTL(), SetVOMSAuthz(), ShrinkHardlinkGroup(), SingleCatalogUploadCallback(), SwapNestedCatalog(), and TouchDirectory().

Here is the caller graph for this function:

void catalog::WritableCatalogManager::TouchDirectory	(	const DirectoryEntryBase &	entry,
		const XattrList &	xattrs,
		const std::string &	directory_path
	)

Update entry meta data (mode, owner, ...). CVMFS specific meta data (i.e. nested catalog transition points) are NOT changed by this method, although transition points intrinsics are taken into account, to keep nested catalogs consistent.

Parameters

entry	the directory entry to be touched
path	the path of the directory entry to be touched

Definition at line 676 of file catalog_mgr_rw.cc.

Referenced by swissknife::IngestSQL::do_additions(), and publish::SyncMediator::TouchDirectory().

Here is the call graph for this function:

Here is the caller graph for this function:

Friends And Related Function Documentation

friend class CatalogBalancer< WritableCatalogManager >

friend

Definition at line 76 of file catalog_mgr_rw.h.

friend class VirtualCatalog

friend

Definition at line 79 of file catalog_mgr_rw.h.

Member Data Documentation

const unsigned catalog::WritableCatalogManager::balance_weight_

private

Defines the threshold that will be used to balance a catalog that has been overflowed. Its value should be lesser than max_weight_ and greater than min_weight. By default it is set to max_weight / 2.

Definition at line 315 of file catalog_mgr_rw.h.

Referenced by catalog::CatalogBalancer< CatalogMgrT >::PartitionOptimally().

pthread_mutex_t* catalog::WritableCatalogManager::catalog_download_lock_

private

Definition at line 272 of file catalog_mgr_rw.h.

Referenced by CatalogDownloadCallback(), and LoadCatalogs().

std::unordered_map<std::string, Catalog*> catalog::WritableCatalogManager::catalog_download_map_

private

Definition at line 273 of file catalog_mgr_rw.h.

Referenced by CatalogDownloadCallback(), and LoadCatalogs().

CatalogDownloadPipeline* catalog::WritableCatalogManager::catalog_download_pipeline_

private

Definition at line 271 of file catalog_mgr_rw.h.

Referenced by CatalogDownloadCallback(), and LoadCatalogs().

pthread_mutex_t* catalog::WritableCatalogManager::catalog_hash_lock_

private

Definition at line 274 of file catalog_mgr_rw.h.

Referenced by CatalogHashSerializedCallback().

std::map<std::string, shash::Any> catalog::WritableCatalogManager::catalog_hash_map_

private

Definition at line 275 of file catalog_mgr_rw.h.

Referenced by CatalogHashSerializedCallback().

pthread_mutex_t* catalog::WritableCatalogManager::catalog_processing_lock_

private

Definition at line 266 of file catalog_mgr_rw.h.

Referenced by CatalogUploadCallback(), ScheduleCatalogProcessing(), SingleCatalogUploadCallback(), WritableCatalogManager(), and ~WritableCatalogManager().

std::map<std::string, WritableCatalog *> catalog::WritableCatalogManager::catalog_processing_map_

private

Definition at line 267 of file catalog_mgr_rw.h.

Referenced by CatalogUploadCallback(), ScheduleCatalogProcessing(), and SingleCatalogUploadCallback().

XattrList catalog::WritableCatalogManager::empty_xattrs

private

Directories don't have extended attributes at this point.

Definition at line 286 of file catalog_mgr_rw.h.

Referenced by Clone().

bool catalog::WritableCatalogManager::enforce_limits_

private

Definition at line 278 of file catalog_mgr_rw.h.

Referenced by AddFile(), AddHardlinkGroup(), and FinalizeCatalog().

unsigned catalog::WritableCatalogManager::file_mbyte_limit_

private

Definition at line 281 of file catalog_mgr_rw.h.

Referenced by AddFile(), and AddHardlinkGroup().

const bool catalog::WritableCatalogManager::is_balanceable_

private

It indicates whether this catalog manager supports balancing operations

Definition at line 291 of file catalog_mgr_rw.h.

Referenced by IsBalanceable().

const std::string catalog::WritableCatalogManager::kCatalogFilename

staticprivate

Definition at line 260 of file catalog_mgr_rw.h.

const unsigned catalog::WritableCatalogManager::max_weight_

private

Defines the maximum weight an autogenerated catalog can have. If after a publishing operation the catalog's weight is greater than this threshold it will be considered overflowed and will automatically be split in different sub-catalogs.

Definition at line 299 of file catalog_mgr_rw.h.

Referenced by FixWeight().

const unsigned catalog::WritableCatalogManager::min_weight_

private

Defines the minimum weight an autogenerated catalog can have. If after a publishing operation the catalog's weight is lesser than this threshold it will be considered underflowed and will automatically be merged with its parent. This last operation can provoke an overflow in the parent, though.

Definition at line 308 of file catalog_mgr_rw.h.

Referenced by FixWeight(), and catalog::CatalogBalancer< CatalogMgrT >::PartitionOptimally().

unsigned catalog::WritableCatalogManager::nested_kcatalog_limit_

private

Definition at line 279 of file catalog_mgr_rw.h.

Referenced by FinalizeCatalog().

std::list<WritableCatalog*> catalog::WritableCatalogManager::pending_catalogs_

private

Definition at line 270 of file catalog_mgr_rw.h.

Referenced by AddCatalogToQueue(), RemoveSingleCatalogUploadCallback(), and ScheduleReadyCatalogs().

unsigned catalog::WritableCatalogManager::root_kcatalog_limit_

private

Definition at line 280 of file catalog_mgr_rw.h.

Referenced by FinalizeCatalog().

upload::Spooler* catalog::WritableCatalogManager::spooler_

private

Definition at line 264 of file catalog_mgr_rw.h.

Referenced by Commit(), CreateNestedCatalog(), catalog::VirtualCatalog::CreateNestedCatalogMarker(), catalog::CatalogBalancer< CatalogMgrT >::MakeEmptyDirectoryEntryBase(), RemoveSingleCatalogUploadCallback(), ScheduleCatalogProcessing(), SetupSingleCatalogUploadCallback(), SnapshotCatalogs(), and SnapshotCatalogsSerialized().

pthread_mutex_t* catalog::WritableCatalogManager::sync_lock_

private

Definition at line 263 of file catalog_mgr_rw.h.

Referenced by SyncLock(), SyncUnlock(), WritableCatalogManager(), and ~WritableCatalogManager().

The documentation for this class was generated from the following files:

/home/sftnight/jenkins/workspace/CvmfsDoxygen/label/docker-x86_64/cvmfs/cvmfs/catalog_mgr_rw.h
/home/sftnight/jenkins/workspace/CvmfsDoxygen/label/docker-x86_64/cvmfs/cvmfs/catalog_mgr_rw.cc

Classes

Public Member Functions

Static Public Member Functions

Protected Member Functions

Private Member Functions

Private Attributes

Static Private Attributes

Friends

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Friends And Related Function Documentation

Member Data Documentation