#include <catalog_traversal.h>

Inheritance diagram for swissknife::CatalogTraversal< ObjectFetcherT >:

Collaboration diagram for swissknife::CatalogTraversal< ObjectFetcherT >:

Classes
struct	TraversalContext

Public Types
typedef CatalogTraversalBase < ObjectFetcherT >	Base

typedef ObjectFetcherT	ObjectFetcherTN

typedef ObjectFetcherT::CatalogTN	CatalogTN

typedef ObjectFetcherT::HistoryTN	HistoryTN

typedef CatalogTraversalData < CatalogTN >	CallbackDataTN

typedef Base::Parameters	Parameters

typedef Base::TraversalType	TraversalType

typedef Base::CatalogJob	CatalogJob

Public Types inherited from swissknife::CatalogTraversalBase< ObjectFetcherT >
enum	TraversalType { kBreadthFirst, kDepthFirst }

typedef ObjectFetcherT	ObjectFetcherTN

typedef ObjectFetcherT::CatalogTN	CatalogTN

typedef ObjectFetcherT::HistoryTN	HistoryTN

typedef CatalogTraversalData < CatalogTN >	CallbackDataTN

Public Types inherited from Observable< CatalogTraversalData< ObjectFetcherT::CatalogTN > >
typedef Callbackable < CatalogTraversalData < ObjectFetcherT::CatalogTN > >::CallbackTN *	CallbackPtr

Public Types inherited from Callbackable< CatalogTraversalData< ObjectFetcherT::CatalogTN > >
typedef CallbackBase < CatalogTraversalData < ObjectFetcherT::CatalogTN > >	CallbackTN

Public Member Functions
	CatalogTraversal (const Parameters &params)

bool	Traverse (const TraversalType type=Base::kBreadthFirst)

bool	Traverse (const shash::Any &root_catalog_hash, const TraversalType type=Base::kBreadthFirst)

bool	TraverseList (const std::vector< shash::Any > &catalog_list, const TraversalType type=Base::kBreadthFirst)

bool	TraverseRevision (const shash::Any &root_catalog_hash, const TraversalType type=Base::kBreadthFirst)

Public Member Functions inherited from swissknife::CatalogTraversalBase< ObjectFetcherT >
	CatalogTraversalBase (const Parameters &params)

virtual bool	TraverseNamedSnapshots (const TraversalType type=kBreadthFirst)

void	SetCatalogInfoShim (CatalogTraversalInfoShim< CatalogTN > *shim)

Public Member Functions inherited from Observable< CatalogTraversalData< ObjectFetcherT::CatalogTN > >
virtual	~Observable ()

CallbackPtr	RegisterListener (typename BoundClosure< CatalogTraversalData< ObjectFetcherT::CatalogTN >, DelegateT, ClosureDataT >::CallbackMethod method, DelegateT *delegate, ClosureDataT data)

CallbackPtr	RegisterListener (typename BoundCallback< CatalogTraversalData< ObjectFetcherT::CatalogTN >, DelegateT >::CallbackMethod method, DelegateT *delegate)

CallbackPtr	RegisterListener (typename Callback< CatalogTraversalData< ObjectFetcherT::CatalogTN > >::CallbackFunction fn)

void	UnregisterListener (CallbackPtr callback_object)

void	UnregisterListeners ()

Protected Types
typedef std::set< shash::Any >	HashSet

typedef std::stack< CatalogJob >	CatalogJobStack

Protected Types inherited from swissknife::CatalogTraversalBase< ObjectFetcherT >
typedef std::set< shash::Any >	HashSet

Protected Types inherited from Observable< CatalogTraversalData< ObjectFetcherT::CatalogTN > >
typedef std::set< CallbackPtr >	Callbacks

Protected Member Functions
bool	DoTraverse (TraversalContext *ctx)

void	PushReferencedCatalogs (CatalogJob job, TraversalContext ctx)

unsigned int	PushPreviousRevision (const CatalogJob &job, TraversalContext *ctx)

unsigned int	PushNestedCatalogs (const CatalogJob &job, TraversalContext *ctx)

void	Push (const shash::Any &root_catalog_hash, TraversalContext *ctx)

bool	YieldToListeners (CatalogJob job, TraversalContext ctx)

bool	ShouldBeSkipped (const CatalogJob &job)

void	MarkAsVisited (const CatalogJob &job)

Protected Member Functions inherited from swissknife::CatalogTraversalBase< ObjectFetcherT >
bool	PrepareCatalog (CatalogJob *job)

bool	ReopenCatalog (CatalogJob *job)

bool	CloseCatalog (const bool unlink_db, CatalogJob *job)

shash::Any	GetRepositoryRootCatalogHash ()

bool	IsBelowPruningThresholds (const CatalogJob &job, const uint64_t history_depth, const time_t timestamp_threshold)

Protected Member Functions inherited from Observable< CatalogTraversalData< ObjectFetcherT::CatalogTN > >
void	RegisterListener (CallbackPtr callback_object)

	Observable ()

void	NotifyListeners (const CatalogTraversalData< ObjectFetcherT::CatalogTN > &parameter)

Protected Attributes
HashSet	visited_catalogs_

Protected Attributes inherited from swissknife::CatalogTraversalBase< ObjectFetcherT >
ObjectFetcherT *	object_fetcher_

CatalogTraversalInfoShim < CatalogTN >	catalog_info_default_shim_

CatalogTraversalInfoShim < CatalogTN > *	catalog_info_shim_

const uint64_t	default_history_depth_

const time_t	default_timestamp_threshold_

const bool	no_close_

const bool	ignore_load_failure_

const bool	no_repeat_history_

LogFacilities	error_sink_

Private Member Functions
bool	Yield (CatalogJob *job)

void	PostponeYield (CatalogJob job, TraversalContext ctx)

bool	HandlePostponedYields (const CatalogJob &job, TraversalContext *ctx)

void	Push (const CatalogJob &job, TraversalContext *ctx)

CatalogJob	Pop (TraversalContext *ctx)

Additional Inherited Members
Static Public Member Functions inherited from Callbackable< CatalogTraversalData< ObjectFetcherT::CatalogTN > >
static CallbackTN *	MakeClosure (typename BoundClosure< CatalogTraversalData< ObjectFetcherT::CatalogTN >, DelegateT, ClosureDataT >::CallbackMethod method, DelegateT *delegate, const ClosureDataT &closure_data)

static CallbackTN *	MakeCallback (typename BoundCallback< CatalogTraversalData< ObjectFetcherT::CatalogTN >, DelegateT >::CallbackMethod method, DelegateT *delegate)

static CallbackTN *	MakeCallback (typename Callback< CatalogTraversalData< ObjectFetcherT::CatalogTN > >::CallbackFunction function)

Detailed Description

template<class ObjectFetcherT>
class swissknife::CatalogTraversal< ObjectFetcherT >

This class traverses the catalog hierarchy of a CVMFS repository recursively. Also historic catalog trees can be traversed. The user needs to specify a callback which is called for each catalog on the way.

CatalogTraversal<> can be configured and used in various ways: -> Historic catalog traversal -> Prune catalogs below a certain history level -> Prune catalogs older than a certain threshold timestamp -> Never traverse a certain catalog twice -> Breadth First Traversal or Depth First Traversal -> Optional catalog memory management (no_close) -> Use all Named Snapshots of a repository as traversal entry point -> Traverse starting from a provided catalog -> Traverse catalogs that were previously skipped -> Produce various flavours of catalogs (writable, mocked, ...)

Breadth First Traversal Strategy Catalogs are handed out to the user identical as they are traversed. Say: From top to bottom. When you would simply print each received catalog the result would be a nice representation of the catalog tree. This method is more efficient, because catalogs are opened, processed and thrown away directly afterwards.

Depth First Traversal Strategy The user gets the catalog tree starting from the leaf nodes. Say: From bottom to top. A user can assume that he got all children or historical ancestors of a catalog before. This method climbs down the full catalog tree and hands it out 'in reverse order'. Thus, catalogs on the way are opened, checked for their descendants and closed. Once all children and historical ancestors are processed, it is re-opened and handed out to the user. Note: This method needs more disk space to temporarily store downloaded but not yet processed catalogs.

Note: Since all CVMFS catalog files together can grow to several gigabytes in file size, each catalog is loaded, processed and removed immediately afterwards. Except if no_close is specified, which allows the user to choose when a catalog should be closed. Keep in mind, that a user is responsible for both deletion of the delivered catalog objects as well as unlinking of the catalog database file.

Parameters

ObjectFetcherT Strategy Pattern implementation that defines how to retrieve catalogs from various backend storage types. Furthermore the ObjectFetcherT::CatalogTN is the type of catalog to be instantiated by CatalogTraversal<>.

CAUTION: the CatalogTN* pointer passed into the callback becomes invalid directly after the callback method returns, unless you create the CatalogTraversal object with no_close = true.

Definition at line 495 of file catalog_traversal.h.

Member Typedef Documentation

template<class ObjectFetcherT>

typedef CatalogTraversalBase<ObjectFetcherT> swissknife::CatalogTraversal< ObjectFetcherT >::Base

Definition at line 499 of file catalog_traversal.h.

template<class ObjectFetcherT>

typedef CatalogTraversalData<CatalogTN> swissknife::CatalogTraversal< ObjectFetcherT >::CallbackDataTN

Definition at line 503 of file catalog_traversal.h.

template<class ObjectFetcherT>

typedef Base::CatalogJob swissknife::CatalogTraversal< ObjectFetcherT >::CatalogJob

Definition at line 506 of file catalog_traversal.h.

template<class ObjectFetcherT>

typedef std::stack<CatalogJob> swissknife::CatalogTraversal< ObjectFetcherT >::CatalogJobStack

protected

Definition at line 510 of file catalog_traversal.h.

template<class ObjectFetcherT>

typedef ObjectFetcherT::CatalogTN swissknife::CatalogTraversal< ObjectFetcherT >::CatalogTN

Definition at line 501 of file catalog_traversal.h.

template<class ObjectFetcherT>

typedef std::set<shash::Any> swissknife::CatalogTraversal< ObjectFetcherT >::HashSet

protected

Definition at line 509 of file catalog_traversal.h.

template<class ObjectFetcherT>

typedef ObjectFetcherT::HistoryTN swissknife::CatalogTraversal< ObjectFetcherT >::HistoryTN

Definition at line 502 of file catalog_traversal.h.

template<class ObjectFetcherT>

typedef ObjectFetcherT swissknife::CatalogTraversal< ObjectFetcherT >::ObjectFetcherTN

Definition at line 500 of file catalog_traversal.h.

template<class ObjectFetcherT>

typedef Base::Parameters swissknife::CatalogTraversal< ObjectFetcherT >::Parameters

Definition at line 504 of file catalog_traversal.h.

template<class ObjectFetcherT>

typedef Base::TraversalType swissknife::CatalogTraversal< ObjectFetcherT >::TraversalType

Definition at line 505 of file catalog_traversal.h.

Constructor & Destructor Documentation

template<class ObjectFetcherT>

swissknife::CatalogTraversal< ObjectFetcherT >::CatalogTraversal ( const Parameters & params )

inlineexplicit

Constructs a new catalog traversal engine based on the construction parameters described in struct ConstructionParams.

Definition at line 542 of file catalog_traversal.h.

Member Function Documentation

template<class ObjectFetcherT>

bool swissknife::CatalogTraversal< ObjectFetcherT >::DoTraverse ( TraversalContext * ctx )

inlineprotected

This controls the actual traversal. Using a stack to traverse down the catalog hierarchy. This method implements the traversal itself, but not in which way catalogs are handed out to the user code.

Each catalog is processed in these steps: 1.) Pop the next catalog from the stack Catalogs are always traversed from latest to oldest revision and from root to leaf nested catalogs 2.) Prepare the catalog for traversing 2.1.) Check if it was visited before 2.2.) Fetch the catalog database from the backend storage This might fail and produce an error. For root catalogs this error can be ignored (might be garbage collected before) 2.3.) Open the catalog database 2.4.) Check if the catalog is older than the timestamp threshold After these steps the catalog is opened either opened and ready for the traversal to continue, or it was marked for ignore (job.ignore) 3.) Check if the catalog is marked to be ignored Catalog might not be loadable (sweeped root catalog) or is too old Note: ignored catalogs can still trigger postponed yields 4.) Mark the catalog as visited to be able to skip it later on 5.) Find and push referencing catalogs This pushes all descendents of the current catalog onto the stack. Note that this is dependent on the strategy (depth or breadth first) and on the history threshold (see history_depth). 6.) Hand the catalog out to the user code Depending on the traversal strategy (depth of breadths first) this might immediately yield zero to N catalogs to the user code.

Note: If anything unexpected goes wrong during the traversal process, it is aborted immediately.

Parameters

ctx	the traversal context that steers the whole traversal process

Returns: true on successful traversal and false on abort

Definition at line 625 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::Traverse(), and swissknife::CatalogTraversal< ObjectFetcherT >::TraverseRevision().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>

bool swissknife::CatalogTraversal< ObjectFetcherT >::HandlePostponedYields	(	const CatalogJob &	job,
		TraversalContext *	ctx
	)

inlineprivate

Determines if there are postponed yields that can be set free based on the catalog currently being yielded

Note: the CatalogJob being handed into this method does not necessarily have an open Catalog attached to it.

Parameters

ctx	the TraversalContext
job	the catalog job that was just yielded

Returns: true on successful execution

Definition at line 849 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::DoTraverse(), and swissknife::CatalogTraversal< ObjectFetcherT >::YieldToListeners().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>

void swissknife::CatalogTraversal< ObjectFetcherT >::MarkAsVisited ( const CatalogJob & job )

inlineprotected

Definition at line 783 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::DoTraverse().

Here is the caller graph for this function:

template<class ObjectFetcherT>

CatalogJob swissknife::CatalogTraversal< ObjectFetcherT >::Pop ( TraversalContext * ctx )

inlineprivate

Definition at line 881 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::DoTraverse().

Here is the caller graph for this function:

template<class ObjectFetcherT>

void swissknife::CatalogTraversal< ObjectFetcherT >::PostponeYield	(	CatalogJob *	job,
		TraversalContext *	ctx
	)

inlineprivate

Pushes a catalog to the callback_stack for later yielding Note: this is only used for the Depth First Traversal strategy!

Definition at line 827 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::YieldToListeners().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>

void swissknife::CatalogTraversal< ObjectFetcherT >::Push	(	const shash::Any &	root_catalog_hash,
		TraversalContext *	ctx
	)

inlineprotected

Definition at line 744 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::PushNestedCatalogs(), swissknife::CatalogTraversal< ObjectFetcherT >::PushPreviousRevision(), swissknife::CatalogTraversal< ObjectFetcherT >::Traverse(), and swissknife::CatalogTraversal< ObjectFetcherT >::TraverseRevision().

Here is the caller graph for this function:

template<class ObjectFetcherT>

void swissknife::CatalogTraversal< ObjectFetcherT >::Push	(	const CatalogJob &	job,
		TraversalContext *	ctx
	)

inlineprivate

Definition at line 877 of file catalog_traversal.h.

template<class ObjectFetcherT>

unsigned int swissknife::CatalogTraversal< ObjectFetcherT >::PushNestedCatalogs	(	const CatalogJob &	job,
		TraversalContext *	ctx
	)

inlineprotected

Pushes all the referenced nested catalogs.

Returns: the number of catalogs pushed on the processing stack

Definition at line 723 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::PushReferencedCatalogs().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>

unsigned int swissknife::CatalogTraversal< ObjectFetcherT >::PushPreviousRevision	(	const CatalogJob &	job,
		TraversalContext *	ctx
	)

inlineprotected

Pushes the previous revision of a (root) catalog.

Returns: the number of catalogs pushed on the processing stack

Definition at line 692 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::PushReferencedCatalogs().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>

void swissknife::CatalogTraversal< ObjectFetcherT >::PushReferencedCatalogs	(	CatalogJob *	job,
		TraversalContext *	ctx
	)

inlineprotected

Definition at line 665 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::DoTraverse().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>

bool swissknife::CatalogTraversal< ObjectFetcherT >::ShouldBeSkipped ( const CatalogJob & job )

inlineprotected

Checks the traversal history if the given catalog was traversed or at least seen before. If 'no_repeat_history' is not set this is always 'false'.

Parameters

job	the job to be checked against the traversal history

Returns: true if the specified catalog was hit before

Definition at line 779 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::DoTraverse().

Here is the caller graph for this function:

template<class ObjectFetcherT>

bool swissknife::CatalogTraversal< ObjectFetcherT >::Traverse ( const TraversalType type = Base::kBreadthFirst )

inlinevirtual

Starts the traversal process. After calling this methods CatalogTraversal will go through all catalogs and call the registered callback methods for each found catalog. If something goes wrong in the process, the traversal will be cancelled.

Parameters

type	breadths or depth first traversal

Returns: true, when all catalogs were successfully processed. On failure the traversal is cancelled and false is returned.

Implements swissknife::CatalogTraversalBase< ObjectFetcherT >.

Definition at line 546 of file catalog_traversal.h.

Referenced by swissknife::CommandMigrate::LoadCatalogs(), swissknife::CommandListCatalogs::Run(), and swissknife::CommandFileStats::Run().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>

bool swissknife::CatalogTraversal< ObjectFetcherT >::Traverse	(	const shash::Any &	root_catalog_hash,
		const TraversalType	type = `Base::kBreadthFirst`
	)

inlinevirtual

Starts the traversal process at the catalog pointed to by the given hash

Parameters

root_catalog_hash	the entry point into the catalog traversal
type	breadths or depth first traversal

Returns: true when catalogs were successfully traversed

Implements swissknife::CatalogTraversalBase< ObjectFetcherT >.

Definition at line 554 of file catalog_traversal.h.

Here is the call graph for this function:

template<class ObjectFetcherT>

bool swissknife::CatalogTraversal< ObjectFetcherT >::TraverseList	(	const std::vector< shash::Any > &	catalog_list,
		const TraversalType	type = `Base::kBreadthFirst`
	)

inlinevirtual

Traverse a list of revisions represented by root catalogs from first to last. DO NOT traverse previous revisions based on history and timestamp threshold settings.

Parameters

catalog_list	list of root catalog hashes
type	breadth- or depth- first traversal

Returns: true on success

Implements swissknife::CatalogTraversalBase< ObjectFetcherT >.

Definition at line 565 of file catalog_traversal.h.

Here is the call graph for this function:

template<class ObjectFetcherT>

bool swissknife::CatalogTraversal< ObjectFetcherT >::TraverseRevision	(	const shash::Any &	root_catalog_hash,
		const TraversalType	type = `Base::kBreadthFirst`
	)

inlinevirtual

Starts the traversal process at the catalog pointed to by the given hash but doesn't traverse into predecessor catalog revisions. This overrides the TraversalParameter settings provided at construction.

Parameters

root_catalog_hash	the entry point into the catalog traversal
type	breadths or depth first traversal

Returns: true when catalogs were successfully traversed

Implements swissknife::CatalogTraversalBase< ObjectFetcherT >.

Definition at line 576 of file catalog_traversal.h.

Referenced by swissknife::CommandListReflog::Run(), and swissknife::CatalogTraversal< ObjectFetcherT >::TraverseList().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>

bool swissknife::CatalogTraversal< ObjectFetcherT >::Yield ( CatalogJob * job )

inlineprivate

This actually hands out a catalog to the user code It is not called by DoTraversa() directly but by wrapper functions in order to provide higher level yielding behaviour.

Definition at line 795 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::HandlePostponedYields(), and swissknife::CatalogTraversal< ObjectFetcherT >::YieldToListeners().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>

bool swissknife::CatalogTraversal< ObjectFetcherT >::YieldToListeners	(	CatalogJob *	job,
		TraversalContext *	ctx
	)

inlineprotected

Definition at line 748 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::DoTraverse().

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

template<class ObjectFetcherT>

HashSet swissknife::CatalogTraversal< ObjectFetcherT >::visited_catalogs_

protected

Definition at line 888 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::MarkAsVisited(), and swissknife::CatalogTraversal< ObjectFetcherT >::ShouldBeSkipped().

The documentation for this class was generated from the following file:

/home/sftnight/jenkins/workspace/CvmfsDoxygen/label/docker-x86_64/cvmfs/cvmfs/catalog_traversal.h

Classes

Public Types

Public Member Functions

Protected Types

Protected Member Functions

Protected Attributes

Private Member Functions

Additional Inherited Members

Detailed Description

template<class ObjectFetcherT> class swissknife::CatalogTraversal< ObjectFetcherT >

Member Typedef Documentation

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation

template<class ObjectFetcherT>
class swissknife::CatalogTraversal< ObjectFetcherT >