CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
swissknife::CatalogTraversal< ObjectFetcherT > Class Template Reference

#include <catalog_traversal.h>

Inheritance diagram for swissknife::CatalogTraversal< ObjectFetcherT >:
Collaboration diagram for swissknife::CatalogTraversal< ObjectFetcherT >:

Classes

struct  TraversalContext
 

Public Types

typedef CatalogTraversalBase
< ObjectFetcherT > 
Base
 
typedef ObjectFetcherT ObjectFetcherTN
 
typedef ObjectFetcherT::CatalogTN CatalogTN
 
typedef ObjectFetcherT::HistoryTN HistoryTN
 
typedef CatalogTraversalData
< CatalogTN
CallbackDataTN
 
typedef Base::Parameters Parameters
 
typedef Base::TraversalType TraversalType
 
typedef Base::CatalogJob CatalogJob
 
- Public Types inherited from swissknife::CatalogTraversalBase< ObjectFetcherT >
enum  TraversalType { kBreadthFirst, kDepthFirst }
 
typedef ObjectFetcherT ObjectFetcherTN
 
typedef ObjectFetcherT::CatalogTN CatalogTN
 
typedef ObjectFetcherT::HistoryTN HistoryTN
 
typedef CatalogTraversalData
< CatalogTN
CallbackDataTN
 
- Public Types inherited from Observable< CatalogTraversalData< ObjectFetcherT::CatalogTN > >
typedef Callbackable
< CatalogTraversalData
< ObjectFetcherT::CatalogTN >
>::CallbackTN
CallbackPtr
 
- Public Types inherited from Callbackable< CatalogTraversalData< ObjectFetcherT::CatalogTN > >
typedef CallbackBase
< CatalogTraversalData
< ObjectFetcherT::CatalogTN > > 
CallbackTN
 

Public Member Functions

 CatalogTraversal (const Parameters &params)
 
bool Traverse (const TraversalType type=Base::kBreadthFirst)
 
bool Traverse (const shash::Any &root_catalog_hash, const TraversalType type=Base::kBreadthFirst)
 
bool TraverseList (const std::vector< shash::Any > &catalog_list, const TraversalType type=Base::kBreadthFirst)
 
bool TraverseRevision (const shash::Any &root_catalog_hash, const TraversalType type=Base::kBreadthFirst)
 
- Public Member Functions inherited from swissknife::CatalogTraversalBase< ObjectFetcherT >
 CatalogTraversalBase (const Parameters &params)
 
virtual bool TraverseNamedSnapshots (const TraversalType type=kBreadthFirst)
 
void SetCatalogInfoShim (CatalogTraversalInfoShim< CatalogTN > *shim)
 
- Public Member Functions inherited from Observable< CatalogTraversalData< ObjectFetcherT::CatalogTN > >
virtual ~Observable ()
 
CallbackPtr RegisterListener (typename BoundClosure< CatalogTraversalData< ObjectFetcherT::CatalogTN >, DelegateT, ClosureDataT >::CallbackMethod method, DelegateT *delegate, ClosureDataT data)
 
CallbackPtr RegisterListener (typename BoundCallback< CatalogTraversalData< ObjectFetcherT::CatalogTN >, DelegateT >::CallbackMethod method, DelegateT *delegate)
 
CallbackPtr RegisterListener (typename Callback< CatalogTraversalData< ObjectFetcherT::CatalogTN > >::CallbackFunction fn)
 
void UnregisterListener (CallbackPtr callback_object)
 
void UnregisterListeners ()
 

Protected Types

typedef std::set< shash::AnyHashSet
 
typedef std::stack< CatalogJobCatalogJobStack
 
- Protected Types inherited from swissknife::CatalogTraversalBase< ObjectFetcherT >
typedef std::set< shash::AnyHashSet
 
- Protected Types inherited from Observable< CatalogTraversalData< ObjectFetcherT::CatalogTN > >
typedef std::set< CallbackPtrCallbacks
 

Protected Member Functions

bool DoTraverse (TraversalContext *ctx)
 
void PushReferencedCatalogs (CatalogJob *job, TraversalContext *ctx)
 
unsigned int PushPreviousRevision (const CatalogJob &job, TraversalContext *ctx)
 
unsigned int PushNestedCatalogs (const CatalogJob &job, TraversalContext *ctx)
 
void Push (const shash::Any &root_catalog_hash, TraversalContext *ctx)
 
bool YieldToListeners (CatalogJob *job, TraversalContext *ctx)
 
bool ShouldBeSkipped (const CatalogJob &job)
 
void MarkAsVisited (const CatalogJob &job)
 
- Protected Member Functions inherited from swissknife::CatalogTraversalBase< ObjectFetcherT >
bool PrepareCatalog (CatalogJob *job)
 
bool ReopenCatalog (CatalogJob *job)
 
bool CloseCatalog (const bool unlink_db, CatalogJob *job)
 
shash::Any GetRepositoryRootCatalogHash ()
 
bool IsBelowPruningThresholds (const CatalogJob &job, const unsigned history_depth, const time_t timestamp_threshold)
 
- Protected Member Functions inherited from Observable< CatalogTraversalData< ObjectFetcherT::CatalogTN > >
void RegisterListener (CallbackPtr callback_object)
 
 Observable ()
 
void NotifyListeners (const CatalogTraversalData< ObjectFetcherT::CatalogTN > &parameter)
 

Protected Attributes

HashSet visited_catalogs_
 
- Protected Attributes inherited from swissknife::CatalogTraversalBase< ObjectFetcherT >
ObjectFetcherT * object_fetcher_
 
CatalogTraversalInfoShim
< CatalogTN
catalog_info_default_shim_
 
CatalogTraversalInfoShim
< CatalogTN > * 
catalog_info_shim_
 
const unsigned int default_history_depth_
 
const time_t default_timestamp_threshold_
 
const bool no_close_
 
const bool ignore_load_failure_
 
const bool no_repeat_history_
 
LogFacilities error_sink_
 

Private Member Functions

bool Yield (CatalogJob *job)
 
void PostponeYield (CatalogJob *job, TraversalContext *ctx)
 
bool HandlePostponedYields (const CatalogJob &job, TraversalContext *ctx)
 
void Push (const CatalogJob &job, TraversalContext *ctx)
 
CatalogJob Pop (TraversalContext *ctx)
 

Additional Inherited Members

- Static Public Member Functions inherited from Callbackable< CatalogTraversalData< ObjectFetcherT::CatalogTN > >
static CallbackTNMakeClosure (typename BoundClosure< CatalogTraversalData< ObjectFetcherT::CatalogTN >, DelegateT, ClosureDataT >::CallbackMethod method, DelegateT *delegate, const ClosureDataT &closure_data)
 
static CallbackTNMakeCallback (typename BoundCallback< CatalogTraversalData< ObjectFetcherT::CatalogTN >, DelegateT >::CallbackMethod method, DelegateT *delegate)
 
static CallbackTNMakeCallback (typename Callback< CatalogTraversalData< ObjectFetcherT::CatalogTN > >::CallbackFunction function)
 

Detailed Description

template<class ObjectFetcherT>
class swissknife::CatalogTraversal< ObjectFetcherT >

This class traverses the catalog hierarchy of a CVMFS repository recursively. Also historic catalog trees can be traversed. The user needs to specify a callback which is called for each catalog on the way.

CatalogTraversal<> can be configured and used in various ways: -> Historic catalog traversal -> Prune catalogs below a certain history level -> Prune catalogs older than a certain threshold timestamp -> Never traverse a certain catalog twice -> Breadth First Traversal or Depth First Traversal -> Optional catalog memory management (no_close) -> Use all Named Snapshots of a repository as traversal entry point -> Traverse starting from a provided catalog -> Traverse catalogs that were previously skipped -> Produce various flavours of catalogs (writable, mocked, ...)

Breadth First Traversal Strategy Catalogs are handed out to the user identical as they are traversed. Say: From top to buttom. When you would simply print each received catalog the result would be a nice representation of the catalog tree. This method is more efficient, because catalogs are opened, processed and thrown away directly afterwards.

Depth First Traversal Strategy The user gets the catalog tree starting from the leaf nodes. Say: From bottom to top. A user can assume that he got all children or historical ancestors of a catalog before. This method climbs down the full catalog tree and hands it out 'in reverse order'. Thus, catalogs on the way are opened, checked for their descendants and closed. Once all children and historical ancestors are processed, it is re-opened and handed out to the user. Note: This method needs more disk space to temporarily store downloaded but not yet processed catalogs.

Note: Since all CVMFS catalog files together can grow to several gigabytes in file size, each catalog is loaded, processed and removed immediately afterwards. Except if no_close is specified, which allows the user to choose when a catalog should be closed. Keep in mind, that a user is responsible for both deletion of the delivered catalog objects as well as unlinking of the catalog database file.

Parameters
ObjectFetcherTStrategy Pattern implementation that defines how to retrieve catalogs from various backend storage types. Furthermore the ObjectFetcherT::CatalogTN is the type of catalog to be instantiated by CatalogTraversal<>.

CAUTION: the CatalogTN* pointer passed into the callback becomes invalid directly after the callback method returns, unless you create the CatalogTraversal object with no_close = true.

Definition at line 495 of file catalog_traversal.h.

Member Typedef Documentation

template<class ObjectFetcherT>
typedef CatalogTraversalBase<ObjectFetcherT> swissknife::CatalogTraversal< ObjectFetcherT >::Base

Definition at line 499 of file catalog_traversal.h.

template<class ObjectFetcherT>
typedef CatalogTraversalData<CatalogTN> swissknife::CatalogTraversal< ObjectFetcherT >::CallbackDataTN

Definition at line 503 of file catalog_traversal.h.

template<class ObjectFetcherT>
typedef Base::CatalogJob swissknife::CatalogTraversal< ObjectFetcherT >::CatalogJob

Definition at line 506 of file catalog_traversal.h.

template<class ObjectFetcherT>
typedef std::stack<CatalogJob> swissknife::CatalogTraversal< ObjectFetcherT >::CatalogJobStack
protected

Definition at line 510 of file catalog_traversal.h.

template<class ObjectFetcherT>
typedef ObjectFetcherT::CatalogTN swissknife::CatalogTraversal< ObjectFetcherT >::CatalogTN

Definition at line 501 of file catalog_traversal.h.

template<class ObjectFetcherT>
typedef std::set<shash::Any> swissknife::CatalogTraversal< ObjectFetcherT >::HashSet
protected

Definition at line 509 of file catalog_traversal.h.

template<class ObjectFetcherT>
typedef ObjectFetcherT::HistoryTN swissknife::CatalogTraversal< ObjectFetcherT >::HistoryTN

Definition at line 502 of file catalog_traversal.h.

template<class ObjectFetcherT>
typedef ObjectFetcherT swissknife::CatalogTraversal< ObjectFetcherT >::ObjectFetcherTN

Definition at line 500 of file catalog_traversal.h.

template<class ObjectFetcherT>
typedef Base::Parameters swissknife::CatalogTraversal< ObjectFetcherT >::Parameters

Definition at line 504 of file catalog_traversal.h.

template<class ObjectFetcherT>
typedef Base::TraversalType swissknife::CatalogTraversal< ObjectFetcherT >::TraversalType

Definition at line 505 of file catalog_traversal.h.

Constructor & Destructor Documentation

template<class ObjectFetcherT>
swissknife::CatalogTraversal< ObjectFetcherT >::CatalogTraversal ( const Parameters params)
inlineexplicit

Constructs a new catalog traversal engine based on the construction parameters described in struct ConstructionParams.

Definition at line 542 of file catalog_traversal.h.

Member Function Documentation

template<class ObjectFetcherT>
bool swissknife::CatalogTraversal< ObjectFetcherT >::DoTraverse ( TraversalContext ctx)
inlineprotected

This controls the actual traversal. Using a stack to traverse down the catalog hierarchy. This method implements the traversal itself, but not in which way catalogs are handed out to the user code.

Each catalog is processed in these steps: 1.) Pop the next catalog from the stack Catalogs are always traversed from latest to oldest revision and from root to leaf nested catalogs 2.) Prepare the catalog for traversing 2.1.) Check if it was visited before 2.2.) Fetch the catalog database from the backend storage This might fail and produce an error. For root catalogs this error can be ignored (might be garbage collected before) 2.3.) Open the catalog database 2.4.) Check if the catalog is older than the timestamp threshold After these steps the catalog is opened either opened and ready for the traversal to continue, or it was marked for ignore (job.ignore) 3.) Check if the catalog is marked to be ignored Catalog might not be loadable (sweeped root catalog) or is too old Note: ignored catalogs can still trigger postponed yields 4.) Mark the catalog as visited to be able to skip it later on 5.) Find and push referencing catalogs This pushes all descendents of the current catalog onto the stack. Note that this is dependent on the strategy (depth or breadth first) and on the history threshold (see history_depth). 6.) Hand the catalog out to the user code Depending on the traversal strategy (depth of breadths first) this might immediately yield zero to N catalogs to the user code.

Note: If anything unexpected goes wrong during the traversal process, it is aborted immediately.

Parameters
ctxthe traversal context that steers the whole traversal process
Returns
true on successful traversal and false on abort

Definition at line 625 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::Traverse(), and swissknife::CatalogTraversal< ObjectFetcherT >::TraverseRevision().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>
bool swissknife::CatalogTraversal< ObjectFetcherT >::HandlePostponedYields ( const CatalogJob job,
TraversalContext ctx 
)
inlineprivate

Determines if there are postponed yields that can be set free based on the catalog currently being yielded

Note: the CatalogJob being handed into this method does not necessarily have an open Catalog attached to it.

Parameters
ctxthe TraversalContext
jobthe catalog job that was just yielded
Returns
true on successful execution

Definition at line 849 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::DoTraverse(), and swissknife::CatalogTraversal< ObjectFetcherT >::YieldToListeners().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>
void swissknife::CatalogTraversal< ObjectFetcherT >::MarkAsVisited ( const CatalogJob job)
inlineprotected

Definition at line 783 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::DoTraverse().

Here is the caller graph for this function:

template<class ObjectFetcherT>
CatalogJob swissknife::CatalogTraversal< ObjectFetcherT >::Pop ( TraversalContext ctx)
inlineprivate

Definition at line 881 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::DoTraverse().

Here is the caller graph for this function:

template<class ObjectFetcherT>
void swissknife::CatalogTraversal< ObjectFetcherT >::PostponeYield ( CatalogJob job,
TraversalContext ctx 
)
inlineprivate

Pushes a catalog to the callback_stack for later yielding Note: this is only used for the Depth First Traversal strategy!

Definition at line 827 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::YieldToListeners().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>
void swissknife::CatalogTraversal< ObjectFetcherT >::Push ( const CatalogJob job,
TraversalContext ctx 
)
inlineprivate

Definition at line 877 of file catalog_traversal.h.

template<class ObjectFetcherT>
unsigned int swissknife::CatalogTraversal< ObjectFetcherT >::PushNestedCatalogs ( const CatalogJob job,
TraversalContext ctx 
)
inlineprotected

Pushes all the referenced nested catalogs.

Returns
the number of catalogs pushed on the processing stack

Definition at line 723 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::PushReferencedCatalogs().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>
unsigned int swissknife::CatalogTraversal< ObjectFetcherT >::PushPreviousRevision ( const CatalogJob job,
TraversalContext ctx 
)
inlineprotected

Pushes the previous revision of a (root) catalog.

Returns
the number of catalogs pushed on the processing stack

Definition at line 692 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::PushReferencedCatalogs().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>
void swissknife::CatalogTraversal< ObjectFetcherT >::PushReferencedCatalogs ( CatalogJob job,
TraversalContext ctx 
)
inlineprotected

Definition at line 665 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::DoTraverse().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>
bool swissknife::CatalogTraversal< ObjectFetcherT >::ShouldBeSkipped ( const CatalogJob job)
inlineprotected

Checks the traversal history if the given catalog was traversed or at least seen before. If 'no_repeat_history' is not set this is always 'false'.

Parameters
jobthe job to be checked against the traversal history
Returns
true if the specified catalog was hit before

Definition at line 779 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::DoTraverse().

Here is the caller graph for this function:

template<class ObjectFetcherT>
bool swissknife::CatalogTraversal< ObjectFetcherT >::Traverse ( const TraversalType  type = Base::kBreadthFirst)
inlinevirtual

Starts the traversal process. After calling this methods CatalogTraversal will go through all catalogs and call the registered callback methods for each found catalog. If something goes wrong in the process, the traversal will be cancelled.

Parameters
typebreadths or depth first traversal
Returns
true, when all catalogs were successfully processed. On failure the traversal is cancelled and false is returned.

Implements swissknife::CatalogTraversalBase< ObjectFetcherT >.

Definition at line 546 of file catalog_traversal.h.

Referenced by swissknife::CommandMigrate::LoadCatalogs(), swissknife::CommandListCatalogs::Run(), and swissknife::CommandFileStats::Run().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>
bool swissknife::CatalogTraversal< ObjectFetcherT >::Traverse ( const shash::Any root_catalog_hash,
const TraversalType  type = Base::kBreadthFirst 
)
inlinevirtual

Starts the traversal process at the catalog pointed to by the given hash

Parameters
root_catalog_hashthe entry point into the catalog traversal
typebreadths or depth first traversal
Returns
true when catalogs were successfully traversed

Implements swissknife::CatalogTraversalBase< ObjectFetcherT >.

Definition at line 554 of file catalog_traversal.h.

Here is the call graph for this function:

template<class ObjectFetcherT>
bool swissknife::CatalogTraversal< ObjectFetcherT >::TraverseList ( const std::vector< shash::Any > &  catalog_list,
const TraversalType  type = Base::kBreadthFirst 
)
inlinevirtual

Traverse a list of revisions represented by root catalogs from first to last. DO NOT traverse previous revisions based on history and timestamp threshold settings.

Parameters
catalog_listlist of root catalog hashes
typebreadth- or depth- first traversal
Returns
true on success

Implements swissknife::CatalogTraversalBase< ObjectFetcherT >.

Definition at line 565 of file catalog_traversal.h.

Here is the call graph for this function:

template<class ObjectFetcherT>
bool swissknife::CatalogTraversal< ObjectFetcherT >::TraverseRevision ( const shash::Any root_catalog_hash,
const TraversalType  type = Base::kBreadthFirst 
)
inlinevirtual

Starts the traversal process at the catalog pointed to by the given hash but doesn't traverse into predecessor catalog revisions. This overrides the TraversalParameter settings provided at construction.

Parameters
root_catalog_hashthe entry point into the catalog traversal
typebreadths or depth first traversal
Returns
true when catalogs were successfully traversed

Implements swissknife::CatalogTraversalBase< ObjectFetcherT >.

Definition at line 576 of file catalog_traversal.h.

Referenced by swissknife::CommandListReflog::Run(), and swissknife::CatalogTraversal< ObjectFetcherT >::TraverseList().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>
bool swissknife::CatalogTraversal< ObjectFetcherT >::Yield ( CatalogJob job)
inlineprivate

This actually hands out a catalog to the user code It is not called by DoTraversa() directly but by wrapper functions in order to provide higher level yielding behaviour.

Definition at line 795 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::HandlePostponedYields(), and swissknife::CatalogTraversal< ObjectFetcherT >::YieldToListeners().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>
bool swissknife::CatalogTraversal< ObjectFetcherT >::YieldToListeners ( CatalogJob job,
TraversalContext ctx 
)
inlineprotected

Definition at line 748 of file catalog_traversal.h.

Referenced by swissknife::CatalogTraversal< ObjectFetcherT >::DoTraverse().

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

template<class ObjectFetcherT>
HashSet swissknife::CatalogTraversal< ObjectFetcherT >::visited_catalogs_
protected

The documentation for this class was generated from the following file: