CernVM-FS  2.12.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
swissknife::CatalogTraversalParallel< ObjectFetcherT > Class Template Reference

#include <catalog_traversal_parallel.h>

Inheritance diagram for swissknife::CatalogTraversalParallel< ObjectFetcherT >:
Collaboration diagram for swissknife::CatalogTraversalParallel< ObjectFetcherT >:

Classes

struct  CatalogJob
 

Public Types

typedef CatalogTraversalBase
< ObjectFetcherT > 
Base
 
typedef ObjectFetcherT ObjectFetcherTN
 
typedef ObjectFetcherT::CatalogTN CatalogTN
 
typedef ObjectFetcherT::HistoryTN HistoryTN
 
typedef CatalogTraversalData
< CatalogTN
CallbackDataTN
 
typedef
CatalogTN::NestedCatalogList 
NestedCatalogList
 
typedef Base::Parameters Parameters
 
typedef Base::TraversalType TraversalType
 
typedef std::vector< shash::AnyHashList
 
- Public Types inherited from swissknife::CatalogTraversalBase< ObjectFetcherT >
enum  TraversalType { kBreadthFirst, kDepthFirst }
 
typedef ObjectFetcherT ObjectFetcherTN
 
typedef ObjectFetcherT::CatalogTN CatalogTN
 
typedef ObjectFetcherT::HistoryTN HistoryTN
 
typedef CatalogTraversalData
< CatalogTN
CallbackDataTN
 
- Public Types inherited from Observable< CatalogTraversalData< ObjectFetcherT::CatalogTN > >
typedef Callbackable
< CatalogTraversalData
< ObjectFetcherT::CatalogTN >
>::CallbackTN
CallbackPtr
 
- Public Types inherited from Callbackable< CatalogTraversalData< ObjectFetcherT::CatalogTN > >
typedef CallbackBase
< CatalogTraversalData
< ObjectFetcherT::CatalogTN > > 
CallbackTN
 

Public Member Functions

 CatalogTraversalParallel (const Parameters &params)
 
bool Traverse (const TraversalType type=Base::kBreadthFirst)
 
bool Traverse (const shash::Any &root_catalog_hash, const TraversalType type=Base::kBreadthFirst)
 
bool TraverseList (const HashList &root_catalog_list, const TraversalType type=Base::kBreadthFirst)
 
bool TraverseRevision (const shash::Any &root_catalog_hash, const TraversalType type=Base::kBreadthFirst)
 
- Public Member Functions inherited from swissknife::CatalogTraversalBase< ObjectFetcherT >
 CatalogTraversalBase (const Parameters &params)
 
virtual bool TraverseList (const std::vector< shash::Any > &catalog_list, const TraversalType type=kBreadthFirst)=0
 
virtual bool TraverseNamedSnapshots (const TraversalType type=kBreadthFirst)
 
void SetCatalogInfoShim (CatalogTraversalInfoShim< CatalogTN > *shim)
 
- Public Member Functions inherited from Observable< CatalogTraversalData< ObjectFetcherT::CatalogTN > >
virtual ~Observable ()
 
CallbackPtr RegisterListener (typename BoundClosure< CatalogTraversalData< ObjectFetcherT::CatalogTN >, DelegateT, ClosureDataT >::CallbackMethod method, DelegateT *delegate, ClosureDataT data)
 
CallbackPtr RegisterListener (typename BoundCallback< CatalogTraversalData< ObjectFetcherT::CatalogTN >, DelegateT >::CallbackMethod method, DelegateT *delegate)
 
CallbackPtr RegisterListener (typename Callback< CatalogTraversalData< ObjectFetcherT::CatalogTN > >::CallbackFunction fn)
 
void UnregisterListener (CallbackPtr callback_object)
 
void UnregisterListeners ()
 

Protected Member Functions

bool DoTraverse ()
 
void NotifyFinished ()
 
void PushJob (CatalogJob *job)
 
void PushJobUnlocked (CatalogJob *job)
 
void ProcessJobPre (CatalogJob *job)
 
unsigned int PushNestedCatalogs (CatalogJob *job, const NestedCatalogList &catalog_list)
 
unsigned int PushPreviousRevision (CatalogJob *job)
 
void ProcessJobPost (CatalogJob *job)
 
void FinalizeJob (CatalogJob *job)
 
void OnChildFinished (const int &a, CatalogJob *job)
 
- Protected Member Functions inherited from swissknife::CatalogTraversalBase< ObjectFetcherT >
bool PrepareCatalog (CatalogJob *job)
 
bool ReopenCatalog (CatalogJob *job)
 
bool CloseCatalog (const bool unlink_db, CatalogJob *job)
 
shash::Any GetRepositoryRootCatalogHash ()
 
bool IsBelowPruningThresholds (const CatalogJob &job, const uint64_t history_depth, const time_t timestamp_threshold)
 
- Protected Member Functions inherited from Observable< CatalogTraversalData< ObjectFetcherT::CatalogTN > >
void RegisterListener (CallbackPtr callback_object)
 
 Observable ()
 
void NotifyListeners (const CatalogTraversalData< ObjectFetcherT::CatalogTN > &parameter)
 

Static Protected Member Functions

static uint32_t hasher (const shash::Any &key)
 
static void * MainProcessQueue (void *data)
 

Protected Attributes

unsigned int num_threads_
 
bool serialize_callbacks_
 
uint64_t effective_history_depth_
 
time_t effective_timestamp_threshold_
 
TraversalType effective_traversal_type_
 
pthread_t * threads_process_
 
atomic_int32 num_errors_
 
Tube< CatalogJobpre_job_queue_
 
Tube< CatalogJobpost_job_queue_
 
SmallHashDynamic< shash::Any,
CatalogJob * > 
catalogs_processing_
 
SmallHashDynamic< shash::Any,
bool > 
catalogs_done_
 
pthread_mutex_t catalogs_lock_
 
pthread_mutex_t catalog_callback_lock_
 
- Protected Attributes inherited from swissknife::CatalogTraversalBase< ObjectFetcherT >
ObjectFetcherT * object_fetcher_
 
CatalogTraversalInfoShim
< CatalogTN
catalog_info_default_shim_
 
CatalogTraversalInfoShim
< CatalogTN > * 
catalog_info_shim_
 
const uint64_t default_history_depth_
 
const time_t default_timestamp_threshold_
 
const bool no_close_
 
const bool ignore_load_failure_
 
const bool no_repeat_history_
 
LogFacilities error_sink_
 

Additional Inherited Members

- Static Public Member Functions inherited from Callbackable< CatalogTraversalData< ObjectFetcherT::CatalogTN > >
static CallbackTNMakeClosure (typename BoundClosure< CatalogTraversalData< ObjectFetcherT::CatalogTN >, DelegateT, ClosureDataT >::CallbackMethod method, DelegateT *delegate, const ClosureDataT &closure_data)
 
static CallbackTNMakeCallback (typename BoundCallback< CatalogTraversalData< ObjectFetcherT::CatalogTN >, DelegateT >::CallbackMethod method, DelegateT *delegate)
 
static CallbackTNMakeCallback (typename Callback< CatalogTraversalData< ObjectFetcherT::CatalogTN > >::CallbackFunction function)
 
- Protected Types inherited from swissknife::CatalogTraversalBase< ObjectFetcherT >
typedef std::set< shash::AnyHashSet
 
- Protected Types inherited from Observable< CatalogTraversalData< ObjectFetcherT::CatalogTN > >
typedef std::set< CallbackPtrCallbacks
 

Detailed Description

template<class ObjectFetcherT>
class swissknife::CatalogTraversalParallel< ObjectFetcherT >

This class implements the same functionality as CatalogTraversal, but in parallel. For common functionality, see the documentation of CatalogTraversal. Differences:

  • can choose number of threads
  • traversal types change meaning:
    • depth-first -> parallelized post-order traversal (parents are processed after all children are finished)
    • breadth-first -> same as original, but parallelized

Definition at line 30 of file catalog_traversal_parallel.h.

Member Typedef Documentation

template<class ObjectFetcherT>
typedef CatalogTraversalBase<ObjectFetcherT> swissknife::CatalogTraversalParallel< ObjectFetcherT >::Base

Definition at line 32 of file catalog_traversal_parallel.h.

template<class ObjectFetcherT>
typedef CatalogTraversalData<CatalogTN> swissknife::CatalogTraversalParallel< ObjectFetcherT >::CallbackDataTN

Definition at line 36 of file catalog_traversal_parallel.h.

template<class ObjectFetcherT>
typedef ObjectFetcherT::CatalogTN swissknife::CatalogTraversalParallel< ObjectFetcherT >::CatalogTN

Definition at line 34 of file catalog_traversal_parallel.h.

template<class ObjectFetcherT>
typedef std::vector<shash::Any> swissknife::CatalogTraversalParallel< ObjectFetcherT >::HashList

Definition at line 40 of file catalog_traversal_parallel.h.

template<class ObjectFetcherT>
typedef ObjectFetcherT::HistoryTN swissknife::CatalogTraversalParallel< ObjectFetcherT >::HistoryTN

Definition at line 35 of file catalog_traversal_parallel.h.

template<class ObjectFetcherT>
typedef CatalogTN::NestedCatalogList swissknife::CatalogTraversalParallel< ObjectFetcherT >::NestedCatalogList

Definition at line 37 of file catalog_traversal_parallel.h.

template<class ObjectFetcherT>
typedef ObjectFetcherT swissknife::CatalogTraversalParallel< ObjectFetcherT >::ObjectFetcherTN

Definition at line 33 of file catalog_traversal_parallel.h.

template<class ObjectFetcherT>
typedef Base::Parameters swissknife::CatalogTraversalParallel< ObjectFetcherT >::Parameters

Definition at line 38 of file catalog_traversal_parallel.h.

template<class ObjectFetcherT>
typedef Base::TraversalType swissknife::CatalogTraversalParallel< ObjectFetcherT >::TraversalType

Definition at line 39 of file catalog_traversal_parallel.h.

Constructor & Destructor Documentation

template<class ObjectFetcherT>
swissknife::CatalogTraversalParallel< ObjectFetcherT >::CatalogTraversalParallel ( const Parameters params)
inlineexplicit

Definition at line 42 of file catalog_traversal_parallel.h.

Here is the call graph for this function:

Member Function Documentation

template<class ObjectFetcherT>
bool swissknife::CatalogTraversalParallel< ObjectFetcherT >::DoTraverse ( )
inlineprotected

Definition at line 177 of file catalog_traversal_parallel.h.

Referenced by swissknife::CatalogTraversalParallel< ObjectFetcherT >::Traverse(), and swissknife::CatalogTraversalParallel< ObjectFetcherT >::TraverseList().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>
void swissknife::CatalogTraversalParallel< ObjectFetcherT >::FinalizeJob ( CatalogJob job)
inlineprotected

Definition at line 379 of file catalog_traversal_parallel.h.

Referenced by swissknife::CatalogTraversalParallel< ObjectFetcherT >::ProcessJobPost(), and swissknife::CatalogTraversalParallel< ObjectFetcherT >::ProcessJobPre().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>
static uint32_t swissknife::CatalogTraversalParallel< ObjectFetcherT >::hasher ( const shash::Any key)
inlinestaticprotected

Definition at line 172 of file catalog_traversal_parallel.h.

Referenced by swissknife::CatalogTraversalParallel< ObjectFetcherT >::CatalogTraversalParallel().

Here is the caller graph for this function:

template<class ObjectFetcherT>
static void* swissknife::CatalogTraversalParallel< ObjectFetcherT >::MainProcessQueue ( void *  data)
inlinestaticprotected

Definition at line 203 of file catalog_traversal_parallel.h.

Referenced by swissknife::CatalogTraversalParallel< ObjectFetcherT >::DoTraverse().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>
void swissknife::CatalogTraversalParallel< ObjectFetcherT >::NotifyFinished ( )
inlineprotected
template<class ObjectFetcherT>
void swissknife::CatalogTraversalParallel< ObjectFetcherT >::OnChildFinished ( const int &  a,
CatalogJob job 
)
inlineprotected
template<class ObjectFetcherT>
void swissknife::CatalogTraversalParallel< ObjectFetcherT >::ProcessJobPost ( CatalogJob job)
inlineprotected

Definition at line 354 of file catalog_traversal_parallel.h.

Referenced by swissknife::CatalogTraversalParallel< ObjectFetcherT >::MainProcessQueue(), and swissknife::CatalogTraversalParallel< ObjectFetcherT >::ProcessJobPre().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>
void swissknife::CatalogTraversalParallel< ObjectFetcherT >::ProcessJobPre ( CatalogJob job)
inlineprotected

Definition at line 243 of file catalog_traversal_parallel.h.

Referenced by swissknife::CatalogTraversalParallel< ObjectFetcherT >::MainProcessQueue().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>
void swissknife::CatalogTraversalParallel< ObjectFetcherT >::PushJob ( CatalogJob job)
inlineprotected

Definition at line 233 of file catalog_traversal_parallel.h.

Referenced by swissknife::CatalogTraversalParallel< ObjectFetcherT >::Traverse().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>
unsigned int swissknife::CatalogTraversalParallel< ObjectFetcherT >::PushNestedCatalogs ( CatalogJob job,
const NestedCatalogList catalog_list 
)
inlineprotected

Definition at line 282 of file catalog_traversal_parallel.h.

Referenced by swissknife::CatalogTraversalParallel< ObjectFetcherT >::ProcessJobPre().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>
unsigned int swissknife::CatalogTraversalParallel< ObjectFetcherT >::PushPreviousRevision ( CatalogJob job)
inlineprotected

Pushes the previous revision of a root catalog.

Returns
the number of catalogs pushed on the processing stack

Definition at line 317 of file catalog_traversal_parallel.h.

Referenced by swissknife::CatalogTraversalParallel< ObjectFetcherT >::ProcessJobPre().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>
bool swissknife::CatalogTraversalParallel< ObjectFetcherT >::Traverse ( const TraversalType  type = Base::kBreadthFirst)
inlinevirtual

Starts the traversal process. After calling this methods CatalogTraversal will go through all catalogs and call the registered callback methods for each found catalog. If something goes wrong in the process, the traversal will be cancelled.

Returns
true, when all catalogs were successfully processed. On failure the traversal is cancelled and false is returned.

Implements swissknife::CatalogTraversalBase< ObjectFetcherT >.

Definition at line 87 of file catalog_traversal_parallel.h.

Referenced by swissknife::CatalogTraversalParallel< ObjectFetcherT >::TraverseRevision().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class ObjectFetcherT>
bool swissknife::CatalogTraversalParallel< ObjectFetcherT >::Traverse ( const shash::Any root_catalog_hash,
const TraversalType  type = Base::kBreadthFirst 
)
inlinevirtual

Starts the traversal process at the catalog pointed to by the given hash

Parameters
root_catalog_hashthe entry point into the catalog traversal
Returns
true when catalogs were successfully traversed

Implements swissknife::CatalogTraversalBase< ObjectFetcherT >.

Definition at line 101 of file catalog_traversal_parallel.h.

Here is the call graph for this function:

template<class ObjectFetcherT>
bool swissknife::CatalogTraversalParallel< ObjectFetcherT >::TraverseList ( const HashList root_catalog_list,
const TraversalType  type = Base::kBreadthFirst 
)
inline

Start the traversal process from a list of root catalogs. Same as TraverseRevision function, TraverseList does not traverse into predecessor catalog revisions and ignores TraversalParameter settings.

Definition at line 120 of file catalog_traversal_parallel.h.

Here is the call graph for this function:

template<class ObjectFetcherT>
bool swissknife::CatalogTraversalParallel< ObjectFetcherT >::TraverseRevision ( const shash::Any root_catalog_hash,
const TraversalType  type = Base::kBreadthFirst 
)
inlinevirtual

Starts the traversal process at the catalog pointed to by the given hash but doesn't traverse into predecessor catalog revisions. This overrides the TraversalParameter settings provided at construction.

Parameters
root_catalog_hashthe entry point into the catalog traversal
Returns
true when catalogs were successfully traversed

Implements swissknife::CatalogTraversalBase< ObjectFetcherT >.

Definition at line 159 of file catalog_traversal_parallel.h.

Here is the call graph for this function:

Member Data Documentation

template<class ObjectFetcherT>
pthread_mutex_t swissknife::CatalogTraversalParallel< ObjectFetcherT >::catalog_callback_lock_
protected
template<class ObjectFetcherT>
unsigned int swissknife::CatalogTraversalParallel< ObjectFetcherT >::num_threads_
protected
template<class ObjectFetcherT>
bool swissknife::CatalogTraversalParallel< ObjectFetcherT >::serialize_callbacks_
protected
template<class ObjectFetcherT>
pthread_t* swissknife::CatalogTraversalParallel< ObjectFetcherT >::threads_process_
protected

The documentation for this class was generated from the following file: