CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
IngestionPipeline Class Reference

#include <pipeline.h>

Inheritance diagram for IngestionPipeline:
Collaboration diagram for IngestionPipeline:

Public Member Functions

 IngestionPipeline (upload::AbstractUploader *uploader, const upload::SpoolerDefinition &spooler_definition)
 
 ~IngestionPipeline ()
 
void Spawn ()
 
void Process (IngestionSource *source, bool allow_chunking, shash::Suffix hash_suffix=shash::kSuffixNone)
 
void WaitFor ()
 
void OnFileProcessed (const upload::SpoolerResult &spooler_result)
 
- Public Member Functions inherited from Observable< upload::SpoolerResult >
virtual ~Observable ()
 
CallbackPtr RegisterListener (typename BoundClosure< upload::SpoolerResult, DelegateT, ClosureDataT >::CallbackMethod method, DelegateT *delegate, ClosureDataT data)
 
CallbackPtr RegisterListener (typename BoundCallback< upload::SpoolerResult, DelegateT >::CallbackMethod method, DelegateT *delegate)
 
CallbackPtr RegisterListener (typename Callback< upload::SpoolerResult >::CallbackFunction fn)
 
void UnregisterListener (CallbackPtr callback_object)
 
void UnregisterListeners ()
 

Private Attributes

const zlib::Algorithms compression_algorithm_
 
const shash::Algorithms hash_algorithm_
 
const bool generate_legacy_bulk_chunks_
 
const bool chunking_enabled_
 
const size_t minimal_chunk_size_
 
const size_t average_chunk_size_
 
const size_t maximal_chunk_size_
 
bool spawned_
 
upload::AbstractUploaderuploader_
 
Tube< FileItemtube_counter_
 
Tube< FileItemtube_input_
 
TubeConsumerGroup< FileItemtasks_read_
 
TubeGroup< BlockItemtubes_chunk_
 
TubeConsumerGroup< BlockItemtasks_chunk_
 
TubeGroup< BlockItemtubes_compress_
 
TubeConsumerGroup< BlockItemtasks_compress_
 
TubeGroup< BlockItemtubes_hash_
 
TubeConsumerGroup< BlockItemtasks_hash_
 
TubeGroup< BlockItemtubes_write_
 
TubeConsumerGroup< BlockItemtasks_write_
 
TubeGroup< FileItemtubes_register_
 
TubeConsumerGroup< FileItemtasks_register_
 
ItemAllocator item_allocator_
 

Static Private Attributes

static const uint64_t kMaxPipelineMem = 1024 * 1024 * 1024
 
static const unsigned kMaxFilesInFlight = 8000
 
static const unsigned kNforkRegister = 1
 
static const unsigned kNforkWrite = 1
 
static const unsigned kNforkHash = 2
 
static const unsigned kNforkCompress = 4
 
static const unsigned kNforkChunk = 1
 
static const unsigned kNforkRead = 8
 

Additional Inherited Members

- Public Types inherited from Observable< upload::SpoolerResult >
typedef Callbackable
< upload::SpoolerResult >
::CallbackTN
CallbackPtr
 
- Public Types inherited from Callbackable< upload::SpoolerResult >
typedef CallbackBase
< upload::SpoolerResult
CallbackTN
 
- Static Public Member Functions inherited from Callbackable< upload::SpoolerResult >
static CallbackTNMakeClosure (typename BoundClosure< upload::SpoolerResult, DelegateT, ClosureDataT >::CallbackMethod method, DelegateT *delegate, const ClosureDataT &closure_data)
 
static CallbackTNMakeCallback (typename BoundCallback< upload::SpoolerResult, DelegateT >::CallbackMethod method, DelegateT *delegate)
 
static CallbackTNMakeCallback (typename Callback< upload::SpoolerResult >::CallbackFunction function)
 
- Protected Types inherited from Observable< upload::SpoolerResult >
typedef std::set< CallbackPtrCallbacks
 
- Protected Member Functions inherited from Observable< upload::SpoolerResult >
void RegisterListener (CallbackPtr callback_object)
 
 Observable ()
 
void NotifyListeners (const upload::SpoolerResult &parameter)
 

Detailed Description

Definition at line 24 of file pipeline.h.

Constructor & Destructor Documentation

IngestionPipeline::IngestionPipeline ( upload::AbstractUploader uploader,
const upload::SpoolerDefinition spooler_definition 
)
explicit

Definition at line 26 of file pipeline.cc.

Here is the call graph for this function:

IngestionPipeline::~IngestionPipeline ( )

Definition at line 100 of file pipeline.cc.

Here is the call graph for this function:

Member Function Documentation

void IngestionPipeline::OnFileProcessed ( const upload::SpoolerResult spooler_result)

Definition at line 112 of file pipeline.cc.

Referenced by IngestionPipeline().

Here is the call graph for this function:

Here is the caller graph for this function:

void IngestionPipeline::Process ( IngestionSource source,
bool  allow_chunking,
shash::Suffix  hash_suffix = shash::kSuffixNone 
)

Definition at line 119 of file pipeline.cc.

Here is the call graph for this function:

void IngestionPipeline::Spawn ( )

Definition at line 139 of file pipeline.cc.

Here is the call graph for this function:

void IngestionPipeline::WaitFor ( )

Definition at line 150 of file pipeline.cc.

Here is the call graph for this function:

Member Data Documentation

const size_t IngestionPipeline::average_chunk_size_
private

Definition at line 53 of file pipeline.h.

Referenced by Process().

const bool IngestionPipeline::chunking_enabled_
private

Definition at line 51 of file pipeline.h.

Referenced by Process().

const zlib::Algorithms IngestionPipeline::compression_algorithm_
private

Definition at line 48 of file pipeline.h.

Referenced by Process().

const bool IngestionPipeline::generate_legacy_bulk_chunks_
private

Definition at line 50 of file pipeline.h.

Referenced by Process().

const shash::Algorithms IngestionPipeline::hash_algorithm_
private

Definition at line 49 of file pipeline.h.

Referenced by Process().

ItemAllocator IngestionPipeline::item_allocator_
private

Definition at line 79 of file pipeline.h.

Referenced by IngestionPipeline().

const unsigned IngestionPipeline::kMaxFilesInFlight = 8000
staticprivate

Definition at line 40 of file pipeline.h.

const uint64_t IngestionPipeline::kMaxPipelineMem = 1024 * 1024 * 1024
staticprivate

This file is part of the CernVM File System.

Definition at line 39 of file pipeline.h.

Referenced by IngestionPipeline().

const unsigned IngestionPipeline::kNforkChunk = 1
staticprivate

Definition at line 45 of file pipeline.h.

Referenced by IngestionPipeline().

const unsigned IngestionPipeline::kNforkCompress = 4
staticprivate

Definition at line 44 of file pipeline.h.

Referenced by IngestionPipeline().

const unsigned IngestionPipeline::kNforkHash = 2
staticprivate

Definition at line 43 of file pipeline.h.

Referenced by IngestionPipeline().

const unsigned IngestionPipeline::kNforkRead = 8
staticprivate

Definition at line 46 of file pipeline.h.

Referenced by IngestionPipeline().

const unsigned IngestionPipeline::kNforkRegister = 1
staticprivate

Definition at line 41 of file pipeline.h.

Referenced by IngestionPipeline().

const unsigned IngestionPipeline::kNforkWrite = 1
staticprivate

Definition at line 42 of file pipeline.h.

Referenced by IngestionPipeline().

const size_t IngestionPipeline::maximal_chunk_size_
private

Definition at line 54 of file pipeline.h.

Referenced by Process().

const size_t IngestionPipeline::minimal_chunk_size_
private

Definition at line 52 of file pipeline.h.

Referenced by Process().

bool IngestionPipeline::spawned_
private

Definition at line 56 of file pipeline.h.

Referenced by Spawn(), and ~IngestionPipeline().

TubeConsumerGroup<BlockItem> IngestionPipeline::tasks_chunk_
private

Definition at line 65 of file pipeline.h.

Referenced by IngestionPipeline(), Spawn(), and ~IngestionPipeline().

TubeConsumerGroup<BlockItem> IngestionPipeline::tasks_compress_
private

Definition at line 68 of file pipeline.h.

Referenced by IngestionPipeline(), Spawn(), and ~IngestionPipeline().

TubeConsumerGroup<BlockItem> IngestionPipeline::tasks_hash_
private

Definition at line 71 of file pipeline.h.

Referenced by IngestionPipeline(), Spawn(), and ~IngestionPipeline().

TubeConsumerGroup<FileItem> IngestionPipeline::tasks_read_
private

Definition at line 62 of file pipeline.h.

Referenced by IngestionPipeline(), Spawn(), and ~IngestionPipeline().

TubeConsumerGroup<FileItem> IngestionPipeline::tasks_register_
private

Definition at line 77 of file pipeline.h.

Referenced by IngestionPipeline(), Spawn(), and ~IngestionPipeline().

TubeConsumerGroup<BlockItem> IngestionPipeline::tasks_write_
private

Definition at line 74 of file pipeline.h.

Referenced by IngestionPipeline(), Spawn(), and ~IngestionPipeline().

Tube<FileItem> IngestionPipeline::tube_counter_
private

Definition at line 59 of file pipeline.h.

Referenced by IngestionPipeline(), Process(), and WaitFor().

Tube<FileItem> IngestionPipeline::tube_input_
private

Definition at line 60 of file pipeline.h.

Referenced by IngestionPipeline(), and Process().

TubeGroup<BlockItem> IngestionPipeline::tubes_chunk_
private

Definition at line 64 of file pipeline.h.

Referenced by IngestionPipeline().

TubeGroup<BlockItem> IngestionPipeline::tubes_compress_
private

Definition at line 67 of file pipeline.h.

Referenced by IngestionPipeline().

TubeGroup<BlockItem> IngestionPipeline::tubes_hash_
private

Definition at line 70 of file pipeline.h.

Referenced by IngestionPipeline().

TubeGroup<FileItem> IngestionPipeline::tubes_register_
private

Definition at line 76 of file pipeline.h.

Referenced by IngestionPipeline().

TubeGroup<BlockItem> IngestionPipeline::tubes_write_
private

Definition at line 73 of file pipeline.h.

Referenced by IngestionPipeline().

upload::AbstractUploader* IngestionPipeline::uploader_
private

Definition at line 57 of file pipeline.h.

Referenced by IngestionPipeline().


The documentation for this class was generated from the following files: