IngestionPipeline Class Reference

#include <pipeline.h>

Inheritance diagram for IngestionPipeline:

Collaboration diagram for IngestionPipeline:

Public Member Functions
	IngestionPipeline (upload::AbstractUploader *uploader, const upload::SpoolerDefinition &spooler_definition)
	~IngestionPipeline ()
void	Spawn ()
void	Process (IngestionSource *source, bool allow_chunking, shash::Suffix hash_suffix=shash::kSuffixNone)
void	WaitFor ()
void	OnFileProcessed (const upload::SpoolerResult &spooler_result)
Public Member Functions inherited from Observable< upload::SpoolerResult >
virtual	~Observable ()
CallbackPtr	RegisterListener (typename BoundClosure< upload::SpoolerResult, DelegateT, ClosureDataT >::CallbackMethod method, DelegateT *delegate, ClosureDataT data)
CallbackPtr	RegisterListener (typename BoundCallback< upload::SpoolerResult, DelegateT >::CallbackMethod method, DelegateT *delegate)
CallbackPtr	RegisterListener (typename Callback< upload::SpoolerResult >::CallbackFunction fn)
void	UnregisterListener (CallbackPtr callback_object)
void	UnregisterListeners ()

Private Attributes
const zlib::Algorithms	compression_algorithm_
const shash::Algorithms	hash_algorithm_
const bool	generate_legacy_bulk_chunks_
const bool	chunking_enabled_
const size_t	minimal_chunk_size_
const size_t	average_chunk_size_
const size_t	maximal_chunk_size_
bool	spawned_
upload::AbstractUploader *	uploader_
Tube< FileItem >	tube_ctr_inflight_pre_
Tube< FileItem >	tube_ctr_inflight_post_
Tube< FileItem >	tube_input_
TubeConsumerGroup< FileItem >	tasks_read_
TubeGroup< BlockItem >	tubes_chunk_
TubeConsumerGroup< BlockItem >	tasks_chunk_
TubeGroup< BlockItem >	tubes_compress_
TubeConsumerGroup< BlockItem >	tasks_compress_
TubeGroup< BlockItem >	tubes_hash_
TubeConsumerGroup< BlockItem >	tasks_hash_
TubeGroup< BlockItem >	tubes_write_
TubeConsumerGroup< BlockItem >	tasks_write_
TubeGroup< FileItem >	tubes_register_
TubeConsumerGroup< FileItem >	tasks_register_
ItemAllocator	item_allocator_

Static Private Attributes
static const uint64_t	kMaxPipelineMem = 1024 * 1024 * 1024
static const unsigned	kMaxFilesInFlight = 8000
static const unsigned	kNforkRegister = 1
static const unsigned	kNforkWrite = 1
static const unsigned	kNforkHash = 2
static const unsigned	kNforkCompress = 4
static const unsigned	kNforkChunk = 1
static const unsigned	kNforkRead = 8

Additional Inherited Members
Public Types inherited from Observable< upload::SpoolerResult >
typedef Callbackable < upload::SpoolerResult > ::CallbackTN *	CallbackPtr
Public Types inherited from Callbackable< upload::SpoolerResult >
typedef CallbackBase < upload::SpoolerResult >	CallbackTN
Static Public Member Functions inherited from Callbackable< upload::SpoolerResult >
static CallbackTN *	MakeClosure (typename BoundClosure< upload::SpoolerResult, DelegateT, ClosureDataT >::CallbackMethod method, DelegateT *delegate, const ClosureDataT &closure_data)
static CallbackTN *	MakeCallback (typename BoundCallback< upload::SpoolerResult, DelegateT >::CallbackMethod method, DelegateT *delegate)
static CallbackTN *	MakeCallback (typename Callback< upload::SpoolerResult >::CallbackFunction function)
Protected Types inherited from Observable< upload::SpoolerResult >
typedef std::set< CallbackPtr >	Callbacks
Protected Member Functions inherited from Observable< upload::SpoolerResult >
void	RegisterListener (CallbackPtr callback_object)
	Observable ()
void	NotifyListeners (const upload::SpoolerResult &parameter)

Detailed Description

Definition at line 24 of file pipeline.h.

Constructor & Destructor Documentation

IngestionPipeline::IngestionPipeline ( upload::AbstractUploader *  uploader, const upload::SpoolerDefinition &  spooler_definition  )

explicit

Definition at line 26 of file pipeline.cc.

Here is the call graph for this function:

IngestionPipeline::~IngestionPipeline

(

)

Definition at line 101 of file pipeline.cc.

Here is the call graph for this function:

Member Function Documentation

void IngestionPipeline::OnFileProcessed

(

const upload::SpoolerResult &

spooler_result

)

Definition at line 113 of file pipeline.cc.

Referenced by IngestionPipeline().

Here is the call graph for this function:

Here is the caller graph for this function:

void IngestionPipeline::Process	(	IngestionSource *	source,
		bool	allow_chunking,
		shash::Suffix	hash_suffix = shash::kSuffixNone
	)

Definition at line 120 of file pipeline.cc.

Here is the call graph for this function:

void IngestionPipeline::Spawn

(

)

Definition at line 141 of file pipeline.cc.

Here is the call graph for this function:

void IngestionPipeline::WaitFor

(

)

Definition at line 152 of file pipeline.cc.

Here is the call graph for this function:

Member Data Documentation

const size_t IngestionPipeline::average_chunk_size_

private

Definition at line 53 of file pipeline.h.

Referenced by Process().

const bool IngestionPipeline::chunking_enabled_

private

Definition at line 51 of file pipeline.h.

Referenced by Process().

const zlib::Algorithms IngestionPipeline::compression_algorithm_

private

Definition at line 48 of file pipeline.h.

Referenced by Process().

const bool IngestionPipeline::generate_legacy_bulk_chunks_

private

Definition at line 50 of file pipeline.h.

Referenced by Process().

const shash::Algorithms IngestionPipeline::hash_algorithm_

private

Definition at line 49 of file pipeline.h.

Referenced by Process().

ItemAllocator IngestionPipeline::item_allocator_

private

Definition at line 88 of file pipeline.h.

Referenced by IngestionPipeline().

const unsigned IngestionPipeline::kMaxFilesInFlight = 8000

staticprivate

Definition at line 40 of file pipeline.h.

const uint64_t IngestionPipeline::kMaxPipelineMem = 1024 * 1024 * 1024

staticprivate

This file is part of the CernVM File System.

Definition at line 39 of file pipeline.h.

Referenced by IngestionPipeline().

const unsigned IngestionPipeline::kNforkChunk = 1

staticprivate

Definition at line 45 of file pipeline.h.

Referenced by IngestionPipeline().

const unsigned IngestionPipeline::kNforkCompress = 4

staticprivate

Definition at line 44 of file pipeline.h.

Referenced by IngestionPipeline().

const unsigned IngestionPipeline::kNforkHash = 2

staticprivate

Definition at line 43 of file pipeline.h.

Referenced by IngestionPipeline().

const unsigned IngestionPipeline::kNforkRead = 8

staticprivate

Definition at line 46 of file pipeline.h.

Referenced by IngestionPipeline().

const unsigned IngestionPipeline::kNforkRegister = 1

staticprivate

Definition at line 41 of file pipeline.h.

Referenced by IngestionPipeline().

const unsigned IngestionPipeline::kNforkWrite = 1

staticprivate

Definition at line 42 of file pipeline.h.

Referenced by IngestionPipeline().

const size_t IngestionPipeline::maximal_chunk_size_

private

Definition at line 54 of file pipeline.h.

Referenced by Process().

const size_t IngestionPipeline::minimal_chunk_size_

private

Definition at line 52 of file pipeline.h.

Referenced by Process().

bool IngestionPipeline::spawned_

private

Definition at line 56 of file pipeline.h.

Referenced by Spawn(), and ~IngestionPipeline().

TubeConsumerGroup<BlockItem> IngestionPipeline::tasks_chunk_

private

Definition at line 74 of file pipeline.h.

Referenced by IngestionPipeline(), Spawn(), and ~IngestionPipeline().

TubeConsumerGroup<BlockItem> IngestionPipeline::tasks_compress_

private

Definition at line 77 of file pipeline.h.

Referenced by IngestionPipeline(), Spawn(), and ~IngestionPipeline().

TubeConsumerGroup<BlockItem> IngestionPipeline::tasks_hash_

private

Definition at line 80 of file pipeline.h.

Referenced by IngestionPipeline(), Spawn(), and ~IngestionPipeline().

TubeConsumerGroup<FileItem> IngestionPipeline::tasks_read_

private

Definition at line 71 of file pipeline.h.

Referenced by IngestionPipeline(), Spawn(), and ~IngestionPipeline().

TubeConsumerGroup<FileItem> IngestionPipeline::tasks_register_

private

Definition at line 86 of file pipeline.h.

Referenced by IngestionPipeline(), Spawn(), and ~IngestionPipeline().

TubeConsumerGroup<BlockItem> IngestionPipeline::tasks_write_

private

Definition at line 83 of file pipeline.h.

Referenced by IngestionPipeline(), Spawn(), and ~IngestionPipeline().

Tube<FileItem> IngestionPipeline::tube_ctr_inflight_post_

private

Definition at line 67 of file pipeline.h.

Referenced by IngestionPipeline(), Process(), and WaitFor().

Tube<FileItem> IngestionPipeline::tube_ctr_inflight_pre_

private

Definition at line 64 of file pipeline.h.

Referenced by IngestionPipeline(), and Process().

Tube<FileItem> IngestionPipeline::tube_input_

private

Definition at line 69 of file pipeline.h.

Referenced by IngestionPipeline(), and Process().

TubeGroup<BlockItem> IngestionPipeline::tubes_chunk_

private

Definition at line 73 of file pipeline.h.

Referenced by IngestionPipeline().

TubeGroup<BlockItem> IngestionPipeline::tubes_compress_

private

Definition at line 76 of file pipeline.h.

Referenced by IngestionPipeline().

TubeGroup<BlockItem> IngestionPipeline::tubes_hash_

private

Definition at line 79 of file pipeline.h.

Referenced by IngestionPipeline().

TubeGroup<FileItem> IngestionPipeline::tubes_register_

private

Definition at line 85 of file pipeline.h.

Referenced by IngestionPipeline().

TubeGroup<BlockItem> IngestionPipeline::tubes_write_

private

Definition at line 82 of file pipeline.h.

Referenced by IngestionPipeline().

upload::AbstractUploader* IngestionPipeline::uploader_

private

Definition at line 57 of file pipeline.h.

Referenced by IngestionPipeline().

---

The documentation for this class was generated from the following files:

• /home/sftnight/jenkins/workspace/CvmfsDoxygen/label/docker-x86_64/cvmfs/cvmfs/ingestion/pipeline.h
• /home/sftnight/jenkins/workspace/CvmfsDoxygen/label/docker-x86_64/cvmfs/cvmfs/ingestion/pipeline.cc