#include <upload_facility.h>

Inheritance diagram for upload::AbstractUploader:

Collaboration diagram for upload::AbstractUploader:

Classes
struct	JobStatus

struct	UploadBuffer

struct	UploadJob

Public Member Functions
virtual	~AbstractUploader ()

virtual std::string	name () const =0

virtual bool	Create ()=0

virtual unsigned	GetNumTasks () const

virtual bool	Initialize ()

virtual bool	FinalizeSession (bool commit, const std::string &old_root_hash, const std::string &new_root_hash, const RepositoryTag &tag)

void	TearDown ()

void	UploadFile (const std::string &local_path, const std::string &remote_path, const CallbackTN *callback=NULL)

void	UploadIngestionSource (const std::string &remote_path, IngestionSource source, const CallbackTN callback=NULL)

virtual UploadStreamHandle *	InitStreamedUpload (const CallbackTN *callback)=0

void	ScheduleUpload (UploadStreamHandle handle, UploadBuffer buffer, const CallbackTN callback=NULL)

void	ScheduleCommit (UploadStreamHandle *handle, const shash::Any &content_hash)

void	RemoveAsync (const std::string &file_to_delete)

void	RemoveAsync (const shash::Any &hash_to_delete)

int64_t	GetObjectSize (const shash::Any &hash)

virtual bool	Peek (const std::string &path)=0

virtual bool	Mkdir (const std::string &path)=0

virtual bool	PlaceBootstrappingShortcut (const shash::Any &object)=0

virtual void	WaitForUpload () const

virtual unsigned int	GetNumberOfErrors () const =0

void	InitCounters (perf::StatisticsTemplate *statistics)

Public Member Functions inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
virtual	~PolymorphicConstructionImpl ()

Static Public Member Functions
static void	RegisterPlugins ()

Static Public Member Functions inherited from PolymorphicConstruction< AbstractUploader, SpoolerDefinition >
static IntrospectionData	Introspect ()

Static Public Member Functions inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
static AbstractUploader *	Construct (const SpoolerDefinition &param)

Static Public Member Functions inherited from Callbackable< UploaderResults >
static CallbackTN *	MakeClosure (typename BoundClosure< UploaderResults, DelegateT, ClosureDataT >::CallbackMethod method, DelegateT *delegate, const ClosureDataT &closure_data)

static CallbackTN *	MakeCallback (typename BoundCallback< UploaderResults, DelegateT >::CallbackMethod method, DelegateT *delegate)

static CallbackTN *	MakeCallback (typename Callback< UploaderResults >::CallbackFunction function)

Protected Types
typedef Callbackable < UploaderResults > ::CallbackTN *	CallbackPtr

Protected Types inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
typedef AbstractFactory < AbstractUploader, SpoolerDefinition, void >	Factory

typedef std::vector< Factory * >	RegisteredPlugins

Protected Member Functions
	AbstractUploader (const SpoolerDefinition &spooler_definition)

virtual void	DoUpload (const std::string &remote_path, IngestionSource source, const CallbackTN callback)=0

virtual void	StreamedUpload (UploadStreamHandle handle, UploadBuffer buffer, const CallbackTN callback)=0

virtual void	FinalizeStreamedUpload (UploadStreamHandle *handle, const shash::Any &content_hash)=0

virtual void	DoRemoveAsync (const std::string &file_to_delete)=0

virtual int64_t	DoGetObjectSize (const std::string &file_name)=0

void	Respond (const CallbackTN *callback, const UploaderResults &result) const

int	CreateAndOpenTemporaryChunkFile (std::string *path) const

const SpoolerDefinition &	spooler_definition () const

void	CountUploadedChunks () const

void	DecUploadedChunks () const

void	CountUploadedBytes (int64_t bytes_written) const

void	CountDuplicates () const

void	CountUploadedCatalogs () const

void	CountUploadedCatalogBytes (int64_t bytes_written) const

void	IncJobsInFlight ()

Protected Member Functions inherited from SingleCopy
	SingleCopy ()

Private Attributes
const SpoolerDefinition	spooler_definition_

unsigned	num_upload_tasks_

SynchronizingCounter< int32_t >	jobs_in_flight_

TubeGroup< UploadJob >	tubes_upload_

TubeConsumerGroup< UploadJob >	tasks_upload_

UniquePtr< UploadCounters >	counters_

Friends
class	TaskUpload

Additional Inherited Members
Public Types inherited from PolymorphicConstruction< AbstractUploader, SpoolerDefinition >
typedef std::vector< void >	IntrospectionData

Public Types inherited from Callbackable< UploaderResults >
typedef CallbackBase < UploaderResults >	CallbackTN

Static Protected Member Functions inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
static void	LazilyRegisterPlugins ()

static void	RegisterPlugin ()

Static Protected Attributes inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
static RegisteredPlugins	registered_plugins_

Detailed Description

Abstract base class for all backend upload facilities This class defines an interface and constructs the concrete Uploaders, furthermore it handles callbacks to the outside world to notify users of done upload jobs.

Note: Users could be both the Spooler (when calling Spooler::Upload()) and the IngestionPipeline (when calling Spooler::Process()). We therefore cannot use the Observable template here, since this would forward finished upload jobs to ALL listeners instead of only the owner of the specific job.

Definition at line 83 of file upload_facility.h.

Member Typedef Documentation

typedef Callbackable<UploaderResults>::CallbackTN* upload::AbstractUploader::CallbackPtr

protected

Definition at line 334 of file upload_facility.h.

Constructor & Destructor Documentation

virtual upload::AbstractUploader::~AbstractUploader ( )

inlinevirtual

Definition at line 138 of file upload_facility.h.

Here is the call graph for this function:

upload::AbstractUploader::AbstractUploader ( const SpoolerDefinition & spooler_definition )

explicitprotected

Definition at line 46 of file upload_facility.cc.

Member Function Documentation

void upload::AbstractUploader::CountDuplicates ( ) const

protected

Definition at line 125 of file upload_facility.cc.

Referenced by upload::LocalUploader::FinalizeStreamedUpload(), and upload::S3Uploader::MainCollectResults().

Here is the call graph for this function:

Here is the caller graph for this function:

void upload::AbstractUploader::CountUploadedBytes ( int64_t bytes_written ) const

protected

Definition at line 119 of file upload_facility.cc.

Referenced by upload::LocalUploader::FinalizeStreamedUpload(), upload::S3Uploader::FinalizeStreamedUpload(), upload::GatewayUploader::FinalizeStreamedUpload(), and upload::S3Uploader::MainCollectResults().

Here is the call graph for this function:

Here is the caller graph for this function:

void upload::AbstractUploader::CountUploadedCatalogBytes ( int64_t bytes_written ) const

protected

Definition at line 137 of file upload_facility.cc.

Referenced by upload::LocalUploader::FinalizeStreamedUpload(), upload::S3Uploader::FinalizeStreamedUpload(), and upload::GatewayUploader::FinalizeStreamedUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

void upload::AbstractUploader::CountUploadedCatalogs ( ) const

protected

Definition at line 131 of file upload_facility.cc.

Referenced by upload::LocalUploader::FinalizeStreamedUpload(), upload::S3Uploader::FinalizeStreamedUpload(), and upload::GatewayUploader::FinalizeStreamedUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

void upload::AbstractUploader::CountUploadedChunks ( ) const

protected

Definition at line 107 of file upload_facility.cc.

Referenced by upload::LocalUploader::FinalizeStreamedUpload(), upload::S3Uploader::FinalizeStreamedUpload(), and upload::GatewayUploader::FinalizeStreamedUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

virtual bool upload::AbstractUploader::Create ( )

pure virtual

Initializes a new repository storage area, e.g. create directory layout for local backend or create bucket for S3 backend.

Implemented in upload::GatewayUploader, upload::S3Uploader, and upload::LocalUploader.

int upload::AbstractUploader::CreateAndOpenTemporaryChunkFile ( std::string * path ) const

protected

Creates a temporary file in the backend storage's temporary location For the LocalUploader this usually is the 'txn' directory of the backend storage. Otherwise it is some scratch area.

Parameters

path	pointer to a string that will contain the created file path

Returns: a file descriptor to the opened file

Definition at line 72 of file upload_facility.cc.

Referenced by upload::LocalUploader::InitStreamedUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

void upload::AbstractUploader::DecUploadedChunks ( ) const

protected

Definition at line 113 of file upload_facility.cc.

Referenced by upload::S3Uploader::MainCollectResults().

Here is the call graph for this function:

Here is the caller graph for this function:

virtual int64_t upload::AbstractUploader::DoGetObjectSize ( const std::string & file_name )

protectedpure virtual

Implemented in upload::GatewayUploader, upload::LocalUploader, and upload::S3Uploader.

Referenced by GetObjectSize().

Here is the caller graph for this function:

virtual void upload::AbstractUploader::DoRemoveAsync ( const std::string & file_to_delete )

protectedpure virtual

Implemented in upload::GatewayUploader, upload::S3Uploader, and upload::LocalUploader.

Referenced by RemoveAsync().

Here is the caller graph for this function:

virtual void upload::AbstractUploader::DoUpload	(	const std::string &	remote_path,
		IngestionSource *	source,
		const CallbackTN *	callback
	)

protectedpure virtual

Implementation of plain file upload Public interface: AbstractUploader::Upload()

Parameters

local_path	file to be uploaded
remote_path	destination to be written in the backend
callback	callback to be called on completion

Implemented in upload::GatewayUploader, upload::S3Uploader, and upload::LocalUploader.

Referenced by UploadFile(), and UploadIngestionSource().

Here is the caller graph for this function:

bool upload::AbstractUploader::FinalizeSession	(	bool	commit,
		const std::string &	old_root_hash,
		const std::string &	new_root_hash,
		const RepositoryTag &	tag
	)

virtual

Called during Spooler::WaitForUpload(), to ensure that the upload has finished. If commit == true, then a Commit request is also sent, to apply all the the changes accumulated during the session. "catalog_path" represents the path of the root catalog with the changes. By default it is a noop and returns true;

Reimplemented in upload::GatewayUploader.

Definition at line 64 of file upload_facility.cc.

virtual void upload::AbstractUploader::FinalizeStreamedUpload	(	UploadStreamHandle *	handle,
		const shash::Any &	content_hash
	)

protectedpure virtual

Implementation of streamed upload commit Public interface: AbstractUploader::ScheduleUpload()

Parameters

handle	descendant of UploadStreamHandle specifying the stream
content_hash	the computed content hash of the streamed object

Implemented in upload::GatewayUploader, upload::S3Uploader, and upload::LocalUploader.

Referenced by upload::TaskUpload::Process().

Here is the caller graph for this function:

virtual unsigned int upload::AbstractUploader::GetNumberOfErrors ( ) const

pure virtual

Implemented in upload::LocalUploader, upload::S3Uploader, and upload::GatewayUploader.

Referenced by StatisticsDatabase::UploadStatistics().

Here is the caller graph for this function:

virtual unsigned upload::AbstractUploader::GetNumTasks ( ) const

inlinevirtual

Concrete uploaders might want to use a customized setting for multi-stream writing, for instance one per disk. Note that the S3 backend uses one task but this one task uses internally multiple HTTP streams through curl async I/O.

Definition at line 157 of file upload_facility.h.

Referenced by Initialize().

Here is the caller graph for this function:

int64_t upload::AbstractUploader::GetObjectSize ( const shash::Any & hash )

inline

Get object size based on its content hash

Parameters

hash	the content hash of a file

Definition at line 289 of file upload_facility.h.

Here is the call graph for this function:

void upload::AbstractUploader::IncJobsInFlight ( )

inlineprotected

Used by concrete implementations when they use callbacks where it's not already foreseen, e.g. S3Uploader::Peek().

Definition at line 423 of file upload_facility.h.

Referenced by upload::S3Uploader::Create(), and upload::S3Uploader::Peek().

Here is the caller graph for this function:

void upload::AbstractUploader::InitCounters ( perf::StatisticsTemplate * statistics )

Definition at line 103 of file upload_facility.cc.

bool upload::AbstractUploader::Initialize ( )

virtual

This is called right after the constructor of AbstractUploader or/and its derived class has been executed. You can override that to do additional initialization that cannot be done in the constructor itself.

Returns: true on successful initialization

Reimplemented from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >.

Reimplemented in upload::GatewayUploader.

Definition at line 53 of file upload_facility.cc.

Referenced by upload::GatewayUploader::Initialize().

Here is the call graph for this function:

Here is the caller graph for this function:

virtual UploadStreamHandle* upload::AbstractUploader::InitStreamedUpload ( const CallbackTN * callback )

pure virtual

This method is called before the first data block of a streamed upload is scheduled (see above implementation of UploadStreamHandle for details).

Parameters

callback (optional) this callback will be invoked once this parti- cular streamed upload is committed.

Returns: a pointer to the initialized UploadStreamHandle

Implemented in upload::GatewayUploader, upload::S3Uploader, and upload::LocalUploader.

Referenced by TaskWrite::Process().

Here is the caller graph for this function:

virtual bool upload::AbstractUploader::Mkdir ( const std::string & path )

pure virtual

Make directory in upstream storage. Noop if directory already present.

Parameters

path	relative directory path in the upstream storage

Returns: true if the directory was successfully created or already present

Implemented in upload::S3Uploader, upload::LocalUploader, and upload::GatewayUploader.

virtual std::string upload::AbstractUploader::name ( ) const

pure virtual

A string identifying the uploader type

Implemented in upload::GatewayUploader, upload::S3Uploader, and upload::LocalUploader.

virtual bool upload::AbstractUploader::Peek ( const std::string & path )

pure virtual

Checks if a file is already present in the backend storage. This might be a synchronous operation.

Parameters

path	the path of the file to be checked

Returns: true if the file was found in the backend storage

Implemented in upload::S3Uploader, upload::LocalUploader, and upload::GatewayUploader.

virtual bool upload::AbstractUploader::PlaceBootstrappingShortcut ( const shash::Any & object )

pure virtual

Creates a top-level shortcut to the given data object. This is particularly useful for bootstrapping repositories whose data-directory is secured by a VOMS certificate.

Parameters

object content hash of the object to be exposed on the top-level

Returns: true on success

Implemented in upload::LocalUploader, upload::S3Uploader, and upload::GatewayUploader.

void upload::AbstractUploader::RegisterPlugins ( )

static

Definition at line 40 of file upload_facility.cc.

void upload::AbstractUploader::RemoveAsync ( const std::string & file_to_delete )

inline

Removes a file from the backend storage.

Note: If the file doesn't exist before calling this won't be an error.

Parameters

file_to_delete path to the file to be removed

Definition at line 270 of file upload_facility.h.

Referenced by RemoveAsync(), and StatisticsDatabase::UploadStatistics().

Here is the call graph for this function:

Here is the caller graph for this function:

void upload::AbstractUploader::RemoveAsync ( const shash::Any & hash_to_delete )

inline

Overloaded method used to remove a object based on its content hash.

Parameters

hash_to_delete the content hash of a file to be deleted

Definition at line 280 of file upload_facility.h.

Here is the call graph for this function:

void upload::AbstractUploader::Respond	(	const CallbackTN *	callback,
		const UploaderResults &	result
	)		const

inlineprotected

This notifies the callback that is associated to a finishing job. Please do not call the handed callback yourself in concrete Uploaders!

Note: Since the job is finished after we respond to it, the callback object gets automatically destroyed by this call! Therefore you must not call Respond() twice or use the callback later by any means!

Definition at line 386 of file upload_facility.h.

Referenced by upload::LocalUploader::DoRemoveAsync(), upload::GatewayUploader::DoRemoveAsync(), upload::LocalUploader::DoUpload(), upload::S3Uploader::DoUpload(), upload::GatewayUploader::DoUpload(), upload::LocalUploader::FinalizeStreamedUpload(), upload::GatewayUploader::FinalizeStreamedUpload(), upload::S3Uploader::MainCollectResults(), upload::LocalUploader::StreamedUpload(), upload::S3Uploader::StreamedUpload(), and upload::GatewayUploader::StreamedUpload().

Here is the caller graph for this function:

void upload::AbstractUploader::ScheduleCommit	(	UploadStreamHandle *	handle,
		const shash::Any &	content_hash
	)

inline

This method schedules a commit job as soon as all data blocks of a streamed upload are (successfully) uploaded. Derived classes must override AbstractUploader::FinalizeStreamedUpload() for this to happen.

Parameters

handle	Pointer to a previously acquired UploadStreamHandle
content_hash	the content hash of the full uploaded data Chunk

Definition at line 255 of file upload_facility.h.

Referenced by TaskWrite::Process().

Here is the caller graph for this function:

void upload::AbstractUploader::ScheduleUpload	(	UploadStreamHandle *	handle,
		UploadBuffer	buffer,
		const CallbackTN *	callback = `NULL`
	)

inline

This method schedules a buffer to be uploaded in the context of the given UploadStreamHandle. The actual upload will happen asynchronously by a concrete implementation of AbstractUploader (see AbstractUploader::StreamedUpload()). As soon has the scheduled upload job is complete (either successful or not) the optionally passed callback is supposed to be invoked using AbstractUploader::Respond().

Parameters

handle	Pointer to a previously acquired UploadStreamHandle
buffer	contains the data block to be uploaded
callback	(optional) callback object to be invoked once the given upload is finished (see AbstractUploader::Respond())

Definition at line 238 of file upload_facility.h.

Referenced by TaskWrite::Process().

Here is the caller graph for this function:

const SpoolerDefinition& upload::AbstractUploader::spooler_definition ( ) const

inlineprotected

Definition at line 407 of file upload_facility.h.

Referenced by upload::S3Uploader::DoUpload(), upload::GatewayUploader::DoUpload(), and upload::S3Uploader::InitStreamedUpload().

Here is the caller graph for this function:

virtual void upload::AbstractUploader::StreamedUpload	(	UploadStreamHandle *	handle,
		UploadBuffer	buffer,
		const CallbackTN *	callback
	)

protectedpure virtual

Implementation of a streamed upload step. See public interface for details. Public interface: AbstractUploader::ScheduleUpload()

Parameters

handle	descendant of UploadStreamHandle specifying the stream
buffer	the CharBuffer to be uploaded to the stream
callback	callback to be called on completion

Implemented in upload::GatewayUploader, upload::S3Uploader, and upload::LocalUploader.

Referenced by upload::TaskUpload::Process().

Here is the caller graph for this function:

void upload::AbstractUploader::TearDown ( )

This must be called right before the destruction of the AbstractUploader! You are not supposed to overwrite this method in your concrete Uploader.

Definition at line 97 of file upload_facility.cc.

void upload::AbstractUploader::UploadFile	(	const std::string &	local_path,
		const std::string &	remote_path,
		const CallbackTN *	callback = `NULL`
	)

inline

Uploads the file at the path local_path into the backend storage under the path remote_path. When the upload has finished it calls callback. Note: This method might be implemented in a synchronous way.

Parameters

local_path	path to the file to be uploaded
remote_path	desired path for the file in the backend storage
callback	(optional) gets notified when the upload was finished

Definition at line 194 of file upload_facility.h.

Referenced by StatisticsDatabase::UploadStatistics().

Here is the call graph for this function:

Here is the caller graph for this function:

void upload::AbstractUploader::UploadIngestionSource	(	const std::string &	remote_path,
		IngestionSource *	source,
		const CallbackTN *	callback = `NULL`
	)

inline

Definition at line 204 of file upload_facility.h.

Here is the call graph for this function:

void upload::AbstractUploader::WaitForUpload ( ) const

virtual

Waits until the current upload queue is empty.

Note: This does NOT necessarily mean, that all files are actually uploaded. If new jobs are concurrently scheduled the behavior of this method is not defined (it returns also on intermediately empty queues)

Reimplemented in upload::GatewayUploader.

Definition at line 101 of file upload_facility.cc.

Referenced by StatisticsDatabase::UploadStatistics().

Here is the call graph for this function:

Here is the caller graph for this function:

Friends And Related Function Documentation

friend class TaskUpload

friend

Definition at line 87 of file upload_facility.h.

Referenced by Initialize().

Member Data Documentation

UniquePtr<UploadCounters> upload::AbstractUploader::counters_

mutableprivate

Definition at line 438 of file upload_facility.h.

Referenced by CountDuplicates(), CountUploadedBytes(), CountUploadedCatalogBytes(), CountUploadedCatalogs(), CountUploadedChunks(), DecUploadedChunks(), and InitCounters().

SynchronizingCounter<int32_t> upload::AbstractUploader::jobs_in_flight_

mutableprivate

Definition at line 435 of file upload_facility.h.

Referenced by IncJobsInFlight(), RemoveAsync(), Respond(), ScheduleCommit(), ScheduleUpload(), UploadFile(), UploadIngestionSource(), and WaitForUpload().

unsigned upload::AbstractUploader::num_upload_tasks_

private

Number of threads used for I/O write calls. Effectively this parameter sets the I/O depth. Defaults to 1.

Definition at line 434 of file upload_facility.h.

Referenced by GetNumTasks().

const SpoolerDefinition upload::AbstractUploader::spooler_definition_

private

Definition at line 428 of file upload_facility.h.

Referenced by CreateAndOpenTemporaryChunkFile(), and spooler_definition().

TubeConsumerGroup<UploadJob> upload::AbstractUploader::tasks_upload_

private

Definition at line 437 of file upload_facility.h.

Referenced by Initialize(), TearDown(), and ~AbstractUploader().

TubeGroup<UploadJob> upload::AbstractUploader::tubes_upload_

private

Definition at line 436 of file upload_facility.h.

Referenced by Initialize(), ScheduleCommit(), and ScheduleUpload().

The documentation for this class was generated from the following files:

/home/sftnight/jenkins/workspace/CvmfsDoxygen/label/docker-x86_64/cvmfs/cvmfs/upload_facility.h
/home/sftnight/jenkins/workspace/CvmfsDoxygen/label/docker-x86_64/cvmfs/cvmfs/upload_facility.cc

Classes

Public Member Functions

Static Public Member Functions

Protected Types

Protected Member Functions

Private Attributes

Friends

Additional Inherited Members

Detailed Description

Member Typedef Documentation

Constructor & Destructor Documentation

Member Function Documentation

Friends And Related Function Documentation

Member Data Documentation