#include <upload_gateway.h>

Inheritance diagram for upload::GatewayUploader:

Collaboration diagram for upload::GatewayUploader:

Classes
struct	Config

Public Member Functions
	GatewayUploader (const SpoolerDefinition &spooler_definition)

virtual	~GatewayUploader ()

virtual bool	Initialize ()

virtual bool	Create ()

virtual bool	FinalizeSession (bool commit, const std::string &old_root_hash, const std::string &new_root_hash, const RepositoryTag &tag)

virtual void	WaitForUpload () const

virtual std::string	name () const

virtual bool	Peek (const std::string &path)

virtual bool	Mkdir (const std::string &path)

virtual bool	PlaceBootstrappingShortcut (const shash::Any &object)

virtual unsigned int	GetNumberOfErrors () const

Public Member Functions inherited from upload::AbstractUploader
virtual	~AbstractUploader ()

virtual unsigned	GetNumTasks () const

void	TearDown ()

void	UploadFile (const std::string &local_path, const std::string &remote_path, const CallbackTN *callback=NULL)

void	UploadIngestionSource (const std::string &remote_path, IngestionSource source, const CallbackTN callback=NULL)

void	ScheduleUpload (UploadStreamHandle handle, UploadBuffer buffer, const CallbackTN callback=NULL)

void	ScheduleCommit (UploadStreamHandle *handle, const shash::Any &content_hash)

void	RemoveAsync (const std::string &file_to_delete)

void	RemoveAsync (const shash::Any &hash_to_delete)

int64_t	GetObjectSize (const shash::Any &hash)

void	InitCounters (perf::StatisticsTemplate *statistics)

Public Member Functions inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
virtual	~PolymorphicConstructionImpl ()

Static Public Member Functions
static bool	WillHandle (const SpoolerDefinition &spooler_definition)

static bool	ParseSpoolerDefinition (const SpoolerDefinition &spooler_definition, Config *config)

Static Public Member Functions inherited from upload::AbstractUploader
static void	RegisterPlugins ()

Static Public Member Functions inherited from PolymorphicConstruction< AbstractUploader, SpoolerDefinition >
static IntrospectionData	Introspect ()

Static Public Member Functions inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
static AbstractUploader *	Construct (const SpoolerDefinition &param)

Static Public Member Functions inherited from Callbackable< UploaderResults >
static CallbackTN *	MakeClosure (typename BoundClosure< UploaderResults, DelegateT, ClosureDataT >::CallbackMethod method, DelegateT *delegate, const ClosureDataT &closure_data)

static CallbackTN *	MakeCallback (typename BoundCallback< UploaderResults, DelegateT >::CallbackMethod method, DelegateT *delegate)

static CallbackTN *	MakeCallback (typename Callback< UploaderResults >::CallbackFunction function)

Protected Member Functions
virtual void	DoUpload (const std::string &remote_path, IngestionSource source, const CallbackTN callback)

virtual UploadStreamHandle *	InitStreamedUpload (const CallbackTN *callback)

virtual void	StreamedUpload (UploadStreamHandle handle, UploadBuffer buffer, const CallbackTN callback)

virtual void	FinalizeStreamedUpload (UploadStreamHandle *handle, const shash::Any &content_hash)

virtual void	DoRemoveAsync (const std::string &file_to_delete)

virtual void	ReadSessionTokenFile (const std::string &token_file_name, std::string *token)

virtual bool	ReadKey (const std::string &key_file, std::string key_id, std::string secret)

virtual int64_t	DoGetObjectSize (const std::string &file_name)

Protected Member Functions inherited from upload::AbstractUploader
	AbstractUploader (const SpoolerDefinition &spooler_definition)

void	Respond (const CallbackTN *callback, const UploaderResults &result) const

int	CreateAndOpenTemporaryChunkFile (std::string *path) const

const SpoolerDefinition &	spooler_definition () const

void	CountUploadedChunks () const

void	DecUploadedChunks () const

void	CountUploadedBytes (int64_t bytes_written) const

void	CountDuplicates () const

void	CountUploadedCatalogs () const

void	CountUploadedCatalogBytes (int64_t bytes_written) const

void	IncJobsInFlight ()

Protected Member Functions inherited from SingleCopy
	SingleCopy ()

Private Member Functions
void	BumpErrors () const

Private Attributes
Config	config_

SessionContext *	session_context_

atomic_int32	num_errors_

Additional Inherited Members
Public Types inherited from PolymorphicConstruction< AbstractUploader, SpoolerDefinition >
typedef std::vector< void >	IntrospectionData

Public Types inherited from Callbackable< UploaderResults >
typedef CallbackBase < UploaderResults >	CallbackTN

Protected Types inherited from upload::AbstractUploader
typedef Callbackable < UploaderResults > ::CallbackTN *	CallbackPtr

Protected Types inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
typedef AbstractFactory < AbstractUploader, SpoolerDefinition, void >	Factory

typedef std::vector< Factory * >	RegisteredPlugins

Static Protected Member Functions inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
static void	LazilyRegisterPlugins ()

static void	RegisterPlugin ()

Static Protected Attributes inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
static RegisteredPlugins	registered_plugins_

Detailed Description

Definition at line 25 of file upload_gateway.h.

Constructor & Destructor Documentation

upload::GatewayUploader::GatewayUploader ( const SpoolerDefinition & spooler_definition )

explicit

Definition at line 57 of file upload_gateway.cc.

Here is the call graph for this function:

upload::GatewayUploader::~GatewayUploader ( )

virtual

Definition at line 71 of file upload_gateway.cc.

Member Function Documentation

void upload::GatewayUploader::BumpErrors ( ) const

private

Definition at line 257 of file upload_gateway.cc.

Referenced by DoUpload(), FinalizeStreamedUpload(), and StreamedUpload().

Here is the caller graph for this function:

bool upload::GatewayUploader::Create ( )

virtual

Initializes a new repository storage area, e.g. create directory layout for local backend or create bucket for S3 backend.

Implements upload::AbstractUploader.

Definition at line 77 of file upload_gateway.cc.

Here is the call graph for this function:

int64_t upload::GatewayUploader::DoGetObjectSize ( const std::string & file_name )

protectedvirtual

Implements upload::AbstractUploader.

Definition at line 253 of file upload_gateway.cc.

void upload::GatewayUploader::DoRemoveAsync ( const std::string & file_to_delete )

protectedvirtual

Implements upload::AbstractUploader.

Definition at line 113 of file upload_gateway.cc.

Here is the call graph for this function:

void upload::GatewayUploader::DoUpload	(	const std::string &	remote_path,
		IngestionSource *	source,
		const CallbackTN *	callback
	)

protectedvirtual

Implementation of plain file upload Public interface: AbstractUploader::Upload()

Parameters

local_path	file to be uploaded
remote_path	destination to be written in the backend
callback	callback to be called on completion

Implements upload::AbstractUploader.

Definition at line 133 of file upload_gateway.cc.

Here is the call graph for this function:

bool upload::GatewayUploader::FinalizeSession	(	bool	commit,
		const std::string &	old_root_hash,
		const std::string &	new_root_hash,
		const RepositoryTag &	tag
	)

virtual

Called during Spooler::WaitForUpload(), to ensure that the upload has finished. If commit == true, then a Commit request is also sent, to apply all the the changes accumulated during the session. "catalog_path" represents the path of the root catalog with the changes. By default it is a noop and returns true;

Reimplemented from upload::AbstractUploader.

Definition at line 100 of file upload_gateway.cc.

Here is the call graph for this function:

void upload::GatewayUploader::FinalizeStreamedUpload	(	UploadStreamHandle *	handle,
		const shash::Any &	content_hash
	)

protectedvirtual

Implementation of streamed upload commit Public interface: AbstractUploader::ScheduleUpload()

Parameters

handle	descendant of UploadStreamHandle specifying the stream
content_hash	the computed content hash of the streamed object

Implements upload::AbstractUploader.

Definition at line 198 of file upload_gateway.cc.

Here is the call graph for this function:

unsigned int upload::GatewayUploader::GetNumberOfErrors ( ) const

virtual

Implements upload::AbstractUploader.

Definition at line 129 of file upload_gateway.cc.

bool upload::GatewayUploader::Initialize ( )

virtual

This is called right after the constructor of AbstractUploader or/and its derived class has been executed. You can override that to do additional initialization that cannot be done in the constructor itself.

Returns: true on successful initialization

Reimplemented from upload::AbstractUploader.

Definition at line 83 of file upload_gateway.cc.

Here is the call graph for this function:

UploadStreamHandle * upload::GatewayUploader::InitStreamedUpload ( const CallbackTN * callback )

protectedvirtual

This method is called before the first data block of a streamed upload is scheduled (see above implementation of UploadStreamHandle for details).

Parameters

callback (optional) this callback will be invoked once this parti- cular streamed upload is committed.

Returns: a pointer to the initialized UploadStreamHandle

Implements upload::AbstractUploader.

Definition at line 176 of file upload_gateway.cc.

Here is the call graph for this function:

bool upload::GatewayUploader::Mkdir ( const std::string & path )

virtual

Make directory in upstream storage. Noop if directory already present.

Parameters

path	relative directory path in the upstream storage

Returns: true if the directory was successfully created or already present

Implements upload::AbstractUploader.

Definition at line 121 of file upload_gateway.cc.

std::string upload::GatewayUploader::name ( ) const

virtual

A string identifying the uploader type

Implements upload::AbstractUploader.

Definition at line 111 of file upload_gateway.cc.

bool upload::GatewayUploader::ParseSpoolerDefinition	(	const SpoolerDefinition &	spooler_definition,
		GatewayUploader::Config *	config
	)

static

Definition at line 26 of file upload_gateway.cc.

Referenced by GatewayUploader().

Here is the call graph for this function:

Here is the caller graph for this function:

bool upload::GatewayUploader::Peek ( const std::string & path )

virtual

Checks if a file is already present in the backend storage. This might be a synchronous operation.

Parameters

path	the path of the file to be checked

Returns: true if the file was found in the backend storage

Implements upload::AbstractUploader.

Definition at line 118 of file upload_gateway.cc.

bool upload::GatewayUploader::PlaceBootstrappingShortcut ( const shash::Any & object )

virtual

Creates a top-level shortcut to the given data object. This is particularly useful for bootstrapping repositories whose data-directory is secured by a VOMS certificate.

Parameters

object content hash of the object to be exposed on the top-level

Returns: true on success

Implements upload::AbstractUploader.

Definition at line 125 of file upload_gateway.cc.

bool upload::GatewayUploader::ReadKey	(	const std::string &	key_file,
		std::string *	key_id,
		std::string *	secret
	)

protectedvirtual

Definition at line 248 of file upload_gateway.cc.

Referenced by Initialize().

Here is the call graph for this function:

Here is the caller graph for this function:

void upload::GatewayUploader::ReadSessionTokenFile	(	const std::string &	token_file_name,
		std::string *	token
	)

protectedvirtual

Definition at line 232 of file upload_gateway.cc.

Referenced by Initialize().

Here is the call graph for this function:

Here is the caller graph for this function:

void upload::GatewayUploader::StreamedUpload	(	UploadStreamHandle *	handle,
		UploadBuffer	buffer,
		const CallbackTN *	callback
	)

protectedvirtual

Implementation of a streamed upload step. See public interface for details. Public interface: AbstractUploader::ScheduleUpload()

Parameters

handle	descendant of UploadStreamHandle specifying the stream
buffer	the CharBuffer to be uploaded to the stream
callback	callback to be called on completion

Implements upload::AbstractUploader.

Definition at line 181 of file upload_gateway.cc.

Here is the call graph for this function:

void upload::GatewayUploader::WaitForUpload ( ) const

virtual

Waits until the current upload queue is empty.

Note: This does NOT necessarily mean, that all files are actually uploaded. If new jobs are concurrently scheduled the behavior of this method is not defined (it returns also on intermediately empty queues)

Reimplemented from upload::AbstractUploader.

Definition at line 107 of file upload_gateway.cc.

Here is the call graph for this function:

bool upload::GatewayUploader::WillHandle ( const SpoolerDefinition & spooler_definition )

static

Definition at line 22 of file upload_gateway.cc.

Member Data Documentation

Config upload::GatewayUploader::config_

private

Definition at line 97 of file upload_gateway.h.

Referenced by GatewayUploader(), and Initialize().

atomic_int32 upload::GatewayUploader::num_errors_

mutableprivate

Definition at line 99 of file upload_gateway.h.

Referenced by BumpErrors(), DoRemoveAsync(), GatewayUploader(), and GetNumberOfErrors().

SessionContext* upload::GatewayUploader::session_context_

private

Definition at line 98 of file upload_gateway.h.

Referenced by DoUpload(), FinalizeSession(), FinalizeStreamedUpload(), Initialize(), InitStreamedUpload(), WaitForUpload(), and ~GatewayUploader().

The documentation for this class was generated from the following files:

/home/sftnight/jenkins/workspace/CvmfsDoxygen/label/docker-x86_64/cvmfs/cvmfs/upload_gateway.h
/home/sftnight/jenkins/workspace/CvmfsDoxygen/label/docker-x86_64/cvmfs/cvmfs/upload_gateway.cc

Classes

Public Member Functions

Static Public Member Functions

Protected Member Functions

Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation