#include <upload_s3.h>

Inheritance diagram for upload::S3Uploader:

Collaboration diagram for upload::S3Uploader:

Classes
struct	RequestCtrl

Public Member Functions
	S3Uploader (const SpoolerDefinition &spooler_definition)

virtual	~S3Uploader ()

virtual std::string	name () const

virtual bool	Create ()

virtual void	DoUpload (const std::string &remote_path, IngestionSource source, const CallbackTN callback)

virtual UploadStreamHandle *	InitStreamedUpload (const CallbackTN *callback=NULL)

virtual void	StreamedUpload (UploadStreamHandle handle, UploadBuffer buffer, const CallbackTN callback)

virtual void	FinalizeStreamedUpload (UploadStreamHandle *handle, const shash::Any &content_hash)

virtual void	DoRemoveAsync (const std::string &file_to_delete)

virtual bool	Peek (const std::string &path)

virtual bool	Mkdir (const std::string &path)

virtual bool	PlaceBootstrappingShortcut (const shash::Any &object)

virtual unsigned int	GetNumberOfErrors () const

int64_t	DoGetObjectSize (const std::string &file_name)

s3fanout::S3FanoutManager *	GetS3FanoutManager ()

Public Member Functions inherited from upload::AbstractUploader
virtual	~AbstractUploader ()

virtual unsigned	GetNumTasks () const

virtual bool	Initialize ()

virtual bool	FinalizeSession (bool commit, const std::string &old_root_hash, const std::string &new_root_hash, const RepositoryTag &tag)

void	TearDown ()

void	UploadFile (const std::string &local_path, const std::string &remote_path, const CallbackTN *callback=NULL)

void	UploadIngestionSource (const std::string &remote_path, IngestionSource source, const CallbackTN callback=NULL)

void	ScheduleUpload (UploadStreamHandle handle, UploadBuffer buffer, const CallbackTN callback=NULL)

void	ScheduleCommit (UploadStreamHandle *handle, const shash::Any &content_hash)

void	RemoveAsync (const std::string &file_to_delete)

void	RemoveAsync (const shash::Any &hash_to_delete)

int64_t	GetObjectSize (const shash::Any &hash)

virtual void	WaitForUpload () const

void	InitCounters (perf::StatisticsTemplate *statistics)

Public Member Functions inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
virtual	~PolymorphicConstructionImpl ()

Static Public Member Functions
static bool	WillHandle (const SpoolerDefinition &spooler_definition)

Static Public Member Functions inherited from upload::AbstractUploader
static void	RegisterPlugins ()

Static Public Member Functions inherited from PolymorphicConstruction< AbstractUploader, SpoolerDefinition >
static IntrospectionData	Introspect ()

Static Public Member Functions inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
static AbstractUploader *	Construct (const SpoolerDefinition &param)

Static Public Member Functions inherited from Callbackable< UploaderResults >
static CallbackTN *	MakeClosure (typename BoundClosure< UploaderResults, DelegateT, ClosureDataT >::CallbackMethod method, DelegateT *delegate, const ClosureDataT &closure_data)

static CallbackTN *	MakeCallback (typename BoundCallback< UploaderResults, DelegateT >::CallbackMethod method, DelegateT *delegate)

static CallbackTN *	MakeCallback (typename Callback< UploaderResults >::CallbackFunction function)

Private Member Functions
void	OnReqComplete (const upload::UploaderResults &results, RequestCtrl *ctrl)

bool	ParseSpoolerDefinition (const SpoolerDefinition &spooler_definition)

void	UploadJobInfo (s3fanout::JobInfo *info)

s3fanout::JobInfo *	CreateJobInfo (const std::string &path) const

Static Private Member Functions
static void *	MainCollectResults (void *data)

Private Attributes
UniquePtr < s3fanout::S3FanoutManager >	s3fanout_mgr_

std::string	repository_alias_

std::string	host_name_port_

std::string	host_name_

std::string	region_

std::string	flavor_

std::string	bucket_

bool	dns_buckets_

int	num_parallel_uploads_

unsigned	num_retries_

unsigned	timeout_sec_

std::string	access_key_

std::string	secret_key_

s3fanout::AuthzMethods	authz_method_

bool	peek_before_put_

bool	use_https_

std::string	proxy_

const std::string	temporary_path_

atomic_int32	io_errors_

pthread_t	thread_collect_results_

std::string	x_amz_acl_

Static Private Attributes
static const unsigned	kDefaultPort = 80

static const unsigned	kHttpsPort = 443

static const unsigned	kDefaultNumParallelUploads = 16

static const unsigned	kDefaultNumRetries = 3

static const unsigned	kDefaultTimeoutSec = 60

static const unsigned	kDefaultBackoffInitMs = 100

static const unsigned	kDefaultBackoffMaxMs = 2000

static const unsigned	kInMemoryObjectThreshold = 500 * 1024

Additional Inherited Members
Public Types inherited from PolymorphicConstruction< AbstractUploader, SpoolerDefinition >
typedef std::vector< void >	IntrospectionData

Public Types inherited from Callbackable< UploaderResults >
typedef CallbackBase < UploaderResults >	CallbackTN

Protected Types inherited from upload::AbstractUploader
typedef Callbackable < UploaderResults > ::CallbackTN *	CallbackPtr

Protected Types inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
typedef AbstractFactory < AbstractUploader, SpoolerDefinition, void >	Factory

typedef std::vector< Factory * >	RegisteredPlugins

Protected Member Functions inherited from upload::AbstractUploader
	AbstractUploader (const SpoolerDefinition &spooler_definition)

void	Respond (const CallbackTN *callback, const UploaderResults &result) const

int	CreateAndOpenTemporaryChunkFile (std::string *path) const

const SpoolerDefinition &	spooler_definition () const

void	CountUploadedChunks () const

void	DecUploadedChunks () const

void	CountUploadedBytes (int64_t bytes_written) const

void	CountDuplicates () const

void	CountUploadedCatalogs () const

void	CountUploadedCatalogBytes (int64_t bytes_written) const

void	IncJobsInFlight ()

Protected Member Functions inherited from SingleCopy
	SingleCopy ()

Static Protected Member Functions inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
static void	LazilyRegisterPlugins ()

static void	RegisterPlugin ()

Static Protected Attributes inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
static RegisteredPlugins	registered_plugins_

Detailed Description

The S3Spooler implements the AbstractSpooler interface to push files into a S3 CVMFS repository backend. For a detailed description of the classes interface please have a look into the AbstractSpooler base class.

Definition at line 41 of file upload_s3.h.

Constructor & Destructor Documentation

upload::S3Uploader::S3Uploader ( const SpoolerDefinition & spooler_definition )

explicit

Definition at line 45 of file upload_s3.cc.

Here is the call graph for this function:

upload::S3Uploader::~S3Uploader ( )

virtual

Definition at line 98 of file upload_s3.cc.

Member Function Documentation

bool upload::S3Uploader::Create ( )

virtual

Initializes a new repository storage area, e.g. create directory layout for local backend or create bucket for S3 backend.

Implements upload::AbstractUploader.

Definition at line 231 of file upload_s3.cc.

Here is the call graph for this function:

s3fanout::JobInfo * upload::S3Uploader::CreateJobInfo ( const std::string & path ) const

private

Definition at line 449 of file upload_s3.cc.

Referenced by Create(), DoRemoveAsync(), and Peek().

Here is the call graph for this function:

Here is the caller graph for this function:

int64_t upload::S3Uploader::DoGetObjectSize ( const std::string & file_name )

virtual

Implements upload::AbstractUploader.

Definition at line 510 of file upload_s3.cc.

void upload::S3Uploader::DoRemoveAsync ( const std::string & file_to_delete )

virtual

Implements upload::AbstractUploader.

Definition at line 455 of file upload_s3.cc.

Here is the call graph for this function:

void upload::S3Uploader::DoUpload	(	const std::string &	remote_path,
		IngestionSource *	source,
		const CallbackTN *	callback
	)

virtual

Upload() is not done concurrently in the current implementation of the S3Spooler, since it is a simple move or copy of a file without CPU intensive operation This method calls NotifyListeners and invokes a callback for all registered listeners (see the Observable template for details).

Implements upload::AbstractUploader.

Definition at line 322 of file upload_s3.cc.

Here is the call graph for this function:

void upload::S3Uploader::FinalizeStreamedUpload	(	UploadStreamHandle *	handle,
		const shash::Any &	content_hash
	)

virtual

Implementation of streamed upload commit Public interface: AbstractUploader::ScheduleUpload()

Parameters

handle	descendant of UploadStreamHandle specifying the stream
content_hash	the computed content hash of the streamed object

Implements upload::AbstractUploader.

Definition at line 409 of file upload_s3.cc.

Here is the call graph for this function:

unsigned int upload::S3Uploader::GetNumberOfErrors ( ) const

virtual

Implements upload::AbstractUploader.

Definition at line 262 of file upload_s3.cc.

s3fanout::S3FanoutManager* upload::S3Uploader::GetS3FanoutManager ( )

inline

Definition at line 78 of file upload_s3.h.

Here is the call graph for this function:

UploadStreamHandle * upload::S3Uploader::InitStreamedUpload ( const CallbackTN * callback = NULL )

virtual

This method is called before the first data block of a streamed upload is scheduled (see above implementation of UploadStreamHandle for details).

Parameters

callback (optional) this callback will be invoked once this parti- cular streamed upload is committed.

Returns: a pointer to the initialized UploadStreamHandle

Implements upload::AbstractUploader.

Definition at line 393 of file upload_s3.cc.

Here is the call graph for this function:

void * upload::S3Uploader::MainCollectResults ( void * data )

staticprivate

Worker thread takes care of requesting new jobs and cleaning old ones.

Definition at line 270 of file upload_s3.cc.

Referenced by S3Uploader().

Here is the call graph for this function:

Here is the caller graph for this function:

bool upload::S3Uploader::Mkdir ( const std::string & path )

virtual

Make directory in upstream storage. Noop if directory already present.

Parameters

path	relative directory path in the upstream storage

Returns: true if the directory was successfully created or already present

Implements upload::AbstractUploader.

Definition at line 502 of file upload_s3.cc.

virtual std::string upload::S3Uploader::name ( ) const

inlinevirtual

A string identifying the uploader type

Implements upload::AbstractUploader.

Definition at line 47 of file upload_s3.h.

void upload::S3Uploader::OnReqComplete	(	const upload::UploaderResults &	results,
		RequestCtrl *	ctrl
	)

private

Definition at line 467 of file upload_s3.cc.

Referenced by Create(), DoUpload(), and Peek().

Here is the call graph for this function:

Here is the caller graph for this function:

bool upload::S3Uploader::ParseSpoolerDefinition ( const SpoolerDefinition & spooler_definition )

private

Definition at line 105 of file upload_s3.cc.

Referenced by S3Uploader().

Here is the call graph for this function:

Here is the caller graph for this function:

bool upload::S3Uploader::Peek ( const std::string & path )

virtual

Checks if a file is already present in the backend storage. This might be a synchronous operation.

Parameters

path	the path of the file to be checked

Returns: true if the file was found in the backend storage

Implements upload::AbstractUploader.

Definition at line 483 of file upload_s3.cc.

Here is the call graph for this function:

bool upload::S3Uploader::PlaceBootstrappingShortcut ( const shash::Any & object )

virtual

Creates a top-level shortcut to the given data object. This is particularly useful for bootstrapping repositories whose data-directory is secured by a VOMS certificate.

Parameters

object content hash of the object to be exposed on the top-level

Returns: true on success

Implements upload::AbstractUploader.

Definition at line 505 of file upload_s3.cc.

void upload::S3Uploader::StreamedUpload	(	UploadStreamHandle *	handle,
		UploadBuffer	buffer,
		const CallbackTN *	callback
	)

virtual

Implementation of a streamed upload step. See public interface for details. Public interface: AbstractUploader::ScheduleUpload()

Parameters

handle	descendant of UploadStreamHandle specifying the stream
buffer	the CharBuffer to be uploaded to the stream
callback	callback to be called on completion

Implements upload::AbstractUploader.

Definition at line 399 of file upload_s3.cc.

Here is the call graph for this function:

void upload::S3Uploader::UploadJobInfo ( s3fanout::JobInfo * info )

private

Definition at line 381 of file upload_s3.cc.

Referenced by Create(), DoUpload(), FinalizeStreamedUpload(), and Peek().

Here is the call graph for this function:

Here is the caller graph for this function:

bool upload::S3Uploader::WillHandle ( const SpoolerDefinition & spooler_definition )

static

Definition at line 226 of file upload_s3.cc.

Member Data Documentation

std::string upload::S3Uploader::access_key_

private

Definition at line 127 of file upload_s3.h.

Referenced by ParseSpoolerDefinition(), and S3Uploader().

s3fanout::AuthzMethods upload::S3Uploader::authz_method_

private

Definition at line 129 of file upload_s3.h.

Referenced by ParseSpoolerDefinition(), and S3Uploader().

std::string upload::S3Uploader::bucket_

private

Definition at line 122 of file upload_s3.h.

Referenced by DoRemoveAsync(), ParseSpoolerDefinition(), S3Uploader(), and UploadJobInfo().

bool upload::S3Uploader::dns_buckets_

private

Definition at line 123 of file upload_s3.h.

Referenced by Create(), ParseSpoolerDefinition(), and S3Uploader().

std::string upload::S3Uploader::flavor_

private

Definition at line 121 of file upload_s3.h.

Referenced by ParseSpoolerDefinition(), and S3Uploader().

std::string upload::S3Uploader::host_name_

private

Definition at line 119 of file upload_s3.h.

Referenced by ParseSpoolerDefinition().

std::string upload::S3Uploader::host_name_port_

private

Definition at line 118 of file upload_s3.h.

Referenced by ParseSpoolerDefinition(), S3Uploader(), and UploadJobInfo().

atomic_int32 upload::S3Uploader::io_errors_

mutableprivate

Definition at line 135 of file upload_s3.h.

Referenced by GetNumberOfErrors(), MainCollectResults(), and S3Uploader().

const unsigned upload::S3Uploader::kDefaultBackoffInitMs = 100

staticprivate

Definition at line 88 of file upload_s3.h.

Referenced by S3Uploader().

const unsigned upload::S3Uploader::kDefaultBackoffMaxMs = 2000

staticprivate

Definition at line 89 of file upload_s3.h.

Referenced by S3Uploader().

const unsigned upload::S3Uploader::kDefaultNumParallelUploads = 16

staticprivate

Definition at line 85 of file upload_s3.h.

const unsigned upload::S3Uploader::kDefaultNumRetries = 3

staticprivate

Definition at line 86 of file upload_s3.h.

const unsigned upload::S3Uploader::kDefaultPort = 80

staticprivate

Definition at line 83 of file upload_s3.h.

const unsigned upload::S3Uploader::kDefaultTimeoutSec = 60

staticprivate

Definition at line 87 of file upload_s3.h.

const unsigned upload::S3Uploader::kHttpsPort = 443

staticprivate

Definition at line 84 of file upload_s3.h.

const unsigned upload::S3Uploader::kInMemoryObjectThreshold = 500 * 1024

staticprivate

Definition at line 90 of file upload_s3.h.

Referenced by CreateJobInfo(), DoUpload(), and InitStreamedUpload().

int upload::S3Uploader::num_parallel_uploads_

private

Definition at line 124 of file upload_s3.h.

Referenced by ParseSpoolerDefinition(), and S3Uploader().

unsigned upload::S3Uploader::num_retries_

private

Definition at line 125 of file upload_s3.h.

Referenced by ParseSpoolerDefinition(), and S3Uploader().

bool upload::S3Uploader::peek_before_put_

private

Definition at line 130 of file upload_s3.h.

Referenced by DoUpload(), FinalizeStreamedUpload(), and ParseSpoolerDefinition().

std::string upload::S3Uploader::proxy_

private

Definition at line 132 of file upload_s3.h.

Referenced by ParseSpoolerDefinition(), and S3Uploader().

std::string upload::S3Uploader::region_

private

Definition at line 120 of file upload_s3.h.

Referenced by Create(), ParseSpoolerDefinition(), and S3Uploader().

std::string upload::S3Uploader::repository_alias_

private

Definition at line 117 of file upload_s3.h.

Referenced by DoRemoveAsync(), DoUpload(), FinalizeStreamedUpload(), ParseSpoolerDefinition(), and Peek().

UniquePtr<s3fanout::S3FanoutManager> upload::S3Uploader::s3fanout_mgr_

private

Definition at line 116 of file upload_s3.h.

Referenced by DoRemoveAsync(), GetS3FanoutManager(), MainCollectResults(), S3Uploader(), UploadJobInfo(), and ~S3Uploader().

std::string upload::S3Uploader::secret_key_

private

Definition at line 128 of file upload_s3.h.

Referenced by ParseSpoolerDefinition(), and S3Uploader().

const std::string upload::S3Uploader::temporary_path_

private

Definition at line 134 of file upload_s3.h.

pthread_t upload::S3Uploader::thread_collect_results_

private

Definition at line 136 of file upload_s3.h.

Referenced by S3Uploader(), and ~S3Uploader().

unsigned upload::S3Uploader::timeout_sec_

private

Definition at line 126 of file upload_s3.h.

Referenced by ParseSpoolerDefinition(), and S3Uploader().

bool upload::S3Uploader::use_https_

private

Definition at line 131 of file upload_s3.h.

Referenced by ParseSpoolerDefinition(), and S3Uploader().

std::string upload::S3Uploader::x_amz_acl_

private

Definition at line 138 of file upload_s3.h.

Referenced by ParseSpoolerDefinition(), and S3Uploader().

The documentation for this class was generated from the following files:

/home/sftnight/jenkins/workspace/CvmfsDoxygen/label/docker-x86_64/cvmfs/cvmfs/upload_s3.h
/home/sftnight/jenkins/workspace/CvmfsDoxygen/label/docker-x86_64/cvmfs/cvmfs/upload_s3.cc

Classes

Public Member Functions

Static Public Member Functions

Private Member Functions

Static Private Member Functions

Private Attributes

Static Private Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation