CernVM-FS  2.12.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
upload::S3Uploader Class Reference

#include <upload_s3.h>

Inheritance diagram for upload::S3Uploader:
Collaboration diagram for upload::S3Uploader:

Classes

struct  RequestCtrl
 

Public Member Functions

 S3Uploader (const SpoolerDefinition &spooler_definition)
 
virtual ~S3Uploader ()
 
virtual std::string name () const
 
virtual bool Create ()
 
virtual void DoUpload (const std::string &remote_path, IngestionSource *source, const CallbackTN *callback)
 
virtual UploadStreamHandleInitStreamedUpload (const CallbackTN *callback=NULL)
 
virtual void StreamedUpload (UploadStreamHandle *handle, UploadBuffer buffer, const CallbackTN *callback)
 
virtual void FinalizeStreamedUpload (UploadStreamHandle *handle, const shash::Any &content_hash)
 
virtual void DoRemoveAsync (const std::string &file_to_delete)
 
virtual bool Peek (const std::string &path)
 
virtual bool Mkdir (const std::string &path)
 
virtual bool PlaceBootstrappingShortcut (const shash::Any &object)
 
virtual unsigned int GetNumberOfErrors () const
 
int64_t DoGetObjectSize (const std::string &file_name)
 
s3fanout::S3FanoutManagerGetS3FanoutManager ()
 
- Public Member Functions inherited from upload::AbstractUploader
virtual ~AbstractUploader ()
 
virtual unsigned GetNumTasks () const
 
virtual bool Initialize ()
 
virtual bool FinalizeSession (bool commit, const std::string &old_root_hash, const std::string &new_root_hash, const RepositoryTag &tag)
 
void TearDown ()
 
void UploadFile (const std::string &local_path, const std::string &remote_path, const CallbackTN *callback=NULL)
 
void UploadIngestionSource (const std::string &remote_path, IngestionSource *source, const CallbackTN *callback=NULL)
 
void ScheduleUpload (UploadStreamHandle *handle, UploadBuffer buffer, const CallbackTN *callback=NULL)
 
void ScheduleCommit (UploadStreamHandle *handle, const shash::Any &content_hash)
 
void RemoveAsync (const std::string &file_to_delete)
 
void RemoveAsync (const shash::Any &hash_to_delete)
 
int64_t GetObjectSize (const shash::Any &hash)
 
virtual void WaitForUpload () const
 
void InitCounters (perf::StatisticsTemplate *statistics)
 
- Public Member Functions inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
virtual ~PolymorphicConstructionImpl ()
 

Static Public Member Functions

static bool WillHandle (const SpoolerDefinition &spooler_definition)
 
- Static Public Member Functions inherited from upload::AbstractUploader
static void RegisterPlugins ()
 
- Static Public Member Functions inherited from PolymorphicConstruction< AbstractUploader, SpoolerDefinition >
static IntrospectionData Introspect ()
 
- Static Public Member Functions inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
static AbstractUploader * Construct (const SpoolerDefinition &param)
 
- Static Public Member Functions inherited from Callbackable< UploaderResults >
static CallbackTNMakeClosure (typename BoundClosure< UploaderResults, DelegateT, ClosureDataT >::CallbackMethod method, DelegateT *delegate, const ClosureDataT &closure_data)
 
static CallbackTNMakeCallback (typename BoundCallback< UploaderResults, DelegateT >::CallbackMethod method, DelegateT *delegate)
 
static CallbackTNMakeCallback (typename Callback< UploaderResults >::CallbackFunction function)
 

Private Member Functions

void OnReqComplete (const upload::UploaderResults &results, RequestCtrl *ctrl)
 
bool ParseSpoolerDefinition (const SpoolerDefinition &spooler_definition)
 
void UploadJobInfo (s3fanout::JobInfo *info)
 
s3fanout::JobInfoCreateJobInfo (const std::string &path) const
 

Static Private Member Functions

static void * MainCollectResults (void *data)
 

Private Attributes

UniquePtr
< s3fanout::S3FanoutManager
s3fanout_mgr_
 
std::string repository_alias_
 
std::string host_name_port_
 
std::string host_name_
 
std::string region_
 
std::string flavor_
 
std::string bucket_
 
bool dns_buckets_
 
int num_parallel_uploads_
 
unsigned num_retries_
 
unsigned timeout_sec_
 
std::string access_key_
 
std::string secret_key_
 
s3fanout::AuthzMethods authz_method_
 
bool peek_before_put_
 
bool use_https_
 
std::string proxy_
 
const std::string temporary_path_
 
atomic_int32 io_errors_
 
pthread_t thread_collect_results_
 
std::string x_amz_acl_
 

Static Private Attributes

static const unsigned kDefaultPort = 80
 
static const unsigned kHttpsPort = 443
 
static const unsigned kDefaultNumParallelUploads = 16
 
static const unsigned kDefaultNumRetries = 3
 
static const unsigned kDefaultTimeoutSec = 60
 
static const unsigned kDefaultBackoffInitMs = 100
 
static const unsigned kDefaultBackoffMaxMs = 2000
 
static const unsigned kInMemoryObjectThreshold = 500*1024
 

Additional Inherited Members

- Public Types inherited from PolymorphicConstruction< AbstractUploader, SpoolerDefinition >
typedef std::vector< void > IntrospectionData
 
- Public Types inherited from Callbackable< UploaderResults >
typedef CallbackBase
< UploaderResults > 
CallbackTN
 
- Protected Types inherited from upload::AbstractUploader
typedef Callbackable
< UploaderResults >
::CallbackTN
CallbackPtr
 
- Protected Types inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
typedef AbstractFactory
< AbstractUploader,
SpoolerDefinition, void > 
Factory
 
typedef std::vector< Factory * > RegisteredPlugins
 
- Protected Member Functions inherited from upload::AbstractUploader
 AbstractUploader (const SpoolerDefinition &spooler_definition)
 
void Respond (const CallbackTN *callback, const UploaderResults &result) const
 
int CreateAndOpenTemporaryChunkFile (std::string *path) const
 
const SpoolerDefinitionspooler_definition () const
 
void CountUploadedChunks () const
 
void DecUploadedChunks () const
 
void CountUploadedBytes (int64_t bytes_written) const
 
void CountDuplicates () const
 
void CountUploadedCatalogs () const
 
void CountUploadedCatalogBytes (int64_t bytes_written) const
 
void IncJobsInFlight ()
 
- Protected Member Functions inherited from SingleCopy
 SingleCopy ()
 
- Static Protected Member Functions inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
static void LazilyRegisterPlugins ()
 
static void RegisterPlugin ()
 
- Static Protected Attributes inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
static RegisteredPlugins registered_plugins_
 

Detailed Description

The S3Spooler implements the AbstractSpooler interface to push files into a S3 CVMFS repository backend. For a detailed description of the classes interface please have a look into the AbstractSpooler base class.

Definition at line 43 of file upload_s3.h.

Constructor & Destructor Documentation

upload::S3Uploader::S3Uploader ( const SpoolerDefinition spooler_definition)
explicit

Definition at line 47 of file upload_s3.cc.

Here is the call graph for this function:

upload::S3Uploader::~S3Uploader ( )
virtual

Definition at line 101 of file upload_s3.cc.

Member Function Documentation

bool upload::S3Uploader::Create ( )
virtual

Initializes a new repository storage area, e.g. create directory layout for local backend or create bucket for S3 backend.

Implements upload::AbstractUploader.

Definition at line 239 of file upload_s3.cc.

Here is the call graph for this function:

s3fanout::JobInfo * upload::S3Uploader::CreateJobInfo ( const std::string &  path) const
private

Definition at line 468 of file upload_s3.cc.

Referenced by Create(), DoRemoveAsync(), and Peek().

Here is the call graph for this function:

Here is the caller graph for this function:

int64_t upload::S3Uploader::DoGetObjectSize ( const std::string &  file_name)
virtual

Implements upload::AbstractUploader.

Definition at line 532 of file upload_s3.cc.

void upload::S3Uploader::DoRemoveAsync ( const std::string &  file_to_delete)
virtual

Implements upload::AbstractUploader.

Definition at line 474 of file upload_s3.cc.

Here is the call graph for this function:

void upload::S3Uploader::DoUpload ( const std::string &  remote_path,
IngestionSource source,
const CallbackTN callback 
)
virtual

Upload() is not done concurrently in the current implementation of the S3Spooler, since it is a simple move or copy of a file without CPU intensive operation This method calls NotifyListeners and invokes a callback for all registered listeners (see the Observable template for details).

Implements upload::AbstractUploader.

Definition at line 330 of file upload_s3.cc.

Here is the call graph for this function:

void upload::S3Uploader::FinalizeStreamedUpload ( UploadStreamHandle handle,
const shash::Any content_hash 
)
virtual

Implementation of streamed upload commit Public interface: AbstractUploader::ScheduleUpload()

Parameters
handledescendant of UploadStreamHandle specifying the stream
content_hashthe computed content hash of the streamed object

Implements upload::AbstractUploader.

Definition at line 424 of file upload_s3.cc.

Here is the call graph for this function:

unsigned int upload::S3Uploader::GetNumberOfErrors ( ) const
virtual

Implements upload::AbstractUploader.

Definition at line 269 of file upload_s3.cc.

s3fanout::S3FanoutManager* upload::S3Uploader::GetS3FanoutManager ( )
inline

Definition at line 80 of file upload_s3.h.

Here is the call graph for this function:

UploadStreamHandle * upload::S3Uploader::InitStreamedUpload ( const CallbackTN callback = NULL)
virtual

This method is called before the first data block of a streamed upload is scheduled (see above implementation of UploadStreamHandle for details).

Parameters
callback(optional) this callback will be invoked once this parti- cular streamed upload is committed.
Returns
a pointer to the initialized UploadStreamHandle

Implements upload::AbstractUploader.

Definition at line 406 of file upload_s3.cc.

Here is the call graph for this function:

void * upload::S3Uploader::MainCollectResults ( void *  data)
staticprivate

Worker thread takes care of requesting new jobs and cleaning old ones.

Definition at line 277 of file upload_s3.cc.

Referenced by S3Uploader().

Here is the call graph for this function:

Here is the caller graph for this function:

bool upload::S3Uploader::Mkdir ( const std::string &  path)
virtual

Make directory in upstream storage. Noop if directory already present.

Parameters
pathrelative directory path in the upstream storage
Returns
true if the directory was successfully created or already present

Implements upload::AbstractUploader.

Definition at line 522 of file upload_s3.cc.

virtual std::string upload::S3Uploader::name ( ) const
inlinevirtual

A string identifying the uploader type

Implements upload::AbstractUploader.

Definition at line 49 of file upload_s3.h.

void upload::S3Uploader::OnReqComplete ( const upload::UploaderResults results,
RequestCtrl ctrl 
)
private

Definition at line 486 of file upload_s3.cc.

Referenced by Create(), DoUpload(), and Peek().

Here is the call graph for this function:

Here is the caller graph for this function:

bool upload::S3Uploader::ParseSpoolerDefinition ( const SpoolerDefinition spooler_definition)
private

Definition at line 108 of file upload_s3.cc.

Referenced by S3Uploader().

Here is the call graph for this function:

Here is the caller graph for this function:

bool upload::S3Uploader::Peek ( const std::string &  path)
virtual

Checks if a file is already present in the backend storage. This might be a synchronous operation.

Parameters
paththe path of the file to be checked
Returns
true if the file was found in the backend storage

Implements upload::AbstractUploader.

Definition at line 503 of file upload_s3.cc.

Here is the call graph for this function:

bool upload::S3Uploader::PlaceBootstrappingShortcut ( const shash::Any object)
virtual

Creates a top-level shortcut to the given data object. This is particularly useful for bootstrapping repositories whose data-directory is secured by a VOMS certificate.

Parameters
objectcontent hash of the object to be exposed on the top-level
Returns
true on success

Implements upload::AbstractUploader.

Definition at line 527 of file upload_s3.cc.

void upload::S3Uploader::StreamedUpload ( UploadStreamHandle handle,
UploadBuffer  buffer,
const CallbackTN callback 
)
virtual

Implementation of a streamed upload step. See public interface for details. Public interface: AbstractUploader::ScheduleUpload()

Parameters
handledescendant of UploadStreamHandle specifying the stream
bufferthe CharBuffer to be uploaded to the stream
callbackcallback to be called on completion

Implements upload::AbstractUploader.

Definition at line 412 of file upload_s3.cc.

Here is the call graph for this function:

void upload::S3Uploader::UploadJobInfo ( s3fanout::JobInfo info)
private

Definition at line 392 of file upload_s3.cc.

Referenced by Create(), DoUpload(), FinalizeStreamedUpload(), and Peek().

Here is the call graph for this function:

Here is the caller graph for this function:

bool upload::S3Uploader::WillHandle ( const SpoolerDefinition spooler_definition)
static

Definition at line 234 of file upload_s3.cc.

Member Data Documentation

std::string upload::S3Uploader::access_key_
private

Definition at line 129 of file upload_s3.h.

Referenced by ParseSpoolerDefinition(), and S3Uploader().

s3fanout::AuthzMethods upload::S3Uploader::authz_method_
private

Definition at line 131 of file upload_s3.h.

Referenced by ParseSpoolerDefinition(), and S3Uploader().

std::string upload::S3Uploader::bucket_
private

Definition at line 124 of file upload_s3.h.

Referenced by DoRemoveAsync(), ParseSpoolerDefinition(), S3Uploader(), and UploadJobInfo().

bool upload::S3Uploader::dns_buckets_
private

Definition at line 125 of file upload_s3.h.

Referenced by Create(), ParseSpoolerDefinition(), and S3Uploader().

std::string upload::S3Uploader::flavor_
private

Definition at line 123 of file upload_s3.h.

Referenced by ParseSpoolerDefinition(), and S3Uploader().

std::string upload::S3Uploader::host_name_
private

Definition at line 121 of file upload_s3.h.

Referenced by ParseSpoolerDefinition().

std::string upload::S3Uploader::host_name_port_
private

Definition at line 120 of file upload_s3.h.

Referenced by ParseSpoolerDefinition(), S3Uploader(), and UploadJobInfo().

atomic_int32 upload::S3Uploader::io_errors_
mutableprivate

Definition at line 137 of file upload_s3.h.

Referenced by GetNumberOfErrors(), MainCollectResults(), and S3Uploader().

const unsigned upload::S3Uploader::kDefaultBackoffInitMs = 100
staticprivate

Definition at line 90 of file upload_s3.h.

Referenced by S3Uploader().

const unsigned upload::S3Uploader::kDefaultBackoffMaxMs = 2000
staticprivate

Definition at line 91 of file upload_s3.h.

Referenced by S3Uploader().

const unsigned upload::S3Uploader::kDefaultNumParallelUploads = 16
staticprivate

Definition at line 87 of file upload_s3.h.

const unsigned upload::S3Uploader::kDefaultNumRetries = 3
staticprivate

Definition at line 88 of file upload_s3.h.

const unsigned upload::S3Uploader::kDefaultPort = 80
staticprivate

Definition at line 85 of file upload_s3.h.

const unsigned upload::S3Uploader::kDefaultTimeoutSec = 60
staticprivate

Definition at line 89 of file upload_s3.h.

const unsigned upload::S3Uploader::kHttpsPort = 443
staticprivate

Definition at line 86 of file upload_s3.h.

const unsigned upload::S3Uploader::kInMemoryObjectThreshold = 500*1024
staticprivate

Definition at line 92 of file upload_s3.h.

Referenced by CreateJobInfo(), DoUpload(), and InitStreamedUpload().

int upload::S3Uploader::num_parallel_uploads_
private

Definition at line 126 of file upload_s3.h.

Referenced by ParseSpoolerDefinition(), and S3Uploader().

unsigned upload::S3Uploader::num_retries_
private

Definition at line 127 of file upload_s3.h.

Referenced by ParseSpoolerDefinition(), and S3Uploader().

bool upload::S3Uploader::peek_before_put_
private

Definition at line 132 of file upload_s3.h.

Referenced by DoUpload(), FinalizeStreamedUpload(), and ParseSpoolerDefinition().

std::string upload::S3Uploader::proxy_
private

Definition at line 134 of file upload_s3.h.

Referenced by ParseSpoolerDefinition(), and S3Uploader().

std::string upload::S3Uploader::region_
private

Definition at line 122 of file upload_s3.h.

Referenced by Create(), ParseSpoolerDefinition(), and S3Uploader().

std::string upload::S3Uploader::repository_alias_
private
UniquePtr<s3fanout::S3FanoutManager> upload::S3Uploader::s3fanout_mgr_
private
std::string upload::S3Uploader::secret_key_
private

Definition at line 130 of file upload_s3.h.

Referenced by ParseSpoolerDefinition(), and S3Uploader().

const std::string upload::S3Uploader::temporary_path_
private

Definition at line 136 of file upload_s3.h.

pthread_t upload::S3Uploader::thread_collect_results_
private

Definition at line 138 of file upload_s3.h.

Referenced by S3Uploader(), and ~S3Uploader().

unsigned upload::S3Uploader::timeout_sec_
private

Definition at line 128 of file upload_s3.h.

Referenced by ParseSpoolerDefinition(), and S3Uploader().

bool upload::S3Uploader::use_https_
private

Definition at line 133 of file upload_s3.h.

Referenced by ParseSpoolerDefinition(), and S3Uploader().

std::string upload::S3Uploader::x_amz_acl_
private

Definition at line 140 of file upload_s3.h.

Referenced by ParseSpoolerDefinition(), and S3Uploader().


The documentation for this class was generated from the following files: