![]() |
CernVM-FS
2.13.0
|
#include <upload_s3.h>
Classes | |
struct | RequestCtrl |
Public Member Functions | |
S3Uploader (const SpoolerDefinition &spooler_definition) | |
virtual | ~S3Uploader () |
virtual std::string | name () const |
virtual bool | Create () |
virtual void | DoUpload (const std::string &remote_path, IngestionSource *source, const CallbackTN *callback) |
virtual UploadStreamHandle * | InitStreamedUpload (const CallbackTN *callback=NULL) |
virtual void | StreamedUpload (UploadStreamHandle *handle, UploadBuffer buffer, const CallbackTN *callback) |
virtual void | FinalizeStreamedUpload (UploadStreamHandle *handle, const shash::Any &content_hash) |
virtual void | DoRemoveAsync (const std::string &file_to_delete) |
virtual bool | Peek (const std::string &path) |
virtual bool | Mkdir (const std::string &path) |
virtual bool | PlaceBootstrappingShortcut (const shash::Any &object) |
virtual unsigned int | GetNumberOfErrors () const |
int64_t | DoGetObjectSize (const std::string &file_name) |
s3fanout::S3FanoutManager * | GetS3FanoutManager () |
![]() | |
virtual | ~AbstractUploader () |
virtual unsigned | GetNumTasks () const |
virtual bool | Initialize () |
virtual bool | FinalizeSession (bool commit, const std::string &old_root_hash, const std::string &new_root_hash, const RepositoryTag &tag) |
void | TearDown () |
void | UploadFile (const std::string &local_path, const std::string &remote_path, const CallbackTN *callback=NULL) |
void | UploadIngestionSource (const std::string &remote_path, IngestionSource *source, const CallbackTN *callback=NULL) |
void | ScheduleUpload (UploadStreamHandle *handle, UploadBuffer buffer, const CallbackTN *callback=NULL) |
void | ScheduleCommit (UploadStreamHandle *handle, const shash::Any &content_hash) |
void | RemoveAsync (const std::string &file_to_delete) |
void | RemoveAsync (const shash::Any &hash_to_delete) |
int64_t | GetObjectSize (const shash::Any &hash) |
virtual void | WaitForUpload () const |
void | InitCounters (perf::StatisticsTemplate *statistics) |
![]() | |
virtual | ~PolymorphicConstructionImpl () |
Static Public Member Functions | |
static bool | WillHandle (const SpoolerDefinition &spooler_definition) |
![]() | |
static void | RegisterPlugins () |
![]() | |
static IntrospectionData | Introspect () |
![]() | |
static AbstractUploader * | Construct (const SpoolerDefinition ¶m) |
![]() | |
static CallbackTN * | MakeClosure (typename BoundClosure< UploaderResults, DelegateT, ClosureDataT >::CallbackMethod method, DelegateT *delegate, const ClosureDataT &closure_data) |
static CallbackTN * | MakeCallback (typename BoundCallback< UploaderResults, DelegateT >::CallbackMethod method, DelegateT *delegate) |
static CallbackTN * | MakeCallback (typename Callback< UploaderResults >::CallbackFunction function) |
Private Member Functions | |
void | OnReqComplete (const upload::UploaderResults &results, RequestCtrl *ctrl) |
bool | ParseSpoolerDefinition (const SpoolerDefinition &spooler_definition) |
void | UploadJobInfo (s3fanout::JobInfo *info) |
s3fanout::JobInfo * | CreateJobInfo (const std::string &path) const |
Static Private Member Functions | |
static void * | MainCollectResults (void *data) |
Private Attributes | |
UniquePtr < s3fanout::S3FanoutManager > | s3fanout_mgr_ |
std::string | repository_alias_ |
std::string | host_name_port_ |
std::string | host_name_ |
std::string | region_ |
std::string | flavor_ |
std::string | bucket_ |
bool | dns_buckets_ |
int | num_parallel_uploads_ |
unsigned | num_retries_ |
unsigned | timeout_sec_ |
std::string | access_key_ |
std::string | secret_key_ |
s3fanout::AuthzMethods | authz_method_ |
bool | peek_before_put_ |
bool | use_https_ |
std::string | proxy_ |
const std::string | temporary_path_ |
atomic_int32 | io_errors_ |
pthread_t | thread_collect_results_ |
std::string | x_amz_acl_ |
Static Private Attributes | |
static const unsigned | kDefaultPort = 80 |
static const unsigned | kHttpsPort = 443 |
static const unsigned | kDefaultNumParallelUploads = 16 |
static const unsigned | kDefaultNumRetries = 3 |
static const unsigned | kDefaultTimeoutSec = 60 |
static const unsigned | kDefaultBackoffInitMs = 100 |
static const unsigned | kDefaultBackoffMaxMs = 2000 |
static const unsigned | kInMemoryObjectThreshold = 500 * 1024 |
The S3Spooler implements the AbstractSpooler interface to push files into a S3 CVMFS repository backend. For a detailed description of the classes interface please have a look into the AbstractSpooler base class.
Definition at line 41 of file upload_s3.h.
|
explicit |
|
virtual |
Definition at line 98 of file upload_s3.cc.
|
virtual |
Initializes a new repository storage area, e.g. create directory layout for local backend or create bucket for S3 backend.
Implements upload::AbstractUploader.
Definition at line 231 of file upload_s3.cc.
|
private |
Definition at line 449 of file upload_s3.cc.
Referenced by Create(), DoRemoveAsync(), and Peek().
|
virtual |
Implements upload::AbstractUploader.
Definition at line 509 of file upload_s3.cc.
|
virtual |
Implements upload::AbstractUploader.
Definition at line 455 of file upload_s3.cc.
|
virtual |
Upload() is not done concurrently in the current implementation of the S3Spooler, since it is a simple move or copy of a file without CPU intensive operation This method calls NotifyListeners and invokes a callback for all registered listeners (see the Observable template for details).
Implements upload::AbstractUploader.
Definition at line 322 of file upload_s3.cc.
|
virtual |
Implementation of streamed upload commit Public interface: AbstractUploader::ScheduleUpload()
handle | descendant of UploadStreamHandle specifying the stream |
content_hash | the computed content hash of the streamed object |
Implements upload::AbstractUploader.
Definition at line 409 of file upload_s3.cc.
|
virtual |
Implements upload::AbstractUploader.
Definition at line 262 of file upload_s3.cc.
|
inline |
|
virtual |
This method is called before the first data block of a streamed upload is scheduled (see above implementation of UploadStreamHandle for details).
callback | (optional) this callback will be invoked once this parti- cular streamed upload is committed. |
Implements upload::AbstractUploader.
Definition at line 393 of file upload_s3.cc.
|
staticprivate |
Worker thread takes care of requesting new jobs and cleaning old ones.
Definition at line 270 of file upload_s3.cc.
Referenced by S3Uploader().
|
virtual |
Make directory in upstream storage. Noop if directory already present.
path | relative directory path in the upstream storage |
Implements upload::AbstractUploader.
Definition at line 501 of file upload_s3.cc.
|
inlinevirtual |
A string identifying the uploader type
Implements upload::AbstractUploader.
Definition at line 47 of file upload_s3.h.
|
private |
Definition at line 467 of file upload_s3.cc.
Referenced by Create(), DoUpload(), and Peek().
|
private |
Definition at line 105 of file upload_s3.cc.
Referenced by S3Uploader().
|
virtual |
Checks if a file is already present in the backend storage. This might be a synchronous operation.
path | the path of the file to be checked |
Implements upload::AbstractUploader.
Definition at line 482 of file upload_s3.cc.
|
virtual |
Creates a top-level shortcut to the given data object. This is particularly useful for bootstrapping repositories whose data-directory is secured by a VOMS certificate.
object | content hash of the object to be exposed on the top-level |
Implements upload::AbstractUploader.
Definition at line 504 of file upload_s3.cc.
|
virtual |
Implementation of a streamed upload step. See public interface for details. Public interface: AbstractUploader::ScheduleUpload()
handle | descendant of UploadStreamHandle specifying the stream |
buffer | the CharBuffer to be uploaded to the stream |
callback | callback to be called on completion |
Implements upload::AbstractUploader.
Definition at line 399 of file upload_s3.cc.
|
private |
Definition at line 381 of file upload_s3.cc.
Referenced by Create(), DoUpload(), FinalizeStreamedUpload(), and Peek().
|
static |
Definition at line 226 of file upload_s3.cc.
|
private |
Definition at line 127 of file upload_s3.h.
Referenced by ParseSpoolerDefinition(), and S3Uploader().
|
private |
Definition at line 129 of file upload_s3.h.
Referenced by ParseSpoolerDefinition(), and S3Uploader().
|
private |
Definition at line 122 of file upload_s3.h.
Referenced by DoRemoveAsync(), ParseSpoolerDefinition(), S3Uploader(), and UploadJobInfo().
|
private |
Definition at line 123 of file upload_s3.h.
Referenced by Create(), ParseSpoolerDefinition(), and S3Uploader().
|
private |
Definition at line 121 of file upload_s3.h.
Referenced by ParseSpoolerDefinition(), and S3Uploader().
|
private |
Definition at line 119 of file upload_s3.h.
Referenced by ParseSpoolerDefinition().
|
private |
Definition at line 118 of file upload_s3.h.
Referenced by ParseSpoolerDefinition(), S3Uploader(), and UploadJobInfo().
|
mutableprivate |
Definition at line 135 of file upload_s3.h.
Referenced by GetNumberOfErrors(), MainCollectResults(), and S3Uploader().
|
staticprivate |
Definition at line 88 of file upload_s3.h.
Referenced by S3Uploader().
|
staticprivate |
Definition at line 89 of file upload_s3.h.
Referenced by S3Uploader().
|
staticprivate |
Definition at line 85 of file upload_s3.h.
|
staticprivate |
Definition at line 86 of file upload_s3.h.
|
staticprivate |
Definition at line 83 of file upload_s3.h.
|
staticprivate |
Definition at line 87 of file upload_s3.h.
|
staticprivate |
Definition at line 84 of file upload_s3.h.
|
staticprivate |
Definition at line 90 of file upload_s3.h.
Referenced by CreateJobInfo(), DoUpload(), and InitStreamedUpload().
|
private |
Definition at line 124 of file upload_s3.h.
Referenced by ParseSpoolerDefinition(), and S3Uploader().
|
private |
Definition at line 125 of file upload_s3.h.
Referenced by ParseSpoolerDefinition(), and S3Uploader().
|
private |
Definition at line 130 of file upload_s3.h.
Referenced by DoUpload(), FinalizeStreamedUpload(), and ParseSpoolerDefinition().
|
private |
Definition at line 132 of file upload_s3.h.
Referenced by ParseSpoolerDefinition(), and S3Uploader().
|
private |
Definition at line 120 of file upload_s3.h.
Referenced by Create(), ParseSpoolerDefinition(), and S3Uploader().
|
private |
Definition at line 117 of file upload_s3.h.
Referenced by DoRemoveAsync(), DoUpload(), FinalizeStreamedUpload(), ParseSpoolerDefinition(), and Peek().
|
private |
Definition at line 116 of file upload_s3.h.
Referenced by DoRemoveAsync(), GetS3FanoutManager(), MainCollectResults(), S3Uploader(), UploadJobInfo(), and ~S3Uploader().
|
private |
Definition at line 128 of file upload_s3.h.
Referenced by ParseSpoolerDefinition(), and S3Uploader().
|
private |
Definition at line 134 of file upload_s3.h.
|
private |
Definition at line 136 of file upload_s3.h.
Referenced by S3Uploader(), and ~S3Uploader().
|
private |
Definition at line 126 of file upload_s3.h.
Referenced by ParseSpoolerDefinition(), and S3Uploader().
|
private |
Definition at line 131 of file upload_s3.h.
Referenced by ParseSpoolerDefinition(), and S3Uploader().
|
private |
Definition at line 138 of file upload_s3.h.
Referenced by ParseSpoolerDefinition(), and S3Uploader().