CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
upload::GatewayUploader Class Reference

#include <upload_gateway.h>

Inheritance diagram for upload::GatewayUploader:
Collaboration diagram for upload::GatewayUploader:

Classes

struct  Config
 

Public Member Functions

 GatewayUploader (const SpoolerDefinition &spooler_definition)
 
virtual ~GatewayUploader ()
 
virtual bool Initialize ()
 
virtual bool Create ()
 
virtual bool FinalizeSession (bool commit, const std::string &old_root_hash, const std::string &new_root_hash, const RepositoryTag &tag)
 
virtual void WaitForUpload () const
 
virtual std::string name () const
 
virtual bool Peek (const std::string &path)
 
virtual bool Mkdir (const std::string &path)
 
virtual bool PlaceBootstrappingShortcut (const shash::Any &object)
 
virtual unsigned int GetNumberOfErrors () const
 
- Public Member Functions inherited from upload::AbstractUploader
virtual ~AbstractUploader ()
 
virtual unsigned GetNumTasks () const
 
void TearDown ()
 
void UploadFile (const std::string &local_path, const std::string &remote_path, const CallbackTN *callback=NULL)
 
void UploadIngestionSource (const std::string &remote_path, IngestionSource *source, const CallbackTN *callback=NULL)
 
void ScheduleUpload (UploadStreamHandle *handle, UploadBuffer buffer, const CallbackTN *callback=NULL)
 
void ScheduleCommit (UploadStreamHandle *handle, const shash::Any &content_hash)
 
void RemoveAsync (const std::string &file_to_delete)
 
void RemoveAsync (const shash::Any &hash_to_delete)
 
int64_t GetObjectSize (const shash::Any &hash)
 
void InitCounters (perf::StatisticsTemplate *statistics)
 
- Public Member Functions inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
virtual ~PolymorphicConstructionImpl ()
 

Static Public Member Functions

static bool WillHandle (const SpoolerDefinition &spooler_definition)
 
static bool ParseSpoolerDefinition (const SpoolerDefinition &spooler_definition, Config *config)
 
- Static Public Member Functions inherited from upload::AbstractUploader
static void RegisterPlugins ()
 
- Static Public Member Functions inherited from PolymorphicConstruction< AbstractUploader, SpoolerDefinition >
static IntrospectionData Introspect ()
 
- Static Public Member Functions inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
static AbstractUploader * Construct (const SpoolerDefinition &param)
 
- Static Public Member Functions inherited from Callbackable< UploaderResults >
static CallbackTNMakeClosure (typename BoundClosure< UploaderResults, DelegateT, ClosureDataT >::CallbackMethod method, DelegateT *delegate, const ClosureDataT &closure_data)
 
static CallbackTNMakeCallback (typename BoundCallback< UploaderResults, DelegateT >::CallbackMethod method, DelegateT *delegate)
 
static CallbackTNMakeCallback (typename Callback< UploaderResults >::CallbackFunction function)
 

Protected Member Functions

virtual void DoUpload (const std::string &remote_path, IngestionSource *source, const CallbackTN *callback)
 
virtual UploadStreamHandleInitStreamedUpload (const CallbackTN *callback)
 
virtual void StreamedUpload (UploadStreamHandle *handle, UploadBuffer buffer, const CallbackTN *callback)
 
virtual void FinalizeStreamedUpload (UploadStreamHandle *handle, const shash::Any &content_hash)
 
virtual void DoRemoveAsync (const std::string &file_to_delete)
 
virtual bool ReadSessionTokenFile (const std::string &token_file_name, std::string *token)
 
virtual bool ReadKey (const std::string &key_file, std::string *key_id, std::string *secret)
 
virtual int64_t DoGetObjectSize (const std::string &file_name)
 
- Protected Member Functions inherited from upload::AbstractUploader
 AbstractUploader (const SpoolerDefinition &spooler_definition)
 
void Respond (const CallbackTN *callback, const UploaderResults &result) const
 
int CreateAndOpenTemporaryChunkFile (std::string *path) const
 
const SpoolerDefinitionspooler_definition () const
 
void CountUploadedChunks () const
 
void DecUploadedChunks () const
 
void CountUploadedBytes (int64_t bytes_written) const
 
void CountDuplicates () const
 
void CountUploadedCatalogs () const
 
void CountUploadedCatalogBytes (int64_t bytes_written) const
 
void IncJobsInFlight ()
 
- Protected Member Functions inherited from SingleCopy
 SingleCopy ()
 

Private Member Functions

void BumpErrors () const
 

Private Attributes

Config config_
 
SessionContextsession_context_
 
atomic_int32 num_errors_
 

Additional Inherited Members

- Public Types inherited from PolymorphicConstruction< AbstractUploader, SpoolerDefinition >
typedef std::vector< void > IntrospectionData
 
- Public Types inherited from Callbackable< UploaderResults >
typedef CallbackBase
< UploaderResults > 
CallbackTN
 
- Protected Types inherited from upload::AbstractUploader
typedef Callbackable
< UploaderResults >
::CallbackTN
CallbackPtr
 
- Protected Types inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
typedef AbstractFactory
< AbstractUploader,
SpoolerDefinition, void > 
Factory
 
typedef std::vector< Factory * > RegisteredPlugins
 
- Static Protected Member Functions inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
static void LazilyRegisterPlugins ()
 
static void RegisterPlugin ()
 
- Static Protected Attributes inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
static RegisteredPlugins registered_plugins_
 

Detailed Description

Definition at line 26 of file upload_gateway.h.

Constructor & Destructor Documentation

upload::GatewayUploader::GatewayUploader ( const SpoolerDefinition spooler_definition)
explicit

Definition at line 56 of file upload_gateway.cc.

Here is the call graph for this function:

upload::GatewayUploader::~GatewayUploader ( )
virtual

Definition at line 70 of file upload_gateway.cc.

Member Function Documentation

void upload::GatewayUploader::BumpErrors ( ) const
private

Definition at line 262 of file upload_gateway.cc.

Referenced by DoUpload(), FinalizeStreamedUpload(), and StreamedUpload().

Here is the caller graph for this function:

bool upload::GatewayUploader::Create ( )
virtual

Initializes a new repository storage area, e.g. create directory layout for local backend or create bucket for S3 backend.

Implements upload::AbstractUploader.

Definition at line 76 of file upload_gateway.cc.

int64_t upload::GatewayUploader::DoGetObjectSize ( const std::string &  file_name)
protectedvirtual

Implements upload::AbstractUploader.

Definition at line 258 of file upload_gateway.cc.

void upload::GatewayUploader::DoRemoveAsync ( const std::string &  file_to_delete)
protectedvirtual

Implements upload::AbstractUploader.

Definition at line 114 of file upload_gateway.cc.

Here is the call graph for this function:

void upload::GatewayUploader::DoUpload ( const std::string &  remote_path,
IngestionSource source,
const CallbackTN callback 
)
protectedvirtual

Implementation of plain file upload Public interface: AbstractUploader::Upload()

Parameters
local_pathfile to be uploaded
remote_pathdestination to be written in the backend
callbackcallback to be called on completion

Implements upload::AbstractUploader.

Definition at line 134 of file upload_gateway.cc.

Here is the call graph for this function:

bool upload::GatewayUploader::FinalizeSession ( bool  commit,
const std::string &  old_root_hash,
const std::string &  new_root_hash,
const RepositoryTag tag 
)
virtual

Called during Spooler::WaitForUpload(), to ensure that the upload has finished. If commit == true, then a Commit request is also sent, to apply all the the changes accumulated during the session. "catalog_path" represents the path of the root catalog with the changes. By default it is a noop and returns true;

Reimplemented from upload::AbstractUploader.

Definition at line 101 of file upload_gateway.cc.

Here is the call graph for this function:

void upload::GatewayUploader::FinalizeStreamedUpload ( UploadStreamHandle handle,
const shash::Any content_hash 
)
protectedvirtual

Implemetation of streamed upload commit Public interface: AbstractUploader::ScheduleUpload()

Parameters
handledecendant of UploadStreamHandle specifying the stream
content_hashthe computed content hash of the streamed object

Implements upload::AbstractUploader.

Definition at line 199 of file upload_gateway.cc.

Here is the call graph for this function:

unsigned int upload::GatewayUploader::GetNumberOfErrors ( ) const
virtual

Implements upload::AbstractUploader.

Definition at line 130 of file upload_gateway.cc.

bool upload::GatewayUploader::Initialize ( )
virtual

This is called right after the constructor of AbstractUploader or/and its derived class has been executed. You can override that to do additional initialization that cannot be done in the constructor itself.

Returns
true on successful initialization

Reimplemented from upload::AbstractUploader.

Definition at line 82 of file upload_gateway.cc.

Here is the call graph for this function:

UploadStreamHandle * upload::GatewayUploader::InitStreamedUpload ( const CallbackTN callback)
protectedvirtual

This method is called before the first data block of a streamed upload is scheduled (see above implementation of UploadStreamHandle for details).

Parameters
callback(optional) this callback will be invoked once this parti- cular streamed upload is committed.
Returns
a pointer to the initialized UploadStreamHandle

Implements upload::AbstractUploader.

Definition at line 177 of file upload_gateway.cc.

Here is the call graph for this function:

bool upload::GatewayUploader::Mkdir ( const std::string &  path)
virtual

Make directory in upstream storage. Noop if directory already present.

Parameters
pathrelative directory path in the upstream storage
Returns
true if the directory was successfully created or already present

Implements upload::AbstractUploader.

Definition at line 122 of file upload_gateway.cc.

std::string upload::GatewayUploader::name ( ) const
virtual

A string identifying the uploader type

Implements upload::AbstractUploader.

Definition at line 112 of file upload_gateway.cc.

bool upload::GatewayUploader::ParseSpoolerDefinition ( const SpoolerDefinition spooler_definition,
GatewayUploader::Config config 
)
static

Definition at line 25 of file upload_gateway.cc.

Referenced by GatewayUploader().

Here is the caller graph for this function:

bool upload::GatewayUploader::Peek ( const std::string &  path)
virtual

Checks if a file is already present in the backend storage. This might be a synchronous operation.

Parameters
paththe path of the file to be checked
Returns
true if the file was found in the backend storage

Implements upload::AbstractUploader.

Definition at line 119 of file upload_gateway.cc.

bool upload::GatewayUploader::PlaceBootstrappingShortcut ( const shash::Any object)
virtual

Creates a top-level shortcut to the given data object. This is particularly useful for bootstrapping repositories whose data-directory is secured by a VOMS certificate.

Parameters
objectcontent hash of the object to be exposed on the top-level
Returns
true on success

Implements upload::AbstractUploader.

Definition at line 126 of file upload_gateway.cc.

bool upload::GatewayUploader::ReadKey ( const std::string &  key_file,
std::string *  key_id,
std::string *  secret 
)
protectedvirtual

Definition at line 253 of file upload_gateway.cc.

Referenced by Initialize().

Here is the call graph for this function:

Here is the caller graph for this function:

bool upload::GatewayUploader::ReadSessionTokenFile ( const std::string &  token_file_name,
std::string *  token 
)
protectedvirtual

Definition at line 233 of file upload_gateway.cc.

Referenced by Initialize().

Here is the call graph for this function:

Here is the caller graph for this function:

void upload::GatewayUploader::StreamedUpload ( UploadStreamHandle handle,
UploadBuffer  buffer,
const CallbackTN callback 
)
protectedvirtual

Implementation of a streamed upload step. See public interface for details. Public interface: AbstractUploader::ScheduleUpload()

Parameters
handledecendant of UploadStreamHandle specifying the stream
bufferthe CharBuffer to be uploaded to the stream
callbackcallback to be called on completion

Implements upload::AbstractUploader.

Definition at line 182 of file upload_gateway.cc.

Here is the call graph for this function:

void upload::GatewayUploader::WaitForUpload ( ) const
virtual

Waits until the current upload queue is empty.

Note: This does NOT necessarily mean, that all files are actuall uploaded. If new jobs are concurrently scheduled the behavior of this method is not defined (it returns also on intermediately empty queues)

Reimplemented from upload::AbstractUploader.

Definition at line 108 of file upload_gateway.cc.

Here is the call graph for this function:

bool upload::GatewayUploader::WillHandle ( const SpoolerDefinition spooler_definition)
static

Definition at line 21 of file upload_gateway.cc.

Member Data Documentation

Config upload::GatewayUploader::config_
private

Definition at line 98 of file upload_gateway.h.

Referenced by GatewayUploader(), and Initialize().

atomic_int32 upload::GatewayUploader::num_errors_
mutableprivate

Definition at line 100 of file upload_gateway.h.

Referenced by BumpErrors(), DoRemoveAsync(), GatewayUploader(), and GetNumberOfErrors().

SessionContext* upload::GatewayUploader::session_context_
private

The documentation for this class was generated from the following files: