CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
upload::LocalUploader Class Reference

#include <upload_local.h>

Inheritance diagram for upload::LocalUploader:
Collaboration diagram for upload::LocalUploader:

Public Member Functions

 LocalUploader (const SpoolerDefinition &spooler_definition)
 
virtual std::string name () const
 
virtual bool Create ()
 
void DoUpload (const std::string &remote_path, IngestionSource *source, const CallbackTN *callback)
 
UploadStreamHandleInitStreamedUpload (const CallbackTN *callback)
 
void StreamedUpload (UploadStreamHandle *handle, UploadBuffer buffer, const CallbackTN *callback=NULL)
 
void FinalizeStreamedUpload (UploadStreamHandle *handle, const shash::Any &content_hash)
 
void DoRemoveAsync (const std::string &file_to_delete)
 
bool Peek (const std::string &path)
 
bool Mkdir (const std::string &path)
 
bool PlaceBootstrappingShortcut (const shash::Any &object)
 
unsigned int GetNumberOfErrors () const
 
int64_t DoGetObjectSize (const std::string &file_name)
 
- Public Member Functions inherited from upload::AbstractUploader
virtual ~AbstractUploader ()
 
virtual unsigned GetNumTasks () const
 
virtual bool Initialize ()
 
virtual bool FinalizeSession (bool commit, const std::string &old_root_hash, const std::string &new_root_hash, const RepositoryTag &tag)
 
void TearDown ()
 
void UploadFile (const std::string &local_path, const std::string &remote_path, const CallbackTN *callback=NULL)
 
void UploadIngestionSource (const std::string &remote_path, IngestionSource *source, const CallbackTN *callback=NULL)
 
void ScheduleUpload (UploadStreamHandle *handle, UploadBuffer buffer, const CallbackTN *callback=NULL)
 
void ScheduleCommit (UploadStreamHandle *handle, const shash::Any &content_hash)
 
void RemoveAsync (const std::string &file_to_delete)
 
void RemoveAsync (const shash::Any &hash_to_delete)
 
int64_t GetObjectSize (const shash::Any &hash)
 
virtual void WaitForUpload () const
 
void InitCounters (perf::StatisticsTemplate *statistics)
 
- Public Member Functions inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
virtual ~PolymorphicConstructionImpl ()
 

Static Public Member Functions

static bool WillHandle (const SpoolerDefinition &spooler_definition)
 
- Static Public Member Functions inherited from upload::AbstractUploader
static void RegisterPlugins ()
 
- Static Public Member Functions inherited from PolymorphicConstruction< AbstractUploader, SpoolerDefinition >
static IntrospectionData Introspect ()
 
- Static Public Member Functions inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
static AbstractUploader * Construct (const SpoolerDefinition &param)
 
- Static Public Member Functions inherited from Callbackable< UploaderResults >
static CallbackTNMakeClosure (typename BoundClosure< UploaderResults, DelegateT, ClosureDataT >::CallbackMethod method, DelegateT *delegate, const ClosureDataT &closure_data)
 
static CallbackTNMakeCallback (typename BoundCallback< UploaderResults, DelegateT >::CallbackMethod method, DelegateT *delegate)
 
static CallbackTNMakeCallback (typename Callback< UploaderResults >::CallbackFunction function)
 

Protected Member Functions

int Move (const std::string &local_path, const std::string &remote_path) const
 
- Protected Member Functions inherited from upload::AbstractUploader
 AbstractUploader (const SpoolerDefinition &spooler_definition)
 
void Respond (const CallbackTN *callback, const UploaderResults &result) const
 
int CreateAndOpenTemporaryChunkFile (std::string *path) const
 
const SpoolerDefinitionspooler_definition () const
 
void CountUploadedChunks () const
 
void DecUploadedChunks () const
 
void CountUploadedBytes (int64_t bytes_written) const
 
void CountDuplicates () const
 
void CountUploadedCatalogs () const
 
void CountUploadedCatalogBytes (int64_t bytes_written) const
 
void IncJobsInFlight ()
 
- Protected Member Functions inherited from SingleCopy
 SingleCopy ()
 

Private Attributes

const mode_t backend_file_mode_
 
const mode_t backend_dir_mode_
 
const std::string upstream_path_
 
const std::string temporary_path_
 
atomic_int32 copy_errors_
 

Static Private Attributes

static const mode_t default_backend_file_mode_ = 0666
 
static const mode_t default_backend_dir_mode_ = 0777
 

Additional Inherited Members

- Public Types inherited from PolymorphicConstruction< AbstractUploader, SpoolerDefinition >
typedef std::vector< void > IntrospectionData
 
- Public Types inherited from Callbackable< UploaderResults >
typedef CallbackBase
< UploaderResults > 
CallbackTN
 
- Protected Types inherited from upload::AbstractUploader
typedef Callbackable
< UploaderResults >
::CallbackTN
CallbackPtr
 
- Protected Types inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
typedef AbstractFactory
< AbstractUploader,
SpoolerDefinition, void > 
Factory
 
typedef std::vector< Factory * > RegisteredPlugins
 
- Static Protected Member Functions inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
static void LazilyRegisterPlugins ()
 
static void RegisterPlugin ()
 
- Static Protected Attributes inherited from PolymorphicConstructionImpl< AbstractUploader, SpoolerDefinition, void >
static RegisteredPlugins registered_plugins_
 

Detailed Description

The LocalSpooler implements the AbstractSpooler interface to push files into a local CVMFS repository backend. For a detailed description of the classes interface please have a look into the AbstractSpooler base class.

Definition at line 36 of file upload_local.h.

Constructor & Destructor Documentation

upload::LocalUploader::LocalUploader ( const SpoolerDefinition spooler_definition)
explicit

Definition at line 18 of file upload_local.cc.

Here is the call graph for this function:

Member Function Documentation

bool upload::LocalUploader::Create ( )
virtual

Initializes a new repository storage area, e.g. create directory layout for local backend or create bucket for S3 backend.

Implements upload::AbstractUploader.

Definition at line 39 of file upload_local.cc.

Here is the call graph for this function:

int64_t upload::LocalUploader::DoGetObjectSize ( const std::string &  file_name)
virtual

Implements upload::AbstractUploader.

Definition at line 260 of file upload_local.cc.

Here is the call graph for this function:

void upload::LocalUploader::DoRemoveAsync ( const std::string &  file_to_delete)
virtual

TODO(jblomer): investigate if parallelism increases the GC speed on local disks.

Implements upload::AbstractUploader.

Definition at line 210 of file upload_local.cc.

Here is the call graph for this function:

void upload::LocalUploader::DoUpload ( const std::string &  remote_path,
IngestionSource source,
const CallbackTN callback 
)
virtual

Upload() is not done concurrently in the current implementation of the LocalSpooler, since it is a simple move or copy of a file without CPU intensive operation This method calls NotifyListeners and invokes a callback for all registered listeners (see the Observable template for details).

Implements upload::AbstractUploader.

Definition at line 44 of file upload_local.cc.

Here is the call graph for this function:

void upload::LocalUploader::FinalizeStreamedUpload ( UploadStreamHandle handle,
const shash::Any content_hash 
)
virtual

Implemetation of streamed upload commit Public interface: AbstractUploader::ScheduleUpload()

Parameters
handledecendant of UploadStreamHandle specifying the stream
content_hashthe computed content hash of the streamed object

Implements upload::AbstractUploader.

Definition at line 144 of file upload_local.cc.

Here is the call graph for this function:

unsigned int upload::LocalUploader::GetNumberOfErrors ( ) const
virtual

Determines the number of failed jobs in the LocalCompressionWorker as well as in the Upload() command.

Implements upload::AbstractUploader.

Definition at line 35 of file upload_local.cc.

UploadStreamHandle * upload::LocalUploader::InitStreamedUpload ( const CallbackTN callback)
virtual

This method is called before the first data block of a streamed upload is scheduled (see above implementation of UploadStreamHandle for details).

Parameters
callback(optional) this callback will be invoked once this parti- cular streamed upload is committed.
Returns
a pointer to the initialized UploadStreamHandle

Implements upload::AbstractUploader.

Definition at line 109 of file upload_local.cc.

Here is the call graph for this function:

bool upload::LocalUploader::Mkdir ( const std::string &  path)
virtual

Make directory in upstream storage. Noop if directory already present.

Parameters
pathrelative directory path in the upstream storage
Returns
true if the directory was successfully created or already present

Implements upload::AbstractUploader.

Definition at line 222 of file upload_local.cc.

Here is the call graph for this function:

int upload::LocalUploader::Move ( const std::string &  local_path,
const std::string &  remote_path 
) const
protected

Definition at line 232 of file upload_local.cc.

Referenced by DoUpload(), and FinalizeStreamedUpload().

Here is the caller graph for this function:

virtual std::string upload::LocalUploader::name ( ) const
inlinevirtual

A string identifying the uploader type

Implements upload::AbstractUploader.

Definition at line 47 of file upload_local.h.

bool upload::LocalUploader::Peek ( const std::string &  path)
virtual

Checks if a file is already present in the backend storage. This might be a synchronous operation.

Parameters
paththe path of the file to be checked
Returns
true if the file was found in the backend storage

Implements upload::AbstractUploader.

Definition at line 217 of file upload_local.cc.

Referenced by FinalizeStreamedUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

bool upload::LocalUploader::PlaceBootstrappingShortcut ( const shash::Any object)
virtual

Creates a top-level shortcut to the given data object. This is particularly useful for bootstrapping repositories whose data-directory is secured by a VOMS certificate.

Parameters
objectcontent hash of the object to be exposed on the top-level
Returns
true on success

Implements upload::AbstractUploader.

Definition at line 226 of file upload_local.cc.

Here is the call graph for this function:

void upload::LocalUploader::StreamedUpload ( UploadStreamHandle handle,
UploadBuffer  buffer,
const CallbackTN callback = NULL 
)
virtual

Implementation of a streamed upload step. See public interface for details. Public interface: AbstractUploader::ScheduleUpload()

Parameters
handledecendant of UploadStreamHandle specifying the stream
bufferthe CharBuffer to be uploaded to the stream
callbackcallback to be called on completion

Implements upload::AbstractUploader.

Definition at line 121 of file upload_local.cc.

Here is the call graph for this function:

bool upload::LocalUploader::WillHandle ( const SpoolerDefinition spooler_definition)
static

Definition at line 31 of file upload_local.cc.

Member Data Documentation

const mode_t upload::LocalUploader::backend_dir_mode_
private

Definition at line 41 of file upload_local.h.

Referenced by Create(), and Mkdir().

const mode_t upload::LocalUploader::backend_file_mode_
private

Definition at line 40 of file upload_local.h.

Referenced by Move().

atomic_int32 upload::LocalUploader::copy_errors_
mutableprivate

counts the number of occured errors in Upload()

Definition at line 91 of file upload_local.h.

Referenced by DoRemoveAsync(), DoUpload(), FinalizeStreamedUpload(), GetNumberOfErrors(), InitStreamedUpload(), LocalUploader(), and StreamedUpload().

const mode_t upload::LocalUploader::default_backend_dir_mode_ = 0777
staticprivate

Definition at line 39 of file upload_local.h.

const mode_t upload::LocalUploader::default_backend_file_mode_ = 0666
staticprivate

Definition at line 38 of file upload_local.h.

const std::string upload::LocalUploader::temporary_path_
private

Definition at line 90 of file upload_local.h.

Referenced by DoUpload().

const std::string upload::LocalUploader::upstream_path_
private

The documentation for this class was generated from the following files: