CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
s3fanout::S3FanoutManager Class Reference

#include <s3fanout.h>

Inheritance diagram for s3fanout::S3FanoutManager:
Collaboration diagram for s3fanout::S3FanoutManager:

Classes

struct  S3Config
 

Public Member Functions

 S3FanoutManager (const S3Config &config)
 
 ~S3FanoutManager ()
 
void Spawn ()
 
void PushNewJob (JobInfo *info)
 
void PushCompletedJob (JobInfo *info)
 
JobInfoPopCompletedJob ()
 
const StatisticsGetStatistics ()
 

Static Public Member Functions

static void DetectThrottleIndicator (const std::string &header, JobInfo *info)
 

Static Public Attributes

static const unsigned kDefault429ThrottleMs = 250
 
static const unsigned kMax429ThrottleMs = 10000
 
static const unsigned kThrottleReportIntervalSec = 10
 
static const unsigned kDefaultHTTPPort = 80
 

Protected Types

typedef SynchronizingCounter
< uint32_t > 
Semaphore
 

Private Member Functions

CURL * AcquireCurlHandle () const
 
void ReleaseCurlHandle (JobInfo *info, CURL *handle) const
 
void InitPipeWatchFds ()
 
int InitializeDnsSettings (CURL *handle, std::string remote_host) const
 
void InitializeDnsSettingsCurl (CURL *handle, CURLSH *sharehandle, curl_slist *clist) const
 
Failures InitializeRequest (JobInfo *info, CURL *handle) const
 
void SetUrlOptions (JobInfo *info) const
 
void UpdateStatistics (CURL *handle)
 
bool CanRetry (const JobInfo *info)
 
void Backoff (JobInfo *info)
 
bool VerifyAndFinalize (const int curl_error, JobInfo *info)
 
std::string GetRequestString (const JobInfo &info) const
 
std::string GetContentType (const JobInfo &info) const
 
std::string GetUriEncode (const std::string &val, bool encode_slash) const
 
std::string GetAwsV4SigningKey (const std::string &date) const
 
bool MkPayloadHash (const JobInfo &info, std::string *hex_hash) const
 
bool MkV2Authz (const JobInfo &info, std::vector< std::string > *headers) const
 
bool MkV4Authz (const JobInfo &info, std::vector< std::string > *headers) const
 
bool MkAzureAuthz (const JobInfo &info, std::vector< std::string > *headers) const
 
std::string MkUrl (const std::string &objkey) const
 
std::string MkCompleteHostname ()
 
- Private Member Functions inherited from SingleCopy
 SingleCopy ()
 

Static Private Member Functions

static int CallbackCurlSocket (CURL *easy, curl_socket_t s, int action, void *userp, void *socketp)
 
static void * MainUpload (void *data)
 

Private Attributes

std::vector< s3fanout::JobInfo * > jobs_todo_
 
pthread_mutex_t * jobs_todo_lock_
 
pthread_mutex_t * curl_handle_lock_
 
const S3Config config_
 
std::string complete_hostname_
 
Prng prng_
 
std::set< JobInfo * > * active_requests_
 
std::set< CURL * > * pool_handles_idle_
 
std::set< CURL * > * pool_handles_inuse_
 
std::set< S3FanOutDnsEntry * > * sharehandles_
 
std::map< CURL
*, S3FanOutDnsEntry * > * 
curl_sharehandles_
 
dns::CaresResolverresolver_
 
CURLM * curl_multi_
 
std::string * user_agent_
 
std::pair< std::string,
std::string > 
last_signing_key_
 
pthread_t thread_upload_
 
atomic_int32 multi_threaded_
 
struct pollfd * watch_fds_
 
uint32_t watch_fds_size_
 
uint32_t watch_fds_inuse_
 
uint32_t watch_fds_max_
 
int pipe_terminate_ [2]
 
int pipe_jobs_ [2]
 
int pipe_completed_ [2]
 
bool opt_ipv4_only_
 
unsigned int max_available_jobs_
 
Semaphoreavailable_jobs_
 
Statisticsstatistics_
 
uint64_t timestamp_last_throttle_report_
 
bool is_curl_debug_
 

Static Private Attributes

static const char * kCacheControlCas = "Cache-Control: max-age=259200"
 
static const char * kCacheControlDotCvmfs
 
static const unsigned kLowSpeedLimit = 1024
 

Detailed Description

Definition at line 161 of file s3fanout.h.

Member Typedef Documentation

Definition at line 163 of file s3fanout.h.

Constructor & Destructor Documentation

s3fanout::S3FanoutManager::S3FanoutManager ( const S3Config config)
explicit

Definition at line 1164 of file s3fanout.cc.

Here is the call graph for this function:

s3fanout::S3FanoutManager::~S3FanoutManager ( )

Definition at line 1231 of file s3fanout.cc.

Here is the call graph for this function:

Member Function Documentation

CURL * s3fanout::S3FanoutManager::AcquireCurlHandle ( ) const
private

Gets an idle CURL handle from the pool. Creates a new one and adds it to the pool if necessary.

Definition at line 383 of file s3fanout.cc.

Referenced by MainUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

void s3fanout::S3FanoutManager::Backoff ( JobInfo info)
private

Backoff for retry to introduce a jitter into a upload sequence.

Returns
true if backoff has been performed, false otherwise

Definition at line 1020 of file s3fanout.cc.

Here is the call graph for this function:

int s3fanout::S3FanoutManager::CallbackCurlSocket ( CURL *  easy,
curl_socket_t  s,
int  action,
void *  userp,
void *  socketp 
)
staticprivate

Called when new curl sockets arrive or existing curl sockets depart.

Definition at line 161 of file s3fanout.cc.

Referenced by S3FanoutManager().

Here is the caller graph for this function:

bool s3fanout::S3FanoutManager::CanRetry ( const JobInfo info)
private

Retry if possible and if not already done too often.

Definition at line 1005 of file s3fanout.cc.

void s3fanout::S3FanoutManager::DetectThrottleIndicator ( const std::string &  header,
JobInfo info 
)
static

Parses Retry-After and X-Retry-In headers attached to HTTP 429 responses

Definition at line 39 of file s3fanout.cc.

Here is the call graph for this function:

string s3fanout::S3FanoutManager::GetAwsV4SigningKey ( const std::string &  date) const
private

Definition at line 527 of file s3fanout.cc.

Here is the call graph for this function:

string s3fanout::S3FanoutManager::GetContentType ( const JobInfo info) const
private

Definition at line 826 of file s3fanout.cc.

string s3fanout::S3FanoutManager::GetRequestString ( const JobInfo info) const
private

Definition at line 808 of file s3fanout.cc.

const Statistics & s3fanout::S3FanoutManager::GetStatistics ( )

Definition at line 1291 of file s3fanout.cc.

string s3fanout::S3FanoutManager::GetUriEncode ( const std::string &  val,
bool  encode_slash 
) const
private

Definition at line 497 of file s3fanout.cc.

int s3fanout::S3FanoutManager::InitializeDnsSettings ( CURL *  handle,
std::string  remote_host 
) const
private

Definition at line 673 of file s3fanout.cc.

Here is the call graph for this function:

void s3fanout::S3FanoutManager::InitializeDnsSettingsCurl ( CURL *  handle,
CURLSH *  sharehandle,
curl_slist *  clist 
) const
private

Definition at line 661 of file s3fanout.cc.

Here is the call graph for this function:

Failures s3fanout::S3FanoutManager::InitializeRequest ( JobInfo info,
CURL *  handle 
) const
private

Request parameters set the URL and other options such as timeout and proxy.

Definition at line 850 of file s3fanout.cc.

Referenced by MainUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

void s3fanout::S3FanoutManager::InitPipeWatchFds ( )
private

Definition at line 441 of file s3fanout.cc.

Referenced by MainUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

void * s3fanout::S3FanoutManager::MainUpload ( void *  data)
staticprivate

Worker thread event loop.

Definition at line 231 of file s3fanout.cc.

Referenced by Spawn().

Here is the call graph for this function:

Here is the caller graph for this function:

bool s3fanout::S3FanoutManager::MkAzureAuthz ( const JobInfo info,
std::vector< std::string > *  headers 
) const
private

The Azure Blob authorization header according to https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key

Definition at line 616 of file s3fanout.cc.

Here is the call graph for this function:

std::string s3fanout::S3FanoutManager::MkCompleteHostname ( )
inlineprivate

Definition at line 258 of file s3fanout.h.

Referenced by S3FanoutManager().

Here is the caller graph for this function:

bool s3fanout::S3FanoutManager::MkPayloadHash ( const JobInfo info,
std::string *  hex_hash 
) const
private

Definition at line 753 of file s3fanout.cc.

Here is the call graph for this function:

std::string s3fanout::S3FanoutManager::MkUrl ( const std::string &  objkey) const
inlineprivate

Definition at line 250 of file s3fanout.h.

bool s3fanout::S3FanoutManager::MkV2Authz ( const JobInfo info,
std::vector< std::string > *  headers 
) const
private

The Amazon AWS 2 authorization header according to http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#ConstructingTheAuthenticationHeader

Definition at line 458 of file s3fanout.cc.

Here is the call graph for this function:

bool s3fanout::S3FanoutManager::MkV4Authz ( const JobInfo info,
std::vector< std::string > *  headers 
) const
private

The Amazon AWS4 authorization header according to http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html

Definition at line 547 of file s3fanout.cc.

Here is the call graph for this function:

JobInfo * s3fanout::S3FanoutManager::PopCompletedJob ( )

Pop completed job

Definition at line 1313 of file s3fanout.cc.

Here is the call graph for this function:

void s3fanout::S3FanoutManager::PushCompletedJob ( JobInfo info)

Push completed job to list of completed jobs

Definition at line 1306 of file s3fanout.cc.

Referenced by MainUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

void s3fanout::S3FanoutManager::PushNewJob ( JobInfo info)

Push new job to be uploaded to the S3 cloud storage.

Definition at line 1298 of file s3fanout.cc.

Here is the call graph for this function:

void s3fanout::S3FanoutManager::ReleaseCurlHandle ( JobInfo info,
CURL *  handle 
) const
private

Definition at line 416 of file s3fanout.cc.

Referenced by MainUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

void s3fanout::S3FanoutManager::SetUrlOptions ( JobInfo info) const
private

Sets the URL specific options such as host to use and timeout.

Definition at line 966 of file s3fanout.cc.

Referenced by MainUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

void s3fanout::S3FanoutManager::Spawn ( )

Spawns the I/O worker thread. No way back except ~S3FanoutManager.

Definition at line 1281 of file s3fanout.cc.

Here is the call graph for this function:

void s3fanout::S3FanoutManager::UpdateStatistics ( CURL *  handle)
private

Adds transfer time and uploaded bytes to the global counters.

Definition at line 994 of file s3fanout.cc.

bool s3fanout::S3FanoutManager::VerifyAndFinalize ( const int  curl_error,
JobInfo info 
)
private

Checks the result of a curl request and implements the failure logic and takes care of cleanup.

Returns
true if request should be repeated, false otherwise

Definition at line 1065 of file s3fanout.cc.

Referenced by MainUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

std::set<JobInfo *>* s3fanout::S3FanoutManager::active_requests_
private

This is not strictly necessary but it helps the debugging

Definition at line 273 of file s3fanout.h.

Referenced by MainUpload(), S3FanoutManager(), and ~S3FanoutManager().

Semaphore* s3fanout::S3FanoutManager::available_jobs_
private
std::string s3fanout::S3FanoutManager::complete_hostname_
private

Definition at line 267 of file s3fanout.h.

Referenced by MkUrl(), and S3FanoutManager().

const S3Config s3fanout::S3FanoutManager::config_
private

Definition at line 266 of file s3fanout.h.

Referenced by MkCompleteHostname(), MkUrl(), and S3FanoutManager().

pthread_mutex_t* s3fanout::S3FanoutManager::curl_handle_lock_
private

Definition at line 224 of file s3fanout.h.

Referenced by S3FanoutManager(), and ~S3FanoutManager().

CURLM* s3fanout::S3FanoutManager::curl_multi_
private

Definition at line 280 of file s3fanout.h.

Referenced by MainUpload(), S3FanoutManager(), and ~S3FanoutManager().

std::map<CURL *, S3FanOutDnsEntry *>* s3fanout::S3FanoutManager::curl_sharehandles_
private

Definition at line 278 of file s3fanout.h.

Referenced by S3FanoutManager(), and ~S3FanoutManager().

bool s3fanout::S3FanoutManager::is_curl_debug_
private

Definition at line 321 of file s3fanout.h.

Referenced by S3FanoutManager().

std::vector<s3fanout::JobInfo*> s3fanout::S3FanoutManager::jobs_todo_
private

Definition at line 222 of file s3fanout.h.

pthread_mutex_t* s3fanout::S3FanoutManager::jobs_todo_lock_
private

Definition at line 223 of file s3fanout.h.

Referenced by S3FanoutManager(), and ~S3FanoutManager().

const char * s3fanout::S3FanoutManager::kCacheControlCas = "Cache-Control: max-age=259200"
staticprivate

Definition at line 215 of file s3fanout.h.

const char * s3fanout::S3FanoutManager::kCacheControlDotCvmfs
staticprivate
Initial value:
=
"Cache-Control: max-age=61"

Definition at line 216 of file s3fanout.h.

const unsigned s3fanout::S3FanoutManager::kDefault429ThrottleMs = 250
static

Definition at line 167 of file s3fanout.h.

const unsigned s3fanout::S3FanoutManager::kDefaultHTTPPort = 80
static

Definition at line 172 of file s3fanout.h.

const unsigned s3fanout::S3FanoutManager::kLowSpeedLimit = 1024
staticprivate

Definition at line 217 of file s3fanout.h.

const unsigned s3fanout::S3FanoutManager::kMax429ThrottleMs = 10000
static

Definition at line 169 of file s3fanout.h.

const unsigned s3fanout::S3FanoutManager::kThrottleReportIntervalSec = 10
static

Definition at line 171 of file s3fanout.h.

std::pair<std::string, std::string> s3fanout::S3FanoutManager::last_signing_key_
mutableprivate

AWS4 signing keys are derived from the secret key, a region and a date. The signing key for current day can be cached.

Definition at line 287 of file s3fanout.h.

unsigned int s3fanout::S3FanoutManager::max_available_jobs_
private

Definition at line 311 of file s3fanout.h.

Referenced by S3FanoutManager().

atomic_int32 s3fanout::S3FanoutManager::multi_threaded_
private

Definition at line 290 of file s3fanout.h.

Referenced by S3FanoutManager(), Spawn(), and ~S3FanoutManager().

bool s3fanout::S3FanoutManager::opt_ipv4_only_
private

Definition at line 309 of file s3fanout.h.

Referenced by S3FanoutManager().

int s3fanout::S3FanoutManager::pipe_completed_[2]
private

Definition at line 307 of file s3fanout.h.

Referenced by PopCompletedJob(), PushCompletedJob(), S3FanoutManager(), and ~S3FanoutManager().

int s3fanout::S3FanoutManager::pipe_jobs_[2]
private

Definition at line 303 of file s3fanout.h.

Referenced by MainUpload(), PushNewJob(), S3FanoutManager(), and ~S3FanoutManager().

int s3fanout::S3FanoutManager::pipe_terminate_[2]
private

Definition at line 299 of file s3fanout.h.

Referenced by S3FanoutManager(), and ~S3FanoutManager().

std::set<CURL *>* s3fanout::S3FanoutManager::pool_handles_idle_
private

Definition at line 275 of file s3fanout.h.

Referenced by S3FanoutManager(), and ~S3FanoutManager().

std::set<CURL *>* s3fanout::S3FanoutManager::pool_handles_inuse_
private

Definition at line 276 of file s3fanout.h.

Referenced by MainUpload(), S3FanoutManager(), and ~S3FanoutManager().

Prng s3fanout::S3FanoutManager::prng_
private

Definition at line 269 of file s3fanout.h.

Referenced by S3FanoutManager().

dns::CaresResolver* s3fanout::S3FanoutManager::resolver_
private

Definition at line 279 of file s3fanout.h.

Referenced by S3FanoutManager().

std::set<S3FanOutDnsEntry *>* s3fanout::S3FanoutManager::sharehandles_
private

Definition at line 277 of file s3fanout.h.

Referenced by S3FanoutManager(), and ~S3FanoutManager().

Statistics* s3fanout::S3FanoutManager::statistics_
private

Definition at line 316 of file s3fanout.h.

Referenced by GetStatistics(), MainUpload(), S3FanoutManager(), and ~S3FanoutManager().

pthread_t s3fanout::S3FanoutManager::thread_upload_
private

Definition at line 289 of file s3fanout.h.

Referenced by S3FanoutManager(), Spawn(), and ~S3FanoutManager().

uint64_t s3fanout::S3FanoutManager::timestamp_last_throttle_report_
private

Definition at line 319 of file s3fanout.h.

Referenced by S3FanoutManager().

std::string* s3fanout::S3FanoutManager::user_agent_
private

Definition at line 281 of file s3fanout.h.

Referenced by S3FanoutManager(), and ~S3FanoutManager().

struct pollfd* s3fanout::S3FanoutManager::watch_fds_
private

Definition at line 292 of file s3fanout.h.

Referenced by CallbackCurlSocket(), MainUpload(), and S3FanoutManager().

uint32_t s3fanout::S3FanoutManager::watch_fds_inuse_
private

Definition at line 294 of file s3fanout.h.

Referenced by CallbackCurlSocket(), MainUpload(), and S3FanoutManager().

uint32_t s3fanout::S3FanoutManager::watch_fds_max_
private

Definition at line 295 of file s3fanout.h.

Referenced by CallbackCurlSocket(), and S3FanoutManager().

uint32_t s3fanout::S3FanoutManager::watch_fds_size_
private

Definition at line 293 of file s3fanout.h.

Referenced by CallbackCurlSocket(), and S3FanoutManager().


The documentation for this class was generated from the following files: