CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
s3fanout::S3FanoutManager Class Reference

#include <s3fanout.h>

Inheritance diagram for s3fanout::S3FanoutManager:
Collaboration diagram for s3fanout::S3FanoutManager:

Classes

struct  S3Config
 

Public Member Functions

 S3FanoutManager (const S3Config &config)
 
 ~S3FanoutManager ()
 
void Spawn ()
 
void PushNewJob (JobInfo *info)
 
void PushCompletedJob (JobInfo *info)
 
JobInfoPopCompletedJob ()
 
const StatisticsGetStatistics ()
 

Static Public Member Functions

static void DetectThrottleIndicator (const std::string &header, JobInfo *info)
 

Static Public Attributes

static const unsigned kDefault429ThrottleMs = 250
 
static const unsigned kMax429ThrottleMs = 10000
 
static const unsigned kThrottleReportIntervalSec = 10
 
static const unsigned kDefaultHTTPPort = 80
 
static const unsigned kDefaultHTTPSPort = 443
 

Protected Types

typedef SynchronizingCounter
< uint32_t > 
Semaphore
 

Private Member Functions

CURL * AcquireCurlHandle () const
 
void ReleaseCurlHandle (JobInfo *info, CURL *handle) const
 
void InitPipeWatchFds ()
 
int InitializeDnsSettings (CURL *handle, std::string remote_host) const
 
void InitializeDnsSettingsCurl (CURL *handle, CURLSH *sharehandle, curl_slist *clist) const
 
Failures InitializeRequest (JobInfo *info, CURL *handle) const
 
void SetUrlOptions (JobInfo *info) const
 
void UpdateStatistics (CURL *handle)
 
bool CanRetry (const JobInfo *info)
 
void Backoff (JobInfo *info)
 
bool VerifyAndFinalize (const int curl_error, JobInfo *info)
 
std::string GetRequestString (const JobInfo &info) const
 
std::string GetContentType (const JobInfo &info) const
 
std::string GetUriEncode (const std::string &val, bool encode_slash) const
 
std::string GetAwsV4SigningKey (const std::string &date) const
 
bool MkPayloadHash (const JobInfo &info, std::string *hex_hash) const
 
bool MkV2Authz (const JobInfo &info, std::vector< std::string > *headers) const
 
bool MkV4Authz (const JobInfo &info, std::vector< std::string > *headers) const
 
bool MkAzureAuthz (const JobInfo &info, std::vector< std::string > *headers) const
 
std::string MkUrl (const std::string &objkey) const
 
std::string MkCompleteHostname ()
 
- Private Member Functions inherited from SingleCopy
 SingleCopy ()
 

Static Private Member Functions

static int CallbackCurlSocket (CURL *easy, curl_socket_t s, int action, void *userp, void *socketp)
 
static void * MainUpload (void *data)
 

Private Attributes

std::vector< s3fanout::JobInfo * > jobs_todo_
 
pthread_mutex_t * jobs_todo_lock_
 
pthread_mutex_t * curl_handle_lock_
 
const S3Config config_
 
std::string complete_hostname_
 
Prng prng_
 
std::set< JobInfo * > * active_requests_
 
std::set< CURL * > * pool_handles_idle_
 
std::set< CURL * > * pool_handles_inuse_
 
std::set< S3FanOutDnsEntry * > * sharehandles_
 
std::map< CURL
*, S3FanOutDnsEntry * > * 
curl_sharehandles_
 
dns::CaresResolverresolver_
 
CURLM * curl_multi_
 
std::string * user_agent_
 
std::pair< std::string,
std::string > 
last_signing_key_
 
pthread_t thread_upload_
 
atomic_int32 multi_threaded_
 
struct pollfd * watch_fds_
 
uint32_t watch_fds_size_
 
uint32_t watch_fds_inuse_
 
uint32_t watch_fds_max_
 
int pipe_terminate_ [2]
 
int pipe_jobs_ [2]
 
int pipe_completed_ [2]
 
bool opt_ipv4_only_
 
unsigned int max_available_jobs_
 
Semaphoreavailable_jobs_
 
Statisticsstatistics_
 
uint64_t timestamp_last_throttle_report_
 
bool is_curl_debug_
 
SslCertificateStore ssl_certificate_store_
 

Static Private Attributes

static const char * kCacheControlCas = "Cache-Control: max-age=259200"
 
static const char * kCacheControlDotCvmfs
 
static const unsigned kLowSpeedLimit = 1024
 

Detailed Description

Definition at line 170 of file s3fanout.h.

Member Typedef Documentation

Definition at line 172 of file s3fanout.h.

Constructor & Destructor Documentation

s3fanout::S3FanoutManager::S3FanoutManager ( const S3Config config)
explicit

Definition at line 1184 of file s3fanout.cc.

Here is the call graph for this function:

s3fanout::S3FanoutManager::~S3FanoutManager ( )

Definition at line 1253 of file s3fanout.cc.

Here is the call graph for this function:

Member Function Documentation

CURL * s3fanout::S3FanoutManager::AcquireCurlHandle ( ) const
private

Gets an idle CURL handle from the pool. Creates a new one and adds it to the pool if necessary.

Definition at line 384 of file s3fanout.cc.

Referenced by MainUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

void s3fanout::S3FanoutManager::Backoff ( JobInfo info)
private

Backoff for retry to introduce a jitter into a upload sequence.

Returns
true if backoff has been performed, false otherwise

Definition at line 1040 of file s3fanout.cc.

Here is the call graph for this function:

int s3fanout::S3FanoutManager::CallbackCurlSocket ( CURL *  easy,
curl_socket_t  s,
int  action,
void *  userp,
void *  socketp 
)
staticprivate

Called when new curl sockets arrive or existing curl sockets depart.

Definition at line 162 of file s3fanout.cc.

Referenced by S3FanoutManager().

Here is the call graph for this function:

Here is the caller graph for this function:

bool s3fanout::S3FanoutManager::CanRetry ( const JobInfo info)
private

Retry if possible and if not already done too often.

Definition at line 1025 of file s3fanout.cc.

void s3fanout::S3FanoutManager::DetectThrottleIndicator ( const std::string &  header,
JobInfo info 
)
static

Parses Retry-After and X-Retry-In headers attached to HTTP 429 responses

Definition at line 40 of file s3fanout.cc.

Here is the call graph for this function:

string s3fanout::S3FanoutManager::GetAwsV4SigningKey ( const std::string &  date) const
private

Definition at line 528 of file s3fanout.cc.

Here is the call graph for this function:

string s3fanout::S3FanoutManager::GetContentType ( const JobInfo info) const
private

Definition at line 831 of file s3fanout.cc.

string s3fanout::S3FanoutManager::GetRequestString ( const JobInfo info) const
private

Definition at line 813 of file s3fanout.cc.

const Statistics & s3fanout::S3FanoutManager::GetStatistics ( )

Definition at line 1313 of file s3fanout.cc.

string s3fanout::S3FanoutManager::GetUriEncode ( const std::string &  val,
bool  encode_slash 
) const
private

Definition at line 498 of file s3fanout.cc.

int s3fanout::S3FanoutManager::InitializeDnsSettings ( CURL *  handle,
std::string  remote_host 
) const
private

Definition at line 678 of file s3fanout.cc.

Here is the call graph for this function:

void s3fanout::S3FanoutManager::InitializeDnsSettingsCurl ( CURL *  handle,
CURLSH *  sharehandle,
curl_slist *  clist 
) const
private

Definition at line 666 of file s3fanout.cc.

Here is the call graph for this function:

Failures s3fanout::S3FanoutManager::InitializeRequest ( JobInfo info,
CURL *  handle 
) const
private

Request parameters set the URL and other options such as timeout and proxy.

Definition at line 855 of file s3fanout.cc.

Referenced by MainUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

void s3fanout::S3FanoutManager::InitPipeWatchFds ( )
private

Definition at line 442 of file s3fanout.cc.

Referenced by MainUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

void * s3fanout::S3FanoutManager::MainUpload ( void *  data)
staticprivate

Worker thread event loop.

Definition at line 232 of file s3fanout.cc.

Referenced by Spawn().

Here is the call graph for this function:

Here is the caller graph for this function:

bool s3fanout::S3FanoutManager::MkAzureAuthz ( const JobInfo info,
std::vector< std::string > *  headers 
) const
private

The Azure Blob authorization header according to https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key

Definition at line 621 of file s3fanout.cc.

Here is the call graph for this function:

std::string s3fanout::S3FanoutManager::MkCompleteHostname ( )
inlineprivate

Definition at line 271 of file s3fanout.h.

Referenced by S3FanoutManager().

Here is the caller graph for this function:

bool s3fanout::S3FanoutManager::MkPayloadHash ( const JobInfo info,
std::string *  hex_hash 
) const
private

Definition at line 758 of file s3fanout.cc.

Here is the call graph for this function:

std::string s3fanout::S3FanoutManager::MkUrl ( const std::string &  objkey) const
inlineprivate

Definition at line 263 of file s3fanout.h.

bool s3fanout::S3FanoutManager::MkV2Authz ( const JobInfo info,
std::vector< std::string > *  headers 
) const
private

The Amazon AWS 2 authorization header according to http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#ConstructingTheAuthenticationHeader

Definition at line 459 of file s3fanout.cc.

Here is the call graph for this function:

bool s3fanout::S3FanoutManager::MkV4Authz ( const JobInfo info,
std::vector< std::string > *  headers 
) const
private

The Amazon AWS4 authorization header according to http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html

Definition at line 548 of file s3fanout.cc.

Here is the call graph for this function:

JobInfo * s3fanout::S3FanoutManager::PopCompletedJob ( )

Pop completed job

Definition at line 1335 of file s3fanout.cc.

Here is the call graph for this function:

void s3fanout::S3FanoutManager::PushCompletedJob ( JobInfo info)

Push completed job to list of completed jobs

Definition at line 1328 of file s3fanout.cc.

Referenced by MainUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

void s3fanout::S3FanoutManager::PushNewJob ( JobInfo info)

Push new job to be uploaded to the S3 cloud storage.

Definition at line 1320 of file s3fanout.cc.

Here is the call graph for this function:

void s3fanout::S3FanoutManager::ReleaseCurlHandle ( JobInfo info,
CURL *  handle 
) const
private

Definition at line 417 of file s3fanout.cc.

Referenced by MainUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

void s3fanout::S3FanoutManager::SetUrlOptions ( JobInfo info) const
private

Sets the URL specific options such as host to use and timeout.

Definition at line 983 of file s3fanout.cc.

Referenced by MainUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

void s3fanout::S3FanoutManager::Spawn ( )

Spawns the I/O worker thread. No way back except ~S3FanoutManager.

Definition at line 1303 of file s3fanout.cc.

Here is the call graph for this function:

void s3fanout::S3FanoutManager::UpdateStatistics ( CURL *  handle)
private

Adds transfer time and uploaded bytes to the global counters.

Definition at line 1014 of file s3fanout.cc.

bool s3fanout::S3FanoutManager::VerifyAndFinalize ( const int  curl_error,
JobInfo info 
)
private

Checks the result of a curl request and implements the failure logic and takes care of cleanup.

Returns
true if request should be repeated, false otherwise

Definition at line 1085 of file s3fanout.cc.

Referenced by MainUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

std::set<JobInfo *>* s3fanout::S3FanoutManager::active_requests_
private

This is not strictly necessary but it helps the debugging

Definition at line 286 of file s3fanout.h.

Referenced by MainUpload(), S3FanoutManager(), and ~S3FanoutManager().

Semaphore* s3fanout::S3FanoutManager::available_jobs_
private
std::string s3fanout::S3FanoutManager::complete_hostname_
private

Definition at line 280 of file s3fanout.h.

Referenced by MkUrl(), and S3FanoutManager().

const S3Config s3fanout::S3FanoutManager::config_
private

Definition at line 279 of file s3fanout.h.

Referenced by MkCompleteHostname(), MkUrl(), and S3FanoutManager().

pthread_mutex_t* s3fanout::S3FanoutManager::curl_handle_lock_
private

Definition at line 237 of file s3fanout.h.

Referenced by S3FanoutManager(), and ~S3FanoutManager().

CURLM* s3fanout::S3FanoutManager::curl_multi_
private

Definition at line 293 of file s3fanout.h.

Referenced by MainUpload(), S3FanoutManager(), and ~S3FanoutManager().

std::map<CURL *, S3FanOutDnsEntry *>* s3fanout::S3FanoutManager::curl_sharehandles_
private

Definition at line 291 of file s3fanout.h.

Referenced by S3FanoutManager(), and ~S3FanoutManager().

bool s3fanout::S3FanoutManager::is_curl_debug_
private

Definition at line 334 of file s3fanout.h.

Referenced by S3FanoutManager().

std::vector<s3fanout::JobInfo*> s3fanout::S3FanoutManager::jobs_todo_
private

Definition at line 235 of file s3fanout.h.

pthread_mutex_t* s3fanout::S3FanoutManager::jobs_todo_lock_
private

Definition at line 236 of file s3fanout.h.

Referenced by S3FanoutManager(), and ~S3FanoutManager().

const char * s3fanout::S3FanoutManager::kCacheControlCas = "Cache-Control: max-age=259200"
staticprivate

Definition at line 228 of file s3fanout.h.

const char * s3fanout::S3FanoutManager::kCacheControlDotCvmfs
staticprivate
Initial value:
=
"Cache-Control: max-age=61"

Definition at line 229 of file s3fanout.h.

const unsigned s3fanout::S3FanoutManager::kDefault429ThrottleMs = 250
static

Definition at line 176 of file s3fanout.h.

const unsigned s3fanout::S3FanoutManager::kDefaultHTTPPort = 80
static

Definition at line 181 of file s3fanout.h.

const unsigned s3fanout::S3FanoutManager::kDefaultHTTPSPort = 443
static

Definition at line 182 of file s3fanout.h.

const unsigned s3fanout::S3FanoutManager::kLowSpeedLimit = 1024
staticprivate

Definition at line 230 of file s3fanout.h.

const unsigned s3fanout::S3FanoutManager::kMax429ThrottleMs = 10000
static

Definition at line 178 of file s3fanout.h.

const unsigned s3fanout::S3FanoutManager::kThrottleReportIntervalSec = 10
static

Definition at line 180 of file s3fanout.h.

std::pair<std::string, std::string> s3fanout::S3FanoutManager::last_signing_key_
mutableprivate

AWS4 signing keys are derived from the secret key, a region and a date. The signing key for current day can be cached.

Definition at line 300 of file s3fanout.h.

unsigned int s3fanout::S3FanoutManager::max_available_jobs_
private

Definition at line 324 of file s3fanout.h.

Referenced by S3FanoutManager().

atomic_int32 s3fanout::S3FanoutManager::multi_threaded_
private

Definition at line 303 of file s3fanout.h.

Referenced by S3FanoutManager(), Spawn(), and ~S3FanoutManager().

bool s3fanout::S3FanoutManager::opt_ipv4_only_
private

Definition at line 322 of file s3fanout.h.

Referenced by S3FanoutManager().

int s3fanout::S3FanoutManager::pipe_completed_[2]
private

Definition at line 320 of file s3fanout.h.

Referenced by PopCompletedJob(), PushCompletedJob(), S3FanoutManager(), and ~S3FanoutManager().

int s3fanout::S3FanoutManager::pipe_jobs_[2]
private

Definition at line 316 of file s3fanout.h.

Referenced by MainUpload(), PushNewJob(), S3FanoutManager(), and ~S3FanoutManager().

int s3fanout::S3FanoutManager::pipe_terminate_[2]
private

Definition at line 312 of file s3fanout.h.

Referenced by S3FanoutManager(), and ~S3FanoutManager().

std::set<CURL *>* s3fanout::S3FanoutManager::pool_handles_idle_
private

Definition at line 288 of file s3fanout.h.

Referenced by S3FanoutManager(), and ~S3FanoutManager().

std::set<CURL *>* s3fanout::S3FanoutManager::pool_handles_inuse_
private

Definition at line 289 of file s3fanout.h.

Referenced by MainUpload(), S3FanoutManager(), and ~S3FanoutManager().

Prng s3fanout::S3FanoutManager::prng_
private

Definition at line 282 of file s3fanout.h.

Referenced by S3FanoutManager().

dns::CaresResolver* s3fanout::S3FanoutManager::resolver_
private

Definition at line 292 of file s3fanout.h.

Referenced by S3FanoutManager().

std::set<S3FanOutDnsEntry *>* s3fanout::S3FanoutManager::sharehandles_
private

Definition at line 290 of file s3fanout.h.

Referenced by S3FanoutManager(), and ~S3FanoutManager().

SslCertificateStore s3fanout::S3FanoutManager::ssl_certificate_store_
private

Carries the path settings for SSL certificates

Definition at line 339 of file s3fanout.h.

Referenced by S3FanoutManager().

Statistics* s3fanout::S3FanoutManager::statistics_
private

Definition at line 329 of file s3fanout.h.

Referenced by GetStatistics(), MainUpload(), S3FanoutManager(), and ~S3FanoutManager().

pthread_t s3fanout::S3FanoutManager::thread_upload_
private

Definition at line 302 of file s3fanout.h.

Referenced by S3FanoutManager(), Spawn(), and ~S3FanoutManager().

uint64_t s3fanout::S3FanoutManager::timestamp_last_throttle_report_
private

Definition at line 332 of file s3fanout.h.

Referenced by S3FanoutManager().

std::string* s3fanout::S3FanoutManager::user_agent_
private

Definition at line 294 of file s3fanout.h.

Referenced by S3FanoutManager(), and ~S3FanoutManager().

struct pollfd* s3fanout::S3FanoutManager::watch_fds_
private

Definition at line 305 of file s3fanout.h.

Referenced by CallbackCurlSocket(), MainUpload(), and S3FanoutManager().

uint32_t s3fanout::S3FanoutManager::watch_fds_inuse_
private

Definition at line 307 of file s3fanout.h.

Referenced by CallbackCurlSocket(), MainUpload(), and S3FanoutManager().

uint32_t s3fanout::S3FanoutManager::watch_fds_max_
private

Definition at line 308 of file s3fanout.h.

Referenced by CallbackCurlSocket(), and S3FanoutManager().

uint32_t s3fanout::S3FanoutManager::watch_fds_size_
private

Definition at line 306 of file s3fanout.h.

Referenced by CallbackCurlSocket(), and S3FanoutManager().


The documentation for this class was generated from the following files: