#include <s3fanout.h>

Inheritance diagram for s3fanout::S3FanoutManager:

Collaboration diagram for s3fanout::S3FanoutManager:

Classes
struct	S3Config

Public Member Functions
	S3FanoutManager (const S3Config &config)

	~S3FanoutManager ()

void	Spawn ()

void	PushNewJob (JobInfo *info)

void	PushCompletedJob (JobInfo *info)

JobInfo *	PopCompletedJob ()

const Statistics &	GetStatistics ()

Static Public Member Functions
static void	DetectThrottleIndicator (const std::string &header, JobInfo *info)

Static Public Attributes
static const unsigned	kDefault429ThrottleMs = 250

static const unsigned	kMax429ThrottleMs = 10000

static const unsigned	kThrottleReportIntervalSec = 10

static const unsigned	kDefaultHTTPPort = 80

static const unsigned	kDefaultHTTPSPort = 443

Protected Types
typedef SynchronizingCounter < uint32_t >	Semaphore

Private Member Functions
CURL *	AcquireCurlHandle () const

void	ReleaseCurlHandle (JobInfo info, CURL handle) const

void	InitPipeWatchFds ()

int	InitializeDnsSettings (CURL *handle, std::string remote_host) const

void	InitializeDnsSettingsCurl (CURL handle, CURLSH sharehandle, curl_slist *clist) const

Failures	InitializeRequest (JobInfo info, CURL handle) const

void	SetUrlOptions (JobInfo *info) const

void	UpdateStatistics (CURL *handle)

bool	CanRetry (const JobInfo *info)

void	Backoff (JobInfo *info)

bool	VerifyAndFinalize (const int curl_error, JobInfo *info)

std::string	GetRequestString (const JobInfo &info) const

std::string	GetContentType (const JobInfo &info) const

std::string	GetUriEncode (const std::string &val, bool encode_slash) const

std::string	GetAwsV4SigningKey (const std::string &date) const

bool	MkPayloadHash (const JobInfo &info, std::string *hex_hash) const

bool	MkV2Authz (const JobInfo &info, std::vector< std::string > *headers) const

bool	MkV4Authz (const JobInfo &info, std::vector< std::string > *headers) const

bool	MkAzureAuthz (const JobInfo &info, std::vector< std::string > *headers) const

std::string	MkUrl (const std::string &objkey) const

std::string	MkCompleteHostname ()

Private Member Functions inherited from SingleCopy
	SingleCopy ()

Static Private Member Functions
static int	CallbackCurlSocket (CURL easy, curl_socket_t s, int action, void userp, void *socketp)

static void *	MainUpload (void *data)

Private Attributes
std::vector< s3fanout::JobInfo * >	jobs_todo_

pthread_mutex_t *	jobs_todo_lock_

pthread_mutex_t *	curl_handle_lock_

const S3Config	config_

std::string	complete_hostname_

Prng	prng_

std::set< JobInfo * > *	active_requests_

std::set< CURL * > *	pool_handles_idle_

std::set< CURL * > *	pool_handles_inuse_

std::set< S3FanOutDnsEntry * > *	sharehandles_

std::map< CURL , S3FanOutDnsEntry > *	curl_sharehandles_

dns::CaresResolver *	resolver_

CURLM *	curl_multi_

std::string *	user_agent_

std::pair< std::string, std::string >	last_signing_key_

pthread_t	thread_upload_

atomic_int32	multi_threaded_

struct pollfd *	watch_fds_

uint32_t	watch_fds_size_

uint32_t	watch_fds_inuse_

uint32_t	watch_fds_max_

int	pipe_terminate_ [2]

int	pipe_jobs_ [2]

int	pipe_completed_ [2]

bool	opt_ipv4_only_

unsigned int	max_available_jobs_

Semaphore *	available_jobs_

Statistics *	statistics_

uint64_t	timestamp_last_throttle_report_

bool	is_curl_debug_

SslCertificateStore	ssl_certificate_store_

Static Private Attributes
static const char *	kCacheControlCas = "Cache-Control: max-age=259200"

static const char *	kCacheControlDotCvmfs

static const unsigned	kLowSpeedLimit = 1024

Detailed Description

Definition at line 170 of file s3fanout.h.

Member Typedef Documentation

typedef SynchronizingCounter<uint32_t> s3fanout::S3FanoutManager::Semaphore

protected

Definition at line 172 of file s3fanout.h.

Constructor & Destructor Documentation

s3fanout::S3FanoutManager::S3FanoutManager ( const S3Config & config )

explicit

Definition at line 1193 of file s3fanout.cc.

Here is the call graph for this function:

s3fanout::S3FanoutManager::~S3FanoutManager ( )

Definition at line 1262 of file s3fanout.cc.

Here is the call graph for this function:

Member Function Documentation

CURL * s3fanout::S3FanoutManager::AcquireCurlHandle ( ) const

private

Gets an idle CURL handle from the pool. Creates a new one and adds it to the pool if necessary.

Definition at line 385 of file s3fanout.cc.

Referenced by MainUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

void s3fanout::S3FanoutManager::Backoff ( JobInfo * info )

private

Backoff for retry to introduce a jitter into a upload sequence.

Returns: true if backoff has been performed, false otherwise

Definition at line 1049 of file s3fanout.cc.

Here is the call graph for this function:

int s3fanout::S3FanoutManager::CallbackCurlSocket	(	CURL *	easy,
		curl_socket_t	s,
		int	action,
		void *	userp,
		void *	socketp
	)

staticprivate

Called when new curl sockets arrive or existing curl sockets depart.

Definition at line 163 of file s3fanout.cc.

Referenced by S3FanoutManager().

Here is the call graph for this function:

Here is the caller graph for this function:

bool s3fanout::S3FanoutManager::CanRetry ( const JobInfo * info )

private

Retry if possible and if not already done too often.

Definition at line 1034 of file s3fanout.cc.

void s3fanout::S3FanoutManager::DetectThrottleIndicator	(	const std::string &	header,
		JobInfo *	info
	)

static

Parses Retry-After and X-Retry-In headers attached to HTTP 429 responses

Definition at line 40 of file s3fanout.cc.

Here is the call graph for this function:

string s3fanout::S3FanoutManager::GetAwsV4SigningKey ( const std::string & date ) const

private

Definition at line 531 of file s3fanout.cc.

Here is the call graph for this function:

string s3fanout::S3FanoutManager::GetContentType ( const JobInfo & info ) const

private

Definition at line 840 of file s3fanout.cc.

string s3fanout::S3FanoutManager::GetRequestString ( const JobInfo & info ) const

private

Definition at line 822 of file s3fanout.cc.

const Statistics & s3fanout::S3FanoutManager::GetStatistics ( )

Definition at line 1322 of file s3fanout.cc.

string s3fanout::S3FanoutManager::GetUriEncode	(	const std::string &	val,
		bool	encode_slash
	)		const

private

Definition at line 501 of file s3fanout.cc.

int s3fanout::S3FanoutManager::InitializeDnsSettings	(	CURL *	handle,
		std::string	remote_host
	)		const

private

Definition at line 687 of file s3fanout.cc.

Here is the call graph for this function:

void s3fanout::S3FanoutManager::InitializeDnsSettingsCurl	(	CURL *	handle,
		CURLSH *	sharehandle,
		curl_slist *	clist
	)		const

private

Definition at line 675 of file s3fanout.cc.

Here is the call graph for this function:

Failures s3fanout::S3FanoutManager::InitializeRequest	(	JobInfo *	info,
		CURL *	handle
	)		const

private

Request parameters set the URL and other options such as timeout and proxy.

Definition at line 864 of file s3fanout.cc.

Referenced by MainUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

void s3fanout::S3FanoutManager::InitPipeWatchFds ( )

private

Definition at line 443 of file s3fanout.cc.

Referenced by MainUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

void * s3fanout::S3FanoutManager::MainUpload ( void * data )

staticprivate

Worker thread event loop.

Definition at line 233 of file s3fanout.cc.

Referenced by Spawn().

Here is the call graph for this function:

Here is the caller graph for this function:

bool s3fanout::S3FanoutManager::MkAzureAuthz	(	const JobInfo &	info,
		std::vector< std::string > *	headers
	)		const

private

The Azure Blob authorization header according to https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key

Definition at line 630 of file s3fanout.cc.

Here is the call graph for this function:

std::string s3fanout::S3FanoutManager::MkCompleteHostname ( )

inlineprivate

Definition at line 273 of file s3fanout.h.

Referenced by S3FanoutManager().

Here is the caller graph for this function:

bool s3fanout::S3FanoutManager::MkPayloadHash	(	const JobInfo &	info,
		std::string *	hex_hash
	)		const

private

Definition at line 767 of file s3fanout.cc.

Here is the call graph for this function:

std::string s3fanout::S3FanoutManager::MkUrl ( const std::string & objkey ) const

inlineprivate

Definition at line 265 of file s3fanout.h.

bool s3fanout::S3FanoutManager::MkV2Authz	(	const JobInfo &	info,
		std::vector< std::string > *	headers
	)		const

private

The Amazon AWS 2 authorization header according to http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#ConstructingTheAuthenticationHeader

Definition at line 460 of file s3fanout.cc.

Here is the call graph for this function:

bool s3fanout::S3FanoutManager::MkV4Authz	(	const JobInfo &	info,
		std::vector< std::string > *	headers
	)		const

private

The Amazon AWS4 authorization header according to http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html

Definition at line 551 of file s3fanout.cc.

Here is the call graph for this function:

JobInfo * s3fanout::S3FanoutManager::PopCompletedJob ( )

Pop completed job

Definition at line 1344 of file s3fanout.cc.

Here is the call graph for this function:

void s3fanout::S3FanoutManager::PushCompletedJob ( JobInfo * info )

Push completed job to list of completed jobs

Definition at line 1337 of file s3fanout.cc.

Referenced by MainUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

void s3fanout::S3FanoutManager::PushNewJob ( JobInfo * info )

Push new job to be uploaded to the S3 cloud storage.

Definition at line 1329 of file s3fanout.cc.

Here is the call graph for this function:

void s3fanout::S3FanoutManager::ReleaseCurlHandle	(	JobInfo *	info,
		CURL *	handle
	)		const

private

Definition at line 418 of file s3fanout.cc.

Referenced by MainUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

void s3fanout::S3FanoutManager::SetUrlOptions ( JobInfo * info ) const

private

Sets the URL specific options such as host to use and timeout.

Definition at line 992 of file s3fanout.cc.

Referenced by MainUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

void s3fanout::S3FanoutManager::Spawn ( )

Spawns the I/O worker thread. No way back except ~S3FanoutManager.

Definition at line 1312 of file s3fanout.cc.

Here is the call graph for this function:

void s3fanout::S3FanoutManager::UpdateStatistics ( CURL * handle )

private

Adds transfer time and uploaded bytes to the global counters.

Definition at line 1023 of file s3fanout.cc.

bool s3fanout::S3FanoutManager::VerifyAndFinalize	(	const int	curl_error,
		JobInfo *	info
	)

private

Checks the result of a curl request and implements the failure logic and takes care of cleanup.

Returns: true if request should be repeated, false otherwise

Definition at line 1094 of file s3fanout.cc.

Referenced by MainUpload().

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

std::set<JobInfo *>* s3fanout::S3FanoutManager::active_requests_

private

This is not strictly necessary but it helps the debugging

Definition at line 288 of file s3fanout.h.

Referenced by MainUpload(), S3FanoutManager(), and ~S3FanoutManager().

Semaphore* s3fanout::S3FanoutManager::available_jobs_

private

Definition at line 327 of file s3fanout.h.

Referenced by CallbackCurlSocket(), MainUpload(), PushNewJob(), S3FanoutManager(), and ~S3FanoutManager().

std::string s3fanout::S3FanoutManager::complete_hostname_

private

Definition at line 282 of file s3fanout.h.

Referenced by MkUrl(), and S3FanoutManager().

const S3Config s3fanout::S3FanoutManager::config_

private

Definition at line 281 of file s3fanout.h.

Referenced by MkCompleteHostname(), MkUrl(), and S3FanoutManager().

pthread_mutex_t* s3fanout::S3FanoutManager::curl_handle_lock_

private

Definition at line 239 of file s3fanout.h.

Referenced by S3FanoutManager(), and ~S3FanoutManager().

CURLM* s3fanout::S3FanoutManager::curl_multi_

private

Definition at line 295 of file s3fanout.h.

Referenced by MainUpload(), S3FanoutManager(), and ~S3FanoutManager().

std::map<CURL *, S3FanOutDnsEntry *>* s3fanout::S3FanoutManager::curl_sharehandles_

private

Definition at line 293 of file s3fanout.h.

Referenced by S3FanoutManager(), and ~S3FanoutManager().

bool s3fanout::S3FanoutManager::is_curl_debug_

private

Definition at line 336 of file s3fanout.h.

Referenced by S3FanoutManager().

std::vector<s3fanout::JobInfo*> s3fanout::S3FanoutManager::jobs_todo_

private

Definition at line 237 of file s3fanout.h.

pthread_mutex_t* s3fanout::S3FanoutManager::jobs_todo_lock_

private

Definition at line 238 of file s3fanout.h.

Referenced by S3FanoutManager(), and ~S3FanoutManager().

const char * s3fanout::S3FanoutManager::kCacheControlCas = "Cache-Control: max-age=259200"

staticprivate

Definition at line 230 of file s3fanout.h.

const char * s3fanout::S3FanoutManager::kCacheControlDotCvmfs

staticprivate

Initial value:

=

"Cache-Control: max-age=61"

Definition at line 231 of file s3fanout.h.

const unsigned s3fanout::S3FanoutManager::kDefault429ThrottleMs = 250

static

Definition at line 176 of file s3fanout.h.

const unsigned s3fanout::S3FanoutManager::kDefaultHTTPPort = 80

static

Definition at line 181 of file s3fanout.h.

const unsigned s3fanout::S3FanoutManager::kDefaultHTTPSPort = 443

static

Definition at line 182 of file s3fanout.h.

const unsigned s3fanout::S3FanoutManager::kLowSpeedLimit = 1024

staticprivate

Definition at line 232 of file s3fanout.h.

const unsigned s3fanout::S3FanoutManager::kMax429ThrottleMs = 10000

static

Definition at line 178 of file s3fanout.h.

const unsigned s3fanout::S3FanoutManager::kThrottleReportIntervalSec = 10

static

Definition at line 180 of file s3fanout.h.

std::pair<std::string, std::string> s3fanout::S3FanoutManager::last_signing_key_

mutableprivate

AWS4 signing keys are derived from the secret key, a region and a date. The signing key for current day can be cached.

Definition at line 302 of file s3fanout.h.

unsigned int s3fanout::S3FanoutManager::max_available_jobs_

private

Definition at line 326 of file s3fanout.h.

Referenced by S3FanoutManager().

atomic_int32 s3fanout::S3FanoutManager::multi_threaded_

private

Definition at line 305 of file s3fanout.h.

Referenced by S3FanoutManager(), Spawn(), and ~S3FanoutManager().

bool s3fanout::S3FanoutManager::opt_ipv4_only_

private

Definition at line 324 of file s3fanout.h.

Referenced by S3FanoutManager().

int s3fanout::S3FanoutManager::pipe_completed_[2]

private

Definition at line 322 of file s3fanout.h.

Referenced by PopCompletedJob(), PushCompletedJob(), S3FanoutManager(), and ~S3FanoutManager().

int s3fanout::S3FanoutManager::pipe_jobs_[2]

private

Definition at line 318 of file s3fanout.h.

Referenced by MainUpload(), PushNewJob(), S3FanoutManager(), and ~S3FanoutManager().

int s3fanout::S3FanoutManager::pipe_terminate_[2]

private

Definition at line 314 of file s3fanout.h.

Referenced by S3FanoutManager(), and ~S3FanoutManager().

std::set<CURL *>* s3fanout::S3FanoutManager::pool_handles_idle_

private

Definition at line 290 of file s3fanout.h.

Referenced by S3FanoutManager(), and ~S3FanoutManager().

std::set<CURL *>* s3fanout::S3FanoutManager::pool_handles_inuse_

private

Definition at line 291 of file s3fanout.h.

Referenced by MainUpload(), S3FanoutManager(), and ~S3FanoutManager().

Prng s3fanout::S3FanoutManager::prng_

private

Definition at line 284 of file s3fanout.h.

Referenced by S3FanoutManager().

dns::CaresResolver* s3fanout::S3FanoutManager::resolver_

private

Definition at line 294 of file s3fanout.h.

Referenced by S3FanoutManager().

std::set<S3FanOutDnsEntry *>* s3fanout::S3FanoutManager::sharehandles_

private

Definition at line 292 of file s3fanout.h.

Referenced by S3FanoutManager(), and ~S3FanoutManager().

SslCertificateStore s3fanout::S3FanoutManager::ssl_certificate_store_

private

Carries the path settings for SSL certificates

Definition at line 341 of file s3fanout.h.

Referenced by S3FanoutManager().

Statistics* s3fanout::S3FanoutManager::statistics_

private

Definition at line 331 of file s3fanout.h.

Referenced by GetStatistics(), MainUpload(), S3FanoutManager(), and ~S3FanoutManager().

pthread_t s3fanout::S3FanoutManager::thread_upload_

private

Definition at line 304 of file s3fanout.h.

Referenced by S3FanoutManager(), Spawn(), and ~S3FanoutManager().

uint64_t s3fanout::S3FanoutManager::timestamp_last_throttle_report_

private

Definition at line 334 of file s3fanout.h.

Referenced by S3FanoutManager().

std::string* s3fanout::S3FanoutManager::user_agent_

private

Definition at line 296 of file s3fanout.h.

Referenced by S3FanoutManager(), and ~S3FanoutManager().

struct pollfd* s3fanout::S3FanoutManager::watch_fds_

private

Definition at line 307 of file s3fanout.h.

Referenced by CallbackCurlSocket(), MainUpload(), and S3FanoutManager().

uint32_t s3fanout::S3FanoutManager::watch_fds_inuse_

private

Definition at line 309 of file s3fanout.h.

Referenced by CallbackCurlSocket(), MainUpload(), and S3FanoutManager().

uint32_t s3fanout::S3FanoutManager::watch_fds_max_

private

Definition at line 310 of file s3fanout.h.

Referenced by CallbackCurlSocket(), and S3FanoutManager().

uint32_t s3fanout::S3FanoutManager::watch_fds_size_

private

Definition at line 308 of file s3fanout.h.

Referenced by CallbackCurlSocket(), and S3FanoutManager().

The documentation for this class was generated from the following files:

/home/sftnight/jenkins/workspace/CvmfsDoxygen/label/docker-x86_64/cvmfs/cvmfs/network/s3fanout.h
/home/sftnight/jenkins/workspace/CvmfsDoxygen/label/docker-x86_64/cvmfs/cvmfs/network/s3fanout.cc

Classes

Public Member Functions

Static Public Member Functions

Static Public Attributes

Protected Types

Private Member Functions

Static Private Member Functions

Private Attributes

Static Private Attributes

Detailed Description

Member Typedef Documentation

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation