#include <download.h>

Collaboration diagram for download::DownloadManager:

Classes
struct	ProxyInfo

Public Types
enum	ProxySetModes { kSetProxyRegular = 0, kSetProxyFallback, kSetProxyBoth }

Public Member Functions
	DownloadManager ()

	~DownloadManager ()

void	Init (const unsigned max_pool_handles, const perf::StatisticsTemplate &statistics)

void	Fini ()

void	Spawn ()

DownloadManager *	Clone (const perf::StatisticsTemplate &statistics)

Failures	Fetch (JobInfo *info)

void	SetCredentialsAttachment (CredentialsAttachment *ca)

std::string	GetDnsServer () const

void	SetDnsServer (const std::string &address)

void	SetDnsParameters (const unsigned retries, const unsigned timeout_ms)

void	SetDnsTtlLimits (const unsigned min_seconds, const unsigned max_seconds)

void	SetIpPreference (const dns::IpPreference preference)

void	SetTimeout (const unsigned seconds_proxy, const unsigned seconds_direct)

void	GetTimeout (unsigned seconds_proxy, unsigned seconds_direct)

void	SetLowSpeedLimit (const unsigned low_speed_limit)

void	SetHostChain (const std::string &host_list)

void	SetHostChain (const std::vector< std::string > &host_list)

void	GetHostInfo (std::vector< std::string > host_chain, std::vector< int > rtt, unsigned *current_host)

void	ProbeHosts ()

bool	ProbeGeo ()

bool	GeoSortServers (std::vector< std::string > servers, std::vector< uint64_t > output_order=NULL)

void	SwitchHost ()

void	SetProxyChain (const std::string &proxy_list, const std::string &fallback_proxy_list, const ProxySetModes set_mode)

void	GetProxyInfo (std::vector< std::vector< ProxyInfo > > proxy_chain, unsigned current_group, unsigned *fallback_group)

std::string	GetProxyList ()

std::string	GetFallbackProxyList ()

void	ShardProxies ()

void	RebalanceProxies ()

void	SwitchProxyGroup ()

void	SetProxyGroupResetDelay (const unsigned seconds)

void	SetHostResetDelay (const unsigned seconds)

void	SetRetryParameters (const unsigned max_retries, const unsigned backoff_init_ms, const unsigned backoff_max_ms)

void	SetMaxIpaddrPerProxy (unsigned limit)

void	SetProxyTemplates (const std::string &direct, const std::string &forced)

void	EnableInfoHeader ()

void	EnableRedirects ()

void	EnableIgnoreSignatureFailures ()

void	EnableHTTPTracing ()

void	AddHTTPTracingHeader (const std::string &header)

void	UseSystemCertificatePath ()

bool	SetShardingPolicy (const ShardingPolicySelector type)

void	SetFailoverIndefinitely ()

void	SetFqrn (const std::string &fqrn)

unsigned	num_hosts ()

dns::IpPreference	opt_ip_preference () const

Static Public Member Functions
static int	ParseHttpCode (const char digits[3])

Static Public Attributes
static const int	kProbeUnprobed = -1

static const int	kProbeDown = -2

static const int	kProbeGeo = -3

static const unsigned	kDnsDefaultRetries = 1

static const unsigned	kDnsDefaultTimeoutMs = 3000

static const unsigned	kProxyMapScale = 16

Private Member Functions
	FRIEND_TEST (T_Download, ValidateGeoReply)

	FRIEND_TEST (T_Download, StripDirect)

bool	StripDirect (const std::string &proxy_list, std::string *cleaned_list)

bool	ValidateGeoReply (const std::string &reply_order, const unsigned expected_size, std::vector< uint64_t > *reply_vals)

void	SwitchHost (JobInfo *info)

void	SwitchProxy (JobInfo *info)

ProxyInfo *	ChooseProxyUnlocked (const shash::Any *hash)

void	UpdateProxiesUnlocked (const std::string &reason)

void	RebalanceProxiesUnlocked (const std::string &reason)

CURL *	AcquireCurlHandle ()

void	ReleaseCurlHandle (CURL *handle)

void	ReleaseCredential (JobInfo *info)

void	InitializeRequest (JobInfo info, CURL handle)

void	SetUrlOptions (JobInfo *info)

bool	ValidateProxyIpsUnlocked (const std::string &url, const dns::Host &host)

void	UpdateStatistics (CURL *handle)

bool	CanRetry (const JobInfo *info)

void	Backoff (JobInfo *info)

void	SetNocache (JobInfo *info)

void	SetRegularCache (JobInfo *info)

bool	VerifyAndFinalize (const int curl_error, JobInfo *info)

void	InitHeaders ()

void	FiniHeaders ()

void	CloneProxyConfig (DownloadManager *clone)

std::vector< ProxyInfo > *	current_proxy_group () const

Static Private Member Functions
static int	CallbackCurlSocket (CURL easy, curl_socket_t s, int action, void userp, void *socketp)

static void *	MainDownload (void *data)

Private Attributes
Prng	prng_

std::set< CURL * > *	pool_handles_idle_

std::set< CURL * > *	pool_handles_inuse_

uint32_t	pool_max_handles_

CURLM *	curl_multi_

HeaderLists *	header_lists_

curl_slist *	default_headers_

char *	user_agent_

pthread_t	thread_download_

atomic_int32	multi_threaded_

UniquePtr< Pipe < kPipeThreadTerminator > >	pipe_terminate_

UniquePtr< Pipe < kPipeDownloadJobs > >	pipe_jobs_

struct pollfd *	watch_fds_

uint32_t	watch_fds_size_

uint32_t	watch_fds_inuse_

uint32_t	watch_fds_max_

pthread_mutex_t *	lock_options_

pthread_mutex_t *	lock_synchronous_mode_

std::string	opt_dns_server_

unsigned	opt_timeout_proxy_

unsigned	opt_timeout_direct_

unsigned	opt_low_speed_limit_

unsigned	opt_max_retries_

unsigned	opt_backoff_init_ms_

unsigned	opt_backoff_max_ms_

bool	enable_info_header_

bool	opt_ipv4_only_

bool	follow_redirects_

bool	ignore_signature_failures_

bool	enable_http_tracing_

std::vector< std::string >	http_tracing_headers_

std::vector< std::string > *	opt_host_chain_

std::vector< int > *	opt_host_chain_rtt_

unsigned	opt_host_chain_current_

std::vector< std::vector < ProxyInfo > > *	opt_proxy_groups_

unsigned	opt_proxy_groups_current_

unsigned	opt_proxy_groups_current_burned_

unsigned	opt_proxy_groups_fallback_

unsigned	opt_num_proxies_

std::string	opt_proxy_list_

std::string	opt_proxy_fallback_list_

std::map< uint32_t, ProxyInfo * >	opt_proxy_map_

std::vector< std::string >	opt_proxy_urls_

bool	opt_proxy_shard_

SharedPtr< ShardingPolicy >	sharding_policy_

SharedPtr< HealthCheck >	health_check_

bool	failover_indefinitely_

std::string	fqrn_

dns::NormalResolver *	resolver_

dns::IpPreference	opt_ip_preference_

std::string	proxy_template_direct_

std::string	proxy_template_forced_

time_t	opt_timestamp_backup_proxies_

time_t	opt_timestamp_failover_proxies_

unsigned	opt_proxy_groups_reset_after_

time_t	opt_timestamp_backup_host_

unsigned	opt_host_reset_after_

CredentialsAttachment *	credentials_attachment_

Counters *	counters_

SslCertificateStore	ssl_certificate_store_

Detailed Description

Note when adding new fields: Clone() probably needs to be adjusted, too. TODO(jblomer): improve ordering of members

Definition at line 116 of file download.h.

Member Enumeration Documentation

enum download::DownloadManager::ProxySetModes

Enumerator
kSetProxyRegular
kSetProxyFallback
kSetProxyBoth

Definition at line 133 of file download.h.

Constructor & Destructor Documentation

download::DownloadManager::DownloadManager ( )

Definition at line 1592 of file download.cc.

Here is the call graph for this function:

download::DownloadManager::~DownloadManager ( )

Definition at line 1655 of file download.cc.

Member Function Documentation

CURL * download::DownloadManager::AcquireCurlHandle ( )

private

Gets an idle CURL handle from the pool. Creates a new one and adds it to the pool if necessary.

Definition at line 821 of file download.cc.

Referenced by MainDownload().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::AddHTTPTracingHeader ( const std::string & header )

Definition at line 2890 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers().

Here is the caller graph for this function:

void download::DownloadManager::Backoff ( JobInfo * info )

private

Backoff for retry to introduce a jitter into a cluster of requesting cvmfs nodes. Retry only when HTTP caching is on.

Returns: true if backoff has been performed, false otherwise

Definition at line 1232 of file download.cc.

Here is the call graph for this function:

int download::DownloadManager::CallbackCurlSocket	(	CURL *	easy,
		curl_socket_t	s,
		int	action,
		void *	userp,
		void *	socketp
	)

staticprivate

Called when new curl sockets arrive or existing curl sockets depart.

Definition at line 438 of file download.cc.

Here is the call graph for this function:

bool download::DownloadManager::CanRetry ( const JobInfo * info )

private

Retry if possible if not on no-cache and if not already done too often.

Definition at line 1216 of file download.cc.

Here is the call graph for this function:

DownloadManager::ProxyInfo * download::DownloadManager::ChooseProxyUnlocked ( const shash::Any * hash )

private

Choose proxy

Definition at line 2716 of file download.cc.

Here is the call graph for this function:

DownloadManager * download::DownloadManager::Clone ( const perf::StatisticsTemplate & statistics )

Creates a copy of the existing download manager. Must only be called in single-threaded stage because it calls curl_global_init().

Definition at line 2916 of file download.cc.

Referenced by MountPoint::SetupExternalDownloadMgr().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::CloneProxyConfig ( DownloadManager * clone )

private

Definition at line 2961 of file download.cc.

Here is the call graph for this function:

std::vector<ProxyInfo>* download::DownloadManager::current_proxy_group ( ) const

inlineprivate

Definition at line 259 of file download.h.

void download::DownloadManager::EnableHTTPTracing ( )

Definition at line 2886 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers().

Here is the caller graph for this function:

void download::DownloadManager::EnableIgnoreSignatureFailures ( )

Definition at line 2882 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers().

Here is the caller graph for this function:

void download::DownloadManager::EnableInfoHeader ( )

Definition at line 2873 of file download.cc.

Referenced by MountPoint::SetupHttpTuning().

Here is the caller graph for this function:

void download::DownloadManager::EnableRedirects ( )

Definition at line 2878 of file download.cc.

Referenced by MountPoint::SetupHttpTuning().

Here is the caller graph for this function:

Failures download::DownloadManager::Fetch ( JobInfo * info )

Downloads data from an insecure outside channel (currently HTTP or file).

Definition at line 1824 of file download.cc.

Referenced by download::AutoProxy(), manifest::DoFetch(), manifest::DoVerify(), HttpObjectFetcher< CatalogT, HistoryT, ReflogT >::Download(), swissknife::CommandCheck::DownloadPiece(), publish::DownloadRootObjects(), swissknife::CommandCheck::Exists(), cvmfs::Fetcher::Fetch(), swissknife::Assistant::FetchObject(), swissknife::CommandCheck::FetchPath(), publish::IsMasterReplica(), swissknife::MainWorker(), and StreamingCacheManager::Stream().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::Fini ( )

Definition at line 1744 of file download.cc.

Referenced by download::MainResolveProxyDescription(), publish::Repository(), MountPoint::~MountPoint(), and publish::~Repository().

Here is the caller graph for this function:

void download::DownloadManager::FiniHeaders ( )

private

Definition at line 1685 of file download.cc.

download::DownloadManager::FRIEND_TEST	(	T_Download	,
		ValidateGeoReply
	)

private

download::DownloadManager::FRIEND_TEST	(	T_Download	,
		StripDirect
	)

private

bool download::DownloadManager::GeoSortServers	(	std::vector< std::string > *	servers,
		std::vector< uint64_t > *	output_order = `NULL`
	)

Definition at line 2244 of file download.cc.

Referenced by MountPoint::SetupExternalDownloadMgr().

Here is the call graph for this function:

Here is the caller graph for this function:

std::string download::DownloadManager::GetDnsServer ( ) const

Gets the DNS sever.

Definition at line 1925 of file download.cc.

Referenced by TalkManager::MainResponder().

Here is the caller graph for this function:

string download::DownloadManager::GetFallbackProxyList ( )

Definition at line 2708 of file download.cc.

void download::DownloadManager::GetHostInfo	(	std::vector< std::string > *	host_chain,
		std::vector< int > *	rtt,
		unsigned *	current_host
	)

Retrieves the currently set chain of hosts, their round trip times, and the currently used host.

Definition at line 2057 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers(), TalkManager::FormatHostInfo(), ExternalHostMagicXattr::GetValue(), ExternalURLMagicXattr::GetValue(), download::ParsePac(), and MountPoint::SetupExternalDownloadMgr().

Here is the caller graph for this function:

void download::DownloadManager::GetProxyInfo	(	std::vector< std::vector< ProxyInfo > > *	proxy_chain,
		unsigned *	current_group,
		unsigned *	fallback_group
	)

Retrieves the proxy chain, optionally the currently active load-balancing group, and optionally the index of the first fallback proxy group. If there are no fallback proxies, the index will equal the size of the proxy chain.

Definition at line 2679 of file download.cc.

Referenced by TalkManager::FormatProxyInfo(), ProxyMagicXattr::GetValue(), and ListProxy().

Here is the call graph for this function:

Here is the caller graph for this function:

string download::DownloadManager::GetProxyList ( )

Definition at line 2704 of file download.cc.

void download::DownloadManager::GetTimeout	(	unsigned *	seconds_proxy,
		unsigned *	seconds_direct
	)

Receives the currently active timeout values.

Definition at line 2013 of file download.cc.

Referenced by TimeoutMagicXattr::GetValue(), TimeoutDirectMagicXattr::GetValue(), TalkManager::MainResponder(), and MountPoint::SetupExternalDownloadMgr().

Here is the caller graph for this function:

void download::DownloadManager::Init	(	const unsigned	max_pool_handles,
		const perf::StatisticsTemplate &	statistics
	)

Definition at line 1692 of file download.cc.

Referenced by Clone(), MountPoint::CreateDownloadManagers(), download::MainResolveProxyDescription(), and publish::Repository().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::InitHeaders ( )

private

Definition at line 1662 of file download.cc.

Here is the call graph for this function:

void download::DownloadManager::InitializeRequest	(	JobInfo *	info,
		CURL *	handle
	)

private

HTTP request options: set the URL and other options such as timeout and proxy.

Definition at line 862 of file download.cc.

Referenced by MainDownload().

Here is the call graph for this function:

Here is the caller graph for this function:

void * download::DownloadManager::MainDownload ( void * data )

staticprivate

Worker thread event loop. Waits on new JobInfo structs on a pipe.

Definition at line 518 of file download.cc.

Here is the call graph for this function:

unsigned download::DownloadManager::num_hosts ( )

inline

Definition at line 220 of file download.h.

Referenced by manifest::Fetch().

Here is the caller graph for this function:

dns::IpPreference download::DownloadManager::opt_ip_preference ( ) const

inline

Definition at line 225 of file download.h.

Referenced by ResolvConfEventHandler::SetDnsAddress().

Here is the caller graph for this function:

int download::DownloadManager::ParseHttpCode ( const char digits[3] )

static

-1 of digits is not a valid Http return code

Definition at line 422 of file download.cc.

bool download::DownloadManager::ProbeGeo ( )

Uses the Geo-API of Stratum 1s to let any of them order the list of servers and fallback proxies (if any). Tries at most three random Stratum 1s before giving up. If you change the host list in between by SetHostChain() or the fallback proxy list by SetProxyChain(), they will be overwritten by this function.

Definition at line 2334 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers(), and TalkManager::MainResponder().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::ProbeHosts ( )

Orders the hostlist according to RTT of downloading .cvmfschecksum. Sets the current host to the best-responsive host. If you change the host list in between by SetHostChain(), it will be overwritten by this function.

Definition at line 2196 of file download.cc.

Referenced by TalkManager::MainResponder().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::RebalanceProxies ( )

Definition at line 2804 of file download.cc.

Referenced by TalkManager::MainResponder().

Here is the caller graph for this function:

void download::DownloadManager::RebalanceProxiesUnlocked ( const std::string & reason )

private

Selects a new random proxy in the current load-balancing group. Resets the "burned" counter.

Definition at line 2794 of file download.cc.

void download::DownloadManager::ReleaseCredential ( JobInfo * info )

private

Frees the storage associated with the authz attachment from the job

Definition at line 1284 of file download.cc.

Here is the call graph for this function:

void download::DownloadManager::ReleaseCurlHandle ( CURL * handle )

private

Definition at line 844 of file download.cc.

Referenced by MainDownload().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::SetCredentialsAttachment ( CredentialsAttachment * ca )

Used by the client to connect the authz session manager to the download manager.

Definition at line 1917 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers().

Here is the caller graph for this function:

void download::DownloadManager::SetDnsParameters	(	const unsigned	retries,
		const unsigned	timeout_ms
	)

Sets the DNS query timeout parameters.

Definition at line 1951 of file download.cc.

Referenced by Clone(), and MountPoint::SetupDnsTuning().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::SetDnsServer ( const std::string & address )

Sets a DNS server. Only for testing as it cannot be reverted to the system default.

Definition at line 1933 of file download.cc.

Referenced by Clone(), TalkManager::MainResponder(), ResolvConfEventHandler::SetDnsAddress(), and MountPoint::SetupDnsTuning().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::SetDnsTtlLimits	(	const unsigned	min_seconds,
		const unsigned	max_seconds
	)

Definition at line 1969 of file download.cc.

Referenced by Clone(), and MountPoint::SetupDnsTuning().

Here is the caller graph for this function:

void download::DownloadManager::SetFailoverIndefinitely ( )

Definition at line 2908 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers().

Here is the caller graph for this function:

void download::DownloadManager::SetFqrn ( const std::string & fqrn )

inline

Definition at line 218 of file download.h.

Referenced by MountPoint::CreateDownloadManagers().

Here is the caller graph for this function:

void download::DownloadManager::SetHostChain ( const std::string & host_list )

Referenced by MountPoint::CreateDownloadManagers(), download::MainResolveProxyDescription(), TalkManager::MainResponder(), and MountPoint::SetupExternalDownloadMgr().

Here is the caller graph for this function:

void download::DownloadManager::SetHostChain ( const std::vector< std::string > & host_list )

Definition at line 2031 of file download.cc.

void download::DownloadManager::SetHostResetDelay ( const unsigned seconds )

Definition at line 2837 of file download.cc.

Referenced by MountPoint::SetupHttpTuning().

Here is the caller graph for this function:

void download::DownloadManager::SetIpPreference ( const dns::IpPreference preference )

Definition at line 1979 of file download.cc.

Referenced by MountPoint::SetupDnsTuning().

Here is the caller graph for this function:

void download::DownloadManager::SetLowSpeedLimit ( const unsigned low_speed_limit )

Sets contains the average transfer speed in bytes per second that the transfer should be below during CURLOPT_LOW_SPEED_TIME seconds for libcurl to consider it to be too slow and abort. Only effective for new connections.

Definition at line 2004 of file download.cc.

Referenced by MountPoint::SetupHttpTuning().

Here is the caller graph for this function:

void download::DownloadManager::SetMaxIpaddrPerProxy ( unsigned limit )

Definition at line 2857 of file download.cc.

Referenced by Clone(), and MountPoint::SetupDnsTuning().

Here is the caller graph for this function:

void download::DownloadManager::SetNocache ( JobInfo * info )

private

Definition at line 1257 of file download.cc.

Here is the call graph for this function:

void download::DownloadManager::SetProxyChain	(	const std::string &	proxy_list,
		const std::string &	fallback_proxy_list,
		const ProxySetModes	set_mode
	)

Parses a list of ';'- and '|'-separated proxy servers and fallback proxy servers for the proxy groups. The empty string for both removes the proxy chain. The set_mode parameter can be used to set either proxies (leaving fallback proxies unchanged) or fallback proxies (leaving regular proxies unchanged) or both.

Definition at line 2526 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers(), TalkManager::MainResponder(), publish::Repository(), and MountPoint::SetupExternalDownloadMgr().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::SetProxyGroupResetDelay ( const unsigned seconds )

Definition at line 2827 of file download.cc.

Referenced by MountPoint::SetupHttpTuning().

Here is the caller graph for this function:

void download::DownloadManager::SetProxyTemplates	(	const std::string &	direct,
		const std::string &	forced
	)

Definition at line 2863 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers().

Here is the caller graph for this function:

void download::DownloadManager::SetRegularCache ( JobInfo * info )

private

Reverse operation of SetNocache. Makes sure that "no-cache" header disappears from the list of headers to let proxies work normally.

Definition at line 1271 of file download.cc.

Here is the call graph for this function:

void download::DownloadManager::SetRetryParameters	(	const unsigned	max_retries,
		const unsigned	backoff_init_ms,
		const unsigned	backoff_max_ms
	)

Definition at line 2846 of file download.cc.

Referenced by MountPoint::SetupHttpTuning().

Here is the caller graph for this function:

bool download::DownloadManager::SetShardingPolicy ( const ShardingPolicySelector type )

Definition at line 2898 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers().

Here is the caller graph for this function:

void download::DownloadManager::SetTimeout	(	const unsigned	seconds_proxy,
		const unsigned	seconds_direct
	)

Sets two timeout values for proxied and for direct connections, respectively. The timeout counts for all sorts of connection phases, DNS, HTTP connect, etc.

Definition at line 1990 of file download.cc.

Referenced by TalkManager::MainResponder(), MountPoint::SetupExternalDownloadMgr(), and MountPoint::SetupHttpTuning().

Here is the caller graph for this function:

void download::DownloadManager::SetUrlOptions ( JobInfo * info )

private

Sets the URL specific options such as host to use and timeout. It might also set an error code, in which case the further processing should react on.

Definition at line 948 of file download.cc.

Referenced by MainDownload().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::ShardProxies ( )

Enable proxy sharding

Definition at line 2785 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers().

Here is the caller graph for this function:

void download::DownloadManager::Spawn ( )

Spawns the I/O worker thread and switches the module in multi-threaded mode. No way back except Fini(); Init();

Definition at line 1804 of file download.cc.

Referenced by LibContext::EnableMultiThreaded(), and Spawn().

Here is the call graph for this function:

Here is the caller graph for this function:

bool download::DownloadManager::StripDirect	(	const std::string &	proxy_list,
		std::string *	cleaned_list
	)

private

Removes DIRECT from a list of ';' and '|' separated proxies.

Returns: true if DIRECT was present, false otherwise

Definition at line 2486 of file download.cc.

Here is the call graph for this function:

void download::DownloadManager::SwitchHost ( )

Definition at line 2185 of file download.cc.

Referenced by manifest::Fetch(), and TalkManager::MainResponder().

Here is the caller graph for this function:

void download::DownloadManager::SwitchHost ( JobInfo * info )

private

Switches to the next host in the chain. If info is set, switch only if the current host is identical to the one used by info, otherwise another transfer has already done the switch.

Definition at line 2144 of file download.cc.

Here is the call graph for this function:

void download::DownloadManager::SwitchProxy ( JobInfo * info )

private

Jumps to the next proxy in the ring of forward proxy servers. Selects one randomly from a load-balancing group.

Allow for the fact that the proxy may have already been failed by another transfer, or that the proxy may no longer be part of the current load-balancing group.

Definition at line 2077 of file download.cc.

Here is the call graph for this function:

void download::DownloadManager::SwitchProxyGroup ( )

Switches to the next load-balancing group of proxy servers.

Definition at line 2813 of file download.cc.

Referenced by TalkManager::MainResponder().

Here is the caller graph for this function:

void download::DownloadManager::UpdateProxiesUnlocked ( const std::string & reason )

private

Update currently selected proxy

Definition at line 2730 of file download.cc.

Referenced by CloneProxyConfig().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::UpdateStatistics ( CURL * handle )

private

Adds transfer time and downloaded bytes to the global counters.

Definition at line 1198 of file download.cc.

Here is the call graph for this function:

void download::DownloadManager::UseSystemCertificatePath ( )

Definition at line 2894 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers(), and publish::Repository().

Here is the caller graph for this function:

bool download::DownloadManager::ValidateGeoReply	(	const std::string &	reply_order,
		const unsigned	expected_size,
		std::vector< uint64_t > *	reply_vals
	)

private

Validates a string of the form "1,4,2,3" representing in which order the the expected_size number of hosts should be put for optimal geographic proximity. Returns false if the reply_order string is invalid, otherwise fills in the reply_vals array with zero-based order indexes (e.g. [0,3,1,2]) and returns true.

Definition at line 2446 of file download.cc.

Here is the call graph for this function:

bool download::DownloadManager::ValidateProxyIpsUnlocked	(	const std::string &	url,
		const dns::Host &	host
	)

private

Checks if the name resolving information is still up to date. The host object should be one from the current load-balance group. If the information changed, gather new set of resolved IPs and, if different, exchange them in the load-balance group on the fly. In the latter case, also rebalance the proxies. The options mutex needs to be open.

Returns true if proxies may have changed.

Definition at line 1134 of file download.cc.

Here is the call graph for this function:

bool download::DownloadManager::VerifyAndFinalize	(	const int	curl_error,
		JobInfo *	info
	)

private

Checks the result of a curl download and implements the failure logic, such as changing the proxy server. Takes care of cleanup.

Returns: true if another download should be performed, false otherwise

Definition at line 1300 of file download.cc.

Referenced by MainDownload().

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

Counters* download::DownloadManager::counters_

private

Writes and reads should be atomic because reading happens in a different thread than writing.

Definition at line 428 of file download.h.

Referenced by MainDownload().

CredentialsAttachment* download::DownloadManager::credentials_attachment_

private

Definition at line 422 of file download.h.

Referenced by Clone().

CURLM* download::DownloadManager::curl_multi_

private

Definition at line 268 of file download.h.

Referenced by MainDownload().

curl_slist* download::DownloadManager::default_headers_

private

Definition at line 270 of file download.h.

bool download::DownloadManager::enable_http_tracing_

private

Definition at line 302 of file download.h.

Referenced by Clone().

bool download::DownloadManager::enable_info_header_

private

Definition at line 292 of file download.h.

Referenced by Clone().

bool download::DownloadManager::failover_indefinitely_

private

Endless retries for a failed download (hard failures will result in abort)

Definition at line 375 of file download.h.

Referenced by Clone().

bool download::DownloadManager::follow_redirects_

private

Definition at line 294 of file download.h.

Referenced by Clone().

std::string download::DownloadManager::fqrn_

private

Repo name. Needed for the re-try logic if a download was unsuccessful

Definition at line 379 of file download.h.

Referenced by Clone(), and SetFqrn().

HeaderLists* download::DownloadManager::header_lists_

private

Definition at line 269 of file download.h.

SharedPtr<HealthCheck> download::DownloadManager::health_check_

private

Health check for the proxies

Health check is shared between all download managers. As such shared pointers are used to allow for proper clean-up afterwards in Fini() (We cannot assume the order in which the download managers are stopped)

Definition at line 371 of file download.h.

Referenced by Clone().

std::vector<std::string> download::DownloadManager::http_tracing_headers_

private

Definition at line 303 of file download.h.

Referenced by Clone().

bool download::DownloadManager::ignore_signature_failures_

private

Ignore signature failures during download. In general it is a bad idea to do this!

Definition at line 300 of file download.h.

Referenced by Clone().

const unsigned download::DownloadManager::kDnsDefaultRetries = 1

static

Definition at line 153 of file download.h.

Referenced by MountPoint::SetupDnsTuning().

const unsigned download::DownloadManager::kDnsDefaultTimeoutMs = 3000

static

Definition at line 154 of file download.h.

Referenced by MountPoint::SetupDnsTuning().

const int download::DownloadManager::kProbeDown = -2

static

The rtt to a stratum 1 could not be determined because the stratum 1 was unreachable.

Definition at line 147 of file download.h.

Referenced by TalkManager::FormatHostInfo().

const int download::DownloadManager::kProbeGeo = -3

static

The stratum 1 server was put in order according to a Geo-API result

Definition at line 151 of file download.h.

Referenced by TalkManager::FormatHostInfo().

const int download::DownloadManager::kProbeUnprobed = -1

static

No attempt was made to order stratum 1 servers

Definition at line 142 of file download.h.

Referenced by TalkManager::FormatHostInfo().

const unsigned download::DownloadManager::kProxyMapScale = 16

static

Definition at line 155 of file download.h.

pthread_mutex_t* download::DownloadManager::lock_options_

private

Definition at line 283 of file download.h.

pthread_mutex_t* download::DownloadManager::lock_synchronous_mode_

private

Definition at line 284 of file download.h.

atomic_int32 download::DownloadManager::multi_threaded_

private

Definition at line 274 of file download.h.

unsigned download::DownloadManager::opt_backoff_init_ms_

private

Definition at line 290 of file download.h.

Referenced by Clone().

unsigned download::DownloadManager::opt_backoff_max_ms_

private

Definition at line 291 of file download.h.

Referenced by Clone().

std::string download::DownloadManager::opt_dns_server_

private

Definition at line 285 of file download.h.

std::vector<std::string>* download::DownloadManager::opt_host_chain_

private

Definition at line 306 of file download.h.

Referenced by Clone(), and num_hosts().

unsigned download::DownloadManager::opt_host_chain_current_

private

Definition at line 312 of file download.h.

std::vector<int>* download::DownloadManager::opt_host_chain_rtt_

private

Created by SetHostChain(), filled by probe_hosts. Contains time to get .cvmfschecksum in ms. -1 is unprobed, -2 is error.

Definition at line 311 of file download.h.

Referenced by Clone().

unsigned download::DownloadManager::opt_host_reset_after_

private

Definition at line 420 of file download.h.

Referenced by Clone().

dns::IpPreference download::DownloadManager::opt_ip_preference_

private

If a proxy has IPv4 and IPv6 addresses, which one to prefer

Definition at line 389 of file download.h.

Referenced by Clone(), and opt_ip_preference().

bool download::DownloadManager::opt_ipv4_only_

private

Definition at line 293 of file download.h.

unsigned download::DownloadManager::opt_low_speed_limit_

private

Definition at line 288 of file download.h.

Referenced by Clone().

unsigned download::DownloadManager::opt_max_retries_

private

Definition at line 289 of file download.h.

Referenced by Clone().

unsigned download::DownloadManager::opt_num_proxies_

private

Overall number of proxies summed over all the groups.

Definition at line 333 of file download.h.

Referenced by CloneProxyConfig().

std::string download::DownloadManager::opt_proxy_fallback_list_

private

The original proxy fallback list provided to SetProxyChain.

Definition at line 341 of file download.h.

Referenced by CloneProxyConfig().

std::vector< std::vector<ProxyInfo> >* download::DownloadManager::opt_proxy_groups_

private

Definition at line 315 of file download.h.

Referenced by CloneProxyConfig(), and current_proxy_group().

unsigned download::DownloadManager::opt_proxy_groups_current_

private

The current load-balancing group (first dimension in opt_proxy_groups_).

Definition at line 319 of file download.h.

Referenced by CloneProxyConfig(), and current_proxy_group().

unsigned download::DownloadManager::opt_proxy_groups_current_burned_

private

Number of proxy servers that failed within current load-balance group. Between 0 and (*opt_proxy_groups_)[opt_proxy_groups_current_].size().

Definition at line 324 of file download.h.

Referenced by CloneProxyConfig().

unsigned download::DownloadManager::opt_proxy_groups_fallback_

private

The index of the first fallback proxy group. If there are none, it is set to the number of regular proxy groups.

Definition at line 329 of file download.h.

Referenced by CloneProxyConfig().

unsigned download::DownloadManager::opt_proxy_groups_reset_after_

private

Definition at line 412 of file download.h.

Referenced by Clone().

std::string download::DownloadManager::opt_proxy_list_

private

The original proxy list provided to SetProxyChain.

Definition at line 337 of file download.h.

Referenced by CloneProxyConfig().

std::map<uint32_t, ProxyInfo *> download::DownloadManager::opt_proxy_map_

private

Load-balancing map of currently active proxies

Definition at line 345 of file download.h.

bool download::DownloadManager::opt_proxy_shard_

private

Shard requests across multiple proxies via consistent hashing

Definition at line 353 of file download.h.

Referenced by CloneProxyConfig().

std::vector<std::string> download::DownloadManager::opt_proxy_urls_

private

Sorted list of currently active proxy URLs (for log messages)

Definition at line 349 of file download.h.

unsigned download::DownloadManager::opt_timeout_direct_

private

Definition at line 287 of file download.h.

Referenced by Clone().

unsigned download::DownloadManager::opt_timeout_proxy_

private

Definition at line 286 of file download.h.

Referenced by Clone().

time_t download::DownloadManager::opt_timestamp_backup_host_

private

Similarly to proxy group reset, we'd also like to reset the host after a failover. Host outages can last longer and might come with a separate reset delay.

Definition at line 419 of file download.h.

time_t download::DownloadManager::opt_timestamp_backup_proxies_

private

More than one proxy group can be considered as group of primary proxies followed by backup proxy groups, e.g. at another site. If opt_proxy_groups_reset_after_ is > 0, cvmfs will reset its proxy group to the first one after opt_proxy_groups_reset_after_ seconds are elapsed.

Definition at line 410 of file download.h.

time_t download::DownloadManager::opt_timestamp_failover_proxies_

private

Definition at line 411 of file download.h.

UniquePtr<Pipe<kPipeDownloadJobs> > download::DownloadManager::pipe_jobs_

private

Definition at line 277 of file download.h.

Referenced by MainDownload().

UniquePtr<Pipe<kPipeThreadTerminator> > download::DownloadManager::pipe_terminate_

private

Definition at line 275 of file download.h.

Referenced by MainDownload().

std::set<CURL *>* download::DownloadManager::pool_handles_idle_

private

Definition at line 265 of file download.h.

std::set<CURL *>* download::DownloadManager::pool_handles_inuse_

private

Definition at line 266 of file download.h.

Referenced by MainDownload().

uint32_t download::DownloadManager::pool_max_handles_

private

Definition at line 267 of file download.h.

Prng download::DownloadManager::prng_

private

Definition at line 264 of file download.h.

std::string download::DownloadManager::proxy_template_direct_

private

Used to replace @ in the Geo-API calls to order Stratum 1 servers, in case the active proxy is DIRECT (no proxy). Should be a UUID identifying the host.

Definition at line 396 of file download.h.

Referenced by Clone().

std::string download::DownloadManager::proxy_template_forced_

private

Used to force a value for @ in the Geo-API calls to order Stratum 1 servers. If empty, the fully qualified domain name of the active proxy server is used.

Definition at line 402 of file download.h.

Referenced by Clone().

dns::NormalResolver* download::DownloadManager::resolver_

private

Used to resolve proxy addresses (host addresses are resolved by the proxy).

Definition at line 384 of file download.h.

SharedPtr<ShardingPolicy> download::DownloadManager::sharding_policy_

private

Sharding policy deciding which proxy should be chosen for each download request

Sharding policy is shared between all download managers. As such shared pointers are used to allow for proper clean-up afterwards in Fini() (We cannot assume the order in which the download managers are stopped)

Definition at line 363 of file download.h.

Referenced by Clone().

SslCertificateStore download::DownloadManager::ssl_certificate_store_

private

Carries the path settings for SSL certificates

Definition at line 433 of file download.h.

Referenced by Clone().

pthread_t download::DownloadManager::thread_download_

private

Definition at line 273 of file download.h.

char* download::DownloadManager::user_agent_

private

Definition at line 271 of file download.h.

struct pollfd* download::DownloadManager::watch_fds_

private

Definition at line 278 of file download.h.

Referenced by CallbackCurlSocket(), and MainDownload().

uint32_t download::DownloadManager::watch_fds_inuse_

private

Definition at line 280 of file download.h.

Referenced by CallbackCurlSocket(), and MainDownload().

uint32_t download::DownloadManager::watch_fds_max_

private

Definition at line 281 of file download.h.

Referenced by CallbackCurlSocket().

uint32_t download::DownloadManager::watch_fds_size_

private

Definition at line 279 of file download.h.

Referenced by CallbackCurlSocket(), and MainDownload().

The documentation for this class was generated from the following files:

/home/sftnight/jenkins/workspace/CvmfsDoxygen/label/docker-x86_64/cvmfs/cvmfs/network/download.h
/home/sftnight/jenkins/workspace/CvmfsDoxygen/label/docker-x86_64/cvmfs/cvmfs/network/download.cc

Classes

Public Types

Public Member Functions

Static Public Member Functions

Static Public Attributes

Private Member Functions

Static Private Member Functions

Private Attributes

Detailed Description

Member Enumeration Documentation

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation