CernVM-FS  2.13.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Classes | Public Types | Public Member Functions | Static Public Member Functions | Static Public Attributes | Private Member Functions | Static Private Member Functions | Private Attributes | List of all members
download::DownloadManager Class Reference

#include <download.h>

Collaboration diagram for download::DownloadManager:
[legend]

Classes

struct  HostInfo
 
struct  ProxyInfo
 

Public Types

enum  ProxySetModes { kSetProxyRegular = 0, kSetProxyFallback, kSetProxyBoth }
 

Public Member Functions

 DownloadManager (const unsigned max_pool_handles, const perf::StatisticsTemplate &statistics, const std::string &name="standard")
 
 ~DownloadManager ()
 
void Spawn ()
 
DownloadManager * Clone (const perf::StatisticsTemplate &statistics, const std::string &cloned_name)
 
Failures Fetch (JobInfo *info)
 
void SetCredentialsAttachment (CredentialsAttachment *ca)
 
std::string GetDnsServer () const
 
void SetDnsServer (const std::string &address)
 
void SetDnsParameters (const unsigned retries, const unsigned timeout_ms)
 
void SetDnsTtlLimits (const unsigned min_seconds, const unsigned max_seconds)
 
void SetIpPreference (const dns::IpPreference preference)
 
void SetTimeout (const unsigned seconds_proxy, const unsigned seconds_direct)
 
void GetTimeout (unsigned *seconds_proxy, unsigned *seconds_direct)
 
void SetLowSpeedLimit (const unsigned low_speed_limit)
 
void SetMetalinkChain (const std::string &metalink_list)
 
void SetMetalinkChain (const std::vector< std::string > &metalink_list)
 
void GetMetalinkInfo (std::vector< std::string > *metalink_chain, unsigned *current_metalink)
 
void SwitchMetalink ()
 
bool CheckMetalinkChain (const time_t now)
 
void SetHostChain (const std::string &host_list)
 
void SetHostChain (const std::vector< std::string > &host_list)
 
void GetHostInfo (std::vector< std::string > *host_chain, std::vector< int > *rtt, unsigned *current_host)
 
void ProbeHosts ()
 
bool ProbeGeo ()
 
bool GeoSortServers (std::vector< std::string > *servers, std::vector< uint64_t > *output_order=NULL)
 
void SwitchHost ()
 
void SetProxyChain (const std::string &proxy_list, const std::string &fallback_proxy_list, const ProxySetModes set_mode)
 
void GetProxyInfo (std::vector< std::vector< ProxyInfo > > *proxy_chain, unsigned *current_group, unsigned *fallback_group)
 
std::string GetProxyList ()
 
std::string GetFallbackProxyList ()
 
void ShardProxies ()
 
void RebalanceProxies ()
 
void SwitchProxyGroup ()
 
void SetProxyGroupResetDelay (const unsigned seconds)
 
void SetMetalinkResetDelay (const unsigned seconds)
 
void SetHostResetDelay (const unsigned seconds)
 
void SetRetryParameters (const unsigned max_retries, const unsigned backoff_init_ms, const unsigned backoff_max_ms)
 
void SetMaxIpaddrPerProxy (unsigned limit)
 
void SetProxyTemplates (const std::string &direct, const std::string &forced)
 
void EnableInfoHeader ()
 
void EnableRedirects ()
 
void EnableIgnoreSignatureFailures ()
 
void EnableHTTPTracing ()
 
void AddHTTPTracingHeader (const std::string &header)
 
void UseSystemCertificatePath ()
 
bool SetShardingPolicy (const ShardingPolicySelector type)
 
void SetFailoverIndefinitely ()
 
void SetFqrn (const std::string &fqrn)
 
unsigned num_hosts ()
 
unsigned num_metalinks ()
 
dns::IpPreference opt_ip_preference () const
 

Static Public Member Functions

static int ParseHttpCode (const char digits[3])
 

Static Public Attributes

static const int kProbeUnprobed = -1
 
static const int kProbeDown = -2
 
static const int kProbeGeo = -3
 
static const unsigned kDnsDefaultRetries = 1
 
static const unsigned kDnsDefaultTimeoutMs = 3000
 
static const unsigned kProxyMapScale = 16
 

Private Member Functions

 FRIEND_TEST (T_Download, ValidateGeoReply)
 
 FRIEND_TEST (T_Download, StripDirect)
 
 FRIEND_TEST (T_Download, EscapeUrl)
 
bool StripDirect (const std::string &proxy_list, std::string *cleaned_list)
 
bool ValidateGeoReply (const std::string &reply_order, const unsigned expected_size, std::vector< uint64_t > *reply_vals)
 
void SwitchHostInfo (const std::string &typ, HostInfo &info, JobInfo *jobinfo)
 
void SwitchMetalink (JobInfo *info)
 
void SwitchHost (JobInfo *info)
 
void SwitchProxy (JobInfo *info)
 
ProxyInfo * ChooseProxyUnlocked (const shash::Any *hash)
 
void UpdateProxiesUnlocked (const std::string &reason)
 
void RebalanceProxiesUnlocked (const std::string &reason)
 
CURL * AcquireCurlHandle ()
 
void ReleaseCurlHandle (CURL *handle)
 
void ReleaseCredential (JobInfo *info)
 
void InitializeRequest (JobInfo *info, CURL *handle)
 
void SetUrlOptions (JobInfo *info)
 
bool ValidateProxyIpsUnlocked (const std::string &url, const dns::Host &host)
 
void UpdateStatistics (CURL *handle)
 
bool CanRetry (const JobInfo *info)
 
void Backoff (JobInfo *info)
 
void SetNocache (JobInfo *info)
 
void SetRegularCache (JobInfo *info)
 
void ProcessLink (JobInfo *info)
 
bool VerifyAndFinalize (const int curl_error, JobInfo *info)
 
void InitHeaders ()
 
void CloneProxyConfig (DownloadManager *clone)
 
void CheckHostInfoReset (const std::string &typ, HostInfo &info, JobInfo *jobinfo, time_t &now)
 
bool EscapeUrlChar (unsigned char input, char output[3])
 
std::string EscapeUrl (const int64_t jobinfo_id, const std::string &url)
 
unsigned EscapeHeader (const std::string &header, char *escaped_buf, size_t buf_size)
 
std::vector< ProxyInfo > * current_proxy_group () const
 

Static Private Member Functions

static int CallbackCurlSocket (CURL *easy, curl_socket_t s, int action, void *userp, void *socketp)
 
static void * MainDownload (void *data)
 

Private Attributes

Prng prng_
 
std::set< CURL * > * pool_handles_idle_
 
std::set< CURL * > * pool_handles_inuse_
 
uint32_t pool_max_handles_
 
CURLM * curl_multi_
 
HeaderLists * header_lists_
 
curl_slist * default_headers_
 
char * user_agent_
 
pthread_t thread_download_
 
atomic_int32 multi_threaded_
 
UniquePtr< Pipe
< kPipeThreadTerminator > > 		pipe_terminate_
 
UniquePtr< Pipe
< kPipeDownloadJobs > > 		pipe_jobs_
 
struct pollfd * watch_fds_
 
uint32_t watch_fds_size_
 
uint32_t watch_fds_inuse_
 
uint32_t watch_fds_max_
 
pthread_mutex_t * lock_options_
 
pthread_mutex_t * lock_synchronous_mode_
 
std::string opt_dns_server_
 
unsigned opt_timeout_proxy_
 
unsigned opt_timeout_direct_
 
unsigned opt_low_speed_limit_
 
unsigned opt_max_retries_
 
unsigned opt_backoff_init_ms_
 
unsigned opt_backoff_max_ms_
 
bool enable_info_header_
 
bool opt_ipv4_only_
 
bool follow_redirects_
 
bool ignore_signature_failures_
 
bool enable_http_tracing_
 
std::vector< std::string > http_tracing_headers_
 
HostInfo opt_metalink_
 
time_t opt_metalink_timestamp_link_
 
HostInfo opt_host_
 
std::vector< int > * opt_host_chain_rtt_
 
std::vector< std::vector
< ProxyInfo > > * 		opt_proxy_groups_
 
unsigned opt_proxy_groups_current_
 
unsigned opt_proxy_groups_current_burned_
 
unsigned opt_proxy_groups_fallback_
 
unsigned opt_num_proxies_
 
std::string opt_proxy_list_
 
std::string opt_proxy_fallback_list_
 
std::map< uint32_t, ProxyInfo * > opt_proxy_map_
 
std::vector< std::string > opt_proxies_
 
bool opt_proxy_shard_
 
SharedPtr< ShardingPolicy > sharding_policy_
 
SharedPtr< HealthCheck > health_check_
 
bool failover_indefinitely_
 
std::string fqrn_
 
std::string name_
 
dns::NormalResolver * resolver_
 
dns::IpPreference opt_ip_preference_
 
std::string proxy_template_direct_
 
std::string proxy_template_forced_
 
time_t opt_timestamp_backup_proxies_
 
time_t opt_timestamp_failover_proxies_
 
unsigned opt_proxy_groups_reset_after_
 
CredentialsAttachment * credentials_attachment_
 
Counters * counters_
 
SslCertificateStore ssl_certificate_store_
 

Detailed Description

Note when adding new fields: Clone() probably needs to be adjusted, too. TODO(jblomer): improve ordering of members

Definition at line 119 of file download.h.

Member Enumeration Documentation

enum download::DownloadManager::ProxySetModes
Enumerator
kSetProxyRegular 
kSetProxyFallback 
kSetProxyBoth 

Definition at line 152 of file download.h.

Constructor & Destructor Documentation

download::DownloadManager::DownloadManager ( const unsigned  max_pool_handles,
const perf::StatisticsTemplate &  statistics,
const std::string &  name = "standard" 
)

Definition at line 1880 of file download.cc.

Referenced by Clone().

Here is the call graph for this function:

Here is the caller graph for this function:

download::DownloadManager::~DownloadManager ( )

Definition at line 1804 of file download.cc.

Here is the call graph for this function:

Member Function Documentation

CURL * download::DownloadManager::AcquireCurlHandle ( )
private

Gets an idle CURL handle from the pool. Creates a new one and adds it to the pool if necessary.

Definition at line 877 of file download.cc.

Referenced by Fetch(), and MainDownload().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::AddHTTPTracingHeader ( const std::string &  header)

Definition at line 3171 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers().

Here is the caller graph for this function:

void download::DownloadManager::Backoff ( JobInfo *  info)
private

Backoff for retry to introduce a jitter into a cluster of requesting cvmfs nodes. Retry only when HTTP caching is on.

Returns
true if backoff has been performed, false otherwise

Definition at line 1326 of file download.cc.

Here is the call graph for this function:

int download::DownloadManager::CallbackCurlSocket ( CURL *  easy,
curl_socket_t  s,
int  action,
void *  userp,
void *  socketp 
)
staticprivate

Called when new curl sockets arrive or existing curl sockets depart.

Definition at line 489 of file download.cc.

Referenced by DownloadManager().

Here is the call graph for this function:

Here is the caller graph for this function:

bool download::DownloadManager::CanRetry ( const JobInfo *  info)
private

Retry if possible if not on no-cache and if not already done too often.

Definition at line 1310 of file download.cc.

Here is the call graph for this function:

void download::DownloadManager::CheckHostInfoReset ( const std::string &  typ,
HostInfo &  info,
JobInfo *  jobinfo,
time_t &  now 
)
private

Definition at line 1001 of file download.cc.

Here is the call graph for this function:

bool download::DownloadManager::CheckMetalinkChain ( const time_t  now)

Definition at line 2437 of file download.cc.

DownloadManager::ProxyInfo * download::DownloadManager::ChooseProxyUnlocked ( const shash::Any *  hash)
private

Choose proxy

Definition at line 2986 of file download.cc.

Here is the call graph for this function:

DownloadManager * download::DownloadManager::Clone ( const perf::StatisticsTemplate &  statistics,
const std::string &  cloned_name 
)

Creates a copy of the existing download manager. Must only be called in single-threaded stage because it calls curl_global_init().

Definition at line 3200 of file download.cc.

Referenced by MountPoint::SetupExternalDownloadMgr().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::CloneProxyConfig ( DownloadManager *  clone)
private

Definition at line 3247 of file download.cc.

Referenced by Clone().

Here is the call graph for this function:

Here is the caller graph for this function:

std::vector<ProxyInfo>* download::DownloadManager::current_proxy_group ( ) const
inlineprivate

Definition at line 299 of file download.h.

Referenced by SwitchProxy(), and UpdateProxiesUnlocked().

Here is the caller graph for this function:

void download::DownloadManager::EnableHTTPTracing ( )

Definition at line 3169 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers().

Here is the caller graph for this function:

void download::DownloadManager::EnableIgnoreSignatureFailures ( )

Definition at line 3165 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers().

Here is the caller graph for this function:

void download::DownloadManager::EnableInfoHeader ( )

Definition at line 3160 of file download.cc.

Referenced by MountPoint::SetupHttpTuning().

Here is the caller graph for this function:

void download::DownloadManager::EnableRedirects ( )

Definition at line 3163 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers(), MountPoint::SetupExternalDownloadMgr(), and MountPoint::SetupHttpTuning().

Here is the caller graph for this function:

unsigned download::DownloadManager::EscapeHeader ( const std::string &  header,
char *  escaped_buf,
size_t  buf_size 
)
private

escaped array needs to be sufficiently large. Its size is calculated by passing NULL to EscapeHeader.

Definition at line 442 of file download.cc.

Referenced by Fetch().

Here is the caller graph for this function:

string download::DownloadManager::EscapeUrl ( const int64_t  jobinfo_id,
const std::string &  url 
)
private

Escape special chars from the URL, except for ':' and '/', which should keep their meaning.

Definition at line 420 of file download.cc.

Here is the call graph for this function:

bool download::DownloadManager::EscapeUrlChar ( unsigned char  input,
char  output[3] 
)
private

Definition at line 397 of file download.cc.

Failures download::DownloadManager::Fetch ( JobInfo *  info)

Downloads data from an insecure outside channel (currently HTTP or file).

Definition at line 1984 of file download.cc.

Referenced by download::AutoProxy(), manifest::DoFetch(), manifest::DoVerify(), HttpObjectFetcher< CatalogT, HistoryT, ReflogT >::Download(), swissknife::CommandCheck::DownloadPiece(), publish::DownloadRootObjects(), swissknife::CommandCheck::Exists(), cvmfs::Fetcher::Fetch(), swissknife::Assistant::FetchObject(), swissknife::CommandCheck::FetchPath(), GeoSortServers(), publish::IsMasterReplica(), catalog::SimpleCatalogManager::LoadCatalogByHash(), swissknife::MainWorker(), ProbeHosts(), and StreamingCacheManager::Stream().

Here is the call graph for this function:

Here is the caller graph for this function:

download::DownloadManager::FRIEND_TEST ( T_Download  ,
ValidateGeoReply   
)
private
download::DownloadManager::FRIEND_TEST ( T_Download  ,
StripDirect   
)
private
download::DownloadManager::FRIEND_TEST ( T_Download  ,
EscapeUrl   
)
private
bool download::DownloadManager::GeoSortServers ( std::vector< std::string > *  servers,
std::vector< uint64_t > *  output_order = NULL 
)

Definition at line 2506 of file download.cc.

Referenced by ProbeGeo(), and MountPoint::SetupExternalDownloadMgr().

Here is the call graph for this function:

Here is the caller graph for this function:

std::string download::DownloadManager::GetDnsServer ( ) const

Gets the DNS sever.

Definition at line 2104 of file download.cc.

Referenced by TalkManager::MainResponder().

Here is the caller graph for this function:

string download::DownloadManager::GetFallbackProxyList ( )

Definition at line 2979 of file download.cc.

void download::DownloadManager::GetHostInfo ( std::vector< std::string > *  host_chain,
std::vector< int > *  rtt,
unsigned *  current_host 
)

Retrieves the currently set chain of hosts, their round trip times, and the currently used host.

Definition at line 2269 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers(), ExternalHostMagicXattr::FinalizeValue(), HostMagicXattr::FinalizeValue(), HostListMagicXattr::FinalizeValue(), ExternalURLMagicXattr::FinalizeValue(), TalkManager::FormatHostInfo(), GeoSortServers(), download::ParsePac(), ProbeGeo(), ProbeHosts(), and MountPoint::SetupExternalDownloadMgr().

Here is the caller graph for this function:

void download::DownloadManager::GetMetalinkInfo ( std::vector< std::string > *  metalink_chain,
unsigned *  current_metalink 
)

Retrieves the currently set chain of metalink hosts and the currently used metalink host.

Definition at line 2221 of file download.cc.

Referenced by TalkManager::FormatMetalinkInfo().

Here is the caller graph for this function:

void download::DownloadManager::GetProxyInfo ( std::vector< std::vector< ProxyInfo > > *  proxy_chain,
unsigned *  current_group,
unsigned *  fallback_group 
)

Retrieves the proxy chain, optionally the currently active load-balancing group, and optionally the index of the first fallback proxy group. If there are no fallback proxies, the index will equal the size of the proxy chain.

Definition at line 2954 of file download.cc.

Referenced by ProxyMagicXattr::FinalizeValue(), TalkManager::FormatProxyInfo(), ListProxy(), and ProbeGeo().

Here is the call graph for this function:

Here is the caller graph for this function:

string download::DownloadManager::GetProxyList ( )

Definition at line 2977 of file download.cc.

void download::DownloadManager::GetTimeout ( unsigned *  seconds_proxy,
unsigned *  seconds_direct 
)

Receives the currently active timeout values.

Definition at line 2184 of file download.cc.

Referenced by ExternalTimeoutMagicXattr::FinalizeValue(), TimeoutMagicXattr::FinalizeValue(), TimeoutDirectMagicXattr::FinalizeValue(), TalkManager::MainResponder(), and MountPoint::SetupExternalDownloadMgr().

Here is the caller graph for this function:

void download::DownloadManager::InitHeaders ( )
private

Definition at line 1857 of file download.cc.

Referenced by DownloadManager().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::InitializeRequest ( JobInfo *  info,
CURL *  handle 
)
private

HTTP request options: set the URL and other options such as timeout and proxy.

Definition at line 918 of file download.cc.

Referenced by Fetch(), and MainDownload().

Here is the call graph for this function:

Here is the caller graph for this function:

void * download::DownloadManager::MainDownload ( void *  data)
staticprivate

Worker thread event loop. Waits on new JobInfo structs on a pipe.

Definition at line 568 of file download.cc.

Referenced by Spawn().

Here is the call graph for this function:

Here is the caller graph for this function:

unsigned download::DownloadManager::num_hosts ( )
inline

Definition at line 246 of file download.h.

Referenced by manifest::Fetch().

Here is the caller graph for this function:

unsigned download::DownloadManager::num_metalinks ( )
inline

Definition at line 252 of file download.h.

dns::IpPreference download::DownloadManager::opt_ip_preference ( ) const
inline

Definition at line 258 of file download.h.

Referenced by ResolvConfEventHandler::SetDnsAddress().

Here is the caller graph for this function:

int download::DownloadManager::ParseHttpCode ( const char  digits[3])
static

-1 of digits is not a valid Http return code

Definition at line 473 of file download.cc.

bool download::DownloadManager::ProbeGeo ( )

Uses the Geo-API of Stratum 1s to let any of them order the list of servers and fallback proxies (if any). Tries at most three random Stratum 1s before giving up. If you change the host list in between by SetHostChain() or the fallback proxy list by SetProxyChain(), they will be overwritten by this function.

Definition at line 2607 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers(), and TalkManager::MainResponder().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::ProbeHosts ( )

Orders the hostlist according to RTT of downloading .cvmfschecksum. Sets the current host to the best-responsive host. If you change the host list in between by SetHostChain(), it will be overwritten by this function.

Definition at line 2452 of file download.cc.

Referenced by TalkManager::MainResponder().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::ProcessLink ( JobInfo *  info)
private

Parses Link header and uses it to set a new host chain. See rfc6249.

Definition at line 1406 of file download.cc.

Here is the call graph for this function:

void download::DownloadManager::RebalanceProxies ( )

Definition at line 3084 of file download.cc.

Referenced by TalkManager::MainResponder().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::RebalanceProxiesUnlocked ( const std::string &  reason)
private

Selects a new random proxy in the current load-balancing group. Resets the "burned" counter.

Definition at line 3074 of file download.cc.

Referenced by RebalanceProxies(), ShardProxies(), and SwitchProxyGroup().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::ReleaseCredential ( JobInfo *  info)
private

Frees the storage associated with the authz attachment from the job

Definition at line 1379 of file download.cc.

Here is the call graph for this function:

void download::DownloadManager::ReleaseCurlHandle ( CURL *  handle)
private

Definition at line 900 of file download.cc.

Referenced by Fetch(), and MainDownload().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::SetCredentialsAttachment ( CredentialsAttachment *  ca)

Used by the client to connect the authz session manager to the download manager.

Definition at line 2096 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers().

Here is the caller graph for this function:

void download::DownloadManager::SetDnsParameters ( const unsigned  retries,
const unsigned  timeout_ms 
)

Sets the DNS query timeout parameters.

Definition at line 2129 of file download.cc.

Referenced by Clone(), and MountPoint::SetupDnsTuning().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::SetDnsServer ( const std::string &  address)

Sets a DNS server. Only for testing as it cannot be reverted to the system default.

Definition at line 2110 of file download.cc.

Referenced by Clone(), TalkManager::MainResponder(), ResolvConfEventHandler::SetDnsAddress(), and MountPoint::SetupDnsTuning().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::SetDnsTtlLimits ( const unsigned  min_seconds,
const unsigned  max_seconds 
)

Definition at line 2143 of file download.cc.

Referenced by Clone(), and MountPoint::SetupDnsTuning().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::SetFailoverIndefinitely ( )

Definition at line 3192 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers().

Here is the caller graph for this function:

void download::DownloadManager::SetFqrn ( const std::string &  fqrn)
inline

Definition at line 244 of file download.h.

Referenced by MountPoint::CreateDownloadManagers().

Here is the caller graph for this function:

void download::DownloadManager::SetHostChain ( const std::string &  host_list)

Referenced by MountPoint::CreateDownloadManagers(), download::MainResolveProxyDescription(), TalkManager::MainResponder(), and MountPoint::SetupExternalDownloadMgr().

Here is the caller graph for this function:

void download::DownloadManager::SetHostChain ( const std::vector< std::string > &  host_list)

Definition at line 2244 of file download.cc.

void download::DownloadManager::SetHostResetDelay ( const unsigned  seconds)

Definition at line 3128 of file download.cc.

Referenced by MountPoint::SetupHttpTuning().

Here is the caller graph for this function:

void download::DownloadManager::SetIpPreference ( const dns::IpPreference  preference)

Definition at line 2151 of file download.cc.

Referenced by MountPoint::SetupDnsTuning().

Here is the caller graph for this function:

void download::DownloadManager::SetLowSpeedLimit ( const unsigned  low_speed_limit)

Sets contains the average transfer speed in bytes per second that the transfer should be below during CURLOPT_LOW_SPEED_TIME seconds for libcurl to consider it to be too slow and abort. Only effective for new connections.

Definition at line 2175 of file download.cc.

Referenced by MountPoint::SetupHttpTuning().

Here is the caller graph for this function:

void download::DownloadManager::SetMaxIpaddrPerProxy ( unsigned  limit)

Definition at line 3146 of file download.cc.

Referenced by Clone(), and MountPoint::SetupDnsTuning().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::SetMetalinkChain ( const std::string &  metalink_list)

Referenced by MountPoint::CreateDownloadManagers(), TalkManager::MainResponder(), and MountPoint::SetupExternalDownloadMgr().

Here is the caller graph for this function:

void download::DownloadManager::SetMetalinkChain ( const std::vector< std::string > &  metalink_list)

Definition at line 2201 of file download.cc.

void download::DownloadManager::SetMetalinkResetDelay ( const unsigned  seconds)

Definition at line 3120 of file download.cc.

Referenced by MountPoint::SetupHttpTuning().

Here is the caller graph for this function:

void download::DownloadManager::SetNocache ( JobInfo *  info)
private

Definition at line 1352 of file download.cc.

Here is the call graph for this function:

void download::DownloadManager::SetProxyChain ( const std::string &  proxy_list,
const std::string &  fallback_proxy_list,
const ProxySetModes  set_mode 
)

Parses a list of ';'- and '|'-separated proxy servers and fallback proxy servers for the proxy groups. The empty string for both removes the proxy chain. The set_mode parameter can be used to set either proxies (leaving fallback proxies unchanged) or fallback proxies (leaving regular proxies unchanged) or both.

Definition at line 2795 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers(), TalkManager::MainResponder(), publish::Repository(), and MountPoint::SetupExternalDownloadMgr().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::SetProxyGroupResetDelay ( const unsigned  seconds)

Definition at line 3110 of file download.cc.

Referenced by MountPoint::SetupHttpTuning().

Here is the caller graph for this function:

void download::DownloadManager::SetProxyTemplates ( const std::string &  direct,
const std::string &  forced 
)

Definition at line 3152 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers().

Here is the caller graph for this function:

void download::DownloadManager::SetRegularCache ( JobInfo *  info)
private

Reverse operation of SetNocache. Makes sure that "no-cache" header disappears from the list of headers to let proxies work normally.

Definition at line 1366 of file download.cc.

Here is the call graph for this function:

void download::DownloadManager::SetRetryParameters ( const unsigned  max_retries,
const unsigned  backoff_init_ms,
const unsigned  backoff_max_ms 
)

Definition at line 3136 of file download.cc.

Referenced by MountPoint::SetupHttpTuning().

Here is the caller graph for this function:

bool download::DownloadManager::SetShardingPolicy ( const ShardingPolicySelector  type)

Definition at line 3179 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::SetTimeout ( const unsigned  seconds_proxy,
const unsigned  seconds_direct 
)

Sets two timeout values for proxied and for direct connections, respectively. The timeout counts for all sorts of connection phases, DNS, HTTP connect, etc.

Definition at line 2162 of file download.cc.

Referenced by TalkManager::MainResponder(), MountPoint::SetupExternalDownloadMgr(), and MountPoint::SetupHttpTuning().

Here is the caller graph for this function:

void download::DownloadManager::SetUrlOptions ( JobInfo *  info)
private

Sets the URL specific options such as host to use and timeout. It might also set an error code, in which case the further processing should react on.

Definition at line 1027 of file download.cc.

Referenced by Fetch(), and MainDownload().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::ShardProxies ( )

Enable proxy sharding

Definition at line 3065 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::Spawn ( )

Spawns the I/O worker thread and switches the module in multi-threaded mode. No way back except Fini(); Init();

Definition at line 1963 of file download.cc.

Referenced by LibContext::EnableMultiThreaded(), and Spawn().

Here is the call graph for this function:

Here is the caller graph for this function:

bool download::DownloadManager::StripDirect ( const std::string &  proxy_list,
std::string *  cleaned_list 
)
private

Removes DIRECT from a list of ';' and '|' separated proxies.

Returns
true if DIRECT was present, false otherwise

Definition at line 2757 of file download.cc.

Referenced by SetProxyChain().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::SwitchHost ( )

Definition at line 2427 of file download.cc.

Referenced by manifest::Fetch(), and TalkManager::MainResponder().

Here is the caller graph for this function:

void download::DownloadManager::SwitchHost ( JobInfo *  info)
private

Definition at line 2423 of file download.cc.

Here is the call graph for this function:

void download::DownloadManager::SwitchHostInfo ( const std::string &  typ,
HostInfo &  info,
JobInfo *  jobinfo 
)
private

Switches to the next host in the chain. If jobinfo is set, switch only if the current host is identical to the one used by jobinfo, otherwise another transfer has already done the switch.

Definition at line 2364 of file download.cc.

Referenced by SwitchHost(), and SwitchMetalink().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::SwitchMetalink ( )

Definition at line 2435 of file download.cc.

Referenced by TalkManager::MainResponder().

Here is the caller graph for this function:

void download::DownloadManager::SwitchMetalink ( JobInfo *  info)
private

Definition at line 2430 of file download.cc.

Here is the call graph for this function:

void download::DownloadManager::SwitchProxy ( JobInfo *  info)
private

Jumps to the next proxy in the ring of forward proxy servers. Selects one randomly from a load-balancing group.

Allow for the fact that the proxy may have already been failed by another transfer, or that the proxy may no longer be part of the current load-balancing group.

Definition at line 2294 of file download.cc.

Here is the call graph for this function:

void download::DownloadManager::SwitchProxyGroup ( )

Switches to the next load-balancing group of proxy servers.

Definition at line 3093 of file download.cc.

Referenced by TalkManager::MainResponder().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::UpdateProxiesUnlocked ( const std::string &  reason)
private

Update currently selected proxy

Definition at line 3002 of file download.cc.

Referenced by CloneProxyConfig(), ProbeGeo(), RebalanceProxiesUnlocked(), SetProxyChain(), and SwitchProxy().

Here is the call graph for this function:

Here is the caller graph for this function:

void download::DownloadManager::UpdateStatistics ( CURL *  handle)
private

Adds transfer time and downloaded bytes to the global counters.

Definition at line 1292 of file download.cc.

Here is the call graph for this function:

void download::DownloadManager::UseSystemCertificatePath ( )

Definition at line 3175 of file download.cc.

Referenced by MountPoint::CreateDownloadManagers(), and publish::Repository().

Here is the call graph for this function:

Here is the caller graph for this function:

bool download::DownloadManager::ValidateGeoReply ( const std::string &  reply_order,
const unsigned  expected_size,
std::vector< uint64_t > *  reply_vals 
)
private

Validates a string of the form "1,4,2,3" representing in which order the the expected_size number of hosts should be put for optimal geographic proximity. Returns false if the reply_order string is invalid, otherwise fills in the reply_vals array with zero-based order indexes (e.g. [0,3,1,2]) and returns true.

Definition at line 2719 of file download.cc.

Referenced by GeoSortServers().

Here is the call graph for this function:

Here is the caller graph for this function:

bool download::DownloadManager::ValidateProxyIpsUnlocked ( const std::string &  url,
const dns::Host &  host 
)
private

Checks if the name resolving information is still up to date. The host object should be one from the current load-balance group. If the information changed, gather new set of resolved IPs and, if different, exchange them in the load-balance group on the fly. In the latter case, also rebalance the proxies. The options mutex needs to be open.

Returns true if proxies may have changed.

Definition at line 1227 of file download.cc.

Here is the call graph for this function:

bool download::DownloadManager::VerifyAndFinalize ( const int  curl_error,
JobInfo *  info 
)
private

Checks the result of a curl download and implements the failure logic, such as changing the proxy server. Takes care of cleanup.

Returns
true if another download should be performed, false otherwise

Definition at line 1470 of file download.cc.

Referenced by Fetch(), and MainDownload().

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

Counters* download::DownloadManager::counters_
private

Writes and reads should be atomic because reading happens in a different thread than writing.

Definition at line 470 of file download.h.

Referenced by Fetch(), MainDownload(), SwitchHostInfo(), and SwitchProxy().

CredentialsAttachment* download::DownloadManager::credentials_attachment_
private

Definition at line 464 of file download.h.

Referenced by Clone(), and SetCredentialsAttachment().

CURLM* download::DownloadManager::curl_multi_
private

Definition at line 309 of file download.h.

Referenced by DownloadManager(), and MainDownload().

curl_slist* download::DownloadManager::default_headers_
private

Definition at line 311 of file download.h.

bool download::DownloadManager::enable_http_tracing_
private

Definition at line 343 of file download.h.

Referenced by Clone(), EnableHTTPTracing(), and Fetch().

bool download::DownloadManager::enable_info_header_
private

Definition at line 333 of file download.h.

Referenced by Clone(), EnableInfoHeader(), and Fetch().

bool download::DownloadManager::failover_indefinitely_
private

Endless retries for a failed download (hard failures will result in abort)

Definition at line 419 of file download.h.

Referenced by Clone(), and SetFailoverIndefinitely().

bool download::DownloadManager::follow_redirects_
private

Definition at line 335 of file download.h.

Referenced by Clone(), and EnableRedirects().

std::string download::DownloadManager::fqrn_
private

Repo name. Needed for the re-try logic if a download was unsuccessful Used in sharding policy && Interrupted()

Definition at line 424 of file download.h.

Referenced by Clone(), and SetFqrn().

HeaderLists* download::DownloadManager::header_lists_
private

Definition at line 310 of file download.h.

SharedPtr<HealthCheck> download::DownloadManager::health_check_
private

Health check for the proxies

Health check is shared between all download managers. As such shared pointers are used to allow for proper clean-up afterwards in the destructor (We cannot assume the order in which the download managers are stopped)

Definition at line 415 of file download.h.

Referenced by Clone(), and Spawn().

std::vector<std::string> download::DownloadManager::http_tracing_headers_
private

Definition at line 344 of file download.h.

Referenced by AddHTTPTracingHeader(), and Clone().

bool download::DownloadManager::ignore_signature_failures_
private

Ignore signature failures during download. In general it is a bad idea to do this!

Definition at line 341 of file download.h.

Referenced by Clone(), and EnableIgnoreSignatureFailures().

const unsigned download::DownloadManager::kDnsDefaultRetries = 1
static

Definition at line 172 of file download.h.

Referenced by DownloadManager(), and MountPoint::SetupDnsTuning().

const unsigned download::DownloadManager::kDnsDefaultTimeoutMs = 3000
static

Definition at line 173 of file download.h.

Referenced by DownloadManager(), and MountPoint::SetupDnsTuning().

const int download::DownloadManager::kProbeDown = -2
static

The rtt to a stratum 1 could not be determined because the stratum 1 was unreachable.

Definition at line 166 of file download.h.

Referenced by TalkManager::FormatHostInfo(), and ProbeHosts().

const int download::DownloadManager::kProbeGeo = -3
static

The stratum 1 server was put in order according to a Geo-API result

Definition at line 170 of file download.h.

Referenced by TalkManager::FormatHostInfo(), and ProbeGeo().

const int download::DownloadManager::kProbeUnprobed = -1
static

No attempt was made to order stratum 1 servers

Definition at line 161 of file download.h.

Referenced by TalkManager::FormatHostInfo(), and SetHostChain().

const unsigned download::DownloadManager::kProxyMapScale = 16
static

Definition at line 174 of file download.h.

Referenced by UpdateProxiesUnlocked().

pthread_mutex_t* download::DownloadManager::lock_options_
private

Definition at line 324 of file download.h.

Referenced by DownloadManager(), GeoSortServers(), GetHostInfo(), GetMetalinkInfo(), GetProxyInfo(), GetTimeout(), ProbeGeo(), ProbeHosts(), RebalanceProxies(), SetCredentialsAttachment(), SetDnsParameters(), SetDnsServer(), SetDnsTtlLimits(), SetHostChain(), SetHostResetDelay(), SetIpPreference(), SetLowSpeedLimit(), SetMaxIpaddrPerProxy(), SetMetalinkChain(), SetMetalinkResetDelay(), SetProxyChain(), SetProxyGroupResetDelay(), SetProxyTemplates(), SetRetryParameters(), SetTimeout(), SwitchHostInfo(), SwitchProxy(), and SwitchProxyGroup().

pthread_mutex_t* download::DownloadManager::lock_synchronous_mode_
private

Definition at line 325 of file download.h.

Referenced by DownloadManager(), and Fetch().

atomic_int32 download::DownloadManager::multi_threaded_
private

Definition at line 315 of file download.h.

Referenced by DownloadManager(), Fetch(), and Spawn().

std::string download::DownloadManager::name_
private

Name of the download manager (default is "standard")

Definition at line 429 of file download.h.

Referenced by Fetch(), GeoSortServers(), MainDownload(), ProbeHosts(), SetDnsServer(), SetProxyChain(), SetShardingPolicy(), Spawn(), SwitchHostInfo(), SwitchProxy(), and UpdateProxiesUnlocked().

unsigned download::DownloadManager::opt_backoff_init_ms_
private

Definition at line 331 of file download.h.

Referenced by Clone(), and SetRetryParameters().

unsigned download::DownloadManager::opt_backoff_max_ms_
private

Definition at line 332 of file download.h.

Referenced by Clone(), and SetRetryParameters().

std::string download::DownloadManager::opt_dns_server_
private

Definition at line 326 of file download.h.

Referenced by Clone(), GetDnsServer(), and SetDnsServer().

HostInfo download::DownloadManager::opt_host_
private

Definition at line 351 of file download.h.

Referenced by Clone(), GetHostInfo(), num_hosts(), ProbeGeo(), ProbeHosts(), SetHostChain(), SetHostResetDelay(), SwitchHost(), and UpdateProxiesUnlocked().

std::vector<int>* download::DownloadManager::opt_host_chain_rtt_
private

Created by SetHostChain(), filled by probe_hosts. Contains time to get .cvmfschecksum in ms. -1 is unprobed, -2 is error.

Definition at line 356 of file download.h.

Referenced by Clone(), GetHostInfo(), ProbeGeo(), ProbeHosts(), and SetHostChain().

dns::IpPreference download::DownloadManager::opt_ip_preference_
private

If a proxy has IPv4 and IPv6 addresses, which one to prefer

Definition at line 439 of file download.h.

Referenced by Clone(), opt_ip_preference(), SetIpPreference(), and SetProxyChain().

bool download::DownloadManager::opt_ipv4_only_
private

Definition at line 334 of file download.h.

Referenced by DownloadManager(), and SetDnsParameters().

unsigned download::DownloadManager::opt_low_speed_limit_
private

Definition at line 329 of file download.h.

Referenced by Clone(), and SetLowSpeedLimit().

unsigned download::DownloadManager::opt_max_retries_
private

Definition at line 330 of file download.h.

Referenced by Clone(), and SetRetryParameters().

HostInfo download::DownloadManager::opt_metalink_
private

Definition at line 347 of file download.h.

Referenced by CheckMetalinkChain(), Clone(), GetMetalinkInfo(), num_metalinks(), SetMetalinkChain(), SetMetalinkResetDelay(), and SwitchMetalink().

time_t download::DownloadManager::opt_metalink_timestamp_link_
private

Definition at line 348 of file download.h.

Referenced by CheckMetalinkChain().

unsigned download::DownloadManager::opt_num_proxies_
private

Overall number of proxies summed over all the groups.

Definition at line 377 of file download.h.

Referenced by CloneProxyConfig(), ProbeGeo(), and SetProxyChain().

std::vector<std::string> download::DownloadManager::opt_proxies_
private

Sorted list of currently active proxy URLs (for log messages)

Definition at line 393 of file download.h.

Referenced by UpdateProxiesUnlocked().

std::string download::DownloadManager::opt_proxy_fallback_list_
private

The original proxy fallback list provided to SetProxyChain.

Definition at line 385 of file download.h.

Referenced by CloneProxyConfig(), GetFallbackProxyList(), and SetProxyChain().

std::vector<std::vector<ProxyInfo> >* download::DownloadManager::opt_proxy_groups_
private

Definition at line 359 of file download.h.

Referenced by ChooseProxyUnlocked(), CloneProxyConfig(), current_proxy_group(), GetProxyInfo(), ProbeGeo(), RebalanceProxiesUnlocked(), SetProxyChain(), SwitchProxy(), SwitchProxyGroup(), and UpdateProxiesUnlocked().

unsigned download::DownloadManager::opt_proxy_groups_current_
private

The current load-balancing group (first dimension in opt_proxy_groups_).

Definition at line 363 of file download.h.

Referenced by CloneProxyConfig(), current_proxy_group(), GetProxyInfo(), ProbeGeo(), SetProxyChain(), SwitchProxy(), and SwitchProxyGroup().

unsigned download::DownloadManager::opt_proxy_groups_current_burned_
private

Number of proxy servers that failed within current load-balance group. Between 0 and (*opt_proxy_groups_)[opt_proxy_groups_current_].size().

Definition at line 368 of file download.h.

Referenced by CloneProxyConfig(), ProbeGeo(), RebalanceProxiesUnlocked(), SetProxyChain(), SwitchProxy(), and UpdateProxiesUnlocked().

unsigned download::DownloadManager::opt_proxy_groups_fallback_
private

The index of the first fallback proxy group. If there are none, it is set to the number of regular proxy groups.

Definition at line 373 of file download.h.

Referenced by CloneProxyConfig(), GetProxyInfo(), ProbeGeo(), and SetProxyChain().

unsigned download::DownloadManager::opt_proxy_groups_reset_after_
private

Definition at line 462 of file download.h.

Referenced by Clone(), SetProxyGroupResetDelay(), and SwitchProxy().

std::string download::DownloadManager::opt_proxy_list_
private

The original proxy list provided to SetProxyChain.

Definition at line 381 of file download.h.

Referenced by CloneProxyConfig(), GetProxyList(), and SetProxyChain().

std::map<uint32_t, ProxyInfo *> download::DownloadManager::opt_proxy_map_
private

Load-balancing map of currently active proxies

Definition at line 389 of file download.h.

Referenced by ChooseProxyUnlocked(), ProbeGeo(), SetProxyChain(), and UpdateProxiesUnlocked().

bool download::DownloadManager::opt_proxy_shard_
private

Shard requests across multiple proxies via consistent hashing

Definition at line 397 of file download.h.

Referenced by CloneProxyConfig(), ShardProxies(), and UpdateProxiesUnlocked().

unsigned download::DownloadManager::opt_timeout_direct_
private

Definition at line 328 of file download.h.

Referenced by Clone(), GetTimeout(), and SetTimeout().

unsigned download::DownloadManager::opt_timeout_proxy_
private

Definition at line 327 of file download.h.

Referenced by Clone(), GetTimeout(), and SetTimeout().

time_t download::DownloadManager::opt_timestamp_backup_proxies_
private

More than one proxy group can be considered as group of primary proxies followed by backup proxy groups, e.g. at another site. If opt_proxy_groups_reset_after_ is > 0, cvmfs will reset its proxy group to the first one after opt_proxy_groups_reset_after_ seconds are elapsed.

Definition at line 460 of file download.h.

Referenced by SetProxyChain(), SetProxyGroupResetDelay(), SwitchProxy(), and SwitchProxyGroup().

time_t download::DownloadManager::opt_timestamp_failover_proxies_
private

Definition at line 461 of file download.h.

Referenced by RebalanceProxiesUnlocked(), SetProxyChain(), SetProxyGroupResetDelay(), and SwitchProxy().

UniquePtr<Pipe<kPipeDownloadJobs> > download::DownloadManager::pipe_jobs_
private

Definition at line 318 of file download.h.

Referenced by Fetch(), MainDownload(), and Spawn().

UniquePtr<Pipe<kPipeThreadTerminator> > download::DownloadManager::pipe_terminate_
private

Definition at line 316 of file download.h.

Referenced by MainDownload(), and Spawn().

std::set<CURL *>* download::DownloadManager::pool_handles_idle_
private

Definition at line 306 of file download.h.

std::set<CURL *>* download::DownloadManager::pool_handles_inuse_
private

Definition at line 307 of file download.h.

Referenced by MainDownload().

uint32_t download::DownloadManager::pool_max_handles_
private

Definition at line 308 of file download.h.

Referenced by Clone(), and DownloadManager().

Prng download::DownloadManager::prng_
private

Definition at line 305 of file download.h.

Referenced by DownloadManager(), GeoSortServers(), and UpdateProxiesUnlocked().

std::string download::DownloadManager::proxy_template_direct_
private

Used to replace @ in the Geo-API calls to order Stratum 1 servers, in case the active proxy is DIRECT (no proxy). Should be a UUID identifying the host.

Definition at line 446 of file download.h.

Referenced by Clone(), and SetProxyTemplates().

std::string download::DownloadManager::proxy_template_forced_
private

Used to force a value for @ in the Geo-API calls to order Stratum 1 servers. If empty, the fully qualified domain name of the active proxy server is used.

Definition at line 452 of file download.h.

Referenced by Clone(), and SetProxyTemplates().

dns::NormalResolver* download::DownloadManager::resolver_
private

Used to resolve proxy addresses (host addresses are resolved by the proxy).

Definition at line 434 of file download.h.

Referenced by Clone(), DownloadManager(), SetDnsParameters(), SetDnsServer(), SetDnsTtlLimits(), SetMaxIpaddrPerProxy(), and SetProxyChain().

SharedPtr<ShardingPolicy> download::DownloadManager::sharding_policy_
private

Sharding policy deciding which proxy should be chosen for each download request

Sharding policy is shared between all download managers. As such shared pointers are used to allow for proper clean-up afterwards in the destructor (We cannot assume the order in which the download managers are stopped)

Definition at line 407 of file download.h.

Referenced by Clone(), and SetProxyChain().

SslCertificateStore download::DownloadManager::ssl_certificate_store_
private

Carries the path settings for SSL certificates

Definition at line 475 of file download.h.

Referenced by Clone(), and UseSystemCertificatePath().

pthread_t download::DownloadManager::thread_download_
private

Definition at line 314 of file download.h.

Referenced by Spawn().

char* download::DownloadManager::user_agent_
private

Definition at line 312 of file download.h.

struct pollfd* download::DownloadManager::watch_fds_
private

Definition at line 319 of file download.h.

Referenced by CallbackCurlSocket(), and MainDownload().

uint32_t download::DownloadManager::watch_fds_inuse_
private

Definition at line 321 of file download.h.

Referenced by CallbackCurlSocket(), and MainDownload().

uint32_t download::DownloadManager::watch_fds_max_
private

Definition at line 322 of file download.h.

Referenced by CallbackCurlSocket(), and DownloadManager().

uint32_t download::DownloadManager::watch_fds_size_
private

Definition at line 320 of file download.h.

Referenced by CallbackCurlSocket(), and MainDownload().

The documentation for this class was generated from the following files: