5 #ifndef CVMFS_DOWNLOAD_H_
6 #define CVMFS_DOWNLOAD_H_
19 #include "gtest/gtest_prod.h"
93 texts[1] =
"local I/O failure";
94 texts[2] =
"malformed URL";
95 texts[3] =
"failed to resolve proxy address";
96 texts[4] =
"failed to resolve host address";
97 texts[5] =
"all proxies failed, trying host fail-over";
98 texts[6] =
"proxy connection problem";
99 texts[7] =
"host connection problem";
100 texts[8] =
"proxy returned HTTP error";
101 texts[9] =
"host returned HTTP error";
102 texts[10] =
"corrupted data received";
103 texts[11] =
"resource too big to download";
104 texts[12] =
"unknown network error";
105 texts[13] =
"Unsupported URL in protocol";
106 texts[14] =
"proxy serving data too slowly";
107 texts[15] =
"host serving data too slowly";
108 texts[16] =
"proxy data transfer cut short";
109 texts[17] =
"host data transfer cut short";
110 texts[18] =
"request canceled";
111 texts[19] =
"no text";
137 "Number of transferred bytes");
139 "Transfer time (miliseconds)");
141 "Number of requests");
144 "Number of proxy failovers");
146 "Number of host failovers");
222 JobInfo(
const std::string *u,
const bool c,
const bool ph,
233 JobInfo(
const std::string *u,
const bool c,
const bool ph, FILE *f,
244 JobInfo(
const std::string *u,
const bool c,
const bool ph,
254 JobInfo(
const std::string *u,
const bool c,
const bool ph,
265 JobInfo(
const std::string *u,
const bool ph) {
317 curl_slist *
GetList(
const char *header);
319 void AppendHeader(curl_slist *slist,
const char *header);
320 void CutHeader(
const char *header, curl_slist **slist);
321 void PutList(curl_slist *slist);
322 std::string
Print(curl_slist *slist);
327 bool IsUsed(curl_slist *slist) {
return slist->data != NULL; }
328 curl_slist *
Get(
const char *header);
329 void Put(curl_slist *slist);
346 void **info_data) = 0;
406 void Init(
const unsigned max_pool_handles,
417 void SetDnsTtlLimits(
const unsigned min_seconds,
const unsigned max_seconds);
419 void SetTimeout(
const unsigned seconds_proxy,
const unsigned seconds_direct);
420 void GetTimeout(
unsigned *seconds_proxy,
unsigned *seconds_direct);
423 void SetHostChain(
const std::vector<std::string> &host_list);
424 void GetHostInfo(std::vector<std::string> *host_chain,
425 std::vector<int> *rtt,
unsigned *current_host);
433 std::vector<uint64_t> *output_order = NULL);
436 const std::string &fallback_proxy_list,
438 void GetProxyInfo(std::vector< std::vector<ProxyInfo> > *proxy_chain,
439 unsigned *current_group,
440 unsigned *fallback_group);
449 const unsigned backoff_init_ms,
450 const unsigned backoff_max_ms);
468 void *userp,
void *socketp);
471 bool StripDirect(
const std::string &proxy_list, std::string *cleaned_list);
473 const unsigned expected_size,
474 std::vector<uint64_t> *reply_vals);
640 #endif // CVMFS_DOWNLOAD_H_
unsigned opt_timeout_direct_
bool StripDirect(const std::string &proxy_list, std::string *cleaned_list)
unsigned opt_low_speed_limit_
static const unsigned kDnsDefaultTimeoutMs
unsigned opt_backoff_init_ms_
unsigned char num_used_hosts
unsigned opt_proxy_groups_current_burned_
unsigned opt_proxy_groups_reset_after_
void SetUrlOptions(JobInfo *info)
void ReleaseCredential(JobInfo *info)
std::string opt_proxy_fallback_list_
void SetHostChain(const std::string &host_list)
void SetNocache(JobInfo *info)
unsigned opt_host_reset_after_
void SetLowSpeedLimit(const unsigned low_speed_limit)
std::string proxy_template_direct_
static int ParseHttpCode(const char digits[3])
FRIEND_TEST(T_Download, ValidateGeoReply)
static const int kProbeGeo
unsigned opt_proxy_groups_current_
shash::ContextPtr hash_context
bool ValidateGeoReply(const std::string &reply_order, const unsigned expected_size, std::vector< uint64_t > *reply_vals)
std::vector< ProxyInfo > * current_proxy_group() const
time_t opt_timestamp_backup_proxies_
void SetProxyChain(const std::string &proxy_list, const std::string &fallback_proxy_list, const ProxySetModes set_mode)
std::string GetProxyList()
unsigned int current_host_chain_index
std::set< CURL * > * pool_handles_inuse_
pthread_mutex_t * lock_options_
ProxyInfo * ChooseProxyUnlocked(const shash::Any *hash)
pthread_t thread_download_
ProxyInfo(const dns::Host &host, const std::string &url)
std::string opt_proxy_list_
perf::Counter * sz_transfer_time
std::vector< std::vector< ProxyInfo > > * opt_proxy_groups_
unsigned opt_proxy_groups_fallback_
curl_slist * default_headers_
void ReleaseCurlHandle(CURL *handle)
void SetDnsServer(const std::string &address)
JobInfo(const std::string *u, const bool c, const bool ph, const shash::Any *h)
virtual ~CredentialsAttachment()
static void * MainDownload(void *data)
void SetTimeout(const unsigned seconds_proxy, const unsigned seconds_direct)
perf::Counter * n_retries
std::string opt_dns_server_
uint32_t watch_fds_inuse_
void Backoff(JobInfo *info)
perf::Counter * sz_transferred_bytes
Counter * RegisterTemplated(const std::string &name_minor, const std::string &desc)
void UpdateStatistics(CURL *handle)
void SetDnsTtlLimits(const unsigned min_seconds, const unsigned max_seconds)
void Init(const unsigned max_pool_handles, const perf::StatisticsTemplate &statistics)
std::vector< std::string > opt_proxy_urls_
const char * Code2Ascii(const Failures error)
uint32_t pool_max_handles_
unsigned char num_retries
void GetProxyInfo(std::vector< std::vector< ProxyInfo > > *proxy_chain, unsigned *current_group, unsigned *fallback_group)
void SetProxyGroupResetDelay(const unsigned seconds)
atomic_int32 multi_threaded_
dns::NormalResolver * resolver_
ProxyInfo(const std::string &url)
dns::IpPreference opt_ip_preference_
perf::Counter * n_host_failover
void UpdateProxiesUnlocked(const std::string &reason)
bool IsProxyTransferError(const Failures error)
void SetIpPreference(const dns::IpPreference preference)
perf::Counter * n_requests
JobInfo(const std::string *u, const bool c, const bool ph, const std::string *p, const shash::Any *h)
void SetRetryParameters(const unsigned max_retries, const unsigned backoff_init_ms, const unsigned backoff_max_ms)
void SetRegularCache(JobInfo *info)
void CloneProxyConfig(DownloadManager *clone)
void SetMaxIpaddrPerProxy(unsigned limit)
DownloadManager * Clone(const perf::StatisticsTemplate &statistics)
const shash::Any * expected_hash
CURL * AcquireCurlHandle()
const std::string * extra_info
std::vector< int > * opt_host_chain_rtt_
dns::IpPreference opt_ip_preference() const
cvmfs::Sink * destination_sink
SslCertificateStore ssl_certificate_store_
time_t opt_timestamp_backup_host_
std::string GetFallbackProxyList()
void SetProxyTemplates(const std::string &direct, const std::string &forced)
unsigned opt_backoff_max_ms_
std::string GetDnsServer() const
unsigned opt_num_proxies_
unsigned opt_host_chain_current_
CredentialsAttachment * credentials_attachment_
std::vector< std::string > * opt_host_chain_
struct pollfd * watch_fds_
std::map< uint32_t, ProxyInfo * > opt_proxy_map_
JobInfo(const std::string *u, const bool ph)
void UseSystemCertificatePath()
Failures Fetch(JobInfo *info)
InterruptCue * interrupt_cue
unsigned opt_max_retries_
struct download::JobInfo::@3 destination_mem
bool CanRetry(const JobInfo *info)
perf::Counter * n_proxy_failover
JobInfo(const std::string *u, const bool c, const bool ph, FILE *f, const shash::Any *h)
void GetTimeout(unsigned *seconds_proxy, unsigned *seconds_direct)
std::string proxy_template_forced_
const std::string * destination_path
time_t opt_timestamp_failover_proxies_
void SetDnsParameters(const unsigned retries, const unsigned timeout_ms)
static const int kProbeUnprobed
virtual void ReleaseCurlHandle(CURL *curl_handle, void *info_data)=0
static const unsigned kMaxMemSize
bool IsHostTransferError(const Failures error)
JobInfo(const std::string *u, const bool c, const bool ph, cvmfs::Sink *s, const shash::Any *h)
static const unsigned kDnsDefaultRetries
bool GeoSortServers(std::vector< std::string > *servers, std::vector< uint64_t > *output_order=NULL)
static const int kProbeDown
unsigned char num_used_proxies
Counters(perf::StatisticsTemplate statistics)
static const unsigned kProxyMapScale
void GetHostInfo(std::vector< std::string > *host_chain, std::vector< int > *rtt, unsigned *current_host)
bool ValidateProxyIpsUnlocked(const std::string &url, const dns::Host &host)
unsigned opt_timeout_proxy_
bool VerifyAndFinalize(const int curl_error, JobInfo *info)
void SwitchProxy(JobInfo *info)
void SetCredentialsAttachment(CredentialsAttachment *ca)
std::set< CURL * > * pool_handles_idle_
void RebalanceProxiesUnlocked(const std::string &reason)
pthread_mutex_t * lock_synchronous_mode_
virtual bool ConfigureCurlHandle(CURL *curl_handle, pid_t pid, void **info_data)=0
void InitializeRequest(JobInfo *info, CURL *handle)
static int CallbackCurlSocket(CURL *easy, curl_socket_t s, int action, void *userp, void *socketp)
HeaderLists * header_lists_
void SetHostResetDelay(const unsigned seconds)