5 #ifndef CVMFS_NETWORK_DOWNLOAD_H_
6 #define CVMFS_NETWORK_DOWNLOAD_H_
19 #include "gtest/gtest_prod.h"
95 texts[1] =
"local I/O failure";
96 texts[2] =
"malformed URL";
97 texts[3] =
"failed to resolve proxy address";
98 texts[4] =
"failed to resolve host address";
99 texts[5] =
"all proxies failed, trying host fail-over";
100 texts[6] =
"proxy connection problem";
101 texts[7] =
"host connection problem";
102 texts[8] =
"proxy returned HTTP error";
103 texts[9] =
"host returned HTTP error";
104 texts[10] =
"corrupted data received";
105 texts[11] =
"resource too big to download";
106 texts[12] =
"unknown network error";
107 texts[13] =
"Unsupported URL in protocol";
108 texts[14] =
"proxy serving data too slowly";
109 texts[15] =
"host serving data too slowly";
110 texts[16] =
"proxy data transfer cut short";
111 texts[17] =
"host data transfer cut short";
112 texts[18] =
"request canceled";
113 texts[19] =
"no text";
139 "Number of transferred bytes");
141 "Transfer time (milliseconds)");
143 "Number of requests");
146 "Number of proxy failovers");
148 "Number of host failovers");
224 JobInfo(
const std::string *u,
const bool c,
const bool ph,
235 JobInfo(
const std::string *u,
const bool c,
const bool ph, FILE *f,
246 JobInfo(
const std::string *u,
const bool c,
const bool ph,
256 JobInfo(
const std::string *u,
const bool c,
const bool ph,
267 JobInfo(
const std::string *u,
const bool ph) {
321 curl_slist *
GetList(
const char *header);
323 void AppendHeader(curl_slist *slist,
const char *header);
324 void CutHeader(
const char *header, curl_slist **slist);
325 void PutList(curl_slist *slist);
326 std::string
Print(curl_slist *slist);
331 bool IsUsed(curl_slist *slist) {
return slist->data != NULL; }
332 curl_slist *
Get(
const char *header);
333 void Put(curl_slist *slist);
350 void **info_data) = 0;
410 void Init(
const unsigned max_pool_handles,
421 void SetDnsTtlLimits(
const unsigned min_seconds,
const unsigned max_seconds);
423 void SetTimeout(
const unsigned seconds_proxy,
const unsigned seconds_direct);
424 void GetTimeout(
unsigned *seconds_proxy,
unsigned *seconds_direct);
427 void SetHostChain(
const std::vector<std::string> &host_list);
428 void GetHostInfo(std::vector<std::string> *host_chain,
429 std::vector<int> *rtt,
unsigned *current_host);
437 std::vector<uint64_t> *output_order = NULL);
440 const std::string &fallback_proxy_list,
442 void GetProxyInfo(std::vector< std::vector<ProxyInfo> > *proxy_chain,
443 unsigned *current_group,
444 unsigned *fallback_group);
453 const unsigned backoff_init_ms,
454 const unsigned backoff_max_ms);
472 void *userp,
void *socketp);
475 bool StripDirect(
const std::string &proxy_list, std::string *cleaned_list);
477 const unsigned expected_size,
478 std::vector<uint64_t> *reply_vals);
644 #endif // CVMFS_NETWORK_DOWNLOAD_H_
unsigned opt_timeout_direct_
bool StripDirect(const std::string &proxy_list, std::string *cleaned_list)
unsigned opt_low_speed_limit_
static const unsigned kDnsDefaultTimeoutMs
unsigned opt_backoff_init_ms_
unsigned char num_used_hosts
unsigned opt_proxy_groups_current_burned_
unsigned opt_proxy_groups_reset_after_
void SetUrlOptions(JobInfo *info)
void ReleaseCredential(JobInfo *info)
std::string opt_proxy_fallback_list_
void SetHostChain(const std::string &host_list)
void SetNocache(JobInfo *info)
unsigned opt_host_reset_after_
void SetLowSpeedLimit(const unsigned low_speed_limit)
std::string proxy_template_direct_
static int ParseHttpCode(const char digits[3])
FRIEND_TEST(T_Download, ValidateGeoReply)
static const int kProbeGeo
unsigned opt_proxy_groups_current_
shash::ContextPtr hash_context
bool ValidateGeoReply(const std::string &reply_order, const unsigned expected_size, std::vector< uint64_t > *reply_vals)
std::vector< ProxyInfo > * current_proxy_group() const
time_t opt_timestamp_backup_proxies_
void SetProxyChain(const std::string &proxy_list, const std::string &fallback_proxy_list, const ProxySetModes set_mode)
std::string GetProxyList()
unsigned int current_host_chain_index
std::set< CURL * > * pool_handles_inuse_
pthread_mutex_t * lock_options_
ProxyInfo * ChooseProxyUnlocked(const shash::Any *hash)
pthread_t thread_download_
ProxyInfo(const dns::Host &host, const std::string &url)
std::string opt_proxy_list_
perf::Counter * sz_transfer_time
std::vector< std::vector< ProxyInfo > > * opt_proxy_groups_
unsigned opt_proxy_groups_fallback_
curl_slist * default_headers_
void ReleaseCurlHandle(CURL *handle)
void SetDnsServer(const std::string &address)
JobInfo(const std::string *u, const bool c, const bool ph, const shash::Any *h)
virtual ~CredentialsAttachment()
static void * MainDownload(void *data)
void SetTimeout(const unsigned seconds_proxy, const unsigned seconds_direct)
perf::Counter * n_retries
std::string opt_dns_server_
uint32_t watch_fds_inuse_
void Backoff(JobInfo *info)
perf::Counter * sz_transferred_bytes
Counter * RegisterTemplated(const std::string &name_minor, const std::string &desc)
void UpdateStatistics(CURL *handle)
void SetDnsTtlLimits(const unsigned min_seconds, const unsigned max_seconds)
void Init(const unsigned max_pool_handles, const perf::StatisticsTemplate &statistics)
std::vector< std::string > opt_proxy_urls_
const char * Code2Ascii(const Failures error)
uint32_t pool_max_handles_
unsigned char num_retries
void GetProxyInfo(std::vector< std::vector< ProxyInfo > > *proxy_chain, unsigned *current_group, unsigned *fallback_group)
void SetProxyGroupResetDelay(const unsigned seconds)
atomic_int32 multi_threaded_
dns::NormalResolver * resolver_
ProxyInfo(const std::string &url)
dns::IpPreference opt_ip_preference_
perf::Counter * n_host_failover
void UpdateProxiesUnlocked(const std::string &reason)
bool IsProxyTransferError(const Failures error)
void SetIpPreference(const dns::IpPreference preference)
perf::Counter * n_requests
JobInfo(const std::string *u, const bool c, const bool ph, const std::string *p, const shash::Any *h)
void SetRetryParameters(const unsigned max_retries, const unsigned backoff_init_ms, const unsigned backoff_max_ms)
void SetRegularCache(JobInfo *info)
void CloneProxyConfig(DownloadManager *clone)
void SetMaxIpaddrPerProxy(unsigned limit)
DownloadManager * Clone(const perf::StatisticsTemplate &statistics)
const shash::Any * expected_hash
CURL * AcquireCurlHandle()
const std::string * extra_info
std::vector< int > * opt_host_chain_rtt_
dns::IpPreference opt_ip_preference() const
cvmfs::Sink * destination_sink
SslCertificateStore ssl_certificate_store_
time_t opt_timestamp_backup_host_
std::string GetFallbackProxyList()
void SetProxyTemplates(const std::string &direct, const std::string &forced)
unsigned opt_backoff_max_ms_
std::string GetDnsServer() const
unsigned opt_num_proxies_
unsigned opt_host_chain_current_
CredentialsAttachment * credentials_attachment_
std::vector< std::string > * opt_host_chain_
struct pollfd * watch_fds_
std::map< uint32_t, ProxyInfo * > opt_proxy_map_
JobInfo(const std::string *u, const bool ph)
UniquePtr< Pipe< kPipeDownloadJobs > > pipe_jobs_
void UseSystemCertificatePath()
Failures Fetch(JobInfo *info)
InterruptCue * interrupt_cue
unsigned opt_max_retries_
bool CanRetry(const JobInfo *info)
perf::Counter * n_proxy_failover
JobInfo(const std::string *u, const bool c, const bool ph, FILE *f, const shash::Any *h)
void GetTimeout(unsigned *seconds_proxy, unsigned *seconds_direct)
std::string proxy_template_forced_
const std::string * destination_path
time_t opt_timestamp_failover_proxies_
void SetDnsParameters(const unsigned retries, const unsigned timeout_ms)
struct download::JobInfo::@4 destination_mem
UniquePtr< Pipe< kPipeThreadTerminator > > pipe_terminate_
static const int kProbeUnprobed
virtual void ReleaseCurlHandle(CURL *curl_handle, void *info_data)=0
static const unsigned kMaxMemSize
bool IsHostTransferError(const Failures error)
JobInfo(const std::string *u, const bool c, const bool ph, cvmfs::Sink *s, const shash::Any *h)
static const unsigned kDnsDefaultRetries
bool GeoSortServers(std::vector< std::string > *servers, std::vector< uint64_t > *output_order=NULL)
static const int kProbeDown
UniquePtr< Pipe< kPipeDownloadJobsResults > > pipe_job_results
Pipe used for the return value.
unsigned char num_used_proxies
Counters(perf::StatisticsTemplate statistics)
static const unsigned kProxyMapScale
void GetHostInfo(std::vector< std::string > *host_chain, std::vector< int > *rtt, unsigned *current_host)
bool ValidateProxyIpsUnlocked(const std::string &url, const dns::Host &host)
unsigned opt_timeout_proxy_
bool VerifyAndFinalize(const int curl_error, JobInfo *info)
void SwitchProxy(JobInfo *info)
void SetCredentialsAttachment(CredentialsAttachment *ca)
std::set< CURL * > * pool_handles_idle_
void RebalanceProxiesUnlocked(const std::string &reason)
pthread_mutex_t * lock_synchronous_mode_
virtual bool ConfigureCurlHandle(CURL *curl_handle, pid_t pid, void **info_data)=0
void InitializeRequest(JobInfo *info, CURL *handle)
static int CallbackCurlSocket(CURL *easy, curl_socket_t s, int action, void *userp, void *socketp)
HeaderLists * header_lists_
void SetHostResetDelay(const unsigned seconds)