GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/network/download.h
Date: 2024-04-28 02:33:07
Exec Total Coverage
Lines: 20 24 83.3%
Branches: 22 44 50.0%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 */
4
5 #ifndef CVMFS_NETWORK_DOWNLOAD_H_
6 #define CVMFS_NETWORK_DOWNLOAD_H_
7
8 #include <poll.h>
9 #include <pthread.h>
10 #include <stdint.h>
11 #include <unistd.h>
12
13 #include <cstdio>
14 #include <map>
15 #include <set>
16 #include <string>
17 #include <vector>
18
19 #include "gtest/gtest_prod.h"
20
21 #include "compression.h"
22 #include "crypto/hash.h"
23 #include "duplex_curl.h"
24 #include "network/dns.h"
25 #include "network/health_check.h"
26 #include "network/jobinfo.h"
27 #include "network/network_errors.h"
28 #include "network/sharding_policy.h"
29 #include "network/sink.h"
30 #include "ssl.h"
31 #include "statistics.h"
32 #include "util/atomic.h"
33 #include "util/pipe.h"
34 #include "util/pointer.h"
35 #include "util/prng.h"
36 #include "util/shared_ptr.h"
37
38 class InterruptCue;
39
40 namespace download {
41
42 struct Counters {
43 perf::Counter *sz_transferred_bytes;
44 perf::Counter *sz_transfer_time; // measured in milliseconds
45 perf::Counter *n_requests;
46 perf::Counter *n_retries;
47 perf::Counter *n_proxy_failover;
48 perf::Counter *n_host_failover;
49
50 157 explicit Counters(perf::StatisticsTemplate statistics) {
51
3/6
✓ Branch 2 taken 157 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 157 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 157 times.
✗ Branch 10 not taken.
 157 sz_transferred_bytes = statistics.RegisterTemplated("sz_transferred_bytes",
52 "Number of transferred bytes");
53
3/6
✓ Branch 2 taken 157 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 157 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 157 times.
✗ Branch 10 not taken.
 157 sz_transfer_time = statistics.RegisterTemplated("sz_transfer_time",
54 "Transfer time (milliseconds)");
55
3/6
✓ Branch 2 taken 157 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 157 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 157 times.
✗ Branch 10 not taken.
 157 n_requests = statistics.RegisterTemplated("n_requests",
56 "Number of requests");
57
3/6
✓ Branch 2 taken 157 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 157 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 157 times.
✗ Branch 10 not taken.
 157 n_retries = statistics.RegisterTemplated("n_retries", "Number of retries");
58
3/6
✓ Branch 2 taken 157 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 157 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 157 times.
✗ Branch 10 not taken.
 157 n_proxy_failover = statistics.RegisterTemplated("n_proxy_failover",
59 "Number of proxy failovers");
60
3/6
✓ Branch 2 taken 157 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 157 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 157 times.
✗ Branch 10 not taken.
 157 n_host_failover = statistics.RegisterTemplated("n_host_failover",
61 "Number of host failovers");
62 157 }
63 }; // Counters
64
65 /**
66 * Manages blocks of arrays of curl_slist storing header strings. In contrast
67 * to curl's slists, these ones don't take ownership of the header strings.
68 * Overall number of elements is limited as number of concurrent connections
69 * is limited.
70 *
71 * Only use curl_slist objects created in the same HeaderLists instance in this
72 * class
73 */
74 class HeaderLists {
75 FRIEND_TEST(T_HeaderLists, Intrinsics);
76 public:
77 ~HeaderLists();
78 curl_slist *GetList(const char *header);
79 curl_slist *DuplicateList(curl_slist *slist);
80 void AppendHeader(curl_slist *slist, const char *header);
81 void CutHeader(const char *header, curl_slist **slist);
82 void PutList(curl_slist *slist);
83 std::string Print(curl_slist *slist);
84
85 private:
86 static const unsigned kBlockSize = 4096/sizeof(curl_slist);
87
88 70244 bool IsUsed(curl_slist *slist) { return slist->data != NULL; }
89 curl_slist *Get(const char *header);
90 void Put(curl_slist *slist);
91 void AddBlock();
92
93 std::vector<curl_slist *> blocks_; // List of curl_slist blocks
94 };
95
96
97 /**
98 * Provides hooks to attach per-transfer credentials to curl handles.
99 * Overwritten by the AuthzX509Attachment in authz_curl.cc. Needs to be
100 * thread-safe because it can be potentially used by multiple DownloadManagers.
101 */
102 class CredentialsAttachment {
103 public:
104 80 virtual ~CredentialsAttachment() { }
105 virtual bool ConfigureCurlHandle(CURL *curl_handle,
106 pid_t pid,
107 void **info_data) = 0;
108 virtual void ReleaseCurlHandle(CURL *curl_handle, void *info_data) = 0;
109 };
110
111
112 /**
113 * Note when adding new fields: Clone() probably needs to be adjusted, too.
114 * TODO(jblomer): improve ordering of members
115 */
116 class DownloadManager { // NOLINT(clang-analyzer-optin.performance.Padding)
117 FRIEND_TEST(T_Download, ValidateGeoReply);
118 FRIEND_TEST(T_Download, StripDirect);
119 FRIEND_TEST(T_Download, EscapeUrl);
120
121 public:
122 struct ProxyInfo {
123 ProxyInfo() { }
124
1/2
✓ Branch 2 taken 64 times.
✗ Branch 3 not taken.
 64 explicit ProxyInfo(const std::string &url) : url(url) { }
125 4 ProxyInfo(const dns::Host &host, const std::string &url)
126 4 : host(host)
127
1/2
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
 4 , url(url)
128 4 { }
129 std::string Print();
130 dns::Host host;
131 std::string url;
132 };
133
134 enum ProxySetModes {
135 kSetProxyRegular = 0,
136 kSetProxyFallback,
137 kSetProxyBoth,
138 };
139
140 /**
141 * No attempt was made to order stratum 1 servers
142 */
143 static const int kProbeUnprobed;
144 /**
145 * The rtt to a stratum 1 could not be determined because the stratum 1
146 * was unreachable.
147 */
148 static const int kProbeDown;
149 /**
150 * The stratum 1 server was put in order according to a Geo-API result
151 */
152 static const int kProbeGeo;
153
154 static const unsigned kDnsDefaultRetries = 1;
155 static const unsigned kDnsDefaultTimeoutMs = 3000;
156 static const unsigned kProxyMapScale = 16;
157
158 DownloadManager(const unsigned max_pool_handles,
159 const perf::StatisticsTemplate &statistics,
160 const std::string &name = "standard");
161 ~DownloadManager();
162
163 static int ParseHttpCode(const char digits[3]);
164
165 void Spawn();
166 DownloadManager *Clone(const perf::StatisticsTemplate &statistics,
167 const std::string &cloned_name);
168 Failures Fetch(JobInfo *info);
169
170 void SetCredentialsAttachment(CredentialsAttachment *ca);
171 std::string GetDnsServer() const;
172 void SetDnsServer(const std::string &address);
173 void SetDnsParameters(const unsigned retries, const unsigned timeout_ms);
174 void SetDnsTtlLimits(const unsigned min_seconds, const unsigned max_seconds);
175 void SetIpPreference(const dns::IpPreference preference);
176 void SetTimeout(const unsigned seconds_proxy, const unsigned seconds_direct);
177 void GetTimeout(unsigned *seconds_proxy, unsigned *seconds_direct);
178 void SetLowSpeedLimit(const unsigned low_speed_limit);
179 void SetHostChain(const std::string &host_list);
180 void SetHostChain(const std::vector<std::string> &host_list);
181 void GetHostInfo(std::vector<std::string> *host_chain,
182 std::vector<int> *rtt, unsigned *current_host);
183 void ProbeHosts();
184 bool ProbeGeo();
185 // Sort list of servers using the Geo API. If the output_order
186 // vector is NULL, then the servers vector input is itself sorted.
187 // If it is non-NULL, then servers is left unchanged and the zero-based
188 // ordering is stored into output_order.
189 bool GeoSortServers(std::vector<std::string> *servers,
190 std::vector<uint64_t> *output_order = NULL);
191 void SwitchHost();
192 void SetProxyChain(const std::string &proxy_list,
193 const std::string &fallback_proxy_list,
194 const ProxySetModes set_mode);
195 void GetProxyInfo(std::vector< std::vector<ProxyInfo> > *proxy_chain,
196 unsigned *current_group,
197 unsigned *fallback_group);
198 std::string GetProxyList();
199 std::string GetFallbackProxyList();
200 void ShardProxies();
201 void RebalanceProxies();
202 void SwitchProxyGroup();
203 void SetProxyGroupResetDelay(const unsigned seconds);
204 void SetHostResetDelay(const unsigned seconds);
205 void SetRetryParameters(const unsigned max_retries,
206 const unsigned backoff_init_ms,
207 const unsigned backoff_max_ms);
208 void SetMaxIpaddrPerProxy(unsigned limit);
209 void SetProxyTemplates(const std::string &direct, const std::string &forced);
210 void EnableInfoHeader();
211 void EnableRedirects();
212 void EnableIgnoreSignatureFailures();
213 void EnableHTTPTracing();
214 void AddHTTPTracingHeader(const std::string &header);
215 void UseSystemCertificatePath();
216
217 bool SetShardingPolicy(const ShardingPolicySelector type);
218 void SetFailoverIndefinitely();
219 ✗ void SetFqrn(const std::string &fqrn) { fqrn_ = fqrn; }
220
221 1 unsigned num_hosts() {
222
1/2
✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
 1 if (opt_host_chain_) return opt_host_chain_->size();
223 ✗ return 0;
224 }
225
226 ✗ dns::IpPreference opt_ip_preference() const {
227 ✗ return opt_ip_preference_;
228 }
229
230 private:
231 static int CallbackCurlSocket(CURL *easy, curl_socket_t s, int action,
232 void *userp, void *socketp);
233 static void *MainDownload(void *data);
234
235 bool StripDirect(const std::string &proxy_list, std::string *cleaned_list);
236 bool ValidateGeoReply(const std::string &reply_order,
237 const unsigned expected_size,
238 std::vector<uint64_t> *reply_vals);
239 void SwitchHost(JobInfo *info);
240 void SwitchProxy(JobInfo *info);
241 ProxyInfo *ChooseProxyUnlocked(const shash::Any *hash);
242 void UpdateProxiesUnlocked(const std::string &reason);
243 void RebalanceProxiesUnlocked(const std::string &reason);
244 CURL *AcquireCurlHandle();
245 void ReleaseCurlHandle(CURL *handle);
246 void ReleaseCredential(JobInfo *info);
247 void InitializeRequest(JobInfo *info, CURL *handle);
248 void SetUrlOptions(JobInfo *info);
249 bool ValidateProxyIpsUnlocked(const std::string &url, const dns::Host &host);
250 void UpdateStatistics(CURL *handle);
251 bool CanRetry(const JobInfo *info);
252 void Backoff(JobInfo *info);
253 void SetNocache(JobInfo *info);
254 void SetRegularCache(JobInfo *info);
255 bool VerifyAndFinalize(const int curl_error, JobInfo *info);
256 void InitHeaders();
257 void CloneProxyConfig(DownloadManager *clone);
258
259 bool EscapeUrlChar(unsigned char input, char output[3]);
260 std::string EscapeUrl(const int64_t jobinfo_id, const std::string &url);
261 unsigned EscapeHeader(const std::string &header, char *escaped_buf,
262 size_t buf_size);
263
264 102 inline std::vector<ProxyInfo> *current_proxy_group() const {
265
1/2
✓ Branch 0 taken 102 times.
✗ Branch 1 not taken.
 102 return (opt_proxy_groups_ ?
266 102 &((*opt_proxy_groups_)[opt_proxy_groups_current_]) : NULL);
267 }
268
269 Prng prng_;
270 std::set<CURL *> *pool_handles_idle_;
271 std::set<CURL *> *pool_handles_inuse_;
272 uint32_t pool_max_handles_;
273 CURLM *curl_multi_;
274 HeaderLists *header_lists_;
275 curl_slist *default_headers_;
276 char *user_agent_;
277
278 pthread_t thread_download_;
279 atomic_int32 multi_threaded_;
280 UniquePtr<Pipe<kPipeThreadTerminator> > pipe_terminate_;
281
282 UniquePtr<Pipe<kPipeDownloadJobs> > pipe_jobs_;
283 struct pollfd *watch_fds_;
284 uint32_t watch_fds_size_;
285 uint32_t watch_fds_inuse_;
286 uint32_t watch_fds_max_;
287
288 pthread_mutex_t *lock_options_;
289 pthread_mutex_t *lock_synchronous_mode_;
290 std::string opt_dns_server_;
291 unsigned opt_timeout_proxy_;
292 unsigned opt_timeout_direct_;
293 unsigned opt_low_speed_limit_;
294 unsigned opt_max_retries_;
295 unsigned opt_backoff_init_ms_;
296 unsigned opt_backoff_max_ms_;
297 bool enable_info_header_;
298 bool opt_ipv4_only_;
299 bool follow_redirects_;
300
301 /**
302 * Ignore signature failures during download.
303 * In general it is a bad idea to do this!
304 */
305 bool ignore_signature_failures_;
306
307 bool enable_http_tracing_;
308 std::vector<std::string> http_tracing_headers_;
309
310 // Host list
311 std::vector<std::string> *opt_host_chain_;
312 /**
313 * Created by SetHostChain(), filled by probe_hosts. Contains time to get
314 * .cvmfschecksum in ms. -1 is unprobed, -2 is error.
315 */
316 std::vector<int> *opt_host_chain_rtt_;
317 unsigned opt_host_chain_current_;
318
319 // Proxy list
320 std::vector< std::vector<ProxyInfo> > *opt_proxy_groups_;
321 /**
322 * The current load-balancing group (first dimension in opt_proxy_groups_).
323 */
324 unsigned opt_proxy_groups_current_;
325 /**
326 * Number of proxy servers that failed within current load-balance group.
327 * Between 0 and (*opt_proxy_groups_)[opt_proxy_groups_current_].size().
328 */
329 unsigned opt_proxy_groups_current_burned_;
330 /**
331 * The index of the first fallback proxy group. If there are none,
332 * it is set to the number of regular proxy groups.
333 */
334 unsigned opt_proxy_groups_fallback_;
335 /**
336 * Overall number of proxies summed over all the groups.
337 */
338 unsigned opt_num_proxies_;
339 /**
340 * The original proxy list provided to SetProxyChain.
341 */
342 std::string opt_proxy_list_;
343 /**
344 * The original proxy fallback list provided to SetProxyChain.
345 */
346 std::string opt_proxy_fallback_list_;
347 /**
348 * Load-balancing map of currently active proxies
349 */
350 std::map<uint32_t, ProxyInfo *> opt_proxy_map_;
351 /**
352 * Sorted list of currently active proxy URLs (for log messages)
353 */
354 std::vector<std::string> opt_proxy_urls_;
355 /**
356 * Shard requests across multiple proxies via consistent hashing
357 */
358 bool opt_proxy_shard_;
359
360 /**
361 * Sharding policy deciding which proxy should be chosen for each download
362 * request
363 *
364 * Sharding policy is shared between all download managers. As such shared
365 * pointers are used to allow for proper clean-up afterwards in the destructor
366 * (We cannot assume the order in which the download managers are stopped)
367 */
368 SharedPtr<ShardingPolicy> sharding_policy_;
369 /**
370 * Health check for the proxies
371 *
372 * Health check is shared between all download managers. As such shared
373 * pointers are used to allow for proper clean-up afterwards in the destructor
374 * (We cannot assume the order in which the download managers are stopped)
375 */
376 SharedPtr<HealthCheck> health_check_;
377 /**
378 * Endless retries for a failed download (hard failures will result in abort)
379 */
380 bool failover_indefinitely_;
381 /**
382 * Repo name. Needed for the re-try logic if a download was unsuccessful
383 * Used in sharding policy && Interrupted()
384 */
385 std::string fqrn_;
386
387 /**
388 * Name of the download manager (default is "standard")
389 */
390 std::string name_;
391
392 /**
393 * Used to resolve proxy addresses (host addresses are resolved by the proxy).
394 */
395 dns::NormalResolver *resolver_;
396
397 /**
398 * If a proxy has IPv4 and IPv6 addresses, which one to prefer
399 */
400 dns::IpPreference opt_ip_preference_;
401
402 /**
403 * Used to replace @proxy@ in the Geo-API calls to order Stratum 1 servers,
404 * in case the active proxy is DIRECT (no proxy). Should be a UUID
405 * identifying the host.
406 */
407 std::string proxy_template_direct_;
408 /**
409 * Used to force a value for @proxy@ in the Geo-API calls to order Stratum 1
410 * servers. If empty, the fully qualified domain name of the active proxy
411 * server is used.
412 */
413 std::string proxy_template_forced_;
414
415 /**
416 * More than one proxy group can be considered as group of primary proxies
417 * followed by backup proxy groups, e.g. at another site.
418 * If opt_proxy_groups_reset_after_ is > 0, cvmfs will reset its proxy group
419 * to the first one after opt_proxy_groups_reset_after_ seconds are elapsed.
420 */
421 time_t opt_timestamp_backup_proxies_;
422 time_t opt_timestamp_failover_proxies_; // failover within the same group
423 unsigned opt_proxy_groups_reset_after_;
424
425 /**
426 * Similarly to proxy group reset, we'd also like to reset the host after a
427 * failover. Host outages can last longer and might come with a separate
428 * reset delay.
429 */
430 time_t opt_timestamp_backup_host_;
431 unsigned opt_host_reset_after_;
432
433 CredentialsAttachment *credentials_attachment_;
434
435 /**
436 * Writes and reads should be atomic because reading happens in a different
437 * thread than writing.
438 */
439 Counters *counters_;
440
441 /**
442 * Carries the path settings for SSL certificates
443 */
444 SslCertificateStore ssl_certificate_store_;
445 }; // DownloadManager
446
447 } // namespace download
448
449 #endif // CVMFS_NETWORK_DOWNLOAD_H_
450