GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/network/download.cc
Date: 2026-08-23 02:40:52
Exec Total Coverage
Lines: 878 1774 49.5%
Branches: 674 2285 29.5%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * The download module provides an interface for fetching files via HTTP
5 * and file. It is internally using libcurl and the asynchronous DNS resolver
6 * c-ares. The JobInfo struct describes a single file/url to download and
7 * keeps the state during the several phases of downloading.
8 *
9 * The module starts in single-threaded mode and can be switched to multi-
10 * threaded mode by Spawn(). In multi-threaded mode, the Fetch() function still
11 * blocks but there is a separate I/O thread using asynchronous I/O, which
12 * maintains all concurrent connections simultaneously. As there might be more
13 * than 1024 file descriptors for the CernVM-FS process, the I/O thread uses
14 * poll and the libcurl multi socket interface.
15 *
16 * While downloading, files can be decompressed and the secure hash can be
17 * calculated on the fly.
18 *
19 * The module also implements failure handling. If corrupted data has been
20 * downloaded, the transfer is restarted using HTTP "no-cache" pragma.
21 * A "host chain" can be configured. When a host fails, there is automatic
22 * fail-over to the next host in the chain until all hosts are probed.
23 * Similarly a chain of proxy sets can be configured. Inside a proxy set,
24 * proxies are selected randomly (load-balancing set).
25 */
26
27 // TODO(jblomer): MS for time summing
28
29 #include "download.h"
30
31 #include <alloca.h>
32 #include <errno.h>
33 #include <inttypes.h>
34 #include <poll.h>
35 #include <pthread.h>
36 #include <signal.h>
37 #include <stdint.h>
38 #include <sys/time.h>
39 #include <unistd.h>
40
41 #include <algorithm>
42 #include <cassert>
43 #include <cstdio>
44 #include <cstdlib>
45 #include <cstring>
46 #include <map>
47 #include <set>
48 #include <utility>
49
50 #include "compression/compression.h"
51 #include "crypto/hash.h"
52 #include "duplex_curl.h" // IWYU pragma: keep
53 #include "interrupt.h"
54 #include "network/sink_mem.h"
55 #include "network/sink_path.h"
56 #include "sanitizer.h"
57 #include "ssl.h"
58 #include "util/algorithm.h"
59 #include "util/atomic.h"
60 #include "util/exception.h"
61 #include "util/logging.h"
62 #include "util/posix.h"
63 #include "util/prng.h"
64 #include "util/smalloc.h"
65 #include "util/string.h"
66
67 using namespace std; // NOLINT
68
69 namespace download {
70
71 /**
72 * Returns the status if an interrupt happened for a given repository.
73 *
74 * Used only in case CVMFS_FAILOVER_INDEFINITELY (failover_indefinitely_) is set
75 * where failed downloads are retried indefinitely, unless an interrupt occurred
76 *
77 * @note If you use this functionality you need to change the source code of
78 * e.g. cvmfs_config reload to create a sentinel file. See comment below.
79 *
80 * @return true if an interrupt occurred
81 * false otherwise
82 */
83 ✗ bool Interrupted(const std::string &fqrn, JobInfo *info) {
84 ✗ if (info->allow_failure()) {
85 ✗ return true;
86 }
87
88 ✗ if (!fqrn.empty()) {
89 // it is up to the user the create this sentinel file ("pause_file") if
90 // CVMFS_FAILOVER_INDEFINITELY is used. It must be created during
91 // "cvmfs_config reload" and "cvmfs_config reload $fqrn"
92 ✗ const std::string pause_file = std::string("/var/run/cvmfs/interrupt.")
93 ✗ + fqrn;
94
95 ✗ LogCvmfs(kLogDownload, kLogDebug,
96 "(id %" PRId64 ") Interrupted(): checking for existence of %s",
97 info->id(), pause_file.c_str());
98 ✗ if (FileExists(pause_file)) {
99 ✗ LogCvmfs(kLogDownload, kLogDebug,
100 "(id %" PRId64 ") Interrupt marker found - "
101 "Interrupting current download, this will EIO outstanding IO.",
102 info->id());
103 ✗ if (0 != unlink(pause_file.c_str())) {
104 ✗ LogCvmfs(kLogDownload, kLogDebug,
105 "(id %" PRId64 ") Couldn't delete interrupt marker: errno=%d",
106 ✗ info->id(), errno);
107 }
108 ✗ return true;
109 }
110 }
111 ✗ return false;
112 }
113
114 4715 static Failures PrepareDownloadDestination(JobInfo *info) {
115
3/6
✓ Branch 1 taken 4715 times.
✗ Branch 2 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 4715 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 4715 times.
 4715 if (info->sink() != NULL && !info->sink()->IsValid()) {
116 ✗ cvmfs::PathSink *psink = dynamic_cast<cvmfs::PathSink *>(info->sink());
117 ✗ if (psink != NULL) {
118 ✗ LogCvmfs(kLogDownload, kLogDebug,
119 "(id %" PRId64 ") Failed to open path %s: %s (errno=%d).",
120 ✗ info->id(), psink->path().c_str(), strerror(errno), errno);
121 ✗ return kFailLocalIO;
122 } else {
123 ✗ LogCvmfs(kLogDownload, kLogDebug,
124 "(id %" PRId64 ") "
125 "Failed to create a valid sink: \n %s",
126 ✗ info->id(), info->sink()->Describe().c_str());
127 ✗ return kFailOther;
128 }
129 }
130
131 4715 return kFailOk;
132 }
133
134
135 /**
136 * Called by curl for every HTTP header. Not called for file:// transfers.
137 */
138 13999 static size_t CallbackCurlHeader(void *ptr, size_t size, size_t nmemb,
139 void *info_link) {
140 13999 const size_t num_bytes = size * nmemb;
141
1/2
✓ Branch 2 taken 13999 times.
✗ Branch 3 not taken.
 13999 const string header_line(static_cast<const char *>(ptr), num_bytes);
142 13999 JobInfo *info = static_cast<JobInfo *>(info_link);
143
144 // LogCvmfs(kLogDownload, kLogDebug, "REMOVE-ME: Header callback with %s",
145 // header_line.c_str());
146
147 // Check http status codes
148
4/6
✓ Branch 2 taken 13999 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 13999 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 276 times.
✓ Branch 10 taken 13723 times.
 13999 if (HasPrefix(header_line, "HTTP/1.", false)) {
149
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 276 times.
 276 if (header_line.length() < 10) {
150 ✗ return 0;
151 }
152
153 unsigned i;
154
5/6
✓ Branch 1 taken 552 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 276 times.
✓ Branch 5 taken 276 times.
✓ Branch 6 taken 276 times.
✓ Branch 7 taken 276 times.
 552 for (i = 8; (i < header_line.length()) && (header_line[i] == ' '); ++i) {
155 }
156
157 // Code is initialized to -1
158
1/2
✓ Branch 1 taken 276 times.
✗ Branch 2 not taken.
 276 if (header_line.length() > i + 2) {
159 276 info->SetHttpCode(DownloadManager::ParseHttpCode(&header_line[i]));
160 }
161
162
2/2
✓ Branch 1 taken 207 times.
✓ Branch 2 taken 69 times.
 276 if ((info->http_code() / 100) == 2) {
163 207 return num_bytes;
164
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 69 } else if ((info->http_code() == 301) || (info->http_code() == 302)
165
2/8
✗ Branch 0 not taken.
✓ Branch 1 taken 69 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 69 times.
✗ Branch 9 not taken.
 69 || (info->http_code() == 303) || (info->http_code() == 307)) {
166
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 69 times.
 69 if (!info->follow_redirects()) {
167 ✗ LogCvmfs(kLogDownload, kLogDebug,
168 "(id %" PRId64 ") redirect support not enabled: %s",
169 info->id(), header_line.c_str());
170 ✗ info->SetErrorCode(kFailHostHttp);
171 ✗ return 0;
172 }
173
1/2
✓ Branch 3 taken 69 times.
✗ Branch 4 not taken.
 69 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") http redirect: %s",
174 info->id(), header_line.c_str());
175 // libcurl will handle this because of CURLOPT_FOLLOWLOCATION
176 69 return num_bytes;
177 } else {
178 ✗ LogCvmfs(kLogDownload, kLogDebug,
179 "(id %" PRId64 ") http status error code: %s [%d]", info->id(),
180 header_line.c_str(), info->http_code());
181 ✗ if (((info->http_code() / 100) == 5) || (info->http_code() == 400)
182 ✗ || (info->http_code() == 404)) {
183 // 5XX returned by host
184 // 400: error from the GeoAPI module
185 // 404: the stratum 1 does not have the newest files
186 ✗ info->SetErrorCode(kFailHostHttp);
187 ✗ } else if (info->http_code() == 429) {
188 // 429: rate throttling (we ignore the backoff hint for the time being)
189 ✗ info->SetErrorCode(kFailHostConnection);
190 } else {
191 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostHttp
192 : kFailProxyHttp);
193 }
194 ✗ return 0;
195 }
196 }
197
198 // If needed: allocate space in sink
199
3/4
✓ Branch 2 taken 13723 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 5155 times.
✓ Branch 5 taken 8568 times.
 13723 if (info->sink() != NULL && info->sink()->RequiresReserve()
200
11/20
✓ Branch 1 taken 13723 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 5155 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 5155 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 1634 times.
✓ Branch 11 taken 3521 times.
✓ Branch 12 taken 5155 times.
✓ Branch 13 taken 8568 times.
✗ Branch 14 not taken.
✓ Branch 15 taken 5155 times.
✓ Branch 16 taken 8568 times.
✓ Branch 18 taken 1634 times.
✓ Branch 19 taken 12089 times.
✗ Branch 20 not taken.
✗ Branch 21 not taken.
✗ Branch 23 not taken.
✗ Branch 24 not taken.
 27446 && HasPrefix(header_line, "CONTENT-LENGTH:", true)) {
201 1634 char *tmp = reinterpret_cast<char *>(alloca(num_bytes + 1));
202 1634 uint64_t length = 0;
203 1634 sscanf(header_line.c_str(), "%s %" PRIu64, tmp, &length);
204
1/2
✓ Branch 0 taken 1634 times.
✗ Branch 1 not taken.
 1634 if (length > 0) {
205
2/4
✓ Branch 2 taken 1634 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 1634 times.
 1634 if (!info->sink()->Reserve(length)) {
206 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
207 "(id %" PRId64 ") "
208 "resource %s too large to store in memory (%" PRIu64 ")",
209 info->id(), info->url()->c_str(), length);
210 ✗ info->SetErrorCode(kFailTooBig);
211 ✗ return 0;
212 }
213 } else {
214 // Empty resource
215 ✗ info->sink()->Reserve(0);
216 }
217
4/6
✓ Branch 2 taken 12089 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 12089 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 69 times.
✓ Branch 10 taken 12020 times.
 12089 } else if (HasPrefix(header_line, "LOCATION:", true)) {
218 // This comes along with redirects
219
1/2
✓ Branch 3 taken 69 times.
✗ Branch 4 not taken.
 69 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") %s", info->id(),
220 header_line.c_str());
221
3/6
✓ Branch 2 taken 12020 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 12020 times.
✗ Branch 6 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 12020 times.
 12020 } else if (HasPrefix(header_line, "LINK:", true)) {
222 // This is metalink info
223 ✗ LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") %s", info->id(),
224 header_line.c_str());
225 ✗ std::string link = info->link();
226 ✗ if (link.size() != 0) {
227 // multiple LINK headers are allowed
228 ✗ link = link + ", " + header_line.substr(5);
229 } else {
230 ✗ link = header_line.substr(5);
231 }
232 ✗ info->SetLink(link);
233
3/6
✓ Branch 3 taken 12020 times.
✗ Branch 4 not taken.
✓ Branch 6 taken 12020 times.
✗ Branch 7 not taken.
✗ Branch 10 not taken.
✓ Branch 11 taken 12020 times.
 12020 } else if (HasPrefix(header_line, "X-SQUID-ERROR:", true)) {
234 // Reinterpret host error as proxy error
235 ✗ if (info->error_code() == kFailHostHttp) {
236 ✗ info->SetErrorCode(kFailProxyHttp);
237 }
238
3/6
✓ Branch 2 taken 12020 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 12020 times.
✗ Branch 6 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 12020 times.
 12020 } else if (HasPrefix(header_line, "PROXY-STATUS:", true)) {
239 // Reinterpret host error as proxy error if applicable
240 ✗ if ((info->error_code() == kFailHostHttp)
241 ✗ && (header_line.find("error=") != string::npos)) {
242 ✗ info->SetErrorCode(kFailProxyHttp);
243 }
244 }
245
246 13723 return num_bytes;
247 13999 }
248
249
250 /**
251 * Called by curl for every received data chunk.
252 *
253 * In multi-threaded mode the chunk's hash is updated here (cheap) and the
254 * raw bytes are handed off to the calling Fetch() thread via JobInfo's
255 * data tube; the (relatively expensive) zlib decompression and sink->Write
256 * happen on that caller thread. This lets multiple callers — e.g. main
257 * thread plus N file-bundle prefetch workers — run their decompression in
258 * parallel instead of serializing on this single MainDownload thread.
259 *
260 * In single-threaded mode (no MainDownload thread, e.g. unit tests) the
261 * old inline path is kept.
262 */
263 5330 static size_t CallbackCurlData(void *ptr, size_t size, size_t nmemb,
264 void *info_link) {
265 5330 const size_t num_bytes = size * nmemb;
266 5330 JobInfo *info = static_cast<JobInfo *>(info_link);
267
268 // TODO(heretherebedragons) remove if no error comes up
269 // as this means only jobinfo data request (and not header only)
270 // come here
271
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5330 times.
 5330 assert(info->sink() != NULL);
272
273
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5330 times.
 5330 if (num_bytes == 0)
274 ✗ return 0;
275
276
2/2
✓ Branch 1 taken 3143 times.
✓ Branch 2 taken 2187 times.
 5330 if (info->expected_hash()) {
277 3143 shash::Update(reinterpret_cast<unsigned char *>(ptr), num_bytes,
278 info->hash_context());
279 }
280
281
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5330 times.
 5330 if (info->IsValidDataTube()) {
282 // Parallel-decompress path: copy bytes onto a buffer the caller will own
283 // and enqueue. The caller thread (DownloadManager::Fetch) pops these
284 // elements and runs DecompressZStream2Sink / sink->Write itself.
285 ✗ char *buf = new char[num_bytes];
286 ✗ memcpy(buf, ptr, num_bytes);
287 DataTubeElement *ele = new DataTubeElement(buf, num_bytes,
288 ✗ kActionDecompress);
289 ✗ info->GetDataTubePtr()->EnqueueBack(ele);
290 ✗ return num_bytes;
291 }
292
293
2/2
✓ Branch 1 taken 3123 times.
✓ Branch 2 taken 2207 times.
 5330 if (info->compressed()) {
294 3123 const zlib::StreamStates retval = zlib::DecompressZStream2Sink(
295 ptr, static_cast<int64_t>(num_bytes), info->GetZstreamPtr(),
296 info->sink());
297
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3123 times.
 3123 if (retval == zlib::kStreamDataError) {
298 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
299 "(id %" PRId64 ") failed to decompress %s", info->id(),
300 info->url()->c_str());
301 ✗ info->SetErrorCode(kFailBadData);
302 ✗ return 0;
303
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3123 times.
 3123 } else if (retval == zlib::kStreamIOError) {
304 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
305 "(id %" PRId64 ") decompressing %s, local IO error", info->id(),
306 info->url()->c_str());
307 ✗ info->SetErrorCode(kFailLocalIO);
308 ✗ return 0;
309 }
310 } else {
311 2207 const int64_t written = info->sink()->Write(ptr, num_bytes);
312
2/4
✓ Branch 0 taken 2207 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 2207 times.
 2207 if (written < 0 || static_cast<uint64_t>(written) != num_bytes) {
313 ✗ LogCvmfs(kLogDownload, kLogDebug,
314 "(id %" PRId64 ") "
315 "Failed to perform write of %zu bytes to sink %s with errno %ld",
316 ✗ info->id(), num_bytes, info->sink()->Describe().c_str(),
317 written);
318 }
319 }
320
321 5330 return num_bytes;
322 }
323
324 #ifdef DEBUGMSG
325 8994 static int CallbackCurlDebug(CURL *handle,
326 curl_infotype type,
327 char *data,
328 size_t size,
329 void * /* clientp */) {
330 JobInfo *info;
331
1/2
✓ Branch 1 taken 8994 times.
✗ Branch 2 not taken.
 8994 curl_easy_getinfo(handle, CURLINFO_PRIVATE, &info);
332
333
3/6
✓ Branch 2 taken 8994 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 8994 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 8994 times.
✗ Branch 9 not taken.
 17988 std::string prefix = "(id " + StringifyInt(info->id()) + ") ";
334
4/8
✓ Branch 0 taken 6809 times.
✓ Branch 1 taken 1081 times.
✓ Branch 2 taken 276 times.
✓ Branch 3 taken 828 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
 8994 switch (type) {
335 6809 case CURLINFO_TEXT:
336
1/2
✓ Branch 1 taken 6809 times.
✗ Branch 2 not taken.
 6809 prefix += "{info} ";
337 6809 break;
338 1081 case CURLINFO_HEADER_IN:
339
1/2
✓ Branch 1 taken 1081 times.
✗ Branch 2 not taken.
 1081 prefix += "{header/recv} ";
340 1081 break;
341 276 case CURLINFO_HEADER_OUT:
342
1/2
✓ Branch 1 taken 276 times.
✗ Branch 2 not taken.
 276 prefix += "{header/sent} ";
343 276 break;
344 828 case CURLINFO_DATA_IN:
345
2/2
✓ Branch 0 taken 92 times.
✓ Branch 1 taken 736 times.
 828 if (size < 50) {
346
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 prefix += "{data/recv} ";
347 92 break;
348 } else {
349
1/2
✓ Branch 2 taken 736 times.
✗ Branch 3 not taken.
 736 LogCvmfs(kLogCurl, kLogDebug, "%s{data/recv} <snip>", prefix.c_str());
350 736 return 0;
351 }
352 ✗ case CURLINFO_DATA_OUT:
353 ✗ if (size < 50) {
354 ✗ prefix += "{data/sent} ";
355 ✗ break;
356 } else {
357 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{data/sent} <snip>", prefix.c_str());
358 ✗ return 0;
359 }
360 ✗ case CURLINFO_SSL_DATA_IN:
361 ✗ if (size < 50) {
362 ✗ prefix += "{ssldata/recv} ";
363 ✗ break;
364 } else {
365 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{ssldata/recv} <snip>",
366 prefix.c_str());
367 ✗ return 0;
368 }
369 ✗ case CURLINFO_SSL_DATA_OUT:
370 ✗ if (size < 50) {
371 ✗ prefix += "{ssldata/sent} ";
372 ✗ break;
373 } else {
374 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{ssldata/sent} <snip>",
375 prefix.c_str());
376 ✗ return 0;
377 }
378 ✗ default:
379 // just log the message
380 ✗ break;
381 }
382
383 8258 bool valid_char = true;
384
1/2
✓ Branch 2 taken 8258 times.
✗ Branch 3 not taken.
 8258 std::string msg(data, size);
385
2/2
✓ Branch 1 taken 259592 times.
✓ Branch 2 taken 8258 times.
 267850 for (size_t i = 0; i < msg.length(); ++i) {
386
2/4
✓ Branch 1 taken 259592 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 259592 times.
 259592 if (msg[i] == '\0') {
387 ✗ msg[i] = '~';
388 }
389
390 // verify that char is a valid printable char
391
3/6
✓ Branch 1 taken 259592 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 247194 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 247194 times.
 506786 if ((msg[i] < ' ' || msg[i] > '~')
392
6/8
✓ Branch 0 taken 247194 times.
✓ Branch 1 taken 12398 times.
✓ Branch 3 taken 12398 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 2829 times.
✓ Branch 6 taken 9569 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 259592 times.
 509615 && (msg[i] != 10 /*line feed*/
393
2/4
✓ Branch 1 taken 2829 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 2829 times.
 2829 && msg[i] != 13 /*carriage return*/)) {
394 ✗ valid_char = false;
395 }
396 }
397
398
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 8258 times.
 8258 if (!valid_char) {
399 ✗ msg = "<Non-plaintext sequence>";
400 }
401
402
1/2
✓ Branch 3 taken 8258 times.
✗ Branch 4 not taken.
 8258 LogCvmfs(kLogCurl, kLogDebug, "%s%s", prefix.c_str(),
403
1/2
✓ Branch 1 taken 8258 times.
✗ Branch 2 not taken.
 16516 Trim(msg, true /* trim_newline */).c_str());
404 8258 return 0;
405 8994 }
406 #endif
407
408 //------------------------------------------------------------------------------
409
410
411 const int DownloadManager::kProbeUnprobed = -1;
412 const int DownloadManager::kProbeDown = -2;
413 const int DownloadManager::kProbeGeo = -3;
414
415 /**
416 * Escape special chars from the URL, except for ':' and '/',
417 * which should keep their meaning.
418 */
419 4807 string DownloadManager::EscapeUrl(const int64_t jobinfo_id, const string &url) {
420 4807 string escaped;
421
1/2
✓ Branch 2 taken 4807 times.
✗ Branch 3 not taken.
 4807 escaped.reserve(url.length());
422
423 char escaped_char[3];
424
2/2
✓ Branch 1 taken 403547 times.
✓ Branch 2 taken 4807 times.
 408354 for (unsigned i = 0, s = url.length(); i < s; ++i) {
425
3/4
✓ Branch 2 taken 403547 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 207 times.
✓ Branch 5 taken 403340 times.
 403547 if (JobInfo::EscapeUrlChar(url[i], escaped_char)) {
426
1/2
✓ Branch 1 taken 207 times.
✗ Branch 2 not taken.
 207 escaped.append(escaped_char, 3);
427 } else {
428
1/2
✓ Branch 1 taken 403340 times.
✗ Branch 2 not taken.
 403340 escaped.push_back(escaped_char[0]);
429 }
430 }
431
1/2
✓ Branch 3 taken 4807 times.
✗ Branch 4 not taken.
 4807 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") escaped %s to %s",
432 jobinfo_id, url.c_str(), escaped.c_str());
433
434 9614 return escaped;
435 }
436
437 /**
438 * -1 of digits is not a valid Http return code
439 */
440 391 int DownloadManager::ParseHttpCode(const char digits[3]) {
441 391 int result = 0;
442 391 int factor = 100;
443
2/2
✓ Branch 0 taken 1173 times.
✓ Branch 1 taken 368 times.
 1541 for (int i = 0; i < 3; ++i) {
444
3/4
✓ Branch 0 taken 1173 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 23 times.
✓ Branch 3 taken 1150 times.
 1173 if ((digits[i] < '0') || (digits[i] > '9'))
445 23 return -1;
446 1150 result += (digits[i] - '0') * factor;
447 1150 factor /= 10;
448 }
449 368 return result;
450 }
451
452
453 /**
454 * Called when new curl sockets arrive or existing curl sockets depart.
455 */
456 ✗ int DownloadManager::CallbackCurlSocket(CURL * /* easy */,
457 curl_socket_t s,
458 int action,
459 void *userp,
460 void * /* socketp */) {
461 // LogCvmfs(kLogDownload, kLogDebug, "CallbackCurlSocket called with easy "
462 // "handle %p, socket %d, action %d", easy, s, action);
463 ✗ DownloadManager *download_mgr = static_cast<DownloadManager *>(userp);
464 ✗ if (action == CURL_POLL_NONE)
465 ✗ return 0;
466
467 // Find s in watch_fds_
468 unsigned index;
469
470 // TODO(heretherebedragons) why start at index = 0 and not 2?
471 // fd[0] and fd[1] are fixed?
472 ✗ for (index = 0; index < download_mgr->watch_fds_inuse_; ++index) {
473 ✗ if (download_mgr->watch_fds_[index].fd == s)
474 ✗ break;
475 }
476 // Or create newly
477 ✗ if (index == download_mgr->watch_fds_inuse_) {
478 // Extend array if necessary
479 ✗ if (download_mgr->watch_fds_inuse_ == download_mgr->watch_fds_size_) {
480 ✗ assert(download_mgr->watch_fds_size_ > 0);
481 ✗ download_mgr->watch_fds_size_ *= 2;
482 ✗ download_mgr->watch_fds_ = static_cast<struct pollfd *>(
483 ✗ srealloc(download_mgr->watch_fds_,
484 ✗ download_mgr->watch_fds_size_ * sizeof(struct pollfd)));
485 }
486 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].fd = s;
487 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].events = 0;
488 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].revents = 0;
489 ✗ download_mgr->watch_fds_inuse_++;
490 }
491
492 ✗ switch (action) {
493 ✗ case CURL_POLL_IN:
494 ✗ download_mgr->watch_fds_[index].events = POLLIN | POLLPRI;
495 ✗ break;
496 ✗ case CURL_POLL_OUT:
497 ✗ download_mgr->watch_fds_[index].events = POLLOUT | POLLWRBAND;
498 ✗ break;
499 ✗ case CURL_POLL_INOUT:
500 ✗ download_mgr->watch_fds_[index].events = POLLIN | POLLPRI | POLLOUT
501 | POLLWRBAND;
502 ✗ break;
503 ✗ case CURL_POLL_REMOVE:
504 ✗ if (index < download_mgr->watch_fds_inuse_ - 1) {
505 download_mgr
506 ✗ ->watch_fds_[index] = download_mgr->watch_fds_
507 ✗ [download_mgr->watch_fds_inuse_ - 1];
508 }
509 ✗ download_mgr->watch_fds_inuse_--;
510 // Shrink array if necessary
511 ✗ if ((download_mgr->watch_fds_inuse_ > download_mgr->watch_fds_max_)
512 ✗ && (download_mgr->watch_fds_inuse_
513 ✗ < download_mgr->watch_fds_size_ / 2)) {
514 ✗ download_mgr->watch_fds_size_ /= 2;
515 // LogCvmfs(kLogDownload, kLogDebug, "shrinking watch_fds_ (%d)",
516 // watch_fds_size_);
517 ✗ download_mgr->watch_fds_ = static_cast<struct pollfd *>(
518 ✗ srealloc(download_mgr->watch_fds_,
519 ✗ download_mgr->watch_fds_size_ * sizeof(struct pollfd)));
520 // LogCvmfs(kLogDownload, kLogDebug, "shrinking watch_fds_ done",
521 // watch_fds_size_);
522 }
523 ✗ break;
524 ✗ default:
525 ✗ break;
526 }
527
528 ✗ return 0;
529 }
530
531
532 /**
533 * Worker thread event loop. Waits on new JobInfo structs on a pipe.
534 */
535 ✗ void *DownloadManager::MainDownload(void *data) {
536 ✗ DownloadManager *download_mgr = static_cast<DownloadManager *>(data);
537 ✗ LogCvmfs(kLogDownload, kLogDebug,
538 "download I/O thread of DownloadManager '%s' started",
539 download_mgr->name_.c_str());
540
541 ✗ const int kIdxPipeTerminate = 0;
542 ✗ const int kIdxPipeJobs = 1;
543
544 ✗ download_mgr->watch_fds_ = static_cast<struct pollfd *>(
545 ✗ smalloc(2 * sizeof(struct pollfd)));
546 ✗ download_mgr->watch_fds_size_ = 2;
547 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].fd = download_mgr->pipe_terminate_
548 ✗ ->GetReadFd();
549 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].events = POLLIN | POLLPRI;
550 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].revents = 0;
551 ✗ download_mgr->watch_fds_[kIdxPipeJobs].fd = download_mgr->pipe_jobs_
552 ✗ ->GetReadFd();
553 ✗ download_mgr->watch_fds_[kIdxPipeJobs].events = POLLIN | POLLPRI;
554 ✗ download_mgr->watch_fds_[kIdxPipeJobs].revents = 0;
555 ✗ download_mgr->watch_fds_inuse_ = 2;
556
557 ✗ int still_running = 0;
558 struct timeval timeval_start, timeval_stop;
559 ✗ gettimeofday(&timeval_start, NULL);
560 while (true) {
561 int timeout;
562 ✗ if (still_running) {
563 /* NOTE: The following might degrade the performance for many small files
564 * use case. TODO(jblomer): look into it.
565 // Specify a timeout for polling in ms; this allows us to return
566 // to libcurl once a second so it can look for internal operations
567 // which timed out. libcurl has a more elaborate mechanism
568 // (CURLMOPT_TIMERFUNCTION) that would inform us of the next potential
569 // timeout. TODO(bbockelm) we should switch to that in the future.
570 timeout = 100;
571 */
572 ✗ timeout = 1;
573 } else {
574 ✗ timeout = -1;
575 ✗ gettimeofday(&timeval_stop, NULL);
576 const int64_t delta = static_cast<int64_t>(
577 ✗ 1000 * DiffTimeSeconds(timeval_start, timeval_stop));
578 ✗ perf::Xadd(download_mgr->counters_->sz_transfer_time, delta);
579 }
580 ✗ const int retval = poll(download_mgr->watch_fds_,
581 ✗ download_mgr->watch_fds_inuse_, timeout);
582 ✗ if (retval < 0) {
583 ✗ continue;
584 }
585
586 // Handle timeout
587 ✗ if (retval == 0) {
588 ✗ curl_multi_socket_action(download_mgr->curl_multi_, CURL_SOCKET_TIMEOUT,
589 0, &still_running);
590 }
591
592 // Terminate I/O thread
593 ✗ if (download_mgr->watch_fds_[kIdxPipeTerminate].revents)
594 ✗ break;
595
596 // New job arrives
597 ✗ if (download_mgr->watch_fds_[kIdxPipeJobs].revents) {
598 ✗ download_mgr->watch_fds_[kIdxPipeJobs].revents = 0;
599 JobInfo *info;
600 ✗ download_mgr->pipe_jobs_->Read<JobInfo *>(&info);
601 ✗ if (!still_running) {
602 ✗ gettimeofday(&timeval_start, NULL);
603 }
604 ✗ CURL *handle = download_mgr->AcquireCurlHandle();
605 ✗ download_mgr->InitializeRequest(info, handle);
606 ✗ download_mgr->SetUrlOptions(info);
607 ✗ curl_multi_add_handle(download_mgr->curl_multi_, handle);
608 ✗ curl_multi_socket_action(download_mgr->curl_multi_, CURL_SOCKET_TIMEOUT,
609 0, &still_running);
610 }
611
612 // Activity on curl sockets
613 // Within this loop the curl_multi_socket_action() may cause socket(s)
614 // to be removed from watch_fds_. If a socket is removed it is replaced
615 // by the socket at the end of the array and the inuse count is decreased.
616 // Therefore loop over the array in reverse order.
617 ✗ for (int64_t i = download_mgr->watch_fds_inuse_ - 1; i >= 2; --i) {
618 ✗ if (i >= download_mgr->watch_fds_inuse_) {
619 ✗ continue;
620 }
621 ✗ if (download_mgr->watch_fds_[i].revents) {
622 ✗ int ev_bitmask = 0;
623 ✗ if (download_mgr->watch_fds_[i].revents & (POLLIN | POLLPRI))
624 ✗ ev_bitmask |= CURL_CSELECT_IN;
625 ✗ if (download_mgr->watch_fds_[i].revents & (POLLOUT | POLLWRBAND))
626 ✗ ev_bitmask |= CURL_CSELECT_OUT;
627 ✗ if (download_mgr->watch_fds_[i].revents
628 ✗ & (POLLERR | POLLHUP | POLLNVAL)) {
629 ✗ ev_bitmask |= CURL_CSELECT_ERR;
630 }
631 ✗ download_mgr->watch_fds_[i].revents = 0;
632
633 ✗ curl_multi_socket_action(download_mgr->curl_multi_,
634 ✗ download_mgr->watch_fds_[i].fd,
635 ev_bitmask,
636 &still_running);
637 }
638 }
639
640 // Check if transfers are completed
641 CURLMsg *curl_msg;
642 int msgs_in_queue;
643 ✗ while ((curl_msg = curl_multi_info_read(download_mgr->curl_multi_,
644 &msgs_in_queue))) {
645 ✗ if (curl_msg->msg == CURLMSG_DONE) {
646 ✗ perf::Inc(download_mgr->counters_->n_requests);
647 JobInfo *info;
648 ✗ CURL *easy_handle = curl_msg->easy_handle;
649 ✗ const int curl_error = curl_msg->data.result;
650 ✗ curl_easy_getinfo(easy_handle, CURLINFO_PRIVATE, &info);
651
652 int64_t redir_count;
653 ✗ curl_easy_getinfo(easy_handle, CURLINFO_REDIRECT_COUNT, &redir_count);
654 ✗ LogCvmfs(kLogDownload, kLogDebug,
655 "(manager '%s' - id %" PRId64 ") "
656 "Number of CURL redirects %" PRId64,
657 download_mgr->name_.c_str(), info->id(), redir_count);
658
659 ✗ curl_multi_remove_handle(download_mgr->curl_multi_, easy_handle);
660 ✗ if (download_mgr->VerifyAndFinalize(curl_error, info)) {
661 ✗ curl_multi_add_handle(download_mgr->curl_multi_, easy_handle);
662 ✗ curl_multi_socket_action(download_mgr->curl_multi_,
663 CURL_SOCKET_TIMEOUT,
664 0,
665 &still_running);
666 } else {
667 // Return easy handle into pool and write result back
668 ✗ download_mgr->ReleaseCurlHandle(easy_handle, true /* allow_reuse */);
669
670 ✗ DataTubeElement *ele = new DataTubeElement(kActionStop);
671 ✗ info->GetDataTubePtr()->EnqueueBack(ele);
672 ✗ info->GetPipeJobResultPtr()->Write<download::Failures>(
673 ✗ info->error_code());
674 }
675 }
676 }
677 }
678
679 ✗ for (set<CURL *>::iterator i = download_mgr->pool_handles_inuse_->begin(),
680 ✗ iEnd = download_mgr->pool_handles_inuse_->end();
681 ✗ i != iEnd;
682 ✗ ++i) {
683 ✗ curl_multi_remove_handle(download_mgr->curl_multi_, *i);
684 ✗ curl_easy_cleanup(*i);
685 }
686 ✗ download_mgr->pool_handles_inuse_->clear();
687 ✗ free(download_mgr->watch_fds_);
688
689 ✗ LogCvmfs(kLogDownload, kLogDebug,
690 "download I/O thread of DownloadManager '%s' terminated",
691 download_mgr->name_.c_str());
692 ✗ return NULL;
693 }
694
695
696 //------------------------------------------------------------------------------
697
698
699 4175 HeaderLists::~HeaderLists() {
700
2/2
✓ Branch 1 taken 4224 times.
✓ Branch 2 taken 4175 times.
 8399 for (unsigned i = 0; i < blocks_.size(); ++i) {
701
1/2
✓ Branch 1 taken 4224 times.
✗ Branch 2 not taken.
 4224 delete[] blocks_[i];
702 }
703 4175 blocks_.clear();
704 4175 }
705
706
707 34224 curl_slist *HeaderLists::GetList(const char *header) { return Get(header); }
708
709
710 4715 curl_slist *HeaderLists::DuplicateList(curl_slist *slist) {
711
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4715 times.
 4715 assert(slist);
712 4715 curl_slist *copy = GetList(slist->data);
713 4715 copy->next = slist->next;
714 4715 curl_slist *prev = copy;
715 4715 slist = slist->next;
716
2/2
✓ Branch 0 taken 9430 times.
✓ Branch 1 taken 4715 times.
 14145 while (slist) {
717 9430 curl_slist *new_link = Get(slist->data);
718 9430 new_link->next = slist->next;
719 9430 prev->next = new_link;
720 9430 prev = new_link;
721 9430 slist = slist->next;
722 }
723 4715 return copy;
724 }
725
726
727 8690 void HeaderLists::AppendHeader(curl_slist *slist, const char *header) {
728
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 8690 times.
 8690 assert(slist);
729 8690 curl_slist *new_link = Get(header);
730 8690 new_link->next = NULL;
731
732
2/2
✓ Branch 0 taken 4972 times.
✓ Branch 1 taken 8690 times.
 13662 while (slist->next)
733 4972 slist = slist->next;
734 8690 slist->next = new_link;
735 8690 }
736
737
738 /**
739 * Ensures that a certain header string is _not_ part of slist on return.
740 * Note that if the first header element matches, the returned slist points
741 * to a different value.
742 */
743 245 void HeaderLists::CutHeader(const char *header, curl_slist **slist) {
744
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 245 times.
 245 assert(slist);
745 curl_slist head;
746 245 head.next = *slist;
747 245 curl_slist *prev = &head;
748 245 curl_slist *rover = *slist;
749
2/2
✓ Branch 0 taken 441 times.
✓ Branch 1 taken 245 times.
 686 while (rover) {
750
2/2
✓ Branch 0 taken 196 times.
✓ Branch 1 taken 245 times.
 441 if (strcmp(rover->data, header) == 0) {
751 196 prev->next = rover->next;
752 196 Put(rover);
753 196 rover = prev;
754 }
755 441 prev = rover;
756 441 rover = rover->next;
757 }
758 245 *slist = head.next;
759 245 }
760
761
762 17455 void HeaderLists::PutList(curl_slist *slist) {
763
2/2
✓ Branch 0 taken 27321 times.
✓ Branch 1 taken 17455 times.
 44776 while (slist) {
764 27321 curl_slist *next = slist->next;
765 27321 Put(slist);
766 27321 slist = next;
767 }
768 17455 }
769
770
771 196 string HeaderLists::Print(curl_slist *slist) {
772 196 string verbose;
773
2/2
✓ Branch 0 taken 392 times.
✓ Branch 1 taken 196 times.
 588 while (slist) {
774
3/6
✓ Branch 2 taken 392 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 392 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 392 times.
✗ Branch 9 not taken.
 392 verbose += string(slist->data) + "\n";
775 392 slist = slist->next;
776 }
777 196 return verbose;
778 }
779
780
781 52344 curl_slist *HeaderLists::Get(const char *header) {
782
2/2
✓ Branch 1 taken 48119 times.
✓ Branch 2 taken 4274 times.
 52393 for (unsigned i = 0; i < blocks_.size(); ++i) {
783
2/2
✓ Branch 0 taken 3342791 times.
✓ Branch 1 taken 49 times.
 3342840 for (unsigned j = 0; j < kBlockSize; ++j) {
784
2/2
✓ Branch 2 taken 48070 times.
✓ Branch 3 taken 3294721 times.
 3342791 if (!IsUsed(&(blocks_[i][j]))) {
785 48070 blocks_[i][j].data = const_cast<char *>(header);
786 48070 return &(blocks_[i][j]);
787 }
788 }
789 }
790
791 // All used, new block
792 4274 AddBlock();
793 4274 blocks_[blocks_.size() - 1][0].data = const_cast<char *>(header);
794 4274 return &(blocks_[blocks_.size() - 1][0]);
795 }
796
797
798 1121661 void HeaderLists::Put(curl_slist *slist) {
799 1121661 slist->data = NULL;
800 1121661 slist->next = NULL;
801 1121661 }
802
803
804 4274 void HeaderLists::AddBlock() {
805
1/2
✓ Branch 1 taken 4274 times.
✗ Branch 2 not taken.
 4274 curl_slist *new_block = new curl_slist[kBlockSize];
806
2/2
✓ Branch 0 taken 1094144 times.
✓ Branch 1 taken 4274 times.
 1098418 for (unsigned i = 0; i < kBlockSize; ++i) {
807 1094144 Put(&new_block[i]);
808 }
809
1/2
✓ Branch 1 taken 4274 times.
✗ Branch 2 not taken.
 4274 blocks_.push_back(new_block);
810 4274 }
811
812
813 //------------------------------------------------------------------------------
814
815
816 ✗ string DownloadManager::ProxyInfo::Print() {
817 ✗ if (url == "DIRECT")
818 ✗ return url;
819
820 ✗ string result = url;
821 ✗ const int remaining = static_cast<int>(host.deadline())
822 ✗ - static_cast<int>(time(NULL));
823 ✗ string expinfo = (remaining >= 0) ? "+" : "";
824 ✗ if (abs(remaining) >= 3600) {
825 ✗ expinfo += StringifyInt(remaining / 3600) + "h";
826 ✗ } else if (abs(remaining) >= 60) {
827 ✗ expinfo += StringifyInt(remaining / 60) + "m";
828 } else {
829 ✗ expinfo += StringifyInt(remaining) + "s";
830 }
831 ✗ if (host.status() == dns::kFailOk) {
832 ✗ result += " (" + host.name() + ", " + expinfo + ")";
833 } else {
834 ✗ result += " (:unresolved:, " + expinfo + ")";
835 }
836 ✗ return result;
837 }
838
839
840 /**
841 * Gets an idle CURL handle from the pool. Creates a new one and adds it to
842 * the pool if necessary.
843 */
844 4715 CURL *DownloadManager::AcquireCurlHandle() {
845 CURL *handle;
846
847
1/2
✓ Branch 1 taken 4715 times.
✗ Branch 2 not taken.
 4715 if (pool_handles_idle_->empty()) {
848 // Create a new handle
849
1/2
✓ Branch 1 taken 4715 times.
✗ Branch 2 not taken.
 4715 handle = curl_easy_init();
850
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4715 times.
 4715 assert(handle != NULL);
851
852
1/2
✓ Branch 1 taken 4715 times.
✗ Branch 2 not taken.
 4715 curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1);
853 // curl_easy_setopt(curl_default, CURLOPT_FAILONERROR, 1);
854
1/2
✓ Branch 1 taken 4715 times.
✗ Branch 2 not taken.
 4715 curl_easy_setopt(handle, CURLOPT_HEADERFUNCTION, CallbackCurlHeader);
855
1/2
✓ Branch 1 taken 4715 times.
✗ Branch 2 not taken.
 4715 curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, CallbackCurlData);
856 } else {
857 ✗ handle = *(pool_handles_idle_->begin());
858 ✗ pool_handles_idle_->erase(pool_handles_idle_->begin());
859 }
860
861
1/2
✓ Branch 1 taken 4715 times.
✗ Branch 2 not taken.
 4715 pool_handles_inuse_->insert(handle);
862
863 4715 return handle;
864 }
865
866
867 4715 void DownloadManager::ReleaseCurlHandle(CURL *handle, bool allow_reuse) {
868
1/2
✓ Branch 1 taken 4715 times.
✗ Branch 2 not taken.
 4715 const set<CURL *>::iterator elem = pool_handles_inuse_->find(handle);
869
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 4715 times.
 4715 assert(elem != pool_handles_inuse_->end());
870
871
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 4715 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 4715 times.
✗ Branch 6 not taken.
 4715 if (!allow_reuse || pool_handles_idle_->size() > pool_max_handles_) {
872
1/2
✓ Branch 2 taken 4715 times.
✗ Branch 3 not taken.
 4715 curl_easy_cleanup(*elem);
873 } else {
874 ✗ pool_handles_idle_->insert(*elem);
875 }
876
877
1/2
✓ Branch 1 taken 4715 times.
✗ Branch 2 not taken.
 4715 pool_handles_inuse_->erase(elem);
878 4715 }
879
880
881 /**
882 * HTTP request options: set the URL and other options such as timeout and
883 * proxy.
884 */
885 4715 void DownloadManager::InitializeRequest(JobInfo *info, CURL *handle) {
886 // Initialize internal download state
887 4715 info->SetCurlHandle(handle);
888 4715 info->SetErrorCode(kFailOk);
889 4715 info->SetHttpCode(-1);
890 4715 info->SetFollowRedirects(follow_redirects_);
891 4715 info->SetNumUsedProxies(1);
892 4715 info->SetNumUsedMetalinks(1);
893 4715 info->SetNumUsedHosts(1);
894 4715 info->SetNumRetries(0);
895 4715 info->SetBackoffMs(0);
896 4715 info->SetHeaders(header_lists_->DuplicateList(default_headers_));
897
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4715 times.
 4715 if (info->info_header()) {
898 ✗ header_lists_->AppendHeader(info->headers(), info->info_header());
899 }
900
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4715 times.
 4715 if (enable_http_tracing_) {
901 ✗ for (unsigned int i = 0; i < http_tracing_headers_.size(); i++) {
902 ✗ header_lists_->AppendHeader(info->headers(),
903 ✗ (http_tracing_headers_)[i].c_str());
904 }
905
906 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_pid());
907 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_gid());
908 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_uid());
909
910 ✗ LogCvmfs(kLogDownload, kLogDebug,
911 "(manager '%s' - id %" PRId64 ") "
912 "CURL Header for URL: %s is:\n %s",
913 name_.c_str(), info->id(), info->url()->c_str(),
914 ✗ header_lists_->Print(info->headers()).c_str());
915 }
916
917
2/2
✓ Branch 1 taken 60 times.
✓ Branch 2 taken 4655 times.
 4715 if (info->force_nocache()) {
918 60 SetNocache(info);
919 } else {
920 4655 info->SetNocache(false);
921 }
922
2/2
✓ Branch 1 taken 3223 times.
✓ Branch 2 taken 1492 times.
 4715 if (info->compressed()) {
923 3223 zlib::DecompressInit(info->GetZstreamPtr());
924 }
925
2/2
✓ Branch 1 taken 3243 times.
✓ Branch 2 taken 1472 times.
 4715 if (info->expected_hash()) {
926
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 3243 times.
 3243 assert(info->hash_context().buffer != NULL);
927 3243 shash::Init(info->hash_context());
928 }
929
930
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 4715 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 4715 times.
 4715 if ((info->range_offset() != -1) && (info->range_size())) {
931 char byte_range_array[100];
932 ✗ const int64_t range_lower = static_cast<int64_t>(info->range_offset());
933 ✗ const int64_t range_upper = static_cast<int64_t>(info->range_offset()
934 ✗ + info->range_size() - 1);
935 ✗ if (snprintf(byte_range_array, sizeof(byte_range_array),
936 "%" PRId64 "-%" PRId64, range_lower, range_upper)
937 ✗ == 100) {
938 ✗ PANIC(NULL); // Should be impossible given limits on offset size.
939 }
940 ✗ curl_easy_setopt(handle, CURLOPT_RANGE, byte_range_array);
941 } else {
942 4715 curl_easy_setopt(handle, CURLOPT_RANGE, NULL);
943 }
944
945 // Set curl parameters
946 4715 curl_easy_setopt(handle, CURLOPT_PRIVATE, static_cast<void *>(info));
947 4715 curl_easy_setopt(handle, CURLOPT_WRITEHEADER, static_cast<void *>(info));
948 4715 curl_easy_setopt(handle, CURLOPT_WRITEDATA, static_cast<void *>(info));
949 4715 curl_easy_setopt(handle, CURLOPT_HTTPHEADER, info->headers());
950
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4715 times.
 4715 if (info->head_request()) {
951 ✗ curl_easy_setopt(handle, CURLOPT_NOBODY, 1);
952 } else {
953 4715 curl_easy_setopt(handle, CURLOPT_HTTPGET, 1);
954 }
955
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4715 times.
 4715 if (opt_ipv4_only_) {
956 ✗ curl_easy_setopt(handle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
957 }
958
2/2
✓ Branch 0 taken 1724 times.
✓ Branch 1 taken 2991 times.
 4715 if (follow_redirects_) {
959 1724 curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1);
960 1724 curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 4);
961 }
962 #ifdef DEBUGMSG
963 4715 curl_easy_setopt(handle, CURLOPT_VERBOSE, 1);
964 4715 curl_easy_setopt(handle, CURLOPT_DEBUGFUNCTION, CallbackCurlDebug);
965 #endif
966 4715 }
967
968 9568 void DownloadManager::CheckHostInfoReset(const std::string &typ,
969 HostInfo &info,
970 JobInfo *jobinfo,
971 time_t &now) {
972
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9568 times.
 9568 if (info.timestamp_backup > 0) {
973 ✗ if (now == 0)
974 ✗ now = time(NULL);
975 ✗ if (static_cast<int64_t>(now)
976 ✗ > static_cast<int64_t>(info.timestamp_backup + info.reset_after)) {
977 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
978 "(manager %s - id %" PRId64 ") "
979 "switching %s from %s to %s (reset %s)",
980 name_.c_str(), jobinfo->id(), typ.c_str(),
981 ✗ (*info.chain)[info.current].c_str(), (*info.chain)[0].c_str(),
982 typ.c_str());
983 ✗ info.current = 0;
984 ✗ info.timestamp_backup = 0;
985 }
986 }
987 9568 }
988
989
990 /**
991 * Sets the URL specific options such as host to use and timeout. It might also
992 * set an error code, in which case the further processing should react on.
993 */
994 4784 void DownloadManager::SetUrlOptions(JobInfo *info) {
995 4784 CURL *curl_handle = info->curl_handle();
996 4784 string url_prefix;
997 4784 time_t now = 0;
998
999 4784 const MutexLockGuard m(lock_options_);
1000
1001 // sharding policy
1002
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4784 times.
 4784 if (sharding_policy_.UseCount() > 0) {
1003 ✗ if (info->proxy() != "") {
1004 // proxy already set, so this is a failover event
1005 ✗ perf::Inc(counters_->n_proxy_failover);
1006 }
1007 ✗ info->SetProxy(sharding_policy_->GetNextProxy(
1008 ✗ info->url(), info->proxy(),
1009 ✗ info->range_offset() == -1 ? 0 : info->range_offset()));
1010
1011 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, info->proxy().c_str());
1012 } else { // no sharding policy
1013 // Check if proxy group needs to be reset from backup to primary
1014
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4784 times.
 4784 if (opt_timestamp_backup_proxies_ > 0) {
1015 ✗ now = time(NULL);
1016 ✗ if (static_cast<int64_t>(now) > static_cast<int64_t>(
1017 ✗ opt_timestamp_backup_proxies_ + opt_proxy_groups_reset_after_)) {
1018 ✗ opt_proxy_groups_current_ = 0;
1019 ✗ opt_timestamp_backup_proxies_ = 0;
1020 ✗ RebalanceProxiesUnlocked("Reset proxy group from backup to primary");
1021 }
1022 }
1023 // Check if load-balanced proxies within the group need to be reset
1024
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4784 times.
 4784 if (opt_timestamp_failover_proxies_ > 0) {
1025 ✗ if (now == 0)
1026 ✗ now = time(NULL);
1027 ✗ if (static_cast<int64_t>(now)
1028 ✗ > static_cast<int64_t>(opt_timestamp_failover_proxies_
1029 ✗ + opt_proxy_groups_reset_after_)) {
1030 ✗ RebalanceProxiesUnlocked(
1031 "Reset load-balanced proxies within the active group");
1032 }
1033 }
1034
1035
1/2
✓ Branch 2 taken 4784 times.
✗ Branch 3 not taken.
 4784 ProxyInfo *proxy = ChooseProxyUnlocked(info->expected_hash());
1036
6/6
✓ Branch 0 taken 1738 times.
✓ Branch 1 taken 3046 times.
✓ Branch 3 taken 1646 times.
✓ Branch 4 taken 92 times.
✓ Branch 5 taken 4692 times.
✓ Branch 6 taken 92 times.
 4784 if (!proxy || (proxy->url == "DIRECT")) {
1037
2/4
✓ Branch 2 taken 4692 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 4692 times.
✗ Branch 6 not taken.
 4692 info->SetProxy("DIRECT");
1038
1/2
✓ Branch 2 taken 4692 times.
✗ Branch 3 not taken.
 4692 curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "");
1039 } else {
1040 // Note: inside ValidateProxyIpsUnlocked() we may change the proxy data
1041 // structure, so we must not pass proxy->... (== current_proxy())
1042 // parameters directly
1043
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 const std::string purl = proxy->url;
1044
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 const dns::Host phost = proxy->host;
1045
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 const bool changed = ValidateProxyIpsUnlocked(purl, phost);
1046 // Current proxy may have changed
1047
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 92 times.
 92 if (changed) {
1048 ✗ proxy = ChooseProxyUnlocked(info->expected_hash());
1049 }
1050
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 info->SetProxy(proxy->url);
1051
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 if (proxy->host.status() == dns::kFailOk) {
1052
2/4
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
✓ Branch 6 taken 92 times.
✗ Branch 7 not taken.
 92 curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY,
1053 info->proxy().c_str());
1054 } else {
1055 // We know it can't work, don't even try to download
1056 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "0.0.0.0");
1057 }
1058 92 }
1059 } // end !sharding
1060
1061 // Check if metalink and host chains need to be reset
1062
2/4
✓ Branch 2 taken 4784 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 4784 times.
✗ Branch 6 not taken.
 4784 CheckHostInfoReset("metalink", opt_metalink_, info, now);
1063
2/4
✓ Branch 2 taken 4784 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 4784 times.
✗ Branch 6 not taken.
 4784 CheckHostInfoReset("host", opt_host_, info, now);
1064
1065
1/2
✓ Branch 1 taken 4784 times.
✗ Branch 2 not taken.
 4784 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_LIMIT, opt_low_speed_limit_);
1066
3/6
✓ Branch 1 taken 4784 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 92 times.
✓ Branch 6 taken 4692 times.
 4784 if (info->proxy() != "DIRECT") {
1067
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT, opt_timeout_proxy_);
1068
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_TIME, opt_timeout_proxy_);
1069 } else {
1070
1/2
✓ Branch 1 taken 4692 times.
✗ Branch 2 not taken.
 4692 curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT, opt_timeout_direct_);
1071
1/2
✓ Branch 1 taken 4692 times.
✗ Branch 2 not taken.
 4692 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_TIME, opt_timeout_direct_);
1072 }
1073
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4784 times.
 4784 if (!opt_dns_server_.empty())
1074 ✗ curl_easy_setopt(curl_handle, CURLOPT_DNS_SERVERS, opt_dns_server_.c_str());
1075
1076
2/2
✓ Branch 1 taken 2044 times.
✓ Branch 2 taken 2740 times.
 4784 if (info->probe_hosts()) {
1077
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 2044 times.
 2044 if (CheckMetalinkChain(now)) {
1078 ✗ url_prefix = (*opt_metalink_.chain)[opt_metalink_.current];
1079 ✗ info->SetCurrentMetalinkChainIndex(opt_metalink_.current);
1080 ✗ LogCvmfs(kLogDownload, kLogDebug,
1081 "(manager %s - id %" PRId64 ") "
1082 "reading from metalink %d",
1083 name_.c_str(), info->id(), opt_metalink_.current);
1084
1/2
✓ Branch 0 taken 2044 times.
✗ Branch 1 not taken.
 2044 } else if (opt_host_.chain) {
1085
1/2
✓ Branch 2 taken 2044 times.
✗ Branch 3 not taken.
 2044 url_prefix = (*opt_host_.chain)[opt_host_.current];
1086 2044 info->SetCurrentHostChainIndex(opt_host_.current);
1087
1/2
✓ Branch 3 taken 2044 times.
✗ Branch 4 not taken.
 2044 LogCvmfs(kLogDownload, kLogDebug,
1088 "(manager %s - id %" PRId64 ") "
1089 "reading from host %d",
1090 name_.c_str(), info->id(), opt_host_.current);
1091 }
1092 }
1093
1094
1/2
✓ Branch 2 taken 4784 times.
✗ Branch 3 not taken.
 4784 string url = url_prefix + *(info->url());
1095
1096
1/2
✓ Branch 1 taken 4784 times.
✗ Branch 2 not taken.
 4784 curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, 1L);
1097
2/6
✓ Branch 1 taken 4784 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 4784 times.
 4784 if (url.substr(0, 5) == "https") {
1098 ✗ const bool rvb = ssl_certificate_store_.ApplySslCertificatePath(
1099 curl_handle);
1100 ✗ if (!rvb) {
1101 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1102 "(manager %s - id %" PRId64 ") "
1103 "Failed to set SSL certificate path %s",
1104 name_.c_str(), info->id(),
1105 ✗ ssl_certificate_store_.GetCaPath().c_str());
1106 }
1107 ✗ if (info->pid() != -1) {
1108 ✗ if (credentials_attachment_ == NULL) {
1109 ✗ LogCvmfs(kLogDownload, kLogDebug,
1110 "(manager %s - id %" PRId64 ") "
1111 "uses secure downloads but no credentials attachment set",
1112 name_.c_str(), info->id());
1113 } else {
1114 ✗ const bool retval = credentials_attachment_->ConfigureCurlHandle(
1115 curl_handle, info->pid(), info->GetCredDataPtr());
1116 ✗ if (!retval) {
1117 ✗ LogCvmfs(kLogDownload, kLogDebug,
1118 "(manager %s - id %" PRId64 ") "
1119 "failed attaching credentials",
1120 name_.c_str(), info->id());
1121 }
1122 }
1123 }
1124 // The download manager disables signal handling in the curl library;
1125 // as OpenSSL's implementation of TLS will generate a sigpipe in some
1126 // error paths, we must explicitly disable SIGPIPE here.
1127 // TODO(jblomer): it should be enough to do this once
1128 ✗ signal(SIGPIPE, SIG_IGN);
1129 }
1130
1131
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4784 times.
 4784 if (url.find("@proxy@") != string::npos) {
1132 // This is used in Geo-API requests (only), to replace a portion of the
1133 // URL with the current proxy name for the sake of caching the result.
1134 // Replace the @proxy@ either with a passed in "forced" template (which
1135 // is set from $CVMFS_PROXY_TEMPLATE) if there is one, or a "direct"
1136 // template (which is the uuid) if there's no proxy, or the name of the
1137 // proxy.
1138 ✗ string replacement;
1139 ✗ if (proxy_template_forced_ != "") {
1140 ✗ replacement = proxy_template_forced_;
1141 ✗ } else if (info->proxy() == "DIRECT") {
1142 ✗ replacement = proxy_template_direct_;
1143 } else {
1144 ✗ if (opt_proxy_groups_current_ >= opt_proxy_groups_fallback_) {
1145 // It doesn't make sense to use the fallback proxies in Geo-API requests
1146 // since the fallback proxies are supposed to get sorted, too.
1147 ✗ info->SetProxy("DIRECT");
1148 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "");
1149 ✗ replacement = proxy_template_direct_;
1150 } else {
1151 ✗ replacement = ChooseProxyUnlocked(info->expected_hash())->host.name();
1152 }
1153 }
1154 ✗ replacement = (replacement == "") ? proxy_template_direct_ : replacement;
1155 ✗ LogCvmfs(kLogDownload, kLogDebug,
1156 "(manager %s - id %" PRId64 ") "
1157 "replacing @proxy@ by %s",
1158 name_.c_str(), info->id(), replacement.c_str());
1159 ✗ url = ReplaceAll(url, "@proxy@", replacement);
1160 }
1161
1162 // TODO(heretherebedragons) before removing
1163 // static_cast<cvmfs::MemSink*>(info->sink)->size() == 0
1164 // and just always call info->sink->Reserve()
1165 // we should do a speed check
1166
3/4
✓ Branch 2 taken 4784 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1816 times.
✓ Branch 5 taken 2968 times.
 4784 if ((info->sink() != NULL) && info->sink()->RequiresReserve()
1167
1/2
✓ Branch 2 taken 1816 times.
✗ Branch 3 not taken.
 1816 && (static_cast<cvmfs::MemSink *>(info->sink())->size() == 0)
1168
11/20
✓ Branch 1 taken 4784 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 1816 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 1816 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 1518 times.
✓ Branch 11 taken 298 times.
✓ Branch 12 taken 1816 times.
✓ Branch 13 taken 2968 times.
✗ Branch 14 not taken.
✓ Branch 15 taken 1816 times.
✓ Branch 16 taken 2968 times.
✓ Branch 18 taken 1518 times.
✓ Branch 19 taken 3266 times.
✗ Branch 20 not taken.
✗ Branch 21 not taken.
✗ Branch 23 not taken.
✗ Branch 24 not taken.
 9568 && HasPrefix(url, "file://", false)) {
1169 platform_stat64 stat_buf;
1170 1518 const int retval = platform_stat(url.c_str(), &stat_buf);
1171
1/2
✓ Branch 0 taken 1518 times.
✗ Branch 1 not taken.
 1518 if (retval != 0) {
1172 // this is an error: file does not exist or out of memory
1173 // error is caught in other code section.
1174
1/2
✓ Branch 2 taken 1518 times.
✗ Branch 3 not taken.
 1518 info->sink()->Reserve(64ul * 1024ul);
1175 } else {
1176 ✗ info->sink()->Reserve(stat_buf.st_size);
1177 }
1178 }
1179
1180
2/4
✓ Branch 2 taken 4784 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 4784 times.
✗ Branch 7 not taken.
 4784 curl_easy_setopt(curl_handle, CURLOPT_URL,
1181 EscapeUrl(info->id(), url).c_str());
1182 4784 }
1183
1184
1185 /**
1186 * Checks if the name resolving information is still up to date. The host
1187 * object should be one from the current load-balance group. If the information
1188 * changed, gather new set of resolved IPs and, if different, exchange them in
1189 * the load-balance group on the fly. In the latter case, also rebalance the
1190 * proxies. The options mutex needs to be open.
1191 *
1192 * Returns true if proxies may have changed.
1193 */
1194 92 bool DownloadManager::ValidateProxyIpsUnlocked(const string &url,
1195 const dns::Host &host) {
1196
2/4
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 92 times.
✗ Branch 4 not taken.
 92 if (!host.IsExpired())
1197 92 return false;
1198 ✗ LogCvmfs(kLogDownload, kLogDebug, "(manager '%s') validate DNS entry for %s",
1199 ✗ name_.c_str(), host.name().c_str());
1200
1201 ✗ const unsigned group_idx = opt_proxy_groups_current_;
1202 ✗ dns::Host new_host = resolver_->Resolve(host.name());
1203
1204 ✗ bool update_only = true; // No changes to the list of IP addresses.
1205 ✗ if (new_host.status() != dns::kFailOk) {
1206 // Try again later in case resolving fails.
1207 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1208 "(manager '%s') failed to resolve IP addresses for %s (%d - %s)",
1209 ✗ name_.c_str(), host.name().c_str(), new_host.status(),
1210 dns::Code2Ascii(new_host.status()));
1211 ✗ new_host = dns::Host::ExtendDeadline(host, resolver_->min_ttl());
1212 ✗ } else if (!host.IsEquivalent(new_host)) {
1213 ✗ update_only = false;
1214 }
1215
1216 ✗ if (update_only) {
1217 ✗ for (unsigned i = 0; i < (*opt_proxy_groups_)[group_idx].size(); ++i) {
1218 ✗ if ((*opt_proxy_groups_)[group_idx][i].host.id() == host.id())
1219 ✗ (*opt_proxy_groups_)[group_idx][i].host = new_host;
1220 }
1221 ✗ return false;
1222 }
1223
1224 ✗ assert(new_host.status() == dns::kFailOk);
1225
1226 // Remove old host objects, insert new objects, and rebalance.
1227 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslog,
1228 "(manager '%s') DNS entries for proxy %s changed, adjusting",
1229 ✗ name_.c_str(), host.name().c_str());
1230 ✗ vector<ProxyInfo> *group = current_proxy_group();
1231 ✗ opt_num_proxies_ -= group->size();
1232 ✗ for (unsigned i = 0; i < group->size();) {
1233 ✗ if ((*group)[i].host.id() == host.id()) {
1234 ✗ group->erase(group->begin() + i);
1235 } else {
1236 ✗ i++;
1237 }
1238 }
1239 ✗ vector<ProxyInfo> new_infos;
1240 set<string> const best_addresses = new_host.ViewBestAddresses(
1241 ✗ opt_ip_preference_);
1242 ✗ set<string>::const_iterator iter_ips = best_addresses.begin();
1243 ✗ for (; iter_ips != best_addresses.end(); ++iter_ips) {
1244 ✗ const string url_ip = dns::RewriteUrl(url, *iter_ips);
1245 ✗ new_infos.push_back(ProxyInfo(new_host, url_ip));
1246 }
1247 ✗ group->insert(group->end(), new_infos.begin(), new_infos.end());
1248 ✗ opt_num_proxies_ += new_infos.size();
1249
1250 ✗ const std::string msg = "DNS entries for proxy " + host.name() + " changed";
1251
1252 ✗ RebalanceProxiesUnlocked(msg);
1253 ✗ return true;
1254 }
1255
1256
1257 /**
1258 * Adds transfer time and downloaded bytes to the global counters.
1259 */
1260 4975 void DownloadManager::UpdateStatistics(CURL *handle) {
1261 double val;
1262 int retval;
1263 4975 int64_t sum = 0;
1264
1265
1/2
✓ Branch 1 taken 4975 times.
✗ Branch 2 not taken.
 4975 retval = curl_easy_getinfo(handle, CURLINFO_SIZE_DOWNLOAD, &val);
1266
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4975 times.
 4975 assert(retval == CURLE_OK);
1267 4975 sum += static_cast<int64_t>(val);
1268 /*retval = curl_easy_getinfo(handle, CURLINFO_HEADER_SIZE, &val);
1269 assert(retval == CURLE_OK);
1270 sum += static_cast<int64_t>(val);*/
1271 4975 perf::Xadd(counters_->sz_transferred_bytes, sum);
1272 4975 }
1273
1274
1275 /**
1276 * Retry if possible if not on no-cache and if not already done too often.
1277 */
1278 4975 bool DownloadManager::CanRetry(const JobInfo *info) {
1279 4975 const MutexLockGuard m(lock_options_);
1280 4975 const unsigned max_retries = opt_max_retries_;
1281
1282
2/2
✓ Branch 2 taken 1984 times.
✓ Branch 3 taken 2871 times.
 9830 return !(info->nocache()) && (info->num_retries() < max_retries)
1283
3/4
✓ Branch 0 taken 4855 times.
✓ Branch 1 taken 120 times.
✓ Branch 4 taken 1984 times.
✗ Branch 5 not taken.
 11814 && (IsProxyTransferError(info->error_code())
1284
2/2
✓ Branch 2 taken 131 times.
✓ Branch 3 taken 1853 times.
 1984 || IsHostTransferError(info->error_code()));
1285 4975 }
1286
1287 /**
1288 * Backoff for retry to introduce a jitter into a cluster of requesting
1289 * cvmfs nodes.
1290 * Retry only when HTTP caching is on.
1291 *
1292 * \return true if backoff has been performed, false otherwise
1293 */
1294 131 void DownloadManager::Backoff(JobInfo *info) {
1295 131 unsigned backoff_init_ms = 0;
1296 131 unsigned backoff_max_ms = 0;
1297 {
1298 131 const MutexLockGuard m(lock_options_);
1299 131 backoff_init_ms = opt_backoff_init_ms_;
1300 131 backoff_max_ms = opt_backoff_max_ms_;
1301 131 }
1302
1303 131 info->SetNumRetries(info->num_retries() + 1);
1304 131 perf::Inc(counters_->n_retries);
1305
2/2
✓ Branch 1 taken 51 times.
✓ Branch 2 taken 80 times.
 131 if (info->backoff_ms() == 0) {
1306 51 info->SetBackoffMs(prng_.Next(backoff_init_ms + 1)); // Must be != 0
1307 } else {
1308 80 info->SetBackoffMs(info->backoff_ms() * 2);
1309 }
1310
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 131 times.
 131 if (info->backoff_ms() > backoff_max_ms) {
1311 ✗ info->SetBackoffMs(backoff_max_ms);
1312 }
1313
1314 131 LogCvmfs(kLogDownload, kLogDebug,
1315 "(manager '%s' - id %" PRId64 ") backing off for %d ms",
1316 name_.c_str(), info->id(), info->backoff_ms());
1317 131 SafeSleepMs(info->backoff_ms());
1318 131 }
1319
1320 120 void DownloadManager::SetNocache(JobInfo *info) {
1321
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 120 times.
 120 if (info->nocache())
1322 ✗ return;
1323 120 header_lists_->AppendHeader(info->headers(), "Pragma: no-cache");
1324 120 header_lists_->AppendHeader(info->headers(), "Cache-Control: no-cache");
1325 120 curl_easy_setopt(info->curl_handle(), CURLOPT_HTTPHEADER, info->headers());
1326 120 info->SetNocache(true);
1327 }
1328
1329
1330 /**
1331 * Reverse operation of SetNocache. Makes sure that "no-cache" header
1332 * disappears from the list of headers to let proxies work normally.
1333 */
1334 260 void DownloadManager::SetRegularCache(JobInfo *info) {
1335
1/2
✓ Branch 1 taken 260 times.
✗ Branch 2 not taken.
 260 if (info->nocache() == false)
1336 260 return;
1337 ✗ header_lists_->CutHeader("Pragma: no-cache", info->GetHeadersPtr());
1338 ✗ header_lists_->CutHeader("Cache-Control: no-cache", info->GetHeadersPtr());
1339 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_HTTPHEADER, info->headers());
1340 ✗ info->SetNocache(false);
1341 }
1342
1343
1344 /**
1345 * Frees the storage associated with the authz attachment from the job
1346 */
1347 4784 void DownloadManager::ReleaseCredential(JobInfo *info) {
1348
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4784 times.
 4784 if (info->cred_data()) {
1349 ✗ assert(credentials_attachment_ != NULL); // Someone must have set it
1350 ✗ credentials_attachment_->ReleaseCurlHandle(info->curl_handle(),
1351 info->cred_data());
1352 ✗ info->SetCredData(NULL);
1353 }
1354 4784 }
1355
1356
1357 /* Sort links based on the "pri=" parameter */
1358 ✗ static bool sortlinks(const std::string &s1, const std::string &s2) {
1359 ✗ const size_t pos1 = s1.find("; pri=");
1360 ✗ const size_t pos2 = s2.find("; pri=");
1361 int pri1, pri2;
1362 ✗ if ((pos1 != std::string::npos) && (pos2 != std::string::npos)
1363 ✗ && (sscanf(s1.substr(pos1 + 6).c_str(), "%d", &pri1) == 1)
1364 ✗ && (sscanf(s2.substr(pos2 + 6).c_str(), "%d", &pri2) == 1)) {
1365 ✗ return pri1 < pri2;
1366 }
1367 ✗ return false;
1368 }
1369
1370 /**
1371 * Parses Link header and uses it to set a new host chain.
1372 * See rfc6249.
1373 */
1374 ✗ void DownloadManager::ProcessLink(JobInfo *info) {
1375 ✗ std::vector<std::string> links = SplitString(info->link(), ',');
1376 ✗ if (info->link().find("; pri=") != std::string::npos)
1377 ✗ std::sort(links.begin(), links.end(), sortlinks);
1378
1379 ✗ std::vector<std::string> host_list;
1380
1381 ✗ std::vector<std::string>::const_iterator il = links.begin();
1382 ✗ for (; il != links.end(); ++il) {
1383 ✗ const std::string &link = *il;
1384 ✗ if ((link.find("; rel=duplicate") == std::string::npos)
1385 ✗ && (link.find("; rel=\"duplicate\"") == std::string::npos)) {
1386 ✗ LogCvmfs(kLogDownload, kLogDebug,
1387 "skipping link '%s' because it does not contain rel=duplicate",
1388 link.c_str());
1389 ✗ continue;
1390 }
1391 // ignore depth= field since there's nothing useful we can do with it
1392
1393 ✗ size_t start = link.find('<');
1394 ✗ if (start == std::string::npos) {
1395 ✗ LogCvmfs(
1396 kLogDownload, kLogDebug,
1397 "skipping link '%s' because it does not have a left angle bracket",
1398 link.c_str());
1399 ✗ continue;
1400 }
1401
1402 ✗ start++;
1403 ✗ if ((link.substr(start, 7) != "http://")
1404 ✗ && (link.substr(start, 8) != "https://")) {
1405 ✗ LogCvmfs(kLogDownload, kLogDebug,
1406 "skipping link '%s' of unrecognized url protocol", link.c_str());
1407 ✗ continue;
1408 }
1409
1410 ✗ size_t end = link.find('/', start + 8);
1411 ✗ if (end == std::string::npos)
1412 ✗ end = link.find('>');
1413 ✗ if (end == std::string::npos) {
1414 ✗ LogCvmfs(kLogDownload, kLogDebug,
1415 "skipping link '%s' because no slash in url and no right angle "
1416 "bracket",
1417 link.c_str());
1418 ✗ continue;
1419 }
1420 ✗ const std::string host = link.substr(start, end - start);
1421 ✗ LogCvmfs(kLogDownload, kLogDebug, "adding linked host '%s'", host.c_str());
1422 ✗ host_list.push_back(host);
1423 }
1424
1425 ✗ if (host_list.size() > 0) {
1426 ✗ SetHostChain(host_list);
1427 ✗ opt_metalink_timestamp_link_ = time(NULL);
1428 // Set the index to the first linked host because the protocol includes
1429 // redirecting to the first host
1430 ✗ info->SetCurrentHostChainIndex(0);
1431 // Don't process the same links again if there are retries
1432 ✗ info->SetLink("");
1433 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslog,
1434 "(manager '%s' - id %" PRId64 ") "
1435 "received %d hosts from metalink server, starting with %s",
1436 name_.c_str(), info->id(),
1437 host_list.size(),
1438 ✗ host_list[0].c_str());
1439 }
1440 }
1441
1442
1443 /**
1444 * Checks the result of a curl download and implements the failure logic, such
1445 * as changing the proxy server. Takes care of cleanup.
1446 *
1447 * \return true if another download should be performed, false otherwise
1448 */
1449 4975 bool DownloadManager::VerifyAndFinalize(const int curl_error, JobInfo *info) {
1450
1/2
✓ Branch 6 taken 4975 times.
✗ Branch 7 not taken.
 4975 LogCvmfs(kLogDownload, kLogDebug,
1451 "(manager '%s' - id %" PRId64 ") "
1452 "Verify downloaded url %s, proxy %s (curl error %d)",
1453 name_.c_str(), info->id(), info->url()->c_str(),
1454
1/2
✓ Branch 1 taken 4975 times.
✗ Branch 2 not taken.
 9950 info->proxy().c_str(), curl_error);
1455
1/2
✓ Branch 2 taken 4975 times.
✗ Branch 3 not taken.
 4975 UpdateStatistics(info->curl_handle());
1456
1457 bool was_metalink;
1458 4975 std::string typ;
1459
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4975 times.
 4975 if (info->current_metalink_chain_index() >= 0) {
1460 ✗ if (info->link() != "") {
1461 // process Link header whether or not the redirected URL got an error
1462 ✗ ProcessLink(info);
1463 // The metalink lookup succeeded, so if there was an error it was
1464 // with the redirect
1465 ✗ was_metalink = false;
1466 ✗ typ = "host";
1467 } else {
1468 // no Link headers were received, so the metalink lookup failed
1469 ✗ was_metalink = true;
1470 ✗ typ = "metalink";
1471 }
1472 } else {
1473 4975 was_metalink = false;
1474
1/2
✓ Branch 1 taken 4975 times.
✗ Branch 2 not taken.
 4975 typ = "host";
1475 }
1476
1477 // Verification and error classification
1478
3/14
✓ Branch 0 taken 4513 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 46 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 416 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
 4975 switch (curl_error) {
1479 4513 case CURLE_OK:
1480 // Verify content hash
1481
2/2
✓ Branch 1 taken 3131 times.
✓ Branch 2 taken 1382 times.
 4513 if (info->expected_hash()) {
1482
1/2
✓ Branch 1 taken 3131 times.
✗ Branch 2 not taken.
 3131 shash::Any match_hash;
1483
1/2
✓ Branch 2 taken 3131 times.
✗ Branch 3 not taken.
 3131 shash::Final(info->hash_context(), &match_hash);
1484
3/4
✓ Branch 2 taken 3131 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 120 times.
✓ Branch 5 taken 3011 times.
 3131 if (match_hash != *(info->expected_hash())) {
1485
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 120 times.
 120 if (ignore_signature_failures_) {
1486 ✗ LogCvmfs(
1487 kLogDownload, kLogDebug | kLogSyslogErr,
1488 "(manager '%s' - id %" PRId64 ") "
1489 "ignoring failed hash verification of %s (expected %s, got %s)",
1490 name_.c_str(), info->id(), info->url()->c_str(),
1491 ✗ info->expected_hash()->ToString().c_str(),
1492 ✗ match_hash.ToString().c_str());
1493 } else {
1494
1/2
✓ Branch 7 taken 120 times.
✗ Branch 8 not taken.
 240 LogCvmfs(kLogDownload, kLogDebug,
1495 "(manager '%s' - id %" PRId64 ") "
1496 "hash verification of %s failed (expected %s, got %s)",
1497 name_.c_str(), info->id(), info->url()->c_str(),
1498
1/2
✓ Branch 2 taken 120 times.
✗ Branch 3 not taken.
 240 info->expected_hash()->ToString().c_str(),
1499
1/2
✓ Branch 1 taken 120 times.
✗ Branch 2 not taken.
 240 match_hash.ToString().c_str());
1500 120 info->SetErrorCode(kFailBadData);
1501 120 break;
1502 }
1503 }
1504 }
1505
1506 4393 info->SetErrorCode(kFailOk);
1507 4393 break;
1508 ✗ case CURLE_UNSUPPORTED_PROTOCOL:
1509 ✗ info->SetErrorCode(kFailUnsupportedProtocol);
1510 ✗ break;
1511 46 case CURLE_URL_MALFORMAT:
1512 46 info->SetErrorCode(kFailBadUrl);
1513 46 break;
1514 ✗ case CURLE_COULDNT_RESOLVE_PROXY:
1515 ✗ info->SetErrorCode(kFailProxyResolve);
1516 ✗ break;
1517 ✗ case CURLE_COULDNT_RESOLVE_HOST:
1518 ✗ info->SetErrorCode(kFailHostResolve);
1519 ✗ break;
1520 ✗ case CURLE_OPERATION_TIMEDOUT:
1521 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostTooSlow
1522 : kFailProxyTooSlow);
1523 ✗ break;
1524 ✗ case CURLE_PARTIAL_FILE:
1525 case CURLE_GOT_NOTHING:
1526 case CURLE_RECV_ERROR:
1527 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostShortTransfer
1528 : kFailProxyShortTransfer);
1529 ✗ break;
1530 416 case CURLE_FILE_COULDNT_READ_FILE:
1531 case CURLE_COULDNT_CONNECT:
1532
3/6
✓ Branch 1 taken 416 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 23 times.
✓ Branch 6 taken 393 times.
 416 if (info->proxy() != "DIRECT") {
1533 // This is a guess. Fail-over can still change to switching host
1534 23 info->SetErrorCode(kFailProxyConnection);
1535 } else {
1536 393 info->SetErrorCode(kFailHostConnection);
1537 }
1538 416 break;
1539 ✗ case CURLE_TOO_MANY_REDIRECTS:
1540 ✗ info->SetErrorCode(kFailHostConnection);
1541 ✗ break;
1542 ✗ case CURLE_SSL_CACERT_BADFILE:
1543 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
1544 "(manager '%s' -id %" PRId64 ") "
1545 "Failed to load certificate bundle. "
1546 "X509_CERT_BUNDLE might point to the wrong location.",
1547 name_.c_str(), info->id());
1548 ✗ info->SetErrorCode(kFailHostConnection);
1549 ✗ break;
1550 // As of curl 7.62.0, CURLE_SSL_CACERT is the same as
1551 // CURLE_PEER_FAILED_VERIFICATION
1552 ✗ case CURLE_PEER_FAILED_VERIFICATION:
1553 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
1554 "(manager '%s' - id %" PRId64 ") "
1555 "invalid SSL certificate of remote host. "
1556 "X509_CERT_DIR and/or X509_CERT_BUNDLE might point to the wrong "
1557 "location.",
1558 name_.c_str(), info->id());
1559 ✗ info->SetErrorCode(kFailHostConnection);
1560 ✗ break;
1561 ✗ case CURLE_ABORTED_BY_CALLBACK:
1562 case CURLE_WRITE_ERROR:
1563 // Error set by callback
1564 ✗ break;
1565 ✗ case CURLE_SEND_ERROR:
1566 // The curl error CURLE_SEND_ERROR can be seen when a cache is misbehaving
1567 // and closing connections before the http request send is completed.
1568 // Handle this error, treating it as a short transfer error.
1569 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostShortTransfer
1570 : kFailProxyShortTransfer);
1571 ✗ break;
1572 ✗ default:
1573 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
1574 "(manager '%s' - id %" PRId64 ") "
1575 "unexpected curl error (%d) while trying to fetch %s",
1576 name_.c_str(), info->id(), curl_error, info->url()->c_str());
1577 ✗ info->SetErrorCode(kFailOther);
1578 ✗ break;
1579 }
1580
1581 std::vector<std::string> *host_chain;
1582 unsigned char num_used_hosts;
1583
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4975 times.
 4975 if (was_metalink) {
1584 ✗ host_chain = opt_metalink_.chain;
1585 ✗ num_used_hosts = info->num_used_metalinks();
1586 } else {
1587 4975 host_chain = opt_host_.chain;
1588 4975 num_used_hosts = info->num_used_hosts();
1589 }
1590
1591 // Determination if download should be repeated
1592 4975 bool try_again = false;
1593
1/2
✓ Branch 1 taken 4975 times.
✗ Branch 2 not taken.
 4975 bool same_url_retry = CanRetry(info);
1594
2/2
✓ Branch 1 taken 582 times.
✓ Branch 2 taken 4393 times.
 4975 if (info->error_code() != kFailOk) {
1595 582 const MutexLockGuard m(lock_options_);
1596
2/2
✓ Branch 1 taken 120 times.
✓ Branch 2 taken 462 times.
 582 if (info->error_code() == kFailBadData) {
1597
2/2
✓ Branch 1 taken 60 times.
✓ Branch 2 taken 60 times.
 120 if (!info->nocache()) {
1598 60 try_again = true;
1599 } else {
1600 // Make it a host failure
1601
1/2
✓ Branch 4 taken 60 times.
✗ Branch 5 not taken.
 60 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1602 "(manager '%s' - id %" PRId64 ") "
1603 "data corruption with no-cache header, try another %s",
1604 name_.c_str(), info->id(), typ.c_str());
1605
1606 60 info->SetErrorCode(kFailHostHttp);
1607 }
1608 }
1609 582 if (same_url_retry
1610
5/6
✓ Branch 0 taken 451 times.
✓ Branch 1 taken 131 times.
✓ Branch 3 taken 451 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 200 times.
✓ Branch 6 taken 382 times.
 1033 || (((info->error_code() == kFailHostResolve)
1611
2/2
✓ Branch 2 taken 189 times.
✓ Branch 3 taken 262 times.
 451 || IsHostTransferError(info->error_code())
1612
2/2
✓ Branch 1 taken 60 times.
✓ Branch 2 taken 129 times.
 189 || (info->error_code() == kFailHostHttp))
1613
3/4
✓ Branch 1 taken 193 times.
✓ Branch 2 taken 129 times.
✓ Branch 3 taken 193 times.
✗ Branch 4 not taken.
 322 && info->probe_hosts() && host_chain
1614
2/2
✓ Branch 1 taken 69 times.
✓ Branch 2 taken 124 times.
 193 && (num_used_hosts < host_chain->size()))) {
1615 200 try_again = true;
1616 }
1617 582 if (same_url_retry
1618
5/6
✓ Branch 0 taken 451 times.
✓ Branch 1 taken 131 times.
✓ Branch 3 taken 451 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 154 times.
✓ Branch 6 taken 428 times.
 1033 || (((info->error_code() == kFailProxyResolve)
1619
2/2
✓ Branch 2 taken 428 times.
✓ Branch 3 taken 23 times.
 451 || IsProxyTransferError(info->error_code())
1620
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 428 times.
 428 || (info->error_code() == kFailProxyHttp)))) {
1621
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 154 times.
 154 if (sharding_policy_.UseCount() > 0) { // sharding policy
1622 ✗ try_again = true;
1623 ✗ same_url_retry = false;
1624 } else { // no sharding
1625 154 try_again = true;
1626 // If all proxies failed, do a next round with the next host
1627
4/6
✓ Branch 0 taken 23 times.
✓ Branch 1 taken 131 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 23 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 154 times.
 154 if (!same_url_retry && (info->num_used_proxies() >= opt_num_proxies_)) {
1628 // Check if this can be made a host fail-over
1629 ✗ if (info->probe_hosts() && host_chain
1630 ✗ && (num_used_hosts < host_chain->size())) {
1631 // reset proxy group if not already performed by other handle
1632 ✗ if (opt_proxy_groups_) {
1633 ✗ if ((opt_proxy_groups_current_ > 0)
1634 ✗ || (opt_proxy_groups_current_burned_ > 0)) {
1635 ✗ opt_proxy_groups_current_ = 0;
1636 ✗ opt_timestamp_backup_proxies_ = 0;
1637 ✗ const std::string msg = "reset proxies for " + typ
1638 ✗ + " failover";
1639 ✗ RebalanceProxiesUnlocked(msg);
1640 }
1641 }
1642
1643 // Make it a host failure
1644 ✗ LogCvmfs(kLogDownload, kLogDebug,
1645 "(manager '%s' - id %" PRId64 ") make it a %s failure",
1646 name_.c_str(), info->id(), typ.c_str());
1647 ✗ info->SetNumUsedProxies(1);
1648 ✗ info->SetErrorCode(kFailHostAfterProxy);
1649 } else {
1650 ✗ if (failover_indefinitely_) {
1651 // Instead of giving up, reset the num_used_proxies counter,
1652 // switch proxy and try again
1653 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1654 "(manager '%s' - id %" PRId64 ") "
1655 "VerifyAndFinalize() would fail the download here. "
1656 "Instead switch proxy and retry download. "
1657 "typ=%s "
1658 "info->probe_hosts=%d host_chain=%p num_used_hosts=%d "
1659 "host_chain->size()=%lu same_url_retry=%d "
1660 "info->num_used_proxies=%d opt_num_proxies_=%d",
1661 name_.c_str(), info->id(), typ.c_str(),
1662 ✗ static_cast<int>(info->probe_hosts()), host_chain,
1663 ✗ num_used_hosts, host_chain ? host_chain->size() : -1,
1664 static_cast<int>(same_url_retry),
1665 ✗ info->num_used_proxies(), opt_num_proxies_);
1666 ✗ info->SetNumUsedProxies(1);
1667 ✗ RebalanceProxiesUnlocked(
1668 "download failed - failover indefinitely");
1669 ✗ try_again = !Interrupted(fqrn_, info);
1670 } else {
1671 ✗ try_again = false;
1672 }
1673 }
1674 } // Make a proxy failure a host failure
1675 } // Proxy failure assumed
1676 } // end !sharding
1677 582 }
1678
1679
2/2
✓ Branch 0 taken 283 times.
✓ Branch 1 taken 4692 times.
 4975 if (try_again) {
1680
4/6
✗ Branch 0 not taken.
✓ Branch 1 taken 283 times.
✓ Branch 2 taken 131 times.
✓ Branch 3 taken 152 times.
✓ Branch 7 taken 283 times.
✗ Branch 8 not taken.
 566 LogCvmfs(kLogDownload, kLogDebug,
1681 "(manager '%s' - id %" PRId64 ") "
1682 "Trying again on same curl handle, %s"
1683 "error code %d%s",
1684 name_.c_str(), info->id(),
1685 same_url_retry ? "same url, " : "",
1686 283 info->error_code(),
1687 283 info->nocache() ? ", no cache" : "");
1688 // Reset internal state and destination. In parallel-decompress mode the
1689 // sink and zstream are owned by the caller thread (it pops and decompresses
1690 // the queued chunks). Resetting them here would race the caller and, worse,
1691 // leave the bytes of this failed attempt in the tube to be decompressed
1692 // into the output. Defer the reset by enqueuing an ordered marker; the
1693 // caller discards this attempt's bytes when it pops it (see below).
1694 283 const bool defer_reset = info->IsValidDataTube();
1695
1/2
✓ Branch 0 taken 283 times.
✗ Branch 1 not taken.
 283 if (!defer_reset) {
1696
4/8
✓ Branch 1 taken 283 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 283 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 283 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 283 times.
 283 if (info->sink() != NULL && info->sink()->Reset() != 0) {
1697 ✗ info->SetErrorCode(kFailLocalIO);
1698 ✗ goto verify_and_finalize_stop;
1699 }
1700 }
1701
6/8
✓ Branch 1 taken 23 times.
✓ Branch 2 taken 260 times.
✓ Branch 5 taken 23 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 23 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 23 times.
✓ Branch 10 taken 260 times.
 283 if (info->interrupt_cue() && info->interrupt_cue()->IsCanceled()) {
1702 23 info->SetErrorCode(kFailCanceled);
1703 23 goto verify_and_finalize_stop;
1704 }
1705
1706 // The hash context is updated on this (MainDownload) thread in
1707 // CallbackCurlData(), so it is reset here regardless of the decompress
1708 // mode.
1709
2/2
✓ Branch 1 taken 147 times.
✓ Branch 2 taken 113 times.
 260 if (info->expected_hash()) {
1710
1/2
✓ Branch 2 taken 147 times.
✗ Branch 3 not taken.
 147 shash::Init(info->hash_context());
1711 }
1712
5/6
✓ Branch 1 taken 147 times.
✓ Branch 2 taken 113 times.
✓ Branch 3 taken 147 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 147 times.
✓ Branch 6 taken 113 times.
 260 if (info->compressed() && !defer_reset) {
1713
1/2
✓ Branch 2 taken 147 times.
✗ Branch 3 not taken.
 147 zlib::DecompressInit(info->GetZstreamPtr());
1714 }
1715
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 260 times.
 260 if (defer_reset) {
1716 ✗ info->GetDataTubePtr()->EnqueueBack(new DataTubeElement(kActionReset));
1717 }
1718
1719
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 260 times.
 260 if (sharding_policy_.UseCount() > 0) { // sharding policy
1720 ✗ ReleaseCredential(info);
1721 ✗ SetUrlOptions(info);
1722 } else { // no sharding policy
1723
1/2
✓ Branch 1 taken 260 times.
✗ Branch 2 not taken.
 260 SetRegularCache(info);
1724
1725 // Failure handling
1726 260 bool switch_proxy = false;
1727 260 bool switch_host = false;
1728
2/4
✓ Branch 1 taken 60 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 200 times.
 260 switch (info->error_code()) {
1729 60 case kFailBadData:
1730
1/2
✓ Branch 1 taken 60 times.
✗ Branch 2 not taken.
 60 SetNocache(info);
1731 60 break;
1732 ✗ case kFailProxyResolve:
1733 case kFailProxyHttp:
1734 ✗ switch_proxy = true;
1735 ✗ break;
1736 ✗ case kFailHostResolve:
1737 case kFailHostHttp:
1738 case kFailHostAfterProxy:
1739 ✗ switch_host = true;
1740 ✗ break;
1741 200 default:
1742
2/2
✓ Branch 2 taken 23 times.
✓ Branch 3 taken 177 times.
 200 if (IsProxyTransferError(info->error_code())) {
1743
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 23 times.
 23 if (same_url_retry) {
1744 ✗ Backoff(info);
1745 } else {
1746 23 switch_proxy = true;
1747 }
1748
1/2
✓ Branch 2 taken 177 times.
✗ Branch 3 not taken.
 177 } else if (IsHostTransferError(info->error_code())) {
1749
2/2
✓ Branch 0 taken 131 times.
✓ Branch 1 taken 46 times.
 177 if (same_url_retry) {
1750
1/2
✓ Branch 1 taken 131 times.
✗ Branch 2 not taken.
 131 Backoff(info);
1751 } else {
1752 46 switch_host = true;
1753 }
1754 } else {
1755 // No other errors expected when retrying
1756 ✗ PANIC(NULL);
1757 }
1758 }
1759
2/2
✓ Branch 0 taken 23 times.
✓ Branch 1 taken 237 times.
 260 if (switch_proxy) {
1760
1/2
✓ Branch 1 taken 23 times.
✗ Branch 2 not taken.
 23 ReleaseCredential(info);
1761
1/2
✓ Branch 1 taken 23 times.
✗ Branch 2 not taken.
 23 SwitchProxy(info);
1762 23 info->SetNumUsedProxies(info->num_used_proxies() + 1);
1763
1/2
✓ Branch 1 taken 23 times.
✗ Branch 2 not taken.
 23 SetUrlOptions(info);
1764 }
1765
2/2
✓ Branch 0 taken 46 times.
✓ Branch 1 taken 214 times.
 260 if (switch_host) {
1766
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 ReleaseCredential(info);
1767
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 46 times.
 46 if (was_metalink) {
1768 ✗ SwitchMetalink(info);
1769 ✗ info->SetNumUsedMetalinks(num_used_hosts + 1);
1770 } else {
1771
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 SwitchHost(info);
1772 46 info->SetNumUsedHosts(num_used_hosts + 1);
1773 }
1774
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 SetUrlOptions(info);
1775 }
1776 } // end !sharding
1777
1778
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 260 times.
 260 if (failover_indefinitely_) {
1779 // try again, breaking if there's a cvmfs reload happening and we are in a
1780 // proxy failover. This will EIO the call application.
1781 ✗ return !Interrupted(fqrn_, info);
1782 }
1783 260 return true; // try again
1784 }
1785
1786 4692 verify_and_finalize_stop:
1787 // Finalize, flush destination file.
1788
1/2
✓ Branch 1 taken 4715 times.
✗ Branch 2 not taken.
 4715 ReleaseCredential(info);
1789
1790 // In parallel-decompress mode (data_tube_ valid) the caller has not yet
1791 // pushed bytes through the sink / zstream — those operations happen on
1792 // the caller thread when it pops the queued chunks. Flushing the sink
1793 // and tearing down the zstream here would race the caller's decompress
1794 // and corrupt the output. Defer both to the caller.
1795 4715 const bool defer_finalize = info->IsValidDataTube();
1796
1/2
✓ Branch 0 taken 4715 times.
✗ Branch 1 not taken.
 4715 if (!defer_finalize) {
1797
4/8
✓ Branch 1 taken 4715 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 4715 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 4715 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 4715 times.
 4715 if (info->sink() != NULL && info->sink()->Flush() != 0) {
1798 ✗ info->SetErrorCode(kFailLocalIO);
1799 }
1800
2/2
✓ Branch 1 taken 3223 times.
✓ Branch 2 taken 1492 times.
 4715 if (info->compressed())
1801
1/2
✓ Branch 2 taken 3223 times.
✗ Branch 3 not taken.
 3223 zlib::DecompressFini(info->GetZstreamPtr());
1802 }
1803
1804
1/2
✓ Branch 1 taken 4715 times.
✗ Branch 2 not taken.
 4715 if (info->headers()) {
1805
1/2
✓ Branch 2 taken 4715 times.
✗ Branch 3 not taken.
 4715 header_lists_->PutList(info->headers());
1806 4715 info->SetHeaders(NULL);
1807 }
1808
1809 4715 return false; // stop transfer and return to Fetch()
1810 4975 }
1811
1812 4077 DownloadManager::~DownloadManager() {
1813 // cleaned up fini
1814
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4077 times.
 4077 if (sharding_policy_.UseCount() > 0) {
1815 ✗ sharding_policy_.Reset();
1816 }
1817
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4077 times.
 4077 if (health_check_.UseCount() > 0) {
1818 ✗ if (health_check_.Unique()) {
1819 ✗ LogCvmfs(kLogDownload, kLogDebug,
1820 "(manager '%s') Stopping healthcheck thread", name_.c_str());
1821 ✗ health_check_->StopHealthcheck();
1822 }
1823 ✗ health_check_.Reset();
1824 }
1825
1826
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4077 times.
 4077 if (atomic_xadd32(&multi_threaded_, 0) == 1) {
1827 // Shutdown I/O thread
1828 ✗ pipe_terminate_->Write(kPipeTerminateSignal);
1829 ✗ pthread_join(thread_download_, NULL);
1830 // All handles are removed from the multi stack
1831 ✗ pipe_terminate_.reset();
1832 ✗ pipe_jobs_.reset();
1833 }
1834
1835 8154 for (set<CURL *>::iterator i = pool_handles_idle_->begin(),
1836 4077 iEnd = pool_handles_idle_->end();
1837
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4077 times.
 4077 i != iEnd;
1838 ✗ ++i) {
1839 ✗ curl_easy_cleanup(*i);
1840 }
1841
1842
1/2
✓ Branch 0 taken 4077 times.
✗ Branch 1 not taken.
 4077 delete pool_handles_idle_;
1843
1/2
✓ Branch 0 taken 4077 times.
✗ Branch 1 not taken.
 4077 delete pool_handles_inuse_;
1844 4077 curl_multi_cleanup(curl_multi_);
1845
1846
1/2
✓ Branch 0 taken 4077 times.
✗ Branch 1 not taken.
 4077 delete header_lists_;
1847
1/2
✓ Branch 0 taken 4077 times.
✗ Branch 1 not taken.
 4077 if (user_agent_)
1848 4077 free(user_agent_);
1849
1850
1/2
✓ Branch 0 taken 4077 times.
✗ Branch 1 not taken.
 4077 delete counters_;
1851
2/2
✓ Branch 0 taken 1703 times.
✓ Branch 1 taken 2374 times.
 4077 delete opt_host_.chain;
1852
2/2
✓ Branch 0 taken 1703 times.
✓ Branch 1 taken 2374 times.
 4077 delete opt_host_chain_rtt_;
1853
2/2
✓ Branch 0 taken 1517 times.
✓ Branch 1 taken 2560 times.
 4077 delete opt_proxy_groups_;
1854
1855 4077 curl_global_cleanup();
1856
1/2
✓ Branch 0 taken 4077 times.
✗ Branch 1 not taken.
 4077 delete resolver_;
1857
1858 // old destructor
1859 4077 pthread_mutex_destroy(lock_options_);
1860 4077 pthread_mutex_destroy(lock_synchronous_mode_);
1861 4077 free(lock_options_);
1862 4077 free(lock_synchronous_mode_);
1863 4077 }
1864
1865 4078 void DownloadManager::InitHeaders() {
1866 // User-Agent
1867
1/2
✓ Branch 2 taken 4078 times.
✗ Branch 3 not taken.
 4078 string cernvm_id = "User-Agent: cvmfs ";
1868 #ifdef CVMFS_LIBCVMFS
1869
1/2
✓ Branch 1 taken 4078 times.
✗ Branch 2 not taken.
 4078 cernvm_id += "libcvmfs ";
1870 #else
1871 ✗ cernvm_id += "Fuse ";
1872 #endif
1873
2/4
✓ Branch 2 taken 4078 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 4078 times.
✗ Branch 6 not taken.
 4078 cernvm_id += string(CVMFS_VERSION);
1874
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4078 times.
 4078 if (getenv("CERNVM_UUID") != NULL) {
1875 cernvm_id += " "
1876 ✗ + sanitizer::InputSanitizer("az AZ 09 -")
1877 ✗ .Filter(getenv("CERNVM_UUID"));
1878 }
1879 4078 user_agent_ = strdup(cernvm_id.c_str());
1880
1881
1/2
✓ Branch 1 taken 4078 times.
✗ Branch 2 not taken.
 4078 header_lists_ = new HeaderLists();
1882
1883
1/2
✓ Branch 1 taken 4078 times.
✗ Branch 2 not taken.
 4078 default_headers_ = header_lists_->GetList("Connection: Keep-Alive");
1884
1/2
✓ Branch 1 taken 4078 times.
✗ Branch 2 not taken.
 4078 header_lists_->AppendHeader(default_headers_, "Pragma:");
1885
1/2
✓ Branch 1 taken 4078 times.
✗ Branch 2 not taken.
 4078 header_lists_->AppendHeader(default_headers_, user_agent_);
1886 4078 }
1887
1888 4078 DownloadManager::DownloadManager(const unsigned max_pool_handles,
1889 const perf::StatisticsTemplate &statistics,
1890 4078 const std::string &name)
1891 4078 : prng_(Prng())
1892 4078 , pool_handles_idle_(new set<CURL *>)
1893 4078 , pool_handles_inuse_(new set<CURL *>)
1894 4078 , pool_max_handles_(max_pool_handles)
1895 4078 , pipe_terminate_(nullptr)
1896 4078 , pipe_jobs_(nullptr)
1897 4078 , watch_fds_(nullptr)
1898 4078 , watch_fds_size_(0)
1899 4078 , watch_fds_inuse_(0)
1900 4078 , watch_fds_max_(4 * max_pool_handles)
1901 4078 , opt_timeout_proxy_(5)
1902 4078 , opt_timeout_direct_(10)
1903 4078 , opt_low_speed_limit_(1024)
1904 4078 , opt_max_retries_(0)
1905 4078 , opt_backoff_init_ms_(0)
1906 4078 , opt_backoff_max_ms_(0)
1907 4078 , enable_info_header_(false)
1908 4078 , opt_ipv4_only_(false)
1909 4078 , follow_redirects_(false)
1910 4078 , ignore_signature_failures_(false)
1911 4078 , enable_http_tracing_(false)
1912 4078 , opt_metalink_(NULL, 0, 0, 0)
1913 4078 , opt_metalink_timestamp_link_(0)
1914 4078 , opt_host_(NULL, 0, 0, 0)
1915 4078 , opt_host_chain_rtt_(NULL)
1916 4078 , opt_proxy_groups_(NULL)
1917 4078 , opt_proxy_groups_current_(0)
1918 4078 , opt_proxy_groups_current_burned_(0)
1919 4078 , opt_proxy_groups_fallback_(0)
1920 4078 , opt_num_proxies_(0)
1921 4078 , opt_proxy_shard_(false)
1922 4078 , failover_indefinitely_(false)
1923
1/2
✓ Branch 1 taken 4078 times.
✗ Branch 2 not taken.
 4078 , name_(name)
1924 4078 , opt_ip_preference_(dns::kIpPreferSystem)
1925 4078 , opt_timestamp_backup_proxies_(0)
1926 4078 , opt_timestamp_failover_proxies_(0)
1927 4078 , opt_proxy_groups_reset_after_(0)
1928 4078 , credentials_attachment_(NULL)
1929
4/8
✓ Branch 13 taken 4078 times.
✗ Branch 14 not taken.
✓ Branch 16 taken 4078 times.
✗ Branch 17 not taken.
✓ Branch 19 taken 4078 times.
✗ Branch 20 not taken.
✓ Branch 23 taken 4078 times.
✗ Branch 24 not taken.
 12234 , counters_(new Counters(statistics)) {
1930 4078 atomic_init32(&multi_threaded_);
1931
1932 4078 lock_options_ = reinterpret_cast<pthread_mutex_t *>(
1933 4078 smalloc(sizeof(pthread_mutex_t)));
1934 4078 int retval = pthread_mutex_init(lock_options_, NULL);
1935
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4078 times.
 4078 assert(retval == 0);
1936 4078 lock_synchronous_mode_ = reinterpret_cast<pthread_mutex_t *>(
1937 4078 smalloc(sizeof(pthread_mutex_t)));
1938 4078 retval = pthread_mutex_init(lock_synchronous_mode_, NULL);
1939
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4078 times.
 4078 assert(retval == 0);
1940
1941
1/2
✓ Branch 1 taken 4078 times.
✗ Branch 2 not taken.
 4078 retval = curl_global_init(CURL_GLOBAL_ALL);
1942
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4078 times.
 4078 assert(retval == CURLE_OK);
1943
1944
1/2
✓ Branch 1 taken 4078 times.
✗ Branch 2 not taken.
 4078 InitHeaders();
1945
1946
1/2
✓ Branch 1 taken 4078 times.
✗ Branch 2 not taken.
 4078 curl_multi_ = curl_multi_init();
1947
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4078 times.
 4078 assert(curl_multi_ != NULL);
1948
1/2
✓ Branch 1 taken 4078 times.
✗ Branch 2 not taken.
 4078 curl_multi_setopt(curl_multi_, CURLMOPT_SOCKETFUNCTION, CallbackCurlSocket);
1949
1/2
✓ Branch 1 taken 4078 times.
✗ Branch 2 not taken.
 4078 curl_multi_setopt(curl_multi_, CURLMOPT_SOCKETDATA,
1950 static_cast<void *>(this));
1951
1/2
✓ Branch 1 taken 4078 times.
✗ Branch 2 not taken.
 4078 curl_multi_setopt(curl_multi_, CURLMOPT_MAXCONNECTS, watch_fds_max_);
1952
1/2
✓ Branch 1 taken 4078 times.
✗ Branch 2 not taken.
 4078 curl_multi_setopt(curl_multi_, CURLMOPT_MAX_TOTAL_CONNECTIONS,
1953 pool_max_handles_);
1954
1955 4078 prng_.InitLocaltime();
1956
1957 // Name resolving
1958 4078 if ((getenv("CVMFS_IPV4_ONLY") != NULL)
1959
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 4078 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 4078 times.
 4078 && (strlen(getenv("CVMFS_IPV4_ONLY")) > 0)) {
1960 ✗ opt_ipv4_only_ = true;
1961 }
1962
1/2
✓ Branch 1 taken 4078 times.
✗ Branch 2 not taken.
 4078 resolver_ = dns::NormalResolver::Create(opt_ipv4_only_, kDnsDefaultRetries,
1963 kDnsDefaultTimeoutMs);
1964
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4078 times.
 4078 assert(resolver_);
1965 4078 }
1966
1967 /**
1968 * Spawns the I/O worker thread and switches the module in multi-threaded mode.
1969 * No way back except Fini(); Init();
1970 */
1971 ✗ void DownloadManager::Spawn() {
1972 ✗ pipe_terminate_ = std::unique_ptr<Pipe<kPipeThreadTerminator> >(
1973 ✗ new Pipe<kPipeThreadTerminator>());
1974 ✗ pipe_jobs_ = std::unique_ptr<Pipe<kPipeDownloadJobs> >(
1975 ✗ new Pipe<kPipeDownloadJobs>());
1976
1977 ✗ const int retval = pthread_create(&thread_download_, NULL, MainDownload,
1978 static_cast<void *>(this));
1979 ✗ assert(retval == 0);
1980
1981 ✗ atomic_inc32(&multi_threaded_);
1982
1983 ✗ if (health_check_.UseCount() > 0) {
1984 ✗ LogCvmfs(kLogDownload, kLogDebug,
1985 "(manager '%s') Starting healthcheck thread", name_.c_str());
1986 ✗ health_check_->StartHealthcheck();
1987 }
1988 }
1989
1990
1991 /**
1992 * Downloads data from an insecure outside channel (currently HTTP or file).
1993 */
1994 4715 Failures DownloadManager::Fetch(JobInfo *info) {
1995
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4715 times.
 4715 assert(info != NULL);
1996
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4715 times.
 4715 assert(info->url() != NULL);
1997
1998 Failures result;
1999
1/2
✓ Branch 1 taken 4715 times.
✗ Branch 2 not taken.
 4715 result = PrepareDownloadDestination(info);
2000
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4715 times.
 4715 if (result != kFailOk)
2001 ✗ return result;
2002
2003
2/2
✓ Branch 1 taken 3243 times.
✓ Branch 2 taken 1472 times.
 4715 if (info->expected_hash()) {
2004 3243 const shash::Algorithms algorithm = info->expected_hash()->algorithm;
2005 3243 info->GetHashContextPtr()->algorithm = algorithm;
2006
1/2
✓ Branch 1 taken 3243 times.
✗ Branch 2 not taken.
 3243 info->GetHashContextPtr()->size = shash::GetContextSize(algorithm);
2007 3243 info->GetHashContextPtr()->buffer = alloca(info->hash_context().size);
2008 }
2009
2010 // Clear out any saved Links in case this will be a metalink request and
2011 // previous non-metalink requests sent Link headers
2012
2/4
✓ Branch 2 taken 4715 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 4715 times.
✗ Branch 6 not taken.
 4715 info->SetLink("");
2013
2014 // Prepare cvmfs-info: header, allocate string on the stack
2015 4715 info->SetInfoHeader(NULL);
2016
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4715 times.
 4715 if (enable_info_header_) {
2017 const string header_info = info->GetInfoHeaderContents(
2018 ✗ info_header_template_);
2019 ✗ if (header_info != "") {
2020 ✗ const char * const header_name = "cvmfs-info: ";
2021 ✗ const size_t header_name_len = strlen(header_name);
2022 ✗ const size_t info_len = header_info.length();
2023 ✗ char *buf = static_cast<char *>(alloca(header_name_len + info_len + 1));
2024 ✗ memcpy(buf, header_name, header_name_len);
2025 ✗ memcpy(buf + header_name_len, header_info.c_str(), info_len);
2026 ✗ buf[header_name_len + info_len] = '\0';
2027 ✗ info->SetInfoHeader(buf);
2028 }
2029 }
2030
2031
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4715 times.
 4715 if (enable_http_tracing_) {
2032 ✗ const std::string str_pid = "X-CVMFS-PID: " + StringifyInt(info->pid());
2033 ✗ const std::string str_gid = "X-CVMFS-GID: " + StringifyUint(info->gid());
2034 ✗ const std::string str_uid = "X-CVMFS-UID: " + StringifyUint(info->uid());
2035
2036 // will be auto freed at the end of this function Fetch(JobInfo *info)
2037 ✗ info->SetTracingHeaderPid(static_cast<char *>(alloca(str_pid.size() + 1)));
2038 ✗ info->SetTracingHeaderGid(static_cast<char *>(alloca(str_gid.size() + 1)));
2039 ✗ info->SetTracingHeaderUid(static_cast<char *>(alloca(str_uid.size() + 1)));
2040
2041 ✗ memcpy(info->tracing_header_pid(), str_pid.c_str(), str_pid.size() + 1);
2042 ✗ memcpy(info->tracing_header_gid(), str_gid.c_str(), str_gid.size() + 1);
2043 ✗ memcpy(info->tracing_header_uid(), str_uid.c_str(), str_uid.size() + 1);
2044 }
2045
2046
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4715 times.
 4715 if (atomic_xadd32(&multi_threaded_, 0) == 1) {
2047 ✗ if (!info->IsValidPipeJobResults()) {
2048 ✗ info->CreatePipeJobResults();
2049 }
2050 ✗ if (!info->IsValidDataTube()) {
2051 ✗ info->CreateDataTube();
2052 }
2053
2054 // LogCvmfs(kLogDownload, kLogDebug, "send job to thread, pipe %d %d",
2055 // info->wait_at[0], info->wait_at[1]);
2056 ✗ pipe_jobs_->Write<JobInfo *>(info);
2057
2058 // Decompress / copy chunks on this caller thread (in parallel across
2059 // concurrent Fetch() callers) instead of on the single MainDownload
2060 // thread. Track decompression errors locally and apply them at the end
2061 // — VerifyAndFinalize already ran by the time kActionStop arrives.
2062 ✗ Failures decompress_err = kFailOk;
2063 do {
2064 ✗ DataTubeElement *ele = info->GetDataTubePtr()->PopFront();
2065
2066 ✗ if (ele->action == kActionStop) {
2067 ✗ delete ele;
2068 ✗ break;
2069 }
2070
2071 ✗ if (ele->action == kActionReset) {
2072 // The chunks popped so far belonged to a download attempt that was
2073 // superseded by a retry / host fail-over (VerifyAndFinalize enqueued
2074 // this marker). Discard their — possibly corrupt — decompressed output
2075 // and start fresh so the next attempt's bytes are processed cleanly.
2076 ✗ if (info->compressed()) {
2077 ✗ zlib::DecompressFini(info->GetZstreamPtr());
2078 ✗ zlib::DecompressInit(info->GetZstreamPtr());
2079 }
2080 ✗ if (info->sink() != NULL && info->sink()->Reset() != 0) {
2081 ✗ decompress_err = kFailLocalIO;
2082 } else {
2083 ✗ decompress_err = kFailOk;
2084 }
2085 ✗ delete ele;
2086 ✗ continue;
2087 }
2088
2089 ✗ if (decompress_err == kFailOk && ele->size > 0) {
2090 ✗ if (info->compressed()) {
2091 ✗ const zlib::StreamStates retval = zlib::DecompressZStream2Sink(
2092 ✗ ele->data, static_cast<int64_t>(ele->size), info->GetZstreamPtr(),
2093 info->sink());
2094 ✗ if (retval == zlib::kStreamDataError) {
2095 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
2096 "(id %" PRId64 ") failed to decompress %s", info->id(),
2097 info->url()->c_str());
2098 ✗ decompress_err = kFailBadData;
2099 ✗ } else if (retval == zlib::kStreamIOError) {
2100 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
2101 "(id %" PRId64 ") decompressing %s, local IO error",
2102 info->id(), info->url()->c_str());
2103 ✗ decompress_err = kFailLocalIO;
2104 }
2105 } else {
2106 ✗ const int64_t written = info->sink()->Write(ele->data, ele->size);
2107 ✗ if (written < 0 || static_cast<uint64_t>(written) != ele->size) {
2108 ✗ LogCvmfs(kLogDownload, kLogDebug,
2109 "(id %" PRId64 ") sink write failed for %s (%ld of %zu)",
2110 info->id(), info->url()->c_str(),
2111 static_cast<long>(written), ele->size);
2112 ✗ decompress_err = kFailLocalIO;
2113 }
2114 }
2115 }
2116 ✗ delete ele;
2117 ✗ } while (true);
2118
2119 ✗ info->GetPipeJobResultPtr()->Read<download::Failures>(&result);
2120
2121 // Caller-side finalize (deferred from VerifyAndFinalize). Must run after
2122 // all kActionDecompress chunks have been popped and decompressed.
2123 ✗ if (result == kFailOk && decompress_err == kFailOk) {
2124 ✗ if (info->sink() != NULL && info->sink()->Flush() != 0) {
2125 ✗ decompress_err = kFailLocalIO;
2126 }
2127 }
2128 ✗ if (info->compressed()) {
2129 ✗ zlib::DecompressFini(info->GetZstreamPtr());
2130 }
2131 ✗ if (result == kFailOk && decompress_err != kFailOk) {
2132 ✗ result = decompress_err;
2133 ✗ info->SetErrorCode(decompress_err);
2134 ✗ if (info->sink() != NULL) {
2135 ✗ info->sink()->Purge();
2136 }
2137 }
2138 // LogCvmfs(kLogDownload, kLogDebug, "got result %d", result);
2139 } else {
2140 4715 const MutexLockGuard l(lock_synchronous_mode_);
2141
1/2
✓ Branch 1 taken 4715 times.
✗ Branch 2 not taken.
 4715 CURL *handle = AcquireCurlHandle();
2142
1/2
✓ Branch 1 taken 4715 times.
✗ Branch 2 not taken.
 4715 InitializeRequest(info, handle);
2143
1/2
✓ Branch 1 taken 4715 times.
✗ Branch 2 not taken.
 4715 SetUrlOptions(info);
2144 // curl_easy_setopt(handle, CURLOPT_VERBOSE, 1);
2145 int retval;
2146 do {
2147
1/2
✓ Branch 1 taken 4975 times.
✗ Branch 2 not taken.
 4975 retval = curl_easy_perform(handle);
2148 4975 perf::Inc(counters_->n_requests);
2149 double elapsed;
2150
1/2
✓ Branch 1 taken 4975 times.
✗ Branch 2 not taken.
 4975 if (curl_easy_getinfo(handle, CURLINFO_TOTAL_TIME, &elapsed)
2151
1/2
✓ Branch 0 taken 4975 times.
✗ Branch 1 not taken.
 4975 == CURLE_OK) {
2152 4975 perf::Xadd(counters_->sz_transfer_time,
2153 4975 static_cast<int64_t>(elapsed * 1000));
2154 }
2155
3/4
✓ Branch 1 taken 4975 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 260 times.
✓ Branch 4 taken 4715 times.
 4975 } while (VerifyAndFinalize(retval, info));
2156 4715 result = info->error_code();
2157 // Prevent the handle from being added to the idle list, which
2158 // avoids that it is mistakenly picked up by the multi handle later
2159 // in multi-threaded context. This, in turn, would fail to wait for
2160 // for resolver threads that meanwhile vanished due to a fork()
2161 // in Daemonize()
2162
1/2
✓ Branch 2 taken 4715 times.
✗ Branch 3 not taken.
 4715 ReleaseCurlHandle(info->curl_handle(), false /* allow_reuse */);
2163 4715 }
2164
2165
2/2
✓ Branch 0 taken 322 times.
✓ Branch 1 taken 4393 times.
 4715 if (result != kFailOk) {
2166
1/2
✓ Branch 4 taken 322 times.
✗ Branch 5 not taken.
 322 LogCvmfs(kLogDownload, kLogDebug,
2167 "(manager '%s' - id %" PRId64 ") "
2168 "download failed (error %d - %s)",
2169 name_.c_str(), info->id(), result, Code2Ascii(result));
2170
2171
1/2
✓ Branch 1 taken 322 times.
✗ Branch 2 not taken.
 322 if (info->sink() != NULL) {
2172
1/2
✓ Branch 2 taken 322 times.
✗ Branch 3 not taken.
 322 info->sink()->Purge();
2173 }
2174 }
2175
2176 4715 return result;
2177 }
2178
2179
2180 /**
2181 * Used by the client to connect the authz session manager to the download
2182 * manager.
2183 */
2184 1078 void DownloadManager::SetCredentialsAttachment(CredentialsAttachment *ca) {
2185 1078 const MutexLockGuard m(lock_options_);
2186 1078 credentials_attachment_ = ca;
2187 1078 }
2188
2189 /**
2190 * Gets the DNS sever.
2191 */
2192 ✗ std::string DownloadManager::GetDnsServer() const { return opt_dns_server_; }
2193
2194 /**
2195 * Sets a DNS server. Only for testing as it cannot be reverted to the system
2196 * default.
2197 */
2198 ✗ void DownloadManager::SetDnsServer(const string &address) {
2199 ✗ if (!address.empty()) {
2200 ✗ const MutexLockGuard m(lock_options_);
2201 ✗ opt_dns_server_ = address;
2202 ✗ assert(!opt_dns_server_.empty());
2203
2204 ✗ vector<string> servers;
2205 ✗ servers.push_back(address);
2206 ✗ const bool retval = resolver_->SetResolvers(servers);
2207 ✗ assert(retval);
2208 }
2209 ✗ LogCvmfs(kLogDownload, kLogSyslog, "(manager '%s') set nameserver to %s",
2210 name_.c_str(), address.c_str());
2211 }
2212
2213
2214 /**
2215 * Sets the DNS query timeout parameters.
2216 */
2217 1848 void DownloadManager::SetDnsParameters(const unsigned retries,
2218 const unsigned timeout_ms) {
2219 1848 const MutexLockGuard m(lock_options_);
2220 1848 if ((resolver_->retries() == retries)
2221
3/6
✓ Branch 0 taken 1848 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 1848 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 1848 times.
✗ Branch 6 not taken.
 1848 && (resolver_->timeout_ms() == timeout_ms)) {
2222 1848 return;
2223 }
2224 ✗ delete resolver_;
2225 ✗ resolver_ = NULL;
2226 ✗ resolver_ = dns::NormalResolver::Create(opt_ipv4_only_, retries, timeout_ms);
2227 ✗ assert(resolver_);
2228
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1848 times.
 1848 }
2229
2230
2231 1848 void DownloadManager::SetDnsTtlLimits(const unsigned min_seconds,
2232 const unsigned max_seconds) {
2233 1848 const MutexLockGuard m(lock_options_);
2234 1848 resolver_->set_min_ttl(min_seconds);
2235 1848 resolver_->set_max_ttl(max_seconds);
2236 1848 }
2237
2238
2239 ✗ void DownloadManager::SetIpPreference(dns::IpPreference preference) {
2240 ✗ const MutexLockGuard m(lock_options_);
2241 ✗ opt_ip_preference_ = preference;
2242 }
2243
2244
2245 /**
2246 * Sets two timeout values for proxied and for direct connections, respectively.
2247 * The timeout counts for all sorts of connection phases,
2248 * DNS, HTTP connect, etc.
2249 */
2250 2487 void DownloadManager::SetTimeout(const unsigned seconds_proxy,
2251 const unsigned seconds_direct) {
2252 2487 const MutexLockGuard m(lock_options_);
2253 2487 opt_timeout_proxy_ = seconds_proxy;
2254 2487 opt_timeout_direct_ = seconds_direct;
2255 2487 }
2256
2257
2258 /**
2259 * Sets contains the average transfer speed in bytes per second that the
2260 * transfer should be below during CURLOPT_LOW_SPEED_TIME seconds for libcurl to
2261 * consider it to be too slow and abort. Only effective for new connections.
2262 */
2263 ✗ void DownloadManager::SetLowSpeedLimit(const unsigned low_speed_limit) {
2264 ✗ const MutexLockGuard m(lock_options_);
2265 ✗ opt_low_speed_limit_ = low_speed_limit;
2266 }
2267
2268
2269 /**
2270 * Receives the currently active timeout values.
2271 */
2272 724 void DownloadManager::GetTimeout(unsigned *seconds_proxy,
2273 unsigned *seconds_direct) {
2274 724 const MutexLockGuard m(lock_options_);
2275 724 *seconds_proxy = opt_timeout_proxy_;
2276 724 *seconds_direct = opt_timeout_direct_;
2277 724 }
2278
2279
2280 /**
2281 * Parses a list of ';'-separated hosts for the metalink chain. The empty
2282 * string removes the metalink list.
2283 */
2284 ✗ void DownloadManager::SetMetalinkChain(const string &metalink_list) {
2285 ✗ SetMetalinkChain(SplitString(metalink_list, ';'));
2286 }
2287
2288
2289 ✗ void DownloadManager::SetMetalinkChain(
2290 const std::vector<std::string> &metalink_list) {
2291 ✗ const MutexLockGuard m(lock_options_);
2292 ✗ opt_metalink_.timestamp_backup = 0;
2293 ✗ delete opt_metalink_.chain;
2294 ✗ opt_metalink_.current = 0;
2295
2296 ✗ if (metalink_list.empty()) {
2297 ✗ opt_metalink_.chain = NULL;
2298 ✗ return;
2299 }
2300
2301 ✗ opt_metalink_.chain = new vector<string>(metalink_list);
2302 }
2303
2304
2305 /**
2306 * Retrieves the currently set chain of metalink hosts and the currently
2307 * used metalink host.
2308 */
2309 ✗ void DownloadManager::GetMetalinkInfo(vector<string> *metalink_chain,
2310 unsigned *current_metalink) {
2311 ✗ const MutexLockGuard m(lock_options_);
2312 ✗ if (opt_metalink_.chain) {
2313 ✗ if (current_metalink) {
2314 ✗ *current_metalink = opt_metalink_.current;
2315 }
2316 ✗ if (metalink_chain) {
2317 ✗ *metalink_chain = *opt_metalink_.chain;
2318 }
2319 }
2320 }
2321
2322
2323 /**
2324 * Parses a list of ';'-separated hosts for the host chain. The empty string
2325 * removes the host list.
2326 */
2327 1726 void DownloadManager::SetHostChain(const string &host_list) {
2328
1/2
✓ Branch 2 taken 1726 times.
✗ Branch 3 not taken.
 1726 SetHostChain(SplitString(host_list, ';'));
2329 1726 }
2330
2331
2332 1774 void DownloadManager::SetHostChain(const std::vector<std::string> &host_list) {
2333 1774 const MutexLockGuard m(lock_options_);
2334 1774 opt_host_.timestamp_backup = 0;
2335
2/2
✓ Branch 0 taken 795 times.
✓ Branch 1 taken 979 times.
 1774 delete opt_host_.chain;
2336
2/2
✓ Branch 0 taken 795 times.
✓ Branch 1 taken 979 times.
 1774 delete opt_host_chain_rtt_;
2337 1774 opt_host_.current = 0;
2338
2339
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1774 times.
 1774 if (host_list.empty()) {
2340 ✗ opt_host_.chain = NULL;
2341 ✗ opt_host_chain_rtt_ = NULL;
2342 ✗ return;
2343 }
2344
2345
2/4
✓ Branch 1 taken 1774 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1774 times.
✗ Branch 5 not taken.
 1774 opt_host_.chain = new vector<string>(host_list);
2346 3548 opt_host_chain_rtt_ = new vector<int>(opt_host_.chain->size(),
2347
2/4
✓ Branch 1 taken 1774 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1774 times.
✗ Branch 5 not taken.
 1774 kProbeUnprobed);
2348 // LogCvmfs(kLogDownload, kLogSyslog, "using host %s",
2349 // (*opt_host_.chain)[0].c_str());
2350
1/2
✓ Branch 1 taken 1774 times.
✗ Branch 2 not taken.
 1774 }
2351
2352
2353 /**
2354 * Retrieves the currently set chain of hosts, their round trip times, and the
2355 * currently used host.
2356 */
2357 336 void DownloadManager::GetHostInfo(vector<string> *host_chain, vector<int> *rtt,
2358 unsigned *current_host) {
2359 336 const MutexLockGuard m(lock_options_);
2360
1/2
✓ Branch 0 taken 336 times.
✗ Branch 1 not taken.
 336 if (opt_host_.chain) {
2361
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 336 times.
 336 if (current_host) {
2362 ✗ *current_host = opt_host_.current;
2363 }
2364
1/2
✓ Branch 0 taken 336 times.
✗ Branch 1 not taken.
 336 if (host_chain) {
2365
1/2
✓ Branch 1 taken 336 times.
✗ Branch 2 not taken.
 336 *host_chain = *opt_host_.chain;
2366 }
2367
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 336 times.
 336 if (rtt) {
2368 ✗ *rtt = *opt_host_chain_rtt_;
2369 }
2370 }
2371 336 }
2372
2373
2374 /**
2375 * Jumps to the next proxy in the ring of forward proxy servers.
2376 * Selects one randomly from a load-balancing group.
2377 *
2378 * Allow for the fact that the proxy may have already been failed by
2379 * another transfer, or that the proxy may no longer be part of the
2380 * current load-balancing group.
2381 */
2382 23 void DownloadManager::SwitchProxy(JobInfo *info) {
2383 23 const MutexLockGuard m(lock_options_);
2384
2385
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 23 times.
 23 if (!opt_proxy_groups_) {
2386 ✗ return;
2387 }
2388
2389 // Fail any matching proxies within the current load-balancing group
2390 23 vector<ProxyInfo> *group = current_proxy_group();
2391 23 const unsigned group_size = group->size();
2392 23 unsigned failed = 0;
2393
2/2
✓ Branch 0 taken 23 times.
✓ Branch 1 taken 23 times.
 46 for (unsigned i = 0; i < group_size - opt_proxy_groups_current_burned_; ++i) {
2394
5/14
✓ Branch 0 taken 23 times.
✗ Branch 1 not taken.
✓ Branch 4 taken 23 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 23 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 23 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✓ Branch 12 taken 23 times.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
 23 if (info && (info->proxy() == (*group)[i].url)) {
2395 // Move to list of failed proxies
2396 23 opt_proxy_groups_current_burned_++;
2397
1/2
✓ Branch 2 taken 23 times.
✗ Branch 3 not taken.
 23 swap((*group)[i],
2398 23 (*group)[group_size - opt_proxy_groups_current_burned_]);
2399 23 perf::Inc(counters_->n_proxy_failover);
2400 23 failed++;
2401 }
2402 }
2403
2404 // Do nothing more unless at least one proxy was marked as failed
2405
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 23 times.
 23 if (!failed)
2406 ✗ return;
2407
2408 // If all proxies from the current load-balancing group are burned, switch to
2409 // another group
2410
1/2
✓ Branch 1 taken 23 times.
✗ Branch 2 not taken.
 23 if (opt_proxy_groups_current_burned_ == group->size()) {
2411 23 opt_proxy_groups_current_burned_ = 0;
2412
1/2
✓ Branch 1 taken 23 times.
✗ Branch 2 not taken.
 23 if (opt_proxy_groups_->size() > 1) {
2413 46 opt_proxy_groups_current_ = (opt_proxy_groups_current_ + 1)
2414 23 % opt_proxy_groups_->size();
2415 // Remember the timestamp of switching to backup proxies
2416
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 23 times.
 23 if (opt_proxy_groups_reset_after_ > 0) {
2417 ✗ if (opt_proxy_groups_current_ > 0) {
2418 ✗ if (opt_timestamp_backup_proxies_ == 0)
2419 ✗ opt_timestamp_backup_proxies_ = time(NULL);
2420 // LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2421 // "switched to (another) backup proxy group");
2422 } else {
2423 ✗ opt_timestamp_backup_proxies_ = 0;
2424 // LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2425 // "switched back to primary proxy group");
2426 }
2427 ✗ opt_timestamp_failover_proxies_ = 0;
2428 }
2429 }
2430 } else {
2431 // Record failover time
2432 ✗ if (opt_proxy_groups_reset_after_ > 0) {
2433 ✗ if (opt_timestamp_failover_proxies_ == 0)
2434 ✗ opt_timestamp_failover_proxies_ = time(NULL);
2435 }
2436 }
2437
2438
2/4
✓ Branch 2 taken 23 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 23 times.
✗ Branch 6 not taken.
 23 UpdateProxiesUnlocked("failed proxy");
2439
1/2
✓ Branch 2 taken 23 times.
✗ Branch 3 not taken.
 23 LogCvmfs(kLogDownload, kLogDebug,
2440 "(manager '%s' - id %" PRId64 ") "
2441 "%lu proxies remain in group",
2442 name_.c_str(), info->id(),
2443 23 current_proxy_group()->size() - opt_proxy_groups_current_burned_);
2444
1/2
✓ Branch 1 taken 23 times.
✗ Branch 2 not taken.
 23 }
2445
2446
2447 /**
2448 * Switches to the next host in the chain. If jobinfo is set, switch only if
2449 * the current host is identical to the one used by jobinfo, otherwise another
2450 * transfer has already done the switch.
2451 */
2452 69 void DownloadManager::SwitchHostInfo(const std::string &typ,
2453 HostInfo &info,
2454 JobInfo *jobinfo) {
2455 69 const MutexLockGuard m(lock_options_);
2456
2457
5/6
✓ Branch 0 taken 69 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 23 times.
✓ Branch 4 taken 46 times.
✓ Branch 5 taken 23 times.
✓ Branch 6 taken 46 times.
 69 if (!info.chain || (info.chain->size() == 1)) {
2458 23 return;
2459 }
2460
2461
1/2
✓ Branch 0 taken 46 times.
✗ Branch 1 not taken.
 46 if (jobinfo) {
2462 int lastused;
2463
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 if (typ == "host") {
2464 46 lastused = jobinfo->current_host_chain_index();
2465 } else {
2466 ✗ lastused = jobinfo->current_metalink_chain_index();
2467 }
2468
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 46 times.
 46 if (lastused != info.current) {
2469 ✗ LogCvmfs(kLogDownload, kLogDebug,
2470 "(manager '%s' - id %" PRId64 ")"
2471 "don't switch %s, "
2472 "last used %s: %s, current %s: %s",
2473 name_.c_str(), jobinfo->id(), typ.c_str(), typ.c_str(),
2474 ✗ (*info.chain)[lastused].c_str(), typ.c_str(),
2475 ✗ (*info.chain)[info.current].c_str());
2476 ✗ return;
2477 }
2478 }
2479
2480
1/2
✓ Branch 2 taken 46 times.
✗ Branch 3 not taken.
 46 string reason = "manually triggered";
2481
2/4
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 46 times.
✗ Branch 5 not taken.
 46 string info_id = "(manager '" + name_ + "'";
2482
1/2
✓ Branch 0 taken 46 times.
✗ Branch 1 not taken.
 46 if (jobinfo) {
2483
1/2
✓ Branch 3 taken 46 times.
✗ Branch 4 not taken.
 46 reason = download::Code2Ascii(jobinfo->error_code());
2484
3/6
✓ Branch 2 taken 46 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 46 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 46 times.
✗ Branch 9 not taken.
 46 info_id += " - id " + StringifyInt(jobinfo->id());
2485 }
2486
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 info_id += ")";
2487
2488
1/2
✓ Branch 2 taken 46 times.
✗ Branch 3 not taken.
 46 const std::string old_host = (*info.chain)[info.current];
2489 46 info.current = (info.current + 1) % static_cast<int>(info.chain->size());
2490
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 if (typ == "host") {
2491 46 perf::Inc(counters_->n_host_failover);
2492 } else {
2493 ✗ perf::Inc(counters_->n_metalink_failover);
2494 }
2495
1/3
✓ Branch 6 taken 46 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
 92 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2496 "%s switching %s from %s to %s (%s)", info_id.c_str(), typ.c_str(),
2497 46 old_host.c_str(), (*info.chain)[info.current].c_str(),
2498 reason.c_str());
2499
2500 // Remember the timestamp of switching to backup host
2501
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 46 times.
 46 if (info.reset_after > 0) {
2502 ✗ if (info.current != 0) {
2503 ✗ if (info.timestamp_backup == 0)
2504 ✗ info.timestamp_backup = time(NULL);
2505 } else {
2506 ✗ info.timestamp_backup = 0;
2507 }
2508 }
2509
2/2
✓ Branch 4 taken 46 times.
✓ Branch 5 taken 23 times.
 69 }
2510
2511 69 void DownloadManager::SwitchHost(JobInfo *info) {
2512
2/4
✓ Branch 2 taken 69 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 69 times.
✗ Branch 6 not taken.
 69 SwitchHostInfo("host", opt_host_, info);
2513 69 }
2514
2515 23 void DownloadManager::SwitchHost() { SwitchHost(NULL); }
2516
2517
2518 ✗ void DownloadManager::SwitchMetalink(JobInfo *info) {
2519 ✗ SwitchHostInfo("metalink", opt_metalink_, info);
2520 }
2521
2522
2523 ✗ void DownloadManager::SwitchMetalink() { SwitchMetalink(NULL); }
2524
2525 2044 bool DownloadManager::CheckMetalinkChain(time_t now) {
2526 2044 return (opt_metalink_.chain
2527
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2044 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 2044 && ((opt_metalink_timestamp_link_ == 0)
2528 ✗ || (static_cast<int64_t>((now == 0) ? time(NULL) : now)
2529 ✗ > static_cast<int64_t>(opt_metalink_timestamp_link_
2530
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 2044 + opt_metalink_.reset_after))));
2531 }
2532
2533
2534 /**
2535 * Orders the hostlist according to RTT of downloading .cvmfschecksum.
2536 * Sets the current host to the best-responsive host.
2537 * If you change the host list in between by SetHostChain(), it will be
2538 * overwritten by this function.
2539 */
2540 ✗ void DownloadManager::ProbeHosts() {
2541 ✗ vector<string> host_chain;
2542 ✗ vector<int> host_rtt;
2543 unsigned current_host;
2544
2545 ✗ GetHostInfo(&host_chain, &host_rtt, &current_host);
2546
2547 // Stopwatch, two times to fill caches first
2548 unsigned i, retries;
2549 ✗ string url;
2550
2551 ✗ cvmfs::MemSink memsink;
2552 ✗ JobInfo info(&url, false, false, NULL, &memsink);
2553 ✗ for (retries = 0; retries < 2; ++retries) {
2554 ✗ for (i = 0; i < host_chain.size(); ++i) {
2555 ✗ url = host_chain[i] + "/.cvmfspublished";
2556
2557 struct timeval tv_start, tv_end;
2558 ✗ gettimeofday(&tv_start, NULL);
2559 ✗ const Failures result = Fetch(&info);
2560 ✗ gettimeofday(&tv_end, NULL);
2561 ✗ memsink.Reset();
2562 ✗ if (result == kFailOk) {
2563 ✗ host_rtt[i] = static_cast<int>(DiffTimeSeconds(tv_start, tv_end)
2564 ✗ * 1000);
2565 ✗ LogCvmfs(kLogDownload, kLogDebug,
2566 "(manager '%s' - id %" PRId64 ") "
2567 "probing host %s had %dms rtt",
2568 ✗ name_.c_str(), info.id(), url.c_str(), host_rtt[i]);
2569 } else {
2570 ✗ LogCvmfs(kLogDownload, kLogDebug,
2571 "(manager '%s' - id %" PRId64 ") "
2572 "error while probing host %s: %d %s",
2573 name_.c_str(), info.id(), url.c_str(), result,
2574 Code2Ascii(result));
2575 ✗ host_rtt[i] = INT_MAX;
2576 }
2577 }
2578 }
2579
2580 ✗ SortTeam(&host_rtt, &host_chain);
2581 ✗ for (i = 0; i < host_chain.size(); ++i) {
2582 ✗ if (host_rtt[i] == INT_MAX)
2583 ✗ host_rtt[i] = kProbeDown;
2584 }
2585
2586 ✗ const MutexLockGuard m(lock_options_);
2587 ✗ delete opt_host_.chain;
2588 ✗ delete opt_host_chain_rtt_;
2589 ✗ opt_host_.chain = new vector<string>(host_chain);
2590 ✗ opt_host_chain_rtt_ = new vector<int>(host_rtt);
2591 ✗ opt_host_.current = 0;
2592 }
2593
2594 ✗ bool DownloadManager::GeoSortServers(std::vector<std::string> *servers,
2595 std::vector<uint64_t> *output_order) {
2596 ✗ if (!servers) {
2597 ✗ return false;
2598 }
2599 ✗ if (servers->size() == 1) {
2600 ✗ if (output_order) {
2601 ✗ output_order->clear();
2602 ✗ output_order->push_back(0);
2603 }
2604 ✗ return true;
2605 }
2606
2607 ✗ std::vector<std::string> host_chain;
2608 ✗ GetHostInfo(&host_chain, NULL, NULL);
2609
2610 ✗ std::vector<std::string> server_dns_names;
2611 ✗ server_dns_names.reserve(servers->size());
2612 ✗ for (unsigned i = 0; i < servers->size(); ++i) {
2613 ✗ const std::string host = dns::ExtractHost((*servers)[i]);
2614 ✗ server_dns_names.push_back(host.empty() ? (*servers)[i] : host);
2615 }
2616 ✗ const std::string host_list = JoinStrings(server_dns_names, ",");
2617
2618 ✗ vector<string> host_chain_shuffled;
2619 {
2620 // Protect against concurrent access to prng_
2621 ✗ const MutexLockGuard m(lock_options_);
2622 // Determine random hosts for the Geo-API query
2623 ✗ host_chain_shuffled = Shuffle(host_chain, &prng_);
2624 }
2625 // Request ordered list via Geo-API
2626 ✗ bool success = false;
2627 ✗ const unsigned max_attempts = std::min(host_chain_shuffled.size(), size_t(3));
2628 ✗ vector<uint64_t> geo_order(servers->size());
2629 ✗ for (unsigned i = 0; i < max_attempts; ++i) {
2630 ✗ const string url = host_chain_shuffled[i] + "/api/v1.0/geo/@proxy@/"
2631 ✗ + host_list;
2632 ✗ LogCvmfs(kLogDownload, kLogDebug,
2633 "(manager '%s') requesting ordered server list from %s",
2634 name_.c_str(), url.c_str());
2635 ✗ cvmfs::MemSink memsink;
2636 ✗ JobInfo info(&url, false, false, NULL, &memsink);
2637 ✗ const Failures result = Fetch(&info);
2638 ✗ if (result == kFailOk) {
2639 ✗ const string order(reinterpret_cast<char *>(memsink.data()),
2640 ✗ memsink.pos());
2641 ✗ memsink.Reset();
2642 ✗ const bool retval = ValidateGeoReply(order, servers->size(), &geo_order);
2643 ✗ if (!retval) {
2644 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2645 "(manager '%s') retrieved invalid GeoAPI reply from %s [%s]",
2646 name_.c_str(), url.c_str(), order.c_str());
2647 } else {
2648 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslog,
2649 "(manager '%s') "
2650 "geographic order of servers retrieved from %s",
2651 name_.c_str(),
2652 ✗ dns::ExtractHost(host_chain_shuffled[i]).c_str());
2653 // remove new line at end of "order"
2654 ✗ LogCvmfs(kLogDownload, kLogDebug, "order is %s",
2655 ✗ Trim(order, true /* trim_newline */).c_str());
2656 ✗ success = true;
2657 ✗ break;
2658 }
2659 ✗ } else {
2660 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2661 "(manager '%s') GeoAPI request for %s failed with error %d [%s]",
2662 name_.c_str(), url.c_str(), result, Code2Ascii(result));
2663 }
2664 }
2665 ✗ if (!success) {
2666 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2667 "(manager '%s') "
2668 "failed to retrieve geographic order from stratum 1 servers",
2669 name_.c_str());
2670 ✗ return false;
2671 }
2672
2673 ✗ if (output_order) {
2674 ✗ output_order->swap(geo_order);
2675 } else {
2676 ✗ std::vector<std::string> sorted_servers;
2677 ✗ sorted_servers.reserve(geo_order.size());
2678 ✗ for (unsigned i = 0; i < geo_order.size(); ++i) {
2679 ✗ const uint64_t orderval = geo_order[i];
2680 ✗ sorted_servers.push_back((*servers)[orderval]);
2681 }
2682 ✗ servers->swap(sorted_servers);
2683 }
2684 ✗ return true;
2685 }
2686
2687
2688 /**
2689 * Uses the Geo-API of Stratum 1s to let any of them order the list of servers
2690 * and fallback proxies (if any).
2691 * Tries at most three random Stratum 1s before giving up.
2692 * If you change the host list in between by SetHostChain() or the fallback
2693 * proxy list by SetProxyChain(), they will be overwritten by this function.
2694 */
2695 ✗ bool DownloadManager::ProbeGeo() {
2696 ✗ vector<string> host_chain;
2697 ✗ vector<int> host_rtt;
2698 unsigned current_host;
2699 ✗ vector<vector<ProxyInfo> > proxy_chain;
2700 unsigned fallback_group;
2701
2702 ✗ GetHostInfo(&host_chain, &host_rtt, &current_host);
2703 ✗ GetProxyInfo(&proxy_chain, NULL, &fallback_group);
2704 ✗ if ((host_chain.size() < 2) && ((proxy_chain.size() - fallback_group) < 2))
2705 ✗ return true;
2706
2707 ✗ vector<string> host_names;
2708 ✗ for (unsigned i = 0; i < host_chain.size(); ++i)
2709 ✗ host_names.push_back(dns::ExtractHost(host_chain[i]));
2710 ✗ SortTeam(&host_names, &host_chain);
2711 ✗ const unsigned last_geo_host = host_names.size();
2712
2713 ✗ if ((fallback_group == 0) && (last_geo_host > 1)) {
2714 // There are no non-fallback proxies, which means that the client
2715 // will always use the fallback proxies. Add a keyword separator
2716 // between the hosts and fallback proxies so the geosorting service
2717 // will know to sort the hosts based on the distance from the
2718 // closest fallback proxy rather than the distance from the client.
2719 ✗ host_names.push_back("+PXYSEP+");
2720 }
2721
2722 // Add fallback proxy names to the end of the host list
2723 ✗ const unsigned first_geo_fallback = host_names.size();
2724 ✗ for (unsigned i = fallback_group; i < proxy_chain.size(); ++i) {
2725 // We only take the first fallback proxy name from every group under the
2726 // assumption that load-balanced servers are at the same location
2727 ✗ host_names.push_back(proxy_chain[i][0].host.name());
2728 }
2729
2730 ✗ std::vector<uint64_t> geo_order;
2731 ✗ const bool success = GeoSortServers(&host_names, &geo_order);
2732 ✗ if (!success) {
2733 // GeoSortServers already logged a failure message.
2734 ✗ return false;
2735 }
2736
2737 // Re-install host chain and proxy chain
2738 ✗ const MutexLockGuard m(lock_options_);
2739 ✗ delete opt_host_.chain;
2740 ✗ opt_num_proxies_ = 0;
2741 ✗ opt_host_.chain = new vector<string>(host_chain.size());
2742
2743 // It's possible that opt_proxy_groups_fallback_ might have changed while
2744 // the lock wasn't held
2745 vector<vector<ProxyInfo> > *proxy_groups = new vector<vector<ProxyInfo> >(
2746 ✗ opt_proxy_groups_fallback_ + proxy_chain.size() - fallback_group);
2747 // First copy the non-fallback part of the current proxy chain
2748 ✗ for (unsigned i = 0; i < opt_proxy_groups_fallback_; ++i) {
2749 ✗ (*proxy_groups)[i] = (*opt_proxy_groups_)[i];
2750 ✗ opt_num_proxies_ += (*opt_proxy_groups_)[i].size();
2751 }
2752
2753 // Copy the host chain and fallback proxies by geo order. Array indices
2754 // in geo_order that are smaller than last_geo_host refer to a stratum 1,
2755 // and those indices greater than or equal to first_geo_fallback refer to
2756 // a fallback proxy.
2757 ✗ unsigned hosti = 0;
2758 ✗ unsigned proxyi = opt_proxy_groups_fallback_;
2759 ✗ for (unsigned i = 0; i < geo_order.size(); ++i) {
2760 ✗ const uint64_t orderval = geo_order[i];
2761 ✗ if (orderval < static_cast<uint64_t>(last_geo_host)) {
2762 // LogCvmfs(kLogCvmfs, kLogSyslog, "this is orderval %u at host index
2763 // %u", orderval, hosti);
2764 ✗ (*opt_host_.chain)[hosti++] = host_chain[orderval];
2765 ✗ } else if (orderval >= static_cast<uint64_t>(first_geo_fallback)) {
2766 // LogCvmfs(kLogCvmfs, kLogSyslog,
2767 // "this is orderval %u at proxy index %u, using proxy_chain index %u",
2768 // orderval, proxyi, fallback_group + orderval - first_geo_fallback);
2769 ✗ (*proxy_groups)[proxyi] = proxy_chain[fallback_group + orderval
2770 ✗ - first_geo_fallback];
2771 ✗ opt_num_proxies_ += (*proxy_groups)[proxyi].size();
2772 ✗ proxyi++;
2773 }
2774 }
2775
2776 ✗ opt_proxy_map_.clear();
2777 ✗ delete opt_proxy_groups_;
2778 ✗ opt_proxy_groups_ = proxy_groups;
2779 // In pathological cases, opt_proxy_groups_current_ can be larger now when
2780 // proxies changed in-between.
2781 ✗ if (opt_proxy_groups_current_ > opt_proxy_groups_->size()) {
2782 ✗ if (opt_proxy_groups_->size() == 0) {
2783 ✗ opt_proxy_groups_current_ = 0;
2784 } else {
2785 ✗ opt_proxy_groups_current_ = opt_proxy_groups_->size() - 1;
2786 }
2787 ✗ opt_proxy_groups_current_burned_ = 0;
2788 }
2789
2790 ✗ UpdateProxiesUnlocked("geosort");
2791
2792 ✗ delete opt_host_chain_rtt_;
2793 ✗ opt_host_chain_rtt_ = new vector<int>(host_chain.size(), kProbeGeo);
2794 ✗ opt_host_.current = 0;
2795
2796 ✗ return true;
2797 }
2798
2799
2800 /**
2801 * Validates a string of the form "1,4,2,3" representing in which order the
2802 * the expected_size number of hosts should be put for optimal geographic
2803 * proximity. Returns false if the reply_order string is invalid, otherwise
2804 * fills in the reply_vals array with zero-based order indexes (e.g.
2805 * [0,3,1,2]) and returns true.
2806 */
2807 322 bool DownloadManager::ValidateGeoReply(const string &reply_order,
2808 const unsigned expected_size,
2809 vector<uint64_t> *reply_vals) {
2810
2/2
✓ Branch 1 taken 23 times.
✓ Branch 2 taken 299 times.
 322 if (reply_order.empty())
2811 23 return false;
2812
2/4
✓ Branch 2 taken 299 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 299 times.
✗ Branch 6 not taken.
 598 const sanitizer::InputSanitizer sanitizer("09 , \n");
2813
3/4
✓ Branch 1 taken 299 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 23 times.
✓ Branch 4 taken 276 times.
 299 if (!sanitizer.IsValid(reply_order))
2814 23 return false;
2815
2/4
✓ Branch 2 taken 276 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 276 times.
✗ Branch 6 not taken.
 552 const sanitizer::InputSanitizer strip_newline("09 ,");
2816
1/2
✓ Branch 1 taken 276 times.
✗ Branch 2 not taken.
 276 vector<string> reply_strings = SplitString(strip_newline.Filter(reply_order),
2817
1/2
✓ Branch 1 taken 276 times.
✗ Branch 2 not taken.
 276 ',');
2818 276 vector<uint64_t> tmp_vals;
2819
2/2
✓ Branch 1 taken 529 times.
✓ Branch 2 taken 230 times.
 759 for (unsigned i = 0; i < reply_strings.size(); ++i) {
2820
2/2
✓ Branch 2 taken 46 times.
✓ Branch 3 taken 483 times.
 529 if (reply_strings[i].empty())
2821 46 return false;
2822
2/4
✓ Branch 2 taken 483 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 483 times.
✗ Branch 6 not taken.
 483 tmp_vals.push_back(String2Uint64(reply_strings[i]));
2823 }
2824
2/2
✓ Branch 1 taken 115 times.
✓ Branch 2 taken 115 times.
 230 if (tmp_vals.size() != expected_size)
2825 115 return false;
2826
2827 // Check if tmp_vals contains the number 1..n
2828
1/2
✓ Branch 3 taken 115 times.
✗ Branch 4 not taken.
 115 set<uint64_t> const coverage(tmp_vals.begin(), tmp_vals.end());
2829
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 115 times.
 115 if (coverage.size() != tmp_vals.size())
2830 ✗ return false;
2831
5/6
✓ Branch 2 taken 92 times.
✓ Branch 3 taken 23 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 92 times.
✓ Branch 9 taken 23 times.
✓ Branch 10 taken 92 times.
 115 if ((*coverage.begin() != 1) || (*coverage.rbegin() != coverage.size()))
2832 23 return false;
2833
2834
2/2
✓ Branch 0 taken 207 times.
✓ Branch 1 taken 92 times.
 299 for (unsigned i = 0; i < expected_size; ++i) {
2835 207 (*reply_vals)[i] = tmp_vals[i] - 1;
2836 }
2837 92 return true;
2838 299 }
2839
2840
2841 /**
2842 * Removes DIRECT from a list of ';' and '|' separated proxies.
2843 * \return true if DIRECT was present, false otherwise
2844 */
2845 1747 bool DownloadManager::StripDirect(const string &proxy_list,
2846 string *cleaned_list) {
2847
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1747 times.
 1747 assert(cleaned_list);
2848
2/2
✓ Branch 1 taken 1540 times.
✓ Branch 2 taken 207 times.
 1747 if (proxy_list == "") {
2849
1/2
✓ Branch 1 taken 1540 times.
✗ Branch 2 not taken.
 1540 *cleaned_list = "";
2850 1540 return false;
2851 }
2852 207 bool result = false;
2853
2854
1/2
✓ Branch 1 taken 207 times.
✗ Branch 2 not taken.
 207 vector<string> proxy_groups = SplitString(proxy_list, ';');
2855 207 vector<string> cleaned_groups;
2856
2/2
✓ Branch 1 taken 506 times.
✓ Branch 2 taken 207 times.
 713 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2857
1/2
✓ Branch 2 taken 506 times.
✗ Branch 3 not taken.
 506 vector<string> group = SplitString(proxy_groups[i], '|');
2858 506 vector<string> cleaned;
2859
2/2
✓ Branch 1 taken 851 times.
✓ Branch 2 taken 506 times.
 1357 for (unsigned j = 0; j < group.size(); ++j) {
2860
6/6
✓ Branch 2 taken 621 times.
✓ Branch 3 taken 230 times.
✓ Branch 6 taken 299 times.
✓ Branch 7 taken 322 times.
✓ Branch 8 taken 529 times.
✓ Branch 9 taken 322 times.
 851 if ((group[j] == "DIRECT") || (group[j] == "")) {
2861 529 result = true;
2862 } else {
2863
1/2
✓ Branch 2 taken 322 times.
✗ Branch 3 not taken.
 322 cleaned.push_back(group[j]);
2864 }
2865 }
2866
2/2
✓ Branch 1 taken 161 times.
✓ Branch 2 taken 345 times.
 506 if (!cleaned.empty())
2867
3/6
✓ Branch 2 taken 161 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 161 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 161 times.
✗ Branch 9 not taken.
 161 cleaned_groups.push_back(JoinStrings(cleaned, "|"));
2868 506 }
2869
2870
2/4
✓ Branch 2 taken 207 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 207 times.
✗ Branch 6 not taken.
 207 *cleaned_list = JoinStrings(cleaned_groups, ";");
2871 207 return result;
2872 207 }
2873
2874
2875 /**
2876 * Parses a list of ';'- and '|'-separated proxy servers and fallback proxy
2877 * servers for the proxy groups.
2878 * The empty string for both removes the proxy chain.
2879 * The set_mode parameter can be used to set either proxies (leaving fallback
2880 * proxies unchanged) or fallback proxies (leaving regular proxies unchanged)
2881 * or both.
2882 */
2883 1517 void DownloadManager::SetProxyChain(const string &proxy_list,
2884 const string &fallback_proxy_list,
2885 const ProxySetModes set_mode) {
2886 1517 const MutexLockGuard m(lock_options_);
2887
2888 1517 opt_timestamp_backup_proxies_ = 0;
2889 1517 opt_timestamp_failover_proxies_ = 0;
2890
1/2
✓ Branch 1 taken 1517 times.
✗ Branch 2 not taken.
 1517 string set_proxy_list = opt_proxy_list_;
2891
1/2
✓ Branch 1 taken 1517 times.
✗ Branch 2 not taken.
 1517 string set_proxy_fallback_list = opt_proxy_fallback_list_;
2892 bool contains_direct;
2893
3/4
✓ Branch 0 taken 1517 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1448 times.
✓ Branch 3 taken 69 times.
 1517 if ((set_mode == kSetProxyFallback) || (set_mode == kSetProxyBoth)) {
2894
1/2
✓ Branch 1 taken 1448 times.
✗ Branch 2 not taken.
 1448 opt_proxy_fallback_list_ = fallback_proxy_list;
2895 }
2896
3/4
✓ Branch 0 taken 1448 times.
✓ Branch 1 taken 69 times.
✓ Branch 2 taken 1448 times.
✗ Branch 3 not taken.
 1517 if ((set_mode == kSetProxyRegular) || (set_mode == kSetProxyBoth)) {
2897
1/2
✓ Branch 1 taken 1517 times.
✗ Branch 2 not taken.
 1517 opt_proxy_list_ = proxy_list;
2898 }
2899
1/2
✓ Branch 1 taken 1517 times.
✗ Branch 2 not taken.
 1517 contains_direct = StripDirect(opt_proxy_fallback_list_,
2900 &set_proxy_fallback_list);
2901
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1517 times.
 1517 if (contains_direct) {
2902 ✗ LogCvmfs(kLogDownload, kLogSyslogWarn | kLogDebug,
2903 "(manager '%s') fallback proxies do not support DIRECT, removing",
2904 name_.c_str());
2905 }
2906
1/2
✓ Branch 1 taken 1517 times.
✗ Branch 2 not taken.
 1517 if (set_proxy_fallback_list == "") {
2907
1/2
✓ Branch 1 taken 1517 times.
✗ Branch 2 not taken.
 1517 set_proxy_list = opt_proxy_list_;
2908 } else {
2909 ✗ const bool contains_direct = StripDirect(opt_proxy_list_, &set_proxy_list);
2910 ✗ if (contains_direct) {
2911 ✗ LogCvmfs(kLogDownload, kLogSyslog | kLogDebug,
2912 "(manager '%s') skipping DIRECT proxy to use fallback proxy",
2913 name_.c_str());
2914 }
2915 }
2916
2917 // From this point on, use set_proxy_list and set_fallback_proxy_list as
2918 // effective proxy lists!
2919
2920 1517 opt_proxy_map_.clear();
2921
2/2
✓ Branch 0 taken 724 times.
✓ Branch 1 taken 793 times.
 1517 delete opt_proxy_groups_;
2922
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 1517 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 1517 times.
 1517 if ((set_proxy_list == "") && (set_proxy_fallback_list == "")) {
2923 ✗ opt_proxy_groups_ = NULL;
2924 ✗ opt_proxy_groups_current_ = 0;
2925 ✗ opt_proxy_groups_current_burned_ = 0;
2926 ✗ opt_proxy_groups_fallback_ = 0;
2927 ✗ opt_num_proxies_ = 0;
2928 ✗ return;
2929 }
2930
2931 // Determine number of regular proxy groups (== first fallback proxy group)
2932 1517 opt_proxy_groups_fallback_ = 0;
2933
1/2
✓ Branch 1 taken 1517 times.
✗ Branch 2 not taken.
 1517 if (set_proxy_list != "") {
2934
1/2
✓ Branch 1 taken 1517 times.
✗ Branch 2 not taken.
 1517 opt_proxy_groups_fallback_ = SplitString(set_proxy_list, ';').size();
2935 }
2936
1/2
✓ Branch 2 taken 1517 times.
✗ Branch 3 not taken.
 1517 LogCvmfs(kLogDownload, kLogDebug,
2937 "(manager '%s') "
2938 "first fallback proxy group %u",
2939 name_.c_str(), opt_proxy_groups_fallback_);
2940
2941 // Concatenate regular proxies and fallback proxies, both of which can be
2942 // empty.
2943
1/2
✓ Branch 1 taken 1517 times.
✗ Branch 2 not taken.
 1517 string all_proxy_list = set_proxy_list;
2944
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1517 times.
 1517 if (set_proxy_fallback_list != "") {
2945 ✗ if (all_proxy_list != "")
2946 ✗ all_proxy_list += ";";
2947 ✗ all_proxy_list += set_proxy_fallback_list;
2948 }
2949
1/2
✓ Branch 3 taken 1517 times.
✗ Branch 4 not taken.
 1517 LogCvmfs(kLogDownload, kLogDebug, "(manager '%s') full proxy list %s",
2950 name_.c_str(), all_proxy_list.c_str());
2951
2952 // Resolve server names in provided urls
2953 1517 vector<string> hostnames; // All encountered hostnames
2954 1517 vector<string> proxy_groups;
2955
1/2
✓ Branch 1 taken 1517 times.
✗ Branch 2 not taken.
 1517 if (all_proxy_list != "")
2956
1/2
✓ Branch 1 taken 1517 times.
✗ Branch 2 not taken.
 1517 proxy_groups = SplitString(all_proxy_list, ';');
2957
2/2
✓ Branch 1 taken 1540 times.
✓ Branch 2 taken 1517 times.
 3057 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2958
1/2
✓ Branch 2 taken 1540 times.
✗ Branch 3 not taken.
 1540 vector<string> this_group = SplitString(proxy_groups[i], '|');
2959
2/2
✓ Branch 1 taken 1540 times.
✓ Branch 2 taken 1540 times.
 3080 for (unsigned j = 0; j < this_group.size(); ++j) {
2960
1/2
✓ Branch 2 taken 1540 times.
✗ Branch 3 not taken.
 1540 this_group[j] = dns::AddDefaultScheme(this_group[j]);
2961 // Note: DIRECT strings will be "extracted" to an empty string.
2962
1/2
✓ Branch 2 taken 1540 times.
✗ Branch 3 not taken.
 1540 const string hostname = dns::ExtractHost(this_group[j]);
2963 // Save the hostname. Leave empty (DIRECT) names so indexes will
2964 // match later.
2965
1/2
✓ Branch 1 taken 1540 times.
✗ Branch 2 not taken.
 1540 hostnames.push_back(hostname);
2966 1540 }
2967 1540 }
2968 1517 vector<dns::Host> hosts;
2969
1/2
✓ Branch 3 taken 1517 times.
✗ Branch 4 not taken.
 1517 LogCvmfs(kLogDownload, kLogDebug,
2970 "(manager '%s') "
2971 "resolving %lu proxy addresses",
2972 name_.c_str(), hostnames.size());
2973
1/2
✓ Branch 1 taken 1517 times.
✗ Branch 2 not taken.
 1517 resolver_->ResolveMany(hostnames, &hosts);
2974
2975 // Construct opt_proxy_groups_: traverse proxy list in same order and expand
2976 // names to resolved IP addresses.
2977
1/2
✓ Branch 1 taken 1517 times.
✗ Branch 2 not taken.
 1517 opt_proxy_groups_ = new vector<vector<ProxyInfo> >();
2978 1517 opt_num_proxies_ = 0;
2979 1517 unsigned num_proxy = 0; // Combined i, j counter
2980
2/2
✓ Branch 1 taken 1540 times.
✓ Branch 2 taken 1517 times.
 3057 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2981
1/2
✓ Branch 2 taken 1540 times.
✗ Branch 3 not taken.
 1540 vector<string> this_group = SplitString(proxy_groups[i], '|');
2982 // Construct ProxyInfo objects from proxy string and DNS resolver result for
2983 // every proxy in this_group. One URL can result in multiple ProxyInfo
2984 // objects, one for each IP address.
2985 1540 vector<ProxyInfo> infos;
2986
2/2
✓ Branch 1 taken 1540 times.
✓ Branch 2 taken 1540 times.
 3080 for (unsigned j = 0; j < this_group.size(); ++j, ++num_proxy) {
2987
1/2
✓ Branch 2 taken 1540 times.
✗ Branch 3 not taken.
 1540 this_group[j] = dns::AddDefaultScheme(this_group[j]);
2988
2/2
✓ Branch 2 taken 1448 times.
✓ Branch 3 taken 92 times.
 1540 if (this_group[j] == "DIRECT") {
2989
3/6
✓ Branch 2 taken 1448 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1448 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1448 times.
✗ Branch 9 not taken.
 1448 infos.push_back(ProxyInfo("DIRECT"));
2990 1448 continue;
2991 }
2992
2993
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 92 times.
 92 if (hosts[num_proxy].status() != dns::kFailOk) {
2994 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2995 "(manager '%s') "
2996 "failed to resolve IP addresses for %s (%d - %s)",
2997 ✗ name_.c_str(), hosts[num_proxy].name().c_str(),
2998 ✗ hosts[num_proxy].status(),
2999 ✗ dns::Code2Ascii(hosts[num_proxy].status()));
3000 const dns::Host failed_host = dns::Host::ExtendDeadline(
3001 ✗ hosts[num_proxy], resolver_->min_ttl());
3002 ✗ infos.push_back(ProxyInfo(failed_host, this_group[j]));
3003 ✗ continue;
3004 }
3005
3006 // IPv4 addresses have precedence
3007 92 set<string> const best_addresses = hosts[num_proxy].ViewBestAddresses(
3008
2/4
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 92 times.
✗ Branch 5 not taken.
 92 opt_ip_preference_);
3009 92 set<string>::const_iterator iter_ips = best_addresses.begin();
3010
2/2
✓ Branch 3 taken 92 times.
✓ Branch 4 taken 92 times.
 184 for (; iter_ips != best_addresses.end(); ++iter_ips) {
3011
1/2
✓ Branch 3 taken 92 times.
✗ Branch 4 not taken.
 92 const string url_ip = dns::RewriteUrl(this_group[j], *iter_ips);
3012
2/4
✓ Branch 2 taken 92 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 92 times.
✗ Branch 6 not taken.
 92 infos.push_back(ProxyInfo(hosts[num_proxy], url_ip));
3013
3014
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 92 times.
 92 if (sharding_policy_.UseCount() > 0) {
3015 ✗ sharding_policy_->AddProxy(url_ip);
3016 }
3017 92 }
3018 92 }
3019
1/2
✓ Branch 1 taken 1540 times.
✗ Branch 2 not taken.
 1540 opt_proxy_groups_->push_back(infos);
3020 1540 opt_num_proxies_ += infos.size();
3021 1540 }
3022
1/2
✓ Branch 2 taken 1517 times.
✗ Branch 3 not taken.
 1517 LogCvmfs(kLogDownload, kLogDebug,
3023 "(manager '%s') installed %u proxies in %lu load-balance groups",
3024 1517 name_.c_str(), opt_num_proxies_, opt_proxy_groups_->size());
3025 1517 opt_proxy_groups_current_ = 0;
3026 1517 opt_proxy_groups_current_burned_ = 0;
3027
3028 // Select random start proxy from the first group.
3029
1/2
✓ Branch 1 taken 1517 times.
✗ Branch 2 not taken.
 1517 if (opt_proxy_groups_->size() > 0) {
3030 // Select random start proxy from the first group.
3031
2/4
✓ Branch 2 taken 1517 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1517 times.
✗ Branch 6 not taken.
 1517 UpdateProxiesUnlocked("set random start proxy from the first proxy group");
3032 }
3033
3/6
✓ Branch 5 taken 1517 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1517 times.
✗ Branch 9 not taken.
✓ Branch 11 taken 1517 times.
✗ Branch 12 not taken.
 1517 }
3034
3035
3036 /**
3037 * Retrieves the proxy chain, optionally the currently active load-balancing
3038 * group, and optionally the index of the first fallback proxy group.
3039 * If there are no fallback proxies, the index will equal the size of
3040 * the proxy chain.
3041 */
3042 ✗ void DownloadManager::GetProxyInfo(vector<vector<ProxyInfo> > *proxy_chain,
3043 unsigned *current_group,
3044 unsigned *fallback_group) {
3045 ✗ assert(proxy_chain != NULL);
3046 ✗ const MutexLockGuard m(lock_options_);
3047
3048 ✗ if (!opt_proxy_groups_) {
3049 ✗ const vector<vector<ProxyInfo> > empty_chain;
3050 ✗ *proxy_chain = empty_chain;
3051 ✗ if (current_group != NULL)
3052 ✗ *current_group = 0;
3053 ✗ if (fallback_group != NULL)
3054 ✗ *fallback_group = 0;
3055 ✗ return;
3056 }
3057
3058 ✗ *proxy_chain = *opt_proxy_groups_;
3059 ✗ if (current_group != NULL)
3060 ✗ *current_group = opt_proxy_groups_current_;
3061 ✗ if (fallback_group != NULL)
3062 ✗ *fallback_group = opt_proxy_groups_fallback_;
3063 }
3064
3065 ✗ string DownloadManager::GetProxyList() { return opt_proxy_list_; }
3066
3067 ✗ string DownloadManager::GetFallbackProxyList() {
3068 ✗ return opt_proxy_fallback_list_;
3069 }
3070
3071 /**
3072 * Choose proxy
3073 */
3074 4784 DownloadManager::ProxyInfo *DownloadManager::ChooseProxyUnlocked(
3075 const shash::Any *hash) {
3076
2/2
✓ Branch 0 taken 3046 times.
✓ Branch 1 taken 1738 times.
 4784 if (!opt_proxy_groups_)
3077 3046 return NULL;
3078
3079
2/2
✓ Branch 0 taken 882 times.
✓ Branch 1 taken 856 times.
 1738 const uint32_t key = (hash ? hash->Partial32() : 0);
3080
1/2
✓ Branch 1 taken 1738 times.
✗ Branch 2 not taken.
 1738 const map<uint32_t, ProxyInfo *>::iterator it = opt_proxy_map_.lower_bound(
3081 key);
3082 1738 ProxyInfo *proxy = it->second;
3083
3084 1738 return proxy;
3085 }
3086
3087 /**
3088 * Update currently selected proxy
3089 */
3090 2264 void DownloadManager::UpdateProxiesUnlocked(const string &reason) {
3091
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2264 times.
 2264 if (!opt_proxy_groups_)
3092 ✗ return;
3093
3094 // Identify number of non-burned proxies within the current group
3095 2264 vector<ProxyInfo> *group = current_proxy_group();
3096 2264 const unsigned num_alive = (group->size() - opt_proxy_groups_current_burned_);
3097
2/4
✓ Branch 2 taken 2264 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2264 times.
✗ Branch 6 not taken.
 4528 const string old_proxy = JoinStrings(opt_proxies_, "|");
3098
3099 // Rebuild proxy map and URL list
3100 2264 opt_proxy_map_.clear();
3101 2264 opt_proxies_.clear();
3102 2264 const uint32_t max_key = 0xffffffffUL;
3103
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2264 times.
 2264 if (opt_proxy_shard_) {
3104 // Build a consistent map with multiple entries for each proxy
3105 ✗ for (unsigned i = 0; i < num_alive; ++i) {
3106 ✗ ProxyInfo *proxy = &(*group)[i];
3107 ✗ shash::Any proxy_hash(shash::kSha1);
3108 ✗ HashString(proxy->url, &proxy_hash);
3109 ✗ Prng prng;
3110 ✗ prng.InitSeed(proxy_hash.Partial32());
3111 ✗ for (unsigned j = 0; j < kProxyMapScale; ++j) {
3112 ✗ const std::pair<uint32_t, ProxyInfo *> entry(prng.Next(max_key), proxy);
3113 ✗ opt_proxy_map_.insert(entry);
3114 }
3115 ✗ const std::string proxy_name = proxy->host.name().empty()
3116 ? ""
3117 ✗ : " (" + proxy->host.name() + ")";
3118 ✗ opt_proxies_.push_back(proxy->url + proxy_name);
3119 }
3120 // Ensure lower_bound() finds a value for all keys
3121 ✗ ProxyInfo *first_proxy = opt_proxy_map_.begin()->second;
3122 ✗ const std::pair<uint32_t, ProxyInfo *> last_entry(max_key, first_proxy);
3123 ✗ opt_proxy_map_.insert(last_entry);
3124 } else {
3125 // Build a map with a single entry for one randomly selected proxy
3126 2264 const unsigned select = prng_.Next(num_alive);
3127 2264 ProxyInfo *proxy = &(*group)[select];
3128 2264 const std::pair<uint32_t, ProxyInfo *> entry(max_key, proxy);
3129
1/2
✓ Branch 1 taken 2264 times.
✗ Branch 2 not taken.
 2264 opt_proxy_map_.insert(entry);
3130 2264 const std::string proxy_name = proxy->host.name().empty()
3131 ? ""
3132
9/18
✓ Branch 0 taken 2172 times.
✓ Branch 1 taken 92 times.
✓ Branch 4 taken 2172 times.
✗ Branch 5 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 92 times.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✓ Branch 11 taken 92 times.
✗ Branch 12 not taken.
✓ Branch 13 taken 92 times.
✓ Branch 14 taken 2172 times.
✓ Branch 15 taken 2172 times.
✓ Branch 16 taken 92 times.
✗ Branch 17 not taken.
✗ Branch 18 not taken.
✗ Branch 19 not taken.
✗ Branch 20 not taken.
 2356 : " (" + proxy->host.name() + ")";
3133
2/4
✓ Branch 1 taken 2264 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2264 times.
✗ Branch 5 not taken.
 2264 opt_proxies_.push_back(proxy->url + proxy_name);
3134 2264 }
3135
1/2
✓ Branch 3 taken 2264 times.
✗ Branch 4 not taken.
 2264 sort(opt_proxies_.begin(), opt_proxies_.end());
3136
3137 // Report any change in proxy usage
3138
2/4
✓ Branch 2 taken 2264 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2264 times.
✗ Branch 6 not taken.
 4528 const string new_proxy = JoinStrings(opt_proxies_, "|");
3139 const string curr_host = "Current host: "
3140 2264 + (opt_host_.chain
3141
6/12
✓ Branch 0 taken 2172 times.
✓ Branch 1 taken 92 times.
✓ Branch 4 taken 2172 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 92 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 92 times.
✓ Branch 11 taken 2172 times.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
 4528 ? (*opt_host_.chain)[opt_host_.current]
3142
1/2
✓ Branch 1 taken 2264 times.
✗ Branch 2 not taken.
 2264 : "");
3143
2/2
✓ Branch 1 taken 1540 times.
✓ Branch 2 taken 724 times.
 2264 if (new_proxy != old_proxy) {
3144
4/9
✗ Branch 2 not taken.
✓ Branch 3 taken 1540 times.
✓ Branch 4 taken 1517 times.
✓ Branch 5 taken 23 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 1540 times.
✗ Branch 9 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
 4643 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
3145 "(manager '%s') switching proxy from %s to %s. Reason: %s [%s]",
3146 1563 name_.c_str(), (old_proxy.empty() ? "(none)" : old_proxy.c_str()),
3147 3080 (new_proxy.empty() ? "(none)" : new_proxy.c_str()), reason.c_str(),
3148 curr_host.c_str());
3149 }
3150 2264 }
3151
3152 /**
3153 * Enable proxy sharding
3154 */
3155 ✗ void DownloadManager::ShardProxies() {
3156 ✗ opt_proxy_shard_ = true;
3157 ✗ RebalanceProxiesUnlocked("enable sharding");
3158 }
3159
3160 /**
3161 * Selects a new random proxy in the current load-balancing group. Resets the
3162 * "burned" counter.
3163 */
3164 ✗ void DownloadManager::RebalanceProxiesUnlocked(const string &reason) {
3165 ✗ if (!opt_proxy_groups_)
3166 ✗ return;
3167
3168 ✗ opt_timestamp_failover_proxies_ = 0;
3169 ✗ opt_proxy_groups_current_burned_ = 0;
3170 ✗ UpdateProxiesUnlocked(reason);
3171 }
3172
3173
3174 ✗ void DownloadManager::RebalanceProxies() {
3175 ✗ const MutexLockGuard m(lock_options_);
3176 ✗ RebalanceProxiesUnlocked("rebalance invoked manually");
3177 }
3178
3179
3180 /**
3181 * Switches to the next load-balancing group of proxy servers.
3182 */
3183 ✗ void DownloadManager::SwitchProxyGroup() {
3184 ✗ const MutexLockGuard m(lock_options_);
3185
3186 ✗ if (!opt_proxy_groups_ || (opt_proxy_groups_->size() < 2)) {
3187 ✗ return;
3188 }
3189
3190 ✗ opt_proxy_groups_current_ = (opt_proxy_groups_current_ + 1)
3191 ✗ % opt_proxy_groups_->size();
3192 ✗ opt_timestamp_backup_proxies_ = time(NULL);
3193
3194 const std::string msg = "switch to proxy group "
3195 ✗ + StringifyUint(opt_proxy_groups_current_);
3196 ✗ RebalanceProxiesUnlocked(msg);
3197 }
3198
3199
3200 ✗ void DownloadManager::SetProxyGroupResetDelay(const unsigned seconds) {
3201 ✗ const MutexLockGuard m(lock_options_);
3202 ✗ opt_proxy_groups_reset_after_ = seconds;
3203 ✗ if (opt_proxy_groups_reset_after_ == 0) {
3204 ✗ opt_timestamp_backup_proxies_ = 0;
3205 ✗ opt_timestamp_failover_proxies_ = 0;
3206 }
3207 }
3208
3209
3210 ✗ void DownloadManager::SetMetalinkResetDelay(const unsigned seconds) {
3211 ✗ const MutexLockGuard m(lock_options_);
3212 ✗ opt_metalink_.reset_after = seconds;
3213 ✗ if (opt_metalink_.reset_after == 0)
3214 ✗ opt_metalink_.timestamp_backup = 0;
3215 }
3216
3217
3218 ✗ void DownloadManager::SetHostResetDelay(const unsigned seconds) {
3219 ✗ const MutexLockGuard m(lock_options_);
3220 ✗ opt_host_.reset_after = seconds;
3221 ✗ if (opt_host_.reset_after == 0)
3222 ✗ opt_host_.timestamp_backup = 0;
3223 }
3224
3225
3226 1763 void DownloadManager::SetRetryParameters(const unsigned max_retries,
3227 const unsigned backoff_init_ms,
3228 const unsigned backoff_max_ms) {
3229 1763 const MutexLockGuard m(lock_options_);
3230 1763 opt_max_retries_ = max_retries;
3231 1763 opt_backoff_init_ms_ = backoff_init_ms;
3232 1763 opt_backoff_max_ms_ = backoff_max_ms;
3233 1763 }
3234
3235
3236 770 void DownloadManager::SetMaxIpaddrPerProxy(unsigned limit) {
3237 770 const MutexLockGuard m(lock_options_);
3238 770 resolver_->set_throttle(limit);
3239 770 }
3240
3241
3242 1078 void DownloadManager::SetProxyTemplates(const std::string &direct,
3243 const std::string &forced) {
3244 1078 const MutexLockGuard m(lock_options_);
3245
1/2
✓ Branch 1 taken 1078 times.
✗ Branch 2 not taken.
 1078 proxy_template_direct_ = direct;
3246
1/2
✓ Branch 1 taken 1078 times.
✗ Branch 2 not taken.
 1078 proxy_template_forced_ = forced;
3247 1078 }
3248
3249
3250 ✗ void DownloadManager::EnableInfoHeader() { enable_info_header_ = true; }
3251
3252
3253 754 void DownloadManager::EnableRedirects() { follow_redirects_ = true; }
3254
3255 ✗ void DownloadManager::EnableIgnoreSignatureFailures() {
3256 ✗ ignore_signature_failures_ = true;
3257 }
3258
3259 ✗ void DownloadManager::EnableHTTPTracing() { enable_http_tracing_ = true; }
3260
3261 ✗ void DownloadManager::AddHTTPTracingHeader(const std::string &header) {
3262 ✗ http_tracing_headers_.push_back(header);
3263 }
3264
3265 685 void DownloadManager::UseSystemCertificatePath() {
3266 685 ssl_certificate_store_.UseSystemCertificatePath();
3267 685 }
3268
3269 ✗ bool DownloadManager::SetShardingPolicy(const ShardingPolicySelector type) {
3270 ✗ const bool success = false;
3271 ✗ switch (type) {
3272 default:
3273 ✗ LogCvmfs(
3274 kLogDownload, kLogDebug | kLogSyslogErr,
3275 "(manager '%s') "
3276 "Proposed sharding policy does not exist. Falling back to default",
3277 name_.c_str());
3278 }
3279 ✗ return success;
3280 }
3281
3282 ✗ void DownloadManager::SetFailoverIndefinitely() {
3283 ✗ failover_indefinitely_ = true;
3284 }
3285
3286 /**
3287 * Creates a copy of the existing download manager. Must only be called in
3288 * single-threaded stage because it calls curl_global_init().
3289 */
3290 770 DownloadManager *DownloadManager::Clone(
3291 const perf::StatisticsTemplate &statistics,
3292 const std::string &cloned_name) {
3293 DownloadManager *clone = new DownloadManager(pool_max_handles_, statistics,
3294
1/2
✓ Branch 2 taken 770 times.
✗ Branch 3 not taken.
 770 cloned_name);
3295
3296 770 clone->SetDnsParameters(resolver_->retries(), resolver_->timeout_ms());
3297 770 clone->SetDnsTtlLimits(resolver_->min_ttl(), resolver_->max_ttl());
3298 770 clone->SetMaxIpaddrPerProxy(resolver_->throttle());
3299
3300
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 770 times.
 770 if (!opt_dns_server_.empty())
3301 ✗ clone->SetDnsServer(opt_dns_server_);
3302 770 clone->opt_timeout_proxy_ = opt_timeout_proxy_;
3303 770 clone->opt_timeout_direct_ = opt_timeout_direct_;
3304 770 clone->opt_low_speed_limit_ = opt_low_speed_limit_;
3305 770 clone->opt_max_retries_ = opt_max_retries_;
3306 770 clone->opt_backoff_init_ms_ = opt_backoff_init_ms_;
3307 770 clone->opt_backoff_max_ms_ = opt_backoff_max_ms_;
3308 770 clone->enable_info_header_ = enable_info_header_;
3309 770 clone->enable_http_tracing_ = enable_http_tracing_;
3310 770 clone->http_tracing_headers_ = http_tracing_headers_;
3311 770 clone->follow_redirects_ = follow_redirects_;
3312 770 clone->ignore_signature_failures_ = ignore_signature_failures_;
3313
2/2
✓ Branch 0 taken 724 times.
✓ Branch 1 taken 46 times.
 770 if (opt_host_.chain) {
3314
1/2
✓ Branch 2 taken 724 times.
✗ Branch 3 not taken.
 724 clone->opt_host_.chain = new vector<string>(*opt_host_.chain);
3315
1/2
✓ Branch 2 taken 724 times.
✗ Branch 3 not taken.
 724 clone->opt_host_chain_rtt_ = new vector<int>(*opt_host_chain_rtt_);
3316 }
3317
3318 770 CloneProxyConfig(clone);
3319 770 clone->opt_ip_preference_ = opt_ip_preference_;
3320 770 clone->proxy_template_direct_ = proxy_template_direct_;
3321 770 clone->proxy_template_forced_ = proxy_template_forced_;
3322 770 clone->opt_proxy_groups_reset_after_ = opt_proxy_groups_reset_after_;
3323 770 clone->opt_metalink_.reset_after = opt_metalink_.reset_after;
3324 770 clone->opt_host_.reset_after = opt_host_.reset_after;
3325 770 clone->credentials_attachment_ = credentials_attachment_;
3326 770 clone->ssl_certificate_store_ = ssl_certificate_store_;
3327
3328 770 clone->health_check_ = health_check_;
3329 770 clone->sharding_policy_ = sharding_policy_;
3330 770 clone->failover_indefinitely_ = failover_indefinitely_;
3331 770 clone->fqrn_ = fqrn_;
3332
3333 770 return clone;
3334 }
3335
3336
3337 770 void DownloadManager::CloneProxyConfig(DownloadManager *clone) {
3338 770 clone->opt_proxy_groups_current_ = opt_proxy_groups_current_;
3339 770 clone->opt_proxy_groups_current_burned_ = opt_proxy_groups_current_burned_;
3340 770 clone->opt_proxy_groups_fallback_ = opt_proxy_groups_fallback_;
3341 770 clone->opt_num_proxies_ = opt_num_proxies_;
3342 770 clone->opt_proxy_shard_ = opt_proxy_shard_;
3343 770 clone->opt_proxy_list_ = opt_proxy_list_;
3344 770 clone->opt_proxy_fallback_list_ = opt_proxy_fallback_list_;
3345
2/2
✓ Branch 0 taken 46 times.
✓ Branch 1 taken 724 times.
 770 if (opt_proxy_groups_ == NULL)
3346 46 return;
3347
3348
1/2
✓ Branch 2 taken 724 times.
✗ Branch 3 not taken.
 724 clone->opt_proxy_groups_ = new vector<vector<ProxyInfo> >(*opt_proxy_groups_);
3349
2/4
✓ Branch 2 taken 724 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 724 times.
✗ Branch 6 not taken.
 724 clone->UpdateProxiesUnlocked("cloned");
3350 }
3351
3352 } // namespace download
3353