GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/network/download.cc
Date: 2026-10-11 02:40:02
Exec Total Coverage
Lines: 877 1773 49.5%
Branches: 674 2285 29.5%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * The download module provides an interface for fetching files via HTTP
5 * and file. It is internally using libcurl and the asynchronous DNS resolver
6 * c-ares. The JobInfo struct describes a single file/url to download and
7 * keeps the state during the several phases of downloading.
8 *
9 * The module starts in single-threaded mode and can be switched to multi-
10 * threaded mode by Spawn(). In multi-threaded mode, the Fetch() function still
11 * blocks but there is a separate I/O thread using asynchronous I/O, which
12 * maintains all concurrent connections simultaneously. As there might be more
13 * than 1024 file descriptors for the CernVM-FS process, the I/O thread uses
14 * poll and the libcurl multi socket interface.
15 *
16 * While downloading, files can be decompressed and the secure hash can be
17 * calculated on the fly.
18 *
19 * The module also implements failure handling. If corrupted data has been
20 * downloaded, the transfer is restarted using HTTP "no-cache" pragma.
21 * A "host chain" can be configured. When a host fails, there is automatic
22 * fail-over to the next host in the chain until all hosts are probed.
23 * Similarly a chain of proxy sets can be configured. Inside a proxy set,
24 * proxies are selected randomly (load-balancing set).
25 */
26
27 // TODO(jblomer): MS for time summing
28
29 #include "download.h"
30
31 #include <alloca.h>
32 #include <errno.h>
33 #include <inttypes.h>
34 #include <poll.h>
35 #include <pthread.h>
36 #include <signal.h>
37 #include <stdint.h>
38 #include <sys/time.h>
39 #include <unistd.h>
40
41 #include <algorithm>
42 #include <atomic>
43 #include <cassert>
44 #include <cstdio>
45 #include <cstdlib>
46 #include <cstring>
47 #include <map>
48 #include <set>
49 #include <utility>
50
51 #include "compression/compression.h"
52 #include "crypto/hash.h"
53 #include "duplex_curl.h" // IWYU pragma: keep
54 #include "interrupt.h"
55 #include "network/sink_mem.h"
56 #include "network/sink_path.h"
57 #include "sanitizer.h"
58 #include "ssl.h"
59 #include "util/algorithm.h"
60 #include "util/exception.h"
61 #include "util/logging.h"
62 #include "util/posix.h"
63 #include "util/prng.h"
64 #include "util/smalloc.h"
65 #include "util/string.h"
66
67 using namespace std; // NOLINT
68
69 namespace download {
70
71 /**
72 * Returns the status if an interrupt happened for a given repository.
73 *
74 * Used only in case CVMFS_FAILOVER_INDEFINITELY (failover_indefinitely_) is set
75 * where failed downloads are retried indefinitely, unless an interrupt occurred
76 *
77 * @note If you use this functionality you need to change the source code of
78 * e.g. cvmfs_config reload to create a sentinel file. See comment below.
79 *
80 * @return true if an interrupt occurred
81 * false otherwise
82 */
83 ✗ bool Interrupted(const std::string &fqrn, JobInfo *info) {
84 ✗ if (info->allow_failure()) {
85 ✗ return true;
86 }
87
88 ✗ if (!fqrn.empty()) {
89 // it is up to the user the create this sentinel file ("pause_file") if
90 // CVMFS_FAILOVER_INDEFINITELY is used. It must be created during
91 // "cvmfs_config reload" and "cvmfs_config reload $fqrn"
92 ✗ const std::string pause_file = std::string("/var/run/cvmfs/interrupt.")
93 ✗ + fqrn;
94
95 ✗ LogCvmfs(kLogDownload, kLogDebug,
96 "(id %" PRId64 ") Interrupted(): checking for existence of %s",
97 info->id(), pause_file.c_str());
98 ✗ if (FileExists(pause_file)) {
99 ✗ LogCvmfs(kLogDownload, kLogDebug,
100 "(id %" PRId64 ") Interrupt marker found - "
101 "Interrupting current download, this will EIO outstanding IO.",
102 info->id());
103 ✗ if (0 != unlink(pause_file.c_str())) {
104 ✗ LogCvmfs(kLogDownload, kLogDebug,
105 "(id %" PRId64 ") Couldn't delete interrupt marker: errno=%d",
106 ✗ info->id(), errno);
107 }
108 ✗ return true;
109 }
110 }
111 ✗ return false;
112 }
113
114 5720 static Failures PrepareDownloadDestination(JobInfo *info) {
115
3/6
✓ Branch 1 taken 5720 times.
✗ Branch 2 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 5720 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 5720 times.
 5720 if (info->sink() != NULL && !info->sink()->IsValid()) {
116 ✗ cvmfs::PathSink *psink = dynamic_cast<cvmfs::PathSink *>(info->sink());
117 ✗ if (psink != NULL) {
118 ✗ LogCvmfs(kLogDownload, kLogDebug,
119 "(id %" PRId64 ") Failed to open path %s: %s (errno=%d).",
120 ✗ info->id(), psink->path().c_str(), strerror(errno), errno);
121 ✗ return kFailLocalIO;
122 } else {
123 ✗ LogCvmfs(kLogDownload, kLogDebug,
124 "(id %" PRId64 ") "
125 "Failed to create a valid sink: \n %s",
126 ✗ info->id(), info->sink()->Describe().c_str());
127 ✗ return kFailOther;
128 }
129 }
130
131 5720 return kFailOk;
132 }
133
134
135 /**
136 * Called by curl for every HTTP header. Not called for file:// transfers.
137 */
138 17289 static size_t CallbackCurlHeader(void *ptr, size_t size, size_t nmemb,
139 void *info_link) {
140 17289 const size_t num_bytes = size * nmemb;
141
1/2
✓ Branch 2 taken 17289 times.
✗ Branch 3 not taken.
 17289 const string header_line(static_cast<const char *>(ptr), num_bytes);
142 17289 JobInfo *info = static_cast<JobInfo *>(info_link);
143
144 // LogCvmfs(kLogDownload, kLogDebug, "REMOVE-ME: Header callback with %s",
145 // header_line.c_str());
146
147 // Check http status codes
148
4/6
✓ Branch 2 taken 17289 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 17289 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 432 times.
✓ Branch 10 taken 16857 times.
 17289 if (HasPrefix(header_line, "HTTP/1.", false)) {
149
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 432 times.
 432 if (header_line.length() < 10) {
150 ✗ return 0;
151 }
152
153 unsigned i;
154
5/6
✓ Branch 1 taken 864 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 432 times.
✓ Branch 5 taken 432 times.
✓ Branch 6 taken 432 times.
✓ Branch 7 taken 432 times.
 864 for (i = 8; (i < header_line.length()) && (header_line[i] == ' '); ++i) {
155 }
156
157 // Code is initialized to -1
158
1/2
✓ Branch 1 taken 432 times.
✗ Branch 2 not taken.
 432 if (header_line.length() > i + 2) {
159 432 info->SetHttpCode(DownloadManager::ParseHttpCode(&header_line[i]));
160 }
161
162
2/2
✓ Branch 1 taken 324 times.
✓ Branch 2 taken 108 times.
 432 if ((info->http_code() / 100) == 2) {
163 324 return num_bytes;
164
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 108 } else if ((info->http_code() == 301) || (info->http_code() == 302)
165
2/8
✗ Branch 0 not taken.
✓ Branch 1 taken 108 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 108 times.
✗ Branch 9 not taken.
 108 || (info->http_code() == 303) || (info->http_code() == 307)) {
166
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 108 times.
 108 if (!info->follow_redirects()) {
167 ✗ LogCvmfs(kLogDownload, kLogDebug,
168 "(id %" PRId64 ") redirect support not enabled: %s",
169 info->id(), header_line.c_str());
170 ✗ info->SetErrorCode(kFailHostHttp);
171 ✗ return 0;
172 }
173
1/2
✓ Branch 3 taken 108 times.
✗ Branch 4 not taken.
 108 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") http redirect: %s",
174 info->id(), header_line.c_str());
175 // libcurl will handle this because of CURLOPT_FOLLOWLOCATION
176 108 return num_bytes;
177 } else {
178 ✗ LogCvmfs(kLogDownload, kLogDebug,
179 "(id %" PRId64 ") http status error code: %s [%d]", info->id(),
180 header_line.c_str(), info->http_code());
181 ✗ if (((info->http_code() / 100) == 5) || (info->http_code() == 400)
182 ✗ || (info->http_code() == 404)) {
183 // 5XX returned by host
184 // 400: error from the GeoAPI module
185 // 404: the stratum 1 does not have the newest files
186 ✗ info->SetErrorCode(kFailHostHttp);
187 ✗ } else if (info->http_code() == 429) {
188 // 429: rate throttling (we ignore the backoff hint for the time being)
189 ✗ info->SetErrorCode(kFailHostConnection);
190 } else {
191 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostHttp
192 : kFailProxyHttp);
193 }
194 ✗ return 0;
195 }
196 }
197
198 // If needed: allocate space in sink
199
3/4
✓ Branch 2 taken 16857 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 7785 times.
✓ Branch 5 taken 9072 times.
 16857 if (info->sink() != NULL && info->sink()->RequiresReserve()
200
11/20
✓ Branch 1 taken 16857 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 7785 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 7785 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 2463 times.
✓ Branch 11 taken 5322 times.
✓ Branch 12 taken 7785 times.
✓ Branch 13 taken 9072 times.
✗ Branch 14 not taken.
✓ Branch 15 taken 7785 times.
✓ Branch 16 taken 9072 times.
✓ Branch 18 taken 2463 times.
✓ Branch 19 taken 14394 times.
✗ Branch 20 not taken.
✗ Branch 21 not taken.
✗ Branch 23 not taken.
✗ Branch 24 not taken.
 33714 && HasPrefix(header_line, "CONTENT-LENGTH:", true)) {
201 2463 char *tmp = reinterpret_cast<char *>(alloca(num_bytes + 1));
202 2463 uint64_t length = 0;
203 2463 sscanf(header_line.c_str(), "%s %" PRIu64, tmp, &length);
204
1/2
✓ Branch 0 taken 2463 times.
✗ Branch 1 not taken.
 2463 if (length > 0) {
205
2/4
✓ Branch 2 taken 2463 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 2463 times.
 2463 if (!info->sink()->Reserve(length)) {
206 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
207 "(id %" PRId64 ") "
208 "resource %s too large to store in memory (%" PRIu64 ")",
209 info->id(), info->url()->c_str(), length);
210 ✗ info->SetErrorCode(kFailTooBig);
211 ✗ return 0;
212 }
213 } else {
214 // Empty resource
215 ✗ info->sink()->Reserve(0);
216 }
217
4/6
✓ Branch 2 taken 14394 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 14394 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 108 times.
✓ Branch 10 taken 14286 times.
 14394 } else if (HasPrefix(header_line, "LOCATION:", true)) {
218 // This comes along with redirects
219
1/2
✓ Branch 3 taken 108 times.
✗ Branch 4 not taken.
 108 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") %s", info->id(),
220 header_line.c_str());
221
3/6
✓ Branch 2 taken 14286 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 14286 times.
✗ Branch 6 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 14286 times.
 14286 } else if (HasPrefix(header_line, "LINK:", true)) {
222 // This is metalink info
223 ✗ LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") %s", info->id(),
224 header_line.c_str());
225 ✗ std::string link = info->link();
226 ✗ if (link.size() != 0) {
227 // multiple LINK headers are allowed
228 ✗ link = link + ", " + header_line.substr(5);
229 } else {
230 ✗ link = header_line.substr(5);
231 }
232 ✗ info->SetLink(link);
233
3/6
✓ Branch 3 taken 14286 times.
✗ Branch 4 not taken.
✓ Branch 6 taken 14286 times.
✗ Branch 7 not taken.
✗ Branch 10 not taken.
✓ Branch 11 taken 14286 times.
 14286 } else if (HasPrefix(header_line, "X-SQUID-ERROR:", true)) {
234 // Reinterpret host error as proxy error
235 ✗ if (info->error_code() == kFailHostHttp) {
236 ✗ info->SetErrorCode(kFailProxyHttp);
237 }
238
3/6
✓ Branch 2 taken 14286 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 14286 times.
✗ Branch 6 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 14286 times.
 14286 } else if (HasPrefix(header_line, "PROXY-STATUS:", true)) {
239 // Reinterpret host error as proxy error if applicable
240 ✗ if ((info->error_code() == kFailHostHttp)
241 ✗ && (header_line.find("error=") != string::npos)) {
242 ✗ info->SetErrorCode(kFailProxyHttp);
243 }
244 }
245
246 16857 return num_bytes;
247 17289 }
248
249
250 /**
251 * Called by curl for every received data chunk.
252 *
253 * In multi-threaded mode the chunk's hash is updated here (cheap) and the
254 * raw bytes are handed off to the calling Fetch() thread via JobInfo's
255 * data tube; the (relatively expensive) zlib decompression and sink->Write
256 * happen on that caller thread. This lets multiple callers — e.g. main
257 * thread plus N file-bundle prefetch workers — run their decompression in
258 * parallel instead of serializing on this single MainDownload thread.
259 *
260 * In single-threaded mode (no MainDownload thread, e.g. unit tests) the
261 * old inline path is kept.
262 */
263 6783 static size_t CallbackCurlData(void *ptr, size_t size, size_t nmemb,
264 void *info_link) {
265 6783 const size_t num_bytes = size * nmemb;
266 6783 JobInfo *info = static_cast<JobInfo *>(info_link);
267
268 // TODO(heretherebedragons) remove if no error comes up
269 // as this means only jobinfo data request (and not header only)
270 // come here
271
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6783 times.
 6783 assert(info->sink() != NULL);
272
273
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6783 times.
 6783 if (num_bytes == 0)
274 ✗ return 0;
275
276
2/2
✓ Branch 1 taken 3416 times.
✓ Branch 2 taken 3367 times.
 6783 if (info->expected_hash()) {
277 3416 shash::Update(reinterpret_cast<unsigned char *>(ptr), num_bytes,
278 info->hash_context());
279 }
280
281
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6783 times.
 6783 if (info->IsValidDataTube()) {
282 // Parallel-decompress path: copy bytes onto a buffer the caller will own
283 // and enqueue. The caller thread (DownloadManager::Fetch) pops these
284 // elements and runs DecompressZStream2Sink / sink->Write itself.
285 ✗ char *buf = new char[num_bytes];
286 ✗ memcpy(buf, ptr, num_bytes);
287 DataTubeElement *ele = new DataTubeElement(buf, num_bytes,
288 ✗ kActionDecompress);
289 ✗ info->GetDataTubePtr()->EnqueueBack(ele);
290 ✗ return num_bytes;
291 }
292
293
2/2
✓ Branch 1 taken 3397 times.
✓ Branch 2 taken 3386 times.
 6783 if (info->compressed()) {
294 3397 const zlib::StreamStates retval = zlib::DecompressZStream2Sink(
295 ptr, static_cast<int64_t>(num_bytes), info->GetZstreamPtr(),
296 info->sink());
297
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3397 times.
 3397 if (retval == zlib::kStreamDataError) {
298 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
299 "(id %" PRId64 ") failed to decompress %s", info->id(),
300 info->url()->c_str());
301 ✗ info->SetErrorCode(kFailBadData);
302 ✗ return 0;
303
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3397 times.
 3397 } else if (retval == zlib::kStreamIOError) {
304 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
305 "(id %" PRId64 ") decompressing %s, local IO error", info->id(),
306 info->url()->c_str());
307 ✗ info->SetErrorCode(kFailLocalIO);
308 ✗ return 0;
309 }
310 } else {
311 3386 const int64_t written = info->sink()->Write(ptr, num_bytes);
312
2/4
✓ Branch 0 taken 3386 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 3386 times.
 3386 if (written < 0 || static_cast<uint64_t>(written) != num_bytes) {
313 ✗ LogCvmfs(kLogDownload, kLogDebug,
314 "(id %" PRId64 ") "
315 "Failed to perform write of %zu bytes to sink %s with errno %ld",
316 ✗ info->id(), num_bytes, info->sink()->Describe().c_str(),
317 written);
318 }
319 }
320
321 6783 return num_bytes;
322 }
323
324 #ifdef DEBUGMSG
325 11935 static int CallbackCurlDebug(CURL *handle,
326 curl_infotype type,
327 char *data,
328 size_t size,
329 void * /* clientp */) {
330 JobInfo *info;
331
1/2
✓ Branch 1 taken 11935 times.
✗ Branch 2 not taken.
 11935 curl_easy_getinfo(handle, CURLINFO_PRIVATE, &info);
332
333
3/6
✓ Branch 2 taken 11935 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 11935 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 11935 times.
✗ Branch 9 not taken.
 23870 std::string prefix = "(id " + StringifyInt(info->id()) + ") ";
334
4/8
✓ Branch 0 taken 8515 times.
✓ Branch 1 taken 1692 times.
✓ Branch 2 taken 432 times.
✓ Branch 3 taken 1296 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
 11935 switch (type) {
335 8515 case CURLINFO_TEXT:
336
1/2
✓ Branch 1 taken 8515 times.
✗ Branch 2 not taken.
 8515 prefix += "{info} ";
337 8515 break;
338 1692 case CURLINFO_HEADER_IN:
339
1/2
✓ Branch 1 taken 1692 times.
✗ Branch 2 not taken.
 1692 prefix += "{header/recv} ";
340 1692 break;
341 432 case CURLINFO_HEADER_OUT:
342
1/2
✓ Branch 1 taken 432 times.
✗ Branch 2 not taken.
 432 prefix += "{header/sent} ";
343 432 break;
344 1296 case CURLINFO_DATA_IN:
345
2/2
✓ Branch 0 taken 144 times.
✓ Branch 1 taken 1152 times.
 1296 if (size < 50) {
346
1/2
✓ Branch 1 taken 144 times.
✗ Branch 2 not taken.
 144 prefix += "{data/recv} ";
347 144 break;
348 } else {
349
1/2
✓ Branch 2 taken 1152 times.
✗ Branch 3 not taken.
 1152 LogCvmfs(kLogCurl, kLogDebug, "%s{data/recv} <snip>", prefix.c_str());
350 1152 return 0;
351 }
352 ✗ case CURLINFO_DATA_OUT:
353 ✗ if (size < 50) {
354 ✗ prefix += "{data/sent} ";
355 ✗ break;
356 } else {
357 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{data/sent} <snip>", prefix.c_str());
358 ✗ return 0;
359 }
360 ✗ case CURLINFO_SSL_DATA_IN:
361 ✗ if (size < 50) {
362 ✗ prefix += "{ssldata/recv} ";
363 ✗ break;
364 } else {
365 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{ssldata/recv} <snip>",
366 prefix.c_str());
367 ✗ return 0;
368 }
369 ✗ case CURLINFO_SSL_DATA_OUT:
370 ✗ if (size < 50) {
371 ✗ prefix += "{ssldata/sent} ";
372 ✗ break;
373 } else {
374 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{ssldata/sent} <snip>",
375 prefix.c_str());
376 ✗ return 0;
377 }
378 ✗ default:
379 // just log the message
380 ✗ break;
381 }
382
383 10783 bool valid_char = true;
384
1/2
✓ Branch 2 taken 10783 times.
✗ Branch 3 not taken.
 10783 std::string msg(data, size);
385
2/2
✓ Branch 1 taken 363248 times.
✓ Branch 2 taken 10783 times.
 374031 for (size_t i = 0; i < msg.length(); ++i) {
386
2/4
✓ Branch 1 taken 363248 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 363248 times.
 363248 if (msg[i] == '\0') {
387 ✗ msg[i] = '~';
388 }
389
390 // verify that char is a valid printable char
391
3/6
✓ Branch 1 taken 363248 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 345985 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 345985 times.
 709233 if ((msg[i] < ' ' || msg[i] > '~')
392
6/8
✓ Branch 0 taken 345985 times.
✓ Branch 1 taken 17263 times.
✓ Branch 3 taken 17263 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 4428 times.
✓ Branch 6 taken 12835 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 363248 times.
 713661 && (msg[i] != 10 /*line feed*/
393
2/4
✓ Branch 1 taken 4428 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 4428 times.
 4428 && msg[i] != 13 /*carriage return*/)) {
394 ✗ valid_char = false;
395 }
396 }
397
398
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 10783 times.
 10783 if (!valid_char) {
399 ✗ msg = "<Non-plaintext sequence>";
400 }
401
402
1/2
✓ Branch 3 taken 10783 times.
✗ Branch 4 not taken.
 10783 LogCvmfs(kLogCurl, kLogDebug, "%s%s", prefix.c_str(),
403
1/2
✓ Branch 1 taken 10783 times.
✗ Branch 2 not taken.
 21566 Trim(msg, true /* trim_newline */).c_str());
404 10783 return 0;
405 11935 }
406 #endif
407
408 //------------------------------------------------------------------------------
409
410
411 const int DownloadManager::kProbeUnprobed = -1;
412 const int DownloadManager::kProbeDown = -2;
413 const int DownloadManager::kProbeGeo = -3;
414
415 /**
416 * Escape special chars from the URL, except for ':' and '/',
417 * which should keep their meaning.
418 */
419 5864 string DownloadManager::EscapeUrl(const int64_t jobinfo_id, const string &url) {
420 5864 string escaped;
421
1/2
✓ Branch 2 taken 5864 times.
✗ Branch 3 not taken.
 5864 escaped.reserve(url.length());
422
423 char escaped_char[3];
424
2/2
✓ Branch 1 taken 492442 times.
✓ Branch 2 taken 5864 times.
 498306 for (unsigned i = 0, s = url.length(); i < s; ++i) {
425
3/4
✓ Branch 2 taken 492442 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 324 times.
✓ Branch 5 taken 492118 times.
 492442 if (JobInfo::EscapeUrlChar(url[i], escaped_char)) {
426
1/2
✓ Branch 1 taken 324 times.
✗ Branch 2 not taken.
 324 escaped.append(escaped_char, 3);
427 } else {
428
1/2
✓ Branch 1 taken 492118 times.
✗ Branch 2 not taken.
 492118 escaped.push_back(escaped_char[0]);
429 }
430 }
431
1/2
✓ Branch 3 taken 5864 times.
✗ Branch 4 not taken.
 5864 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") escaped %s to %s",
432 jobinfo_id, url.c_str(), escaped.c_str());
433
434 11728 return escaped;
435 }
436
437 /**
438 * -1 of digits is not a valid Http return code
439 */
440 612 int DownloadManager::ParseHttpCode(const char digits[3]) {
441 612 int result = 0;
442 612 int factor = 100;
443
2/2
✓ Branch 0 taken 1836 times.
✓ Branch 1 taken 576 times.
 2412 for (int i = 0; i < 3; ++i) {
444
3/4
✓ Branch 0 taken 1836 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 36 times.
✓ Branch 3 taken 1800 times.
 1836 if ((digits[i] < '0') || (digits[i] > '9'))
445 36 return -1;
446 1800 result += (digits[i] - '0') * factor;
447 1800 factor /= 10;
448 }
449 576 return result;
450 }
451
452
453 /**
454 * Called when new curl sockets arrive or existing curl sockets depart.
455 */
456 ✗ int DownloadManager::CallbackCurlSocket(CURL * /* easy */,
457 curl_socket_t s,
458 int action,
459 void *userp,
460 void * /* socketp */) {
461 // LogCvmfs(kLogDownload, kLogDebug, "CallbackCurlSocket called with easy "
462 // "handle %p, socket %d, action %d", easy, s, action);
463 ✗ DownloadManager *download_mgr = static_cast<DownloadManager *>(userp);
464 ✗ if (action == CURL_POLL_NONE)
465 ✗ return 0;
466
467 // Find s in watch_fds_
468 unsigned index;
469
470 // TODO(heretherebedragons) why start at index = 0 and not 2?
471 // fd[0] and fd[1] are fixed?
472 ✗ for (index = 0; index < download_mgr->watch_fds_inuse_; ++index) {
473 ✗ if (download_mgr->watch_fds_[index].fd == s)
474 ✗ break;
475 }
476 // Or create newly
477 ✗ if (index == download_mgr->watch_fds_inuse_) {
478 // Extend array if necessary
479 ✗ if (download_mgr->watch_fds_inuse_ == download_mgr->watch_fds_size_) {
480 ✗ assert(download_mgr->watch_fds_size_ > 0);
481 ✗ download_mgr->watch_fds_size_ *= 2;
482 ✗ download_mgr->watch_fds_ = static_cast<struct pollfd *>(
483 ✗ srealloc(download_mgr->watch_fds_,
484 ✗ download_mgr->watch_fds_size_ * sizeof(struct pollfd)));
485 }
486 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].fd = s;
487 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].events = 0;
488 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].revents = 0;
489 ✗ download_mgr->watch_fds_inuse_++;
490 }
491
492 ✗ switch (action) {
493 ✗ case CURL_POLL_IN:
494 ✗ download_mgr->watch_fds_[index].events = POLLIN | POLLPRI;
495 ✗ break;
496 ✗ case CURL_POLL_OUT:
497 ✗ download_mgr->watch_fds_[index].events = POLLOUT | POLLWRBAND;
498 ✗ break;
499 ✗ case CURL_POLL_INOUT:
500 ✗ download_mgr->watch_fds_[index].events = POLLIN | POLLPRI | POLLOUT
501 | POLLWRBAND;
502 ✗ break;
503 ✗ case CURL_POLL_REMOVE:
504 ✗ if (index < download_mgr->watch_fds_inuse_ - 1) {
505 download_mgr
506 ✗ ->watch_fds_[index] = download_mgr->watch_fds_
507 ✗ [download_mgr->watch_fds_inuse_ - 1];
508 }
509 ✗ download_mgr->watch_fds_inuse_--;
510 // Shrink array if necessary
511 ✗ if ((download_mgr->watch_fds_inuse_ > download_mgr->watch_fds_max_)
512 ✗ && (download_mgr->watch_fds_inuse_
513 ✗ < download_mgr->watch_fds_size_ / 2)) {
514 ✗ download_mgr->watch_fds_size_ /= 2;
515 // LogCvmfs(kLogDownload, kLogDebug, "shrinking watch_fds_ (%d)",
516 // watch_fds_size_);
517 ✗ download_mgr->watch_fds_ = static_cast<struct pollfd *>(
518 ✗ srealloc(download_mgr->watch_fds_,
519 ✗ download_mgr->watch_fds_size_ * sizeof(struct pollfd)));
520 // LogCvmfs(kLogDownload, kLogDebug, "shrinking watch_fds_ done",
521 // watch_fds_size_);
522 }
523 ✗ break;
524 ✗ default:
525 ✗ break;
526 }
527
528 ✗ return 0;
529 }
530
531
532 /**
533 * Worker thread event loop. Waits on new JobInfo structs on a pipe.
534 */
535 ✗ void *DownloadManager::MainDownload(void *data) {
536 ✗ DownloadManager *download_mgr = static_cast<DownloadManager *>(data);
537 ✗ LogCvmfs(kLogDownload, kLogDebug,
538 "download I/O thread of DownloadManager '%s' started",
539 download_mgr->name_.c_str());
540
541 ✗ const int kIdxPipeTerminate = 0;
542 ✗ const int kIdxPipeJobs = 1;
543
544 ✗ download_mgr->watch_fds_ = static_cast<struct pollfd *>(
545 ✗ smalloc(2 * sizeof(struct pollfd)));
546 ✗ download_mgr->watch_fds_size_ = 2;
547 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].fd = download_mgr->pipe_terminate_
548 ✗ ->GetReadFd();
549 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].events = POLLIN | POLLPRI;
550 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].revents = 0;
551 ✗ download_mgr->watch_fds_[kIdxPipeJobs].fd = download_mgr->pipe_jobs_
552 ✗ ->GetReadFd();
553 ✗ download_mgr->watch_fds_[kIdxPipeJobs].events = POLLIN | POLLPRI;
554 ✗ download_mgr->watch_fds_[kIdxPipeJobs].revents = 0;
555 ✗ download_mgr->watch_fds_inuse_ = 2;
556
557 ✗ int still_running = 0;
558 struct timeval timeval_start, timeval_stop;
559 ✗ gettimeofday(&timeval_start, NULL);
560 while (true) {
561 int timeout;
562 ✗ if (still_running) {
563 /* NOTE: The following might degrade the performance for many small files
564 * use case. TODO(jblomer): look into it.
565 // Specify a timeout for polling in ms; this allows us to return
566 // to libcurl once a second so it can look for internal operations
567 // which timed out. libcurl has a more elaborate mechanism
568 // (CURLMOPT_TIMERFUNCTION) that would inform us of the next potential
569 // timeout. TODO(bbockelm) we should switch to that in the future.
570 timeout = 100;
571 */
572 ✗ timeout = 1;
573 } else {
574 ✗ timeout = -1;
575 ✗ gettimeofday(&timeval_stop, NULL);
576 const int64_t delta = static_cast<int64_t>(
577 ✗ 1000 * DiffTimeSeconds(timeval_start, timeval_stop));
578 ✗ perf::Xadd(download_mgr->counters_->sz_transfer_time, delta);
579 }
580 ✗ const int retval = poll(download_mgr->watch_fds_,
581 ✗ download_mgr->watch_fds_inuse_, timeout);
582 ✗ if (retval < 0) {
583 ✗ continue;
584 }
585
586 // Handle timeout
587 ✗ if (retval == 0) {
588 ✗ curl_multi_socket_action(download_mgr->curl_multi_, CURL_SOCKET_TIMEOUT,
589 0, &still_running);
590 }
591
592 // Terminate I/O thread
593 ✗ if (download_mgr->watch_fds_[kIdxPipeTerminate].revents)
594 ✗ break;
595
596 // New job arrives
597 ✗ if (download_mgr->watch_fds_[kIdxPipeJobs].revents) {
598 ✗ download_mgr->watch_fds_[kIdxPipeJobs].revents = 0;
599 JobInfo *info;
600 ✗ download_mgr->pipe_jobs_->Read<JobInfo *>(&info);
601 ✗ if (!still_running) {
602 ✗ gettimeofday(&timeval_start, NULL);
603 }
604 ✗ CURL *handle = download_mgr->AcquireCurlHandle();
605 ✗ download_mgr->InitializeRequest(info, handle);
606 ✗ download_mgr->SetUrlOptions(info);
607 ✗ curl_multi_add_handle(download_mgr->curl_multi_, handle);
608 ✗ curl_multi_socket_action(download_mgr->curl_multi_, CURL_SOCKET_TIMEOUT,
609 0, &still_running);
610 }
611
612 // Activity on curl sockets
613 // Within this loop the curl_multi_socket_action() may cause socket(s)
614 // to be removed from watch_fds_. If a socket is removed it is replaced
615 // by the socket at the end of the array and the inuse count is decreased.
616 // Therefore loop over the array in reverse order.
617 ✗ for (int64_t i = download_mgr->watch_fds_inuse_ - 1; i >= 2; --i) {
618 ✗ if (i >= download_mgr->watch_fds_inuse_) {
619 ✗ continue;
620 }
621 ✗ if (download_mgr->watch_fds_[i].revents) {
622 ✗ int ev_bitmask = 0;
623 ✗ if (download_mgr->watch_fds_[i].revents & (POLLIN | POLLPRI))
624 ✗ ev_bitmask |= CURL_CSELECT_IN;
625 ✗ if (download_mgr->watch_fds_[i].revents & (POLLOUT | POLLWRBAND))
626 ✗ ev_bitmask |= CURL_CSELECT_OUT;
627 ✗ if (download_mgr->watch_fds_[i].revents
628 ✗ & (POLLERR | POLLHUP | POLLNVAL)) {
629 ✗ ev_bitmask |= CURL_CSELECT_ERR;
630 }
631 ✗ download_mgr->watch_fds_[i].revents = 0;
632
633 ✗ curl_multi_socket_action(download_mgr->curl_multi_,
634 ✗ download_mgr->watch_fds_[i].fd,
635 ev_bitmask,
636 &still_running);
637 }
638 }
639
640 // Check if transfers are completed
641 CURLMsg *curl_msg;
642 int msgs_in_queue;
643 ✗ while ((curl_msg = curl_multi_info_read(download_mgr->curl_multi_,
644 &msgs_in_queue))) {
645 ✗ if (curl_msg->msg == CURLMSG_DONE) {
646 ✗ perf::Inc(download_mgr->counters_->n_requests);
647 JobInfo *info;
648 ✗ CURL *easy_handle = curl_msg->easy_handle;
649 ✗ const int curl_error = curl_msg->data.result;
650 ✗ curl_easy_getinfo(easy_handle, CURLINFO_PRIVATE, &info);
651
652 int64_t redir_count;
653 ✗ curl_easy_getinfo(easy_handle, CURLINFO_REDIRECT_COUNT, &redir_count);
654 ✗ LogCvmfs(kLogDownload, kLogDebug,
655 "(manager '%s' - id %" PRId64 ") "
656 "Number of CURL redirects %" PRId64,
657 download_mgr->name_.c_str(), info->id(), redir_count);
658
659 ✗ curl_multi_remove_handle(download_mgr->curl_multi_, easy_handle);
660 ✗ if (download_mgr->VerifyAndFinalize(curl_error, info)) {
661 ✗ curl_multi_add_handle(download_mgr->curl_multi_, easy_handle);
662 ✗ curl_multi_socket_action(download_mgr->curl_multi_,
663 CURL_SOCKET_TIMEOUT,
664 0,
665 &still_running);
666 } else {
667 // Return easy handle into pool and write result back
668 ✗ download_mgr->ReleaseCurlHandle(easy_handle, true /* allow_reuse */);
669
670 ✗ DataTubeElement *ele = new DataTubeElement(kActionStop);
671 ✗ info->GetDataTubePtr()->EnqueueBack(ele);
672 ✗ info->GetPipeJobResultPtr()->Write<download::Failures>(
673 ✗ info->error_code());
674 }
675 }
676 }
677 }
678
679 ✗ for (set<CURL *>::iterator i = download_mgr->pool_handles_inuse_->begin(),
680 ✗ iEnd = download_mgr->pool_handles_inuse_->end();
681 ✗ i != iEnd;
682 ✗ ++i) {
683 ✗ curl_multi_remove_handle(download_mgr->curl_multi_, *i);
684 ✗ curl_easy_cleanup(*i);
685 }
686 ✗ download_mgr->pool_handles_inuse_->clear();
687 ✗ free(download_mgr->watch_fds_);
688
689 ✗ LogCvmfs(kLogDownload, kLogDebug,
690 "download I/O thread of DownloadManager '%s' terminated",
691 download_mgr->name_.c_str());
692 ✗ return NULL;
693 }
694
695
696 //------------------------------------------------------------------------------
697
698
699 4690 HeaderLists::~HeaderLists() {
700
2/2
✓ Branch 1 taken 4698 times.
✓ Branch 2 taken 4690 times.
 9388 for (unsigned i = 0; i < blocks_.size(); ++i) {
701
1/2
✓ Branch 1 taken 4698 times.
✗ Branch 2 not taken.
 4698 delete[] blocks_[i];
702 }
703 4690 blocks_.clear();
704 4690 }
705
706
707 14547 curl_slist *HeaderLists::GetList(const char *header) { return Get(header); }
708
709
710 5720 curl_slist *HeaderLists::DuplicateList(curl_slist *slist) {
711
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5720 times.
 5720 assert(slist);
712 5720 curl_slist *copy = GetList(slist->data);
713 5720 copy->next = slist->next;
714 5720 curl_slist *prev = copy;
715 5720 slist = slist->next;
716
2/2
✓ Branch 0 taken 11440 times.
✓ Branch 1 taken 5720 times.
 17160 while (slist) {
717 11440 curl_slist *new_link = Get(slist->data);
718 11440 new_link->next = slist->next;
719 11440 prev->next = new_link;
720 11440 prev = new_link;
721 11440 slist = slist->next;
722 }
723 5720 return copy;
724 }
725
726
727 9796 void HeaderLists::AppendHeader(curl_slist *slist, const char *header) {
728
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9796 times.
 9796 assert(slist);
729 9796 curl_slist *new_link = Get(header);
730 9796 new_link->next = NULL;
731
732
2/2
✓ Branch 0 taken 5718 times.
✓ Branch 1 taken 9796 times.
 15514 while (slist->next)
733 5718 slist = slist->next;
734 9796 slist->next = new_link;
735 9796 }
736
737
738 /**
739 * Ensures that a certain header string is _not_ part of slist on return.
740 * Note that if the first header element matches, the returned slist points
741 * to a different value.
742 */
743 40 void HeaderLists::CutHeader(const char *header, curl_slist **slist) {
744
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 40 times.
 40 assert(slist);
745 curl_slist head;
746 40 head.next = *slist;
747 40 curl_slist *prev = &head;
748 40 curl_slist *rover = *slist;
749
2/2
✓ Branch 0 taken 72 times.
✓ Branch 1 taken 40 times.
 112 while (rover) {
750
2/2
✓ Branch 0 taken 32 times.
✓ Branch 1 taken 40 times.
 72 if (strcmp(rover->data, header) == 0) {
751 32 prev->next = rover->next;
752 32 Put(rover);
753 32 rover = prev;
754 }
755 72 prev = rover;
756 72 rover = rover->next;
757 }
758 40 *slist = head.next;
759 40 }
760
761
762 7800 void HeaderLists::PutList(curl_slist *slist) {
763
2/2
✓ Branch 0 taken 19670 times.
✓ Branch 1 taken 7800 times.
 27470 while (slist) {
764 19670 curl_slist *next = slist->next;
765 19670 Put(slist);
766 19670 slist = next;
767 }
768 7800 }
769
770
771 32 string HeaderLists::Print(curl_slist *slist) {
772 32 string verbose;
773
2/2
✓ Branch 0 taken 64 times.
✓ Branch 1 taken 32 times.
 96 while (slist) {
774
3/6
✓ Branch 2 taken 64 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 64 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 64 times.
✗ Branch 9 not taken.
 64 verbose += string(slist->data) + "\n";
775 64 slist = slist->next;
776 }
777 32 return verbose;
778 }
779
780
781 35783 curl_slist *HeaderLists::Get(const char *header) {
782
2/2
✓ Branch 1 taken 31084 times.
✓ Branch 2 taken 4707 times.
 35791 for (unsigned i = 0; i < blocks_.size(); ++i) {
783
2/2
✓ Branch 0 taken 642752 times.
✓ Branch 1 taken 8 times.
 642760 for (unsigned j = 0; j < kBlockSize; ++j) {
784
2/2
✓ Branch 2 taken 31076 times.
✓ Branch 3 taken 611676 times.
 642752 if (!IsUsed(&(blocks_[i][j]))) {
785 31076 blocks_[i][j].data = const_cast<char *>(header);
786 31076 return &(blocks_[i][j]);
787 }
788 }
789 }
790
791 // All used, new block
792 4707 AddBlock();
793 4707 blocks_[blocks_.size() - 1][0].data = const_cast<char *>(header);
794 4707 return &(blocks_[blocks_.size() - 1][0]);
795 }
796
797
798 1224694 void HeaderLists::Put(curl_slist *slist) {
799 1224694 slist->data = NULL;
800 1224694 slist->next = NULL;
801 1224694 }
802
803
804 4707 void HeaderLists::AddBlock() {
805
1/2
✓ Branch 1 taken 4707 times.
✗ Branch 2 not taken.
 4707 curl_slist *new_block = new curl_slist[kBlockSize];
806
2/2
✓ Branch 0 taken 1204992 times.
✓ Branch 1 taken 4707 times.
 1209699 for (unsigned i = 0; i < kBlockSize; ++i) {
807 1204992 Put(&new_block[i]);
808 }
809
1/2
✓ Branch 1 taken 4707 times.
✗ Branch 2 not taken.
 4707 blocks_.push_back(new_block);
810 4707 }
811
812
813 //------------------------------------------------------------------------------
814
815
816 ✗ string DownloadManager::ProxyInfo::Print() {
817 ✗ if (url == "DIRECT")
818 ✗ return url;
819
820 ✗ string result = url;
821 ✗ const int remaining = static_cast<int>(host.deadline())
822 ✗ - static_cast<int>(time(NULL));
823 ✗ string expinfo = (remaining >= 0) ? "+" : "";
824 ✗ if (abs(remaining) >= 3600) {
825 ✗ expinfo += StringifyInt(remaining / 3600) + "h";
826 ✗ } else if (abs(remaining) >= 60) {
827 ✗ expinfo += StringifyInt(remaining / 60) + "m";
828 } else {
829 ✗ expinfo += StringifyInt(remaining) + "s";
830 }
831 ✗ if (host.status() == dns::kFailOk) {
832 ✗ result += " (" + host.name() + ", " + expinfo + ")";
833 } else {
834 ✗ result += " (:unresolved:, " + expinfo + ")";
835 }
836 ✗ return result;
837 }
838
839
840 /**
841 * Gets an idle CURL handle from the pool. Creates a new one and adds it to
842 * the pool if necessary.
843 */
844 5720 CURL *DownloadManager::AcquireCurlHandle() {
845 CURL *handle;
846
847
1/2
✓ Branch 1 taken 5720 times.
✗ Branch 2 not taken.
 5720 if (pool_handles_idle_->empty()) {
848 // Create a new handle
849
1/2
✓ Branch 1 taken 5720 times.
✗ Branch 2 not taken.
 5720 handle = curl_easy_init();
850
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5720 times.
 5720 assert(handle != NULL);
851
852
1/2
✓ Branch 1 taken 5720 times.
✗ Branch 2 not taken.
 5720 curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1);
853 // curl_easy_setopt(curl_default, CURLOPT_FAILONERROR, 1);
854
1/2
✓ Branch 1 taken 5720 times.
✗ Branch 2 not taken.
 5720 curl_easy_setopt(handle, CURLOPT_HEADERFUNCTION, CallbackCurlHeader);
855
1/2
✓ Branch 1 taken 5720 times.
✗ Branch 2 not taken.
 5720 curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, CallbackCurlData);
856 } else {
857 ✗ handle = *(pool_handles_idle_->begin());
858 ✗ pool_handles_idle_->erase(pool_handles_idle_->begin());
859 }
860
861
1/2
✓ Branch 1 taken 5720 times.
✗ Branch 2 not taken.
 5720 pool_handles_inuse_->insert(handle);
862
863 5720 return handle;
864 }
865
866
867 5720 void DownloadManager::ReleaseCurlHandle(CURL *handle, bool allow_reuse) {
868
1/2
✓ Branch 1 taken 5720 times.
✗ Branch 2 not taken.
 5720 const set<CURL *>::iterator elem = pool_handles_inuse_->find(handle);
869
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 5720 times.
 5720 assert(elem != pool_handles_inuse_->end());
870
871
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 5720 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 5720 times.
✗ Branch 6 not taken.
 5720 if (!allow_reuse || pool_handles_idle_->size() > pool_max_handles_) {
872
1/2
✓ Branch 2 taken 5720 times.
✗ Branch 3 not taken.
 5720 curl_easy_cleanup(*elem);
873 } else {
874 ✗ pool_handles_idle_->insert(*elem);
875 }
876
877
1/2
✓ Branch 1 taken 5720 times.
✗ Branch 2 not taken.
 5720 pool_handles_inuse_->erase(elem);
878 5720 }
879
880
881 /**
882 * HTTP request options: set the URL and other options such as timeout and
883 * proxy.
884 */
885 5720 void DownloadManager::InitializeRequest(JobInfo *info, CURL *handle) {
886 // Initialize internal download state
887 5720 info->SetCurlHandle(handle);
888 5720 info->SetErrorCode(kFailOk);
889 5720 info->SetHttpCode(-1);
890 5720 info->SetFollowRedirects(follow_redirects_);
891 5720 info->SetNumUsedProxies(1);
892 5720 info->SetNumUsedMetalinks(1);
893 5720 info->SetNumUsedHosts(1);
894 5720 info->SetNumRetries(0);
895 5720 info->SetBackoffMs(0);
896 5720 info->SetHeaders(header_lists_->DuplicateList(default_headers_));
897
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5720 times.
 5720 if (info->info_header()) {
898 ✗ header_lists_->AppendHeader(info->headers(), info->info_header());
899 }
900
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5720 times.
 5720 if (enable_http_tracing_) {
901 ✗ for (unsigned int i = 0; i < http_tracing_headers_.size(); i++) {
902 ✗ header_lists_->AppendHeader(info->headers(),
903 ✗ (http_tracing_headers_)[i].c_str());
904 }
905
906 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_pid());
907 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_gid());
908 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_uid());
909
910 ✗ LogCvmfs(kLogDownload, kLogDebug,
911 "(manager '%s' - id %" PRId64 ") "
912 "CURL Header for URL: %s is:\n %s",
913 name_.c_str(), info->id(), info->url()->c_str(),
914 ✗ header_lists_->Print(info->headers()).c_str());
915 }
916
917
2/2
✓ Branch 1 taken 108 times.
✓ Branch 2 taken 5612 times.
 5720 if (info->force_nocache()) {
918 108 SetNocache(info);
919 } else {
920 5612 info->SetNocache(false);
921 }
922
2/2
✓ Branch 1 taken 3533 times.
✓ Branch 2 taken 2187 times.
 5720 if (info->compressed()) {
923 3533 zlib::DecompressInit(info->GetZstreamPtr());
924 }
925
2/2
✓ Branch 1 taken 3552 times.
✓ Branch 2 taken 2168 times.
 5720 if (info->expected_hash()) {
926
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 3552 times.
 3552 assert(info->hash_context().buffer != NULL);
927 3552 shash::Init(info->hash_context());
928 }
929
930
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 5720 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 5720 times.
 5720 if ((info->range_offset() != -1) && (info->range_size())) {
931 char byte_range_array[100];
932 ✗ const int64_t range_lower = static_cast<int64_t>(info->range_offset());
933 ✗ const int64_t range_upper = static_cast<int64_t>(info->range_offset()
934 ✗ + info->range_size() - 1);
935 ✗ if (snprintf(byte_range_array, sizeof(byte_range_array),
936 "%" PRId64 "-%" PRId64, range_lower, range_upper)
937 ✗ == 100) {
938 ✗ PANIC(NULL); // Should be impossible given limits on offset size.
939 }
940 ✗ curl_easy_setopt(handle, CURLOPT_RANGE, byte_range_array);
941 } else {
942 5720 curl_easy_setopt(handle, CURLOPT_RANGE, NULL);
943 }
944
945 // Set curl parameters
946 5720 curl_easy_setopt(handle, CURLOPT_PRIVATE, static_cast<void *>(info));
947 5720 curl_easy_setopt(handle, CURLOPT_WRITEHEADER, static_cast<void *>(info));
948 5720 curl_easy_setopt(handle, CURLOPT_WRITEDATA, static_cast<void *>(info));
949 5720 curl_easy_setopt(handle, CURLOPT_HTTPHEADER, info->headers());
950
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5720 times.
 5720 if (info->head_request()) {
951 ✗ curl_easy_setopt(handle, CURLOPT_NOBODY, 1);
952 } else {
953 5720 curl_easy_setopt(handle, CURLOPT_HTTPGET, 1);
954 }
955
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5720 times.
 5720 if (opt_ipv4_only_) {
956 ✗ curl_easy_setopt(handle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
957 }
958
2/2
✓ Branch 0 taken 1593 times.
✓ Branch 1 taken 4127 times.
 5720 if (follow_redirects_) {
959 1593 curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1);
960 1593 curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 4);
961 }
962 #ifdef DEBUGMSG
963 5720 curl_easy_setopt(handle, CURLOPT_VERBOSE, 1);
964 5720 curl_easy_setopt(handle, CURLOPT_DEBUGFUNCTION, CallbackCurlDebug);
965 #endif
966 5720 }
967
968 11656 void DownloadManager::CheckHostInfoReset(const std::string &typ,
969 HostInfo &info,
970 JobInfo *jobinfo,
971 time_t &now) {
972
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11656 times.
 11656 if (info.timestamp_backup > 0) {
973 ✗ if (now == 0)
974 ✗ now = time(NULL);
975 ✗ if (static_cast<int64_t>(now)
976 ✗ > static_cast<int64_t>(info.timestamp_backup + info.reset_after)) {
977 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
978 "(manager %s - id %" PRId64 ") "
979 "switching %s from %s to %s (reset %s)",
980 name_.c_str(), jobinfo->id(), typ.c_str(),
981 ✗ (*info.chain)[info.current].c_str(), (*info.chain)[0].c_str(),
982 typ.c_str());
983 ✗ info.current = 0;
984 ✗ info.timestamp_backup = 0;
985 }
986 }
987 11656 }
988
989
990 /**
991 * Sets the URL specific options such as host to use and timeout. It might also
992 * set an error code, in which case the further processing should react on.
993 */
994 5828 void DownloadManager::SetUrlOptions(JobInfo *info) {
995 5828 CURL *curl_handle = info->curl_handle();
996 5828 string url_prefix;
997 5828 time_t now = 0;
998
999 5828 const MutexLockGuard m(lock_options_);
1000
1001 // sharding policy
1002
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5828 times.
 5828 if (sharding_policy_.UseCount() > 0) {
1003 ✗ if (info->proxy() != "") {
1004 // proxy already set, so this is a failover event
1005 ✗ perf::Inc(counters_->n_proxy_failover);
1006 }
1007 ✗ info->SetProxy(sharding_policy_->GetNextProxy(
1008 ✗ info->url(), info->proxy(),
1009 ✗ info->range_offset() == -1 ? 0 : info->range_offset()));
1010
1011 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, info->proxy().c_str());
1012 } else { // no sharding policy
1013 // Check if proxy group needs to be reset from backup to primary
1014
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5828 times.
 5828 if (opt_timestamp_backup_proxies_ > 0) {
1015 ✗ now = time(NULL);
1016 ✗ if (static_cast<int64_t>(now) > static_cast<int64_t>(
1017 ✗ opt_timestamp_backup_proxies_ + opt_proxy_groups_reset_after_)) {
1018 ✗ opt_proxy_groups_current_ = 0;
1019 ✗ opt_timestamp_backup_proxies_ = 0;
1020 ✗ RebalanceProxiesUnlocked("Reset proxy group from backup to primary");
1021 }
1022 }
1023 // Check if load-balanced proxies within the group need to be reset
1024
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5828 times.
 5828 if (opt_timestamp_failover_proxies_ > 0) {
1025 ✗ if (now == 0)
1026 ✗ now = time(NULL);
1027 ✗ if (static_cast<int64_t>(now)
1028 ✗ > static_cast<int64_t>(opt_timestamp_failover_proxies_
1029 ✗ + opt_proxy_groups_reset_after_)) {
1030 ✗ RebalanceProxiesUnlocked(
1031 "Reset load-balanced proxies within the active group");
1032 }
1033 }
1034
1035
1/2
✓ Branch 2 taken 5828 times.
✗ Branch 3 not taken.
 5828 ProxyInfo *proxy = ChooseProxyUnlocked(info->expected_hash());
1036
6/6
✓ Branch 0 taken 2031 times.
✓ Branch 1 taken 3797 times.
✓ Branch 3 taken 1887 times.
✓ Branch 4 taken 144 times.
✓ Branch 5 taken 5684 times.
✓ Branch 6 taken 144 times.
 5828 if (!proxy || (proxy->url == "DIRECT")) {
1037
2/4
✓ Branch 2 taken 5684 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 5684 times.
✗ Branch 6 not taken.
 5684 info->SetProxy("DIRECT");
1038
1/2
✓ Branch 2 taken 5684 times.
✗ Branch 3 not taken.
 5684 curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "");
1039 } else {
1040 // Note: inside ValidateProxyIpsUnlocked() we may change the proxy data
1041 // structure, so we must not pass proxy->... (== current_proxy())
1042 // parameters directly
1043
1/2
✓ Branch 1 taken 144 times.
✗ Branch 2 not taken.
 144 const std::string purl = proxy->url;
1044
1/2
✓ Branch 1 taken 144 times.
✗ Branch 2 not taken.
 144 const dns::Host phost = proxy->host;
1045
1/2
✓ Branch 1 taken 144 times.
✗ Branch 2 not taken.
 144 const bool changed = ValidateProxyIpsUnlocked(purl, phost);
1046 // Current proxy may have changed
1047
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 144 times.
 144 if (changed) {
1048 ✗ proxy = ChooseProxyUnlocked(info->expected_hash());
1049 }
1050
1/2
✓ Branch 1 taken 144 times.
✗ Branch 2 not taken.
 144 info->SetProxy(proxy->url);
1051
1/2
✓ Branch 1 taken 144 times.
✗ Branch 2 not taken.
 144 if (proxy->host.status() == dns::kFailOk) {
1052
2/4
✓ Branch 1 taken 144 times.
✗ Branch 2 not taken.
✓ Branch 6 taken 144 times.
✗ Branch 7 not taken.
 144 curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY,
1053 info->proxy().c_str());
1054 } else {
1055 // We know it can't work, don't even try to download
1056 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "0.0.0.0");
1057 }
1058 144 }
1059 } // end !sharding
1060
1061 // Check if metalink and host chains need to be reset
1062
2/4
✓ Branch 2 taken 5828 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 5828 times.
✗ Branch 6 not taken.
 5828 CheckHostInfoReset("metalink", opt_metalink_, info, now);
1063
2/4
✓ Branch 2 taken 5828 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 5828 times.
✗ Branch 6 not taken.
 5828 CheckHostInfoReset("host", opt_host_, info, now);
1064
1065
1/2
✓ Branch 1 taken 5828 times.
✗ Branch 2 not taken.
 5828 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_LIMIT, opt_low_speed_limit_);
1066
3/6
✓ Branch 1 taken 5828 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 144 times.
✓ Branch 6 taken 5684 times.
 5828 if (info->proxy() != "DIRECT") {
1067
1/2
✓ Branch 1 taken 144 times.
✗ Branch 2 not taken.
 144 curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT, opt_timeout_proxy_);
1068
1/2
✓ Branch 1 taken 144 times.
✗ Branch 2 not taken.
 144 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_TIME, opt_timeout_proxy_);
1069 } else {
1070
1/2
✓ Branch 1 taken 5684 times.
✗ Branch 2 not taken.
 5684 curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT, opt_timeout_direct_);
1071
1/2
✓ Branch 1 taken 5684 times.
✗ Branch 2 not taken.
 5684 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_TIME, opt_timeout_direct_);
1072 }
1073
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5828 times.
 5828 if (!opt_dns_server_.empty())
1074 ✗ curl_easy_setopt(curl_handle, CURLOPT_DNS_SERVERS, opt_dns_server_.c_str());
1075
1076
2/2
✓ Branch 1 taken 2471 times.
✓ Branch 2 taken 3357 times.
 5828 if (info->probe_hosts()) {
1077
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 2471 times.
 2471 if (CheckMetalinkChain(now)) {
1078 ✗ url_prefix = (*opt_metalink_.chain)[opt_metalink_.current];
1079 ✗ info->SetCurrentMetalinkChainIndex(opt_metalink_.current);
1080 ✗ LogCvmfs(kLogDownload, kLogDebug,
1081 "(manager %s - id %" PRId64 ") "
1082 "reading from metalink %d",
1083 name_.c_str(), info->id(), opt_metalink_.current);
1084
1/2
✓ Branch 0 taken 2471 times.
✗ Branch 1 not taken.
 2471 } else if (opt_host_.chain) {
1085
1/2
✓ Branch 2 taken 2471 times.
✗ Branch 3 not taken.
 2471 url_prefix = (*opt_host_.chain)[opt_host_.current];
1086 2471 info->SetCurrentHostChainIndex(opt_host_.current);
1087
1/2
✓ Branch 3 taken 2471 times.
✗ Branch 4 not taken.
 2471 LogCvmfs(kLogDownload, kLogDebug,
1088 "(manager %s - id %" PRId64 ") "
1089 "reading from host %d",
1090 name_.c_str(), info->id(), opt_host_.current);
1091 }
1092 }
1093
1094
1/2
✓ Branch 2 taken 5828 times.
✗ Branch 3 not taken.
 5828 string url = url_prefix + *(info->url());
1095
1096
1/2
✓ Branch 1 taken 5828 times.
✗ Branch 2 not taken.
 5828 curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, 1L);
1097
2/6
✓ Branch 1 taken 5828 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 5828 times.
 5828 if (url.substr(0, 5) == "https") {
1098 ✗ const bool rvb = ssl_certificate_store_.ApplySslCertificatePath(
1099 curl_handle);
1100 ✗ if (!rvb) {
1101 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1102 "(manager %s - id %" PRId64 ") "
1103 "Failed to set SSL certificate path %s",
1104 name_.c_str(), info->id(),
1105 ✗ ssl_certificate_store_.GetCaPath().c_str());
1106 }
1107 ✗ if (info->pid() != -1) {
1108 ✗ if (credentials_attachment_ == NULL) {
1109 ✗ LogCvmfs(kLogDownload, kLogDebug,
1110 "(manager %s - id %" PRId64 ") "
1111 "uses secure downloads but no credentials attachment set",
1112 name_.c_str(), info->id());
1113 } else {
1114 ✗ const bool retval = credentials_attachment_->ConfigureCurlHandle(
1115 curl_handle, info->pid(), info->GetCredDataPtr());
1116 ✗ if (!retval) {
1117 ✗ LogCvmfs(kLogDownload, kLogDebug,
1118 "(manager %s - id %" PRId64 ") "
1119 "failed attaching credentials",
1120 name_.c_str(), info->id());
1121 }
1122 }
1123 }
1124 // The download manager disables signal handling in the curl library;
1125 // as OpenSSL's implementation of TLS will generate a sigpipe in some
1126 // error paths, we must explicitly disable SIGPIPE here.
1127 // TODO(jblomer): it should be enough to do this once
1128 ✗ signal(SIGPIPE, SIG_IGN);
1129 }
1130
1131
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5828 times.
 5828 if (url.find("@proxy@") != string::npos) {
1132 // This is used in Geo-API requests (only), to replace a portion of the
1133 // URL with the current proxy name for the sake of caching the result.
1134 // Replace the @proxy@ either with a passed in "forced" template (which
1135 // is set from $CVMFS_PROXY_TEMPLATE) if there is one, or a "direct"
1136 // template (which is the uuid) if there's no proxy, or the name of the
1137 // proxy.
1138 ✗ string replacement;
1139 ✗ if (proxy_template_forced_ != "") {
1140 ✗ replacement = proxy_template_forced_;
1141 ✗ } else if (info->proxy() == "DIRECT") {
1142 ✗ replacement = proxy_template_direct_;
1143 } else {
1144 ✗ if (opt_proxy_groups_current_ >= opt_proxy_groups_fallback_) {
1145 // It doesn't make sense to use the fallback proxies in Geo-API requests
1146 // since the fallback proxies are supposed to get sorted, too.
1147 ✗ info->SetProxy("DIRECT");
1148 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "");
1149 ✗ replacement = proxy_template_direct_;
1150 } else {
1151 ✗ replacement = ChooseProxyUnlocked(info->expected_hash())->host.name();
1152 }
1153 }
1154 ✗ replacement = (replacement == "") ? proxy_template_direct_ : replacement;
1155 ✗ LogCvmfs(kLogDownload, kLogDebug,
1156 "(manager %s - id %" PRId64 ") "
1157 "replacing @proxy@ by %s",
1158 name_.c_str(), info->id(), replacement.c_str());
1159 ✗ url = ReplaceAll(url, "@proxy@", replacement);
1160 }
1161
1162 // TODO(heretherebedragons) before removing
1163 // static_cast<cvmfs::MemSink*>(info->sink)->size() == 0
1164 // and just always call info->sink->Reserve()
1165 // we should do a speed check
1166
3/4
✓ Branch 2 taken 5828 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2668 times.
✓ Branch 5 taken 3160 times.
 5828 if ((info->sink() != NULL) && info->sink()->RequiresReserve()
1167
1/2
✓ Branch 2 taken 2668 times.
✗ Branch 3 not taken.
 2668 && (static_cast<cvmfs::MemSink *>(info->sink())->size() == 0)
1168
11/20
✓ Branch 1 taken 5828 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 2668 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 2668 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 2233 times.
✓ Branch 11 taken 435 times.
✓ Branch 12 taken 2668 times.
✓ Branch 13 taken 3160 times.
✗ Branch 14 not taken.
✓ Branch 15 taken 2668 times.
✓ Branch 16 taken 3160 times.
✓ Branch 18 taken 2233 times.
✓ Branch 19 taken 3595 times.
✗ Branch 20 not taken.
✗ Branch 21 not taken.
✗ Branch 23 not taken.
✗ Branch 24 not taken.
 11656 && HasPrefix(url, "file://", false)) {
1169 platform_stat64 stat_buf;
1170 2233 const int retval = platform_stat(url.c_str(), &stat_buf);
1171
1/2
✓ Branch 0 taken 2233 times.
✗ Branch 1 not taken.
 2233 if (retval != 0) {
1172 // this is an error: file does not exist or out of memory
1173 // error is caught in other code section.
1174
1/2
✓ Branch 2 taken 2233 times.
✗ Branch 3 not taken.
 2233 info->sink()->Reserve(64ul * 1024ul);
1175 } else {
1176 ✗ info->sink()->Reserve(stat_buf.st_size);
1177 }
1178 }
1179
1180
2/4
✓ Branch 2 taken 5828 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 5828 times.
✗ Branch 7 not taken.
 5828 curl_easy_setopt(curl_handle, CURLOPT_URL,
1181 EscapeUrl(info->id(), url).c_str());
1182 5828 }
1183
1184
1185 /**
1186 * Checks if the name resolving information is still up to date. The host
1187 * object should be one from the current load-balance group. If the information
1188 * changed, gather new set of resolved IPs and, if different, exchange them in
1189 * the load-balance group on the fly. In the latter case, also rebalance the
1190 * proxies. The options mutex needs to be open.
1191 *
1192 * Returns true if proxies may have changed.
1193 */
1194 144 bool DownloadManager::ValidateProxyIpsUnlocked(const string &url,
1195 const dns::Host &host) {
1196
2/4
✓ Branch 1 taken 144 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 144 times.
✗ Branch 4 not taken.
 144 if (!host.IsExpired())
1197 144 return false;
1198 ✗ LogCvmfs(kLogDownload, kLogDebug, "(manager '%s') validate DNS entry for %s",
1199 ✗ name_.c_str(), host.name().c_str());
1200
1201 ✗ const unsigned group_idx = opt_proxy_groups_current_;
1202 ✗ dns::Host new_host = resolver_->Resolve(host.name());
1203
1204 ✗ bool update_only = true; // No changes to the list of IP addresses.
1205 ✗ if (new_host.status() != dns::kFailOk) {
1206 // Try again later in case resolving fails.
1207 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1208 "(manager '%s') failed to resolve IP addresses for %s (%d - %s)",
1209 ✗ name_.c_str(), host.name().c_str(), new_host.status(),
1210 dns::Code2Ascii(new_host.status()));
1211 ✗ new_host = dns::Host::ExtendDeadline(host, resolver_->min_ttl());
1212 ✗ } else if (!host.IsEquivalent(new_host)) {
1213 ✗ update_only = false;
1214 }
1215
1216 ✗ if (update_only) {
1217 ✗ for (unsigned i = 0; i < (*opt_proxy_groups_)[group_idx].size(); ++i) {
1218 ✗ if ((*opt_proxy_groups_)[group_idx][i].host.id() == host.id())
1219 ✗ (*opt_proxy_groups_)[group_idx][i].host = new_host;
1220 }
1221 ✗ return false;
1222 }
1223
1224 ✗ assert(new_host.status() == dns::kFailOk);
1225
1226 // Remove old host objects, insert new objects, and rebalance.
1227 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslog,
1228 "(manager '%s') DNS entries for proxy %s changed, adjusting",
1229 ✗ name_.c_str(), host.name().c_str());
1230 ✗ vector<ProxyInfo> *group = current_proxy_group();
1231 ✗ opt_num_proxies_ -= group->size();
1232 ✗ for (unsigned i = 0; i < group->size();) {
1233 ✗ if ((*group)[i].host.id() == host.id()) {
1234 ✗ group->erase(group->begin() + i);
1235 } else {
1236 ✗ i++;
1237 }
1238 }
1239 ✗ vector<ProxyInfo> new_infos;
1240 set<string> const best_addresses = new_host.ViewBestAddresses(
1241 ✗ opt_ip_preference_);
1242 ✗ set<string>::const_iterator iter_ips = best_addresses.begin();
1243 ✗ for (; iter_ips != best_addresses.end(); ++iter_ips) {
1244 ✗ const string url_ip = dns::RewriteUrl(url, *iter_ips);
1245 ✗ new_infos.push_back(ProxyInfo(new_host, url_ip));
1246 }
1247 ✗ group->insert(group->end(), new_infos.begin(), new_infos.end());
1248 ✗ opt_num_proxies_ += new_infos.size();
1249
1250 ✗ const std::string msg = "DNS entries for proxy " + host.name() + " changed";
1251
1252 ✗ RebalanceProxiesUnlocked(msg);
1253 ✗ return true;
1254 }
1255
1256
1257 /**
1258 * Adds transfer time and downloaded bytes to the global counters.
1259 */
1260 6054 void DownloadManager::UpdateStatistics(CURL *handle) {
1261 curl_off_t val;
1262 int retval;
1263 6054 int64_t sum = 0;
1264
1265
1/2
✓ Branch 1 taken 6054 times.
✗ Branch 2 not taken.
 6054 retval = curl_easy_getinfo(handle, CURLINFO_SIZE_DOWNLOAD_T, &val);
1266
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6054 times.
 6054 assert(retval == CURLE_OK);
1267 6054 sum += static_cast<int64_t>(val);
1268 /*retval = curl_easy_getinfo(handle, CURLINFO_HEADER_SIZE, &val);
1269 assert(retval == CURLE_OK);
1270 sum += static_cast<int64_t>(val);*/
1271 6054 perf::Xadd(counters_->sz_transferred_bytes, sum);
1272 6054 }
1273
1274
1275 /**
1276 * Retry if possible if not on no-cache and if not already done too often.
1277 */
1278 6054 bool DownloadManager::CanRetry(const JobInfo *info) {
1279 6054 const MutexLockGuard m(lock_options_);
1280 6054 const unsigned max_retries = opt_max_retries_;
1281
1282
2/2
✓ Branch 2 taken 1618 times.
✓ Branch 3 taken 4237 times.
 11909 return !(info->nocache()) && (info->num_retries() < max_retries)
1283
3/4
✓ Branch 0 taken 5855 times.
✓ Branch 1 taken 199 times.
✓ Branch 4 taken 1618 times.
✗ Branch 5 not taken.
 13527 && (IsProxyTransferError(info->error_code())
1284
2/2
✓ Branch 2 taken 135 times.
✓ Branch 3 taken 1483 times.
 1618 || IsHostTransferError(info->error_code()));
1285 6054 }
1286
1287 /**
1288 * Backoff for retry to introduce a jitter into a cluster of requesting
1289 * cvmfs nodes.
1290 * Retry only when HTTP caching is on.
1291 *
1292 * \return true if backoff has been performed, false otherwise
1293 */
1294 135 void DownloadManager::Backoff(JobInfo *info) {
1295 135 unsigned backoff_init_ms = 0;
1296 135 unsigned backoff_max_ms = 0;
1297 {
1298 135 const MutexLockGuard m(lock_options_);
1299 135 backoff_init_ms = opt_backoff_init_ms_;
1300 135 backoff_max_ms = opt_backoff_max_ms_;
1301 135 }
1302
1303 135 info->SetNumRetries(info->num_retries() + 1);
1304 135 perf::Inc(counters_->n_retries);
1305
2/2
✓ Branch 1 taken 46 times.
✓ Branch 2 taken 89 times.
 135 if (info->backoff_ms() == 0) {
1306 46 info->SetBackoffMs(prng_.Next(backoff_init_ms + 1)); // Must be != 0
1307 } else {
1308 89 info->SetBackoffMs(info->backoff_ms() * 2);
1309 }
1310
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 135 times.
 135 if (info->backoff_ms() > backoff_max_ms) {
1311 ✗ info->SetBackoffMs(backoff_max_ms);
1312 }
1313
1314 135 LogCvmfs(kLogDownload, kLogDebug,
1315 "(manager '%s' - id %" PRId64 ") backing off for %d ms",
1316 name_.c_str(), info->id(), info->backoff_ms());
1317 135 SafeSleepMs(info->backoff_ms());
1318 135 }
1319
1320 199 void DownloadManager::SetNocache(JobInfo *info) {
1321
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 199 times.
 199 if (info->nocache())
1322 ✗ return;
1323 199 header_lists_->AppendHeader(info->headers(), "Pragma: no-cache");
1324 199 header_lists_->AppendHeader(info->headers(), "Cache-Control: no-cache");
1325 199 curl_easy_setopt(info->curl_handle(), CURLOPT_HTTPHEADER, info->headers());
1326 199 info->SetNocache(true);
1327 }
1328
1329
1330 /**
1331 * Reverse operation of SetNocache. Makes sure that "no-cache" header
1332 * disappears from the list of headers to let proxies work normally.
1333 */
1334 334 void DownloadManager::SetRegularCache(JobInfo *info) {
1335
1/2
✓ Branch 1 taken 334 times.
✗ Branch 2 not taken.
 334 if (info->nocache() == false)
1336 334 return;
1337 ✗ header_lists_->CutHeader("Pragma: no-cache", info->GetHeadersPtr());
1338 ✗ header_lists_->CutHeader("Cache-Control: no-cache", info->GetHeadersPtr());
1339 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_HTTPHEADER, info->headers());
1340 ✗ info->SetNocache(false);
1341 }
1342
1343
1344 /**
1345 * Frees the storage associated with the authz attachment from the job
1346 */
1347 5828 void DownloadManager::ReleaseCredential(JobInfo *info) {
1348
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5828 times.
 5828 if (info->cred_data()) {
1349 ✗ assert(credentials_attachment_ != NULL); // Someone must have set it
1350 ✗ credentials_attachment_->ReleaseCurlHandle(info->curl_handle(),
1351 info->cred_data());
1352 ✗ info->SetCredData(NULL);
1353 }
1354 5828 }
1355
1356
1357 /* Sort links based on the "pri=" parameter */
1358 ✗ static bool sortlinks(const std::string &s1, const std::string &s2) {
1359 ✗ const size_t pos1 = s1.find("; pri=");
1360 ✗ const size_t pos2 = s2.find("; pri=");
1361 int pri1, pri2;
1362 ✗ if ((pos1 != std::string::npos) && (pos2 != std::string::npos)
1363 ✗ && (sscanf(s1.substr(pos1 + 6).c_str(), "%d", &pri1) == 1)
1364 ✗ && (sscanf(s2.substr(pos2 + 6).c_str(), "%d", &pri2) == 1)) {
1365 ✗ return pri1 < pri2;
1366 }
1367 ✗ return false;
1368 }
1369
1370 /**
1371 * Parses Link header and uses it to set a new host chain.
1372 * See rfc6249.
1373 */
1374 ✗ void DownloadManager::ProcessLink(JobInfo *info) {
1375 ✗ std::vector<std::string> links = SplitString(info->link(), ',');
1376 ✗ if (info->link().find("; pri=") != std::string::npos)
1377 ✗ std::sort(links.begin(), links.end(), sortlinks);
1378
1379 ✗ std::vector<std::string> host_list;
1380
1381 ✗ std::vector<std::string>::const_iterator il = links.begin();
1382 ✗ for (; il != links.end(); ++il) {
1383 ✗ const std::string &link = *il;
1384 ✗ if ((link.find("; rel=duplicate") == std::string::npos)
1385 ✗ && (link.find("; rel=\"duplicate\"") == std::string::npos)) {
1386 ✗ LogCvmfs(kLogDownload, kLogDebug,
1387 "skipping link '%s' because it does not contain rel=duplicate",
1388 link.c_str());
1389 ✗ continue;
1390 }
1391 // ignore depth= field since there's nothing useful we can do with it
1392
1393 ✗ size_t start = link.find('<');
1394 ✗ if (start == std::string::npos) {
1395 ✗ LogCvmfs(
1396 kLogDownload, kLogDebug,
1397 "skipping link '%s' because it does not have a left angle bracket",
1398 link.c_str());
1399 ✗ continue;
1400 }
1401
1402 ✗ start++;
1403 ✗ if ((link.substr(start, 7) != "http://")
1404 ✗ && (link.substr(start, 8) != "https://")) {
1405 ✗ LogCvmfs(kLogDownload, kLogDebug,
1406 "skipping link '%s' of unrecognized url protocol", link.c_str());
1407 ✗ continue;
1408 }
1409
1410 ✗ size_t end = link.find('/', start + 8);
1411 ✗ if (end == std::string::npos)
1412 ✗ end = link.find('>');
1413 ✗ if (end == std::string::npos) {
1414 ✗ LogCvmfs(kLogDownload, kLogDebug,
1415 "skipping link '%s' because no slash in url and no right angle "
1416 "bracket",
1417 link.c_str());
1418 ✗ continue;
1419 }
1420 ✗ const std::string host = link.substr(start, end - start);
1421 ✗ LogCvmfs(kLogDownload, kLogDebug, "adding linked host '%s'", host.c_str());
1422 ✗ host_list.push_back(host);
1423 }
1424
1425 ✗ if (host_list.size() > 0) {
1426 ✗ SetHostChain(host_list);
1427 ✗ opt_metalink_timestamp_link_ = time(NULL);
1428 // Set the index to the first linked host because the protocol includes
1429 // redirecting to the first host
1430 ✗ info->SetCurrentHostChainIndex(0);
1431 // Don't process the same links again if there are retries
1432 ✗ info->SetLink("");
1433 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslog,
1434 "(manager '%s' - id %" PRId64 ") "
1435 "received %zu hosts from metalink server, starting with %s",
1436 name_.c_str(), info->id(),
1437 host_list.size(),
1438 ✗ host_list[0].c_str());
1439 }
1440 }
1441
1442
1443 /**
1444 * Checks the result of a curl download and implements the failure logic, such
1445 * as changing the proxy server. Takes care of cleanup.
1446 *
1447 * \return true if another download should be performed, false otherwise
1448 */
1449 6054 bool DownloadManager::VerifyAndFinalize(const int curl_error, JobInfo *info) {
1450
1/2
✓ Branch 6 taken 6054 times.
✗ Branch 7 not taken.
 6054 LogCvmfs(kLogDownload, kLogDebug,
1451 "(manager '%s' - id %" PRId64 ") "
1452 "Verify downloaded url %s, proxy %s (curl error %d)",
1453 name_.c_str(), info->id(), info->url()->c_str(),
1454
1/2
✓ Branch 1 taken 6054 times.
✗ Branch 2 not taken.
 12108 info->proxy().c_str(), curl_error);
1455
1/2
✓ Branch 2 taken 6054 times.
✗ Branch 3 not taken.
 6054 UpdateStatistics(info->curl_handle());
1456
1457 bool was_metalink;
1458 6054 std::string typ;
1459
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6054 times.
 6054 if (info->current_metalink_chain_index() >= 0) {
1460 ✗ if (info->link() != "") {
1461 // process Link header whether or not the redirected URL got an error
1462 ✗ ProcessLink(info);
1463 // The metalink lookup succeeded, so if there was an error it was
1464 // with the redirect
1465 ✗ was_metalink = false;
1466 ✗ typ = "host";
1467 } else {
1468 // no Link headers were received, so the metalink lookup failed
1469 ✗ was_metalink = true;
1470 ✗ typ = "metalink";
1471 }
1472 } else {
1473 6054 was_metalink = false;
1474
1/2
✓ Branch 1 taken 6054 times.
✗ Branch 2 not taken.
 6054 typ = "host";
1475 }
1476
1477 // Verification and error classification
1478
3/14
✓ Branch 0 taken 5523 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 38 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 493 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
 6054 switch (curl_error) {
1479 5523 case CURLE_OK:
1480 // Verify content hash
1481
2/2
✓ Branch 1 taken 3416 times.
✓ Branch 2 taken 2107 times.
 5523 if (info->expected_hash()) {
1482
1/2
✓ Branch 1 taken 3416 times.
✗ Branch 2 not taken.
 3416 shash::Any match_hash;
1483
1/2
✓ Branch 2 taken 3416 times.
✗ Branch 3 not taken.
 3416 shash::Final(info->hash_context(), &match_hash);
1484
3/4
✓ Branch 2 taken 3416 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 182 times.
✓ Branch 5 taken 3234 times.
 3416 if (match_hash != *(info->expected_hash())) {
1485
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 182 times.
 182 if (ignore_signature_failures_) {
1486 ✗ LogCvmfs(
1487 kLogDownload, kLogDebug | kLogSyslogErr,
1488 "(manager '%s' - id %" PRId64 ") "
1489 "ignoring failed hash verification of %s (expected %s, got %s)",
1490 name_.c_str(), info->id(), info->url()->c_str(),
1491 ✗ info->expected_hash()->ToString().c_str(),
1492 ✗ match_hash.ToString().c_str());
1493 } else {
1494
1/2
✓ Branch 7 taken 182 times.
✗ Branch 8 not taken.
 364 LogCvmfs(kLogDownload, kLogDebug,
1495 "(manager '%s' - id %" PRId64 ") "
1496 "hash verification of %s failed (expected %s, got %s)",
1497 name_.c_str(), info->id(), info->url()->c_str(),
1498
1/2
✓ Branch 2 taken 182 times.
✗ Branch 3 not taken.
 364 info->expected_hash()->ToString().c_str(),
1499
1/2
✓ Branch 1 taken 182 times.
✗ Branch 2 not taken.
 364 match_hash.ToString().c_str());
1500 182 info->SetErrorCode(kFailBadData);
1501 182 break;
1502 }
1503 }
1504 }
1505
1506 5341 info->SetErrorCode(kFailOk);
1507 5341 break;
1508 ✗ case CURLE_UNSUPPORTED_PROTOCOL:
1509 ✗ info->SetErrorCode(kFailUnsupportedProtocol);
1510 ✗ break;
1511 38 case CURLE_URL_MALFORMAT:
1512 38 info->SetErrorCode(kFailBadUrl);
1513 38 break;
1514 ✗ case CURLE_COULDNT_RESOLVE_PROXY:
1515 ✗ info->SetErrorCode(kFailProxyResolve);
1516 ✗ break;
1517 ✗ case CURLE_COULDNT_RESOLVE_HOST:
1518 ✗ info->SetErrorCode(kFailHostResolve);
1519 ✗ break;
1520 ✗ case CURLE_OPERATION_TIMEDOUT:
1521 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostTooSlow
1522 : kFailProxyTooSlow);
1523 ✗ break;
1524 ✗ case CURLE_PARTIAL_FILE:
1525 case CURLE_GOT_NOTHING:
1526 case CURLE_RECV_ERROR:
1527 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostShortTransfer
1528 : kFailProxyShortTransfer);
1529 ✗ break;
1530 493 case CURLE_FILE_COULDNT_READ_FILE:
1531 case CURLE_COULDNT_CONNECT:
1532
3/6
✓ Branch 1 taken 493 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 36 times.
✓ Branch 6 taken 457 times.
 493 if (info->proxy() != "DIRECT") {
1533 // This is a guess. Fail-over can still change to switching host
1534 36 info->SetErrorCode(kFailProxyConnection);
1535 } else {
1536 457 info->SetErrorCode(kFailHostConnection);
1537 }
1538 493 break;
1539 ✗ case CURLE_TOO_MANY_REDIRECTS:
1540 ✗ info->SetErrorCode(kFailHostConnection);
1541 ✗ break;
1542 ✗ case CURLE_SSL_CACERT_BADFILE:
1543 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
1544 "(manager '%s' -id %" PRId64 ") "
1545 "Failed to load certificate bundle. "
1546 "X509_CERT_BUNDLE might point to the wrong location.",
1547 name_.c_str(), info->id());
1548 ✗ info->SetErrorCode(kFailHostConnection);
1549 ✗ break;
1550 // As of curl 7.62.0, CURLE_SSL_CACERT is the same as
1551 // CURLE_PEER_FAILED_VERIFICATION
1552 ✗ case CURLE_PEER_FAILED_VERIFICATION:
1553 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
1554 "(manager '%s' - id %" PRId64 ") "
1555 "invalid SSL certificate of remote host. "
1556 "X509_CERT_DIR and/or X509_CERT_BUNDLE might point to the wrong "
1557 "location.",
1558 name_.c_str(), info->id());
1559 ✗ info->SetErrorCode(kFailHostConnection);
1560 ✗ break;
1561 ✗ case CURLE_ABORTED_BY_CALLBACK:
1562 case CURLE_WRITE_ERROR:
1563 // Error set by callback
1564 ✗ break;
1565 ✗ case CURLE_SEND_ERROR:
1566 // The curl error CURLE_SEND_ERROR can be seen when a cache is misbehaving
1567 // and closing connections before the http request send is completed.
1568 // Handle this error, treating it as a short transfer error.
1569 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostShortTransfer
1570 : kFailProxyShortTransfer);
1571 ✗ break;
1572 ✗ default:
1573 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
1574 "(manager '%s' - id %" PRId64 ") "
1575 "unexpected curl error (%d) while trying to fetch %s",
1576 name_.c_str(), info->id(), curl_error, info->url()->c_str());
1577 ✗ info->SetErrorCode(kFailOther);
1578 ✗ break;
1579 }
1580
1581 std::vector<std::string> *host_chain;
1582 unsigned char num_used_hosts;
1583
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6054 times.
 6054 if (was_metalink) {
1584 ✗ host_chain = opt_metalink_.chain;
1585 ✗ num_used_hosts = info->num_used_metalinks();
1586 } else {
1587 6054 host_chain = opt_host_.chain;
1588 6054 num_used_hosts = info->num_used_hosts();
1589 }
1590
1591 // Determination if download should be repeated
1592 6054 bool try_again = false;
1593
1/2
✓ Branch 1 taken 6054 times.
✗ Branch 2 not taken.
 6054 bool same_url_retry = CanRetry(info);
1594
2/2
✓ Branch 1 taken 713 times.
✓ Branch 2 taken 5341 times.
 6054 if (info->error_code() != kFailOk) {
1595 713 const MutexLockGuard m(lock_options_);
1596
2/2
✓ Branch 1 taken 182 times.
✓ Branch 2 taken 531 times.
 713 if (info->error_code() == kFailBadData) {
1597
2/2
✓ Branch 1 taken 91 times.
✓ Branch 2 taken 91 times.
 182 if (!info->nocache()) {
1598 91 try_again = true;
1599 } else {
1600 // Make it a host failure
1601
1/2
✓ Branch 4 taken 91 times.
✗ Branch 5 not taken.
 91 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1602 "(manager '%s' - id %" PRId64 ") "
1603 "data corruption with no-cache header, try another %s",
1604 name_.c_str(), info->id(), typ.c_str());
1605
1606 91 info->SetErrorCode(kFailHostHttp);
1607 }
1608 }
1609 713 if (same_url_retry
1610
5/6
✓ Branch 0 taken 578 times.
✓ Branch 1 taken 135 times.
✓ Branch 3 taken 578 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 243 times.
✓ Branch 6 taken 470 times.
 1291 || (((info->error_code() == kFailHostResolve)
1611
2/2
✓ Branch 2 taken 256 times.
✓ Branch 3 taken 322 times.
 578 || IsHostTransferError(info->error_code())
1612
2/2
✓ Branch 1 taken 91 times.
✓ Branch 2 taken 165 times.
 256 || (info->error_code() == kFailHostHttp))
1613
3/4
✓ Branch 1 taken 190 times.
✓ Branch 2 taken 223 times.
✓ Branch 3 taken 190 times.
✗ Branch 4 not taken.
 413 && info->probe_hosts() && host_chain
1614
2/2
✓ Branch 1 taken 108 times.
✓ Branch 2 taken 82 times.
 190 && (num_used_hosts < host_chain->size()))) {
1615 243 try_again = true;
1616 }
1617 713 if (same_url_retry
1618
5/6
✓ Branch 0 taken 578 times.
✓ Branch 1 taken 135 times.
✓ Branch 3 taken 578 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 171 times.
✓ Branch 6 taken 542 times.
 1291 || (((info->error_code() == kFailProxyResolve)
1619
2/2
✓ Branch 2 taken 542 times.
✓ Branch 3 taken 36 times.
 578 || IsProxyTransferError(info->error_code())
1620
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 542 times.
 542 || (info->error_code() == kFailProxyHttp)))) {
1621
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 171 times.
 171 if (sharding_policy_.UseCount() > 0) { // sharding policy
1622 ✗ try_again = true;
1623 ✗ same_url_retry = false;
1624 } else { // no sharding
1625 171 try_again = true;
1626 // If all proxies failed, do a next round with the next host
1627
4/6
✓ Branch 0 taken 36 times.
✓ Branch 1 taken 135 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 36 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 171 times.
 171 if (!same_url_retry && (info->num_used_proxies() >= opt_num_proxies_)) {
1628 // Check if this can be made a host fail-over
1629 ✗ if (info->probe_hosts() && host_chain
1630 ✗ && (num_used_hosts < host_chain->size())) {
1631 // reset proxy group if not already performed by other handle
1632 ✗ if (opt_proxy_groups_) {
1633 ✗ if ((opt_proxy_groups_current_ > 0)
1634 ✗ || (opt_proxy_groups_current_burned_ > 0)) {
1635 ✗ opt_proxy_groups_current_ = 0;
1636 ✗ opt_timestamp_backup_proxies_ = 0;
1637 ✗ const std::string msg = "reset proxies for " + typ
1638 ✗ + " failover";
1639 ✗ RebalanceProxiesUnlocked(msg);
1640 }
1641 }
1642
1643 // Make it a host failure
1644 ✗ LogCvmfs(kLogDownload, kLogDebug,
1645 "(manager '%s' - id %" PRId64 ") make it a %s failure",
1646 name_.c_str(), info->id(), typ.c_str());
1647 ✗ info->SetNumUsedProxies(1);
1648 ✗ info->SetErrorCode(kFailHostAfterProxy);
1649 } else {
1650 ✗ if (failover_indefinitely_) {
1651 // Instead of giving up, reset the num_used_proxies counter,
1652 // switch proxy and try again
1653 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1654 "(manager '%s' - id %" PRId64 ") "
1655 "VerifyAndFinalize() would fail the download here. "
1656 "Instead switch proxy and retry download. "
1657 "typ=%s "
1658 "info->probe_hosts=%d host_chain=%p num_used_hosts=%d "
1659 "host_chain->size()=%lu same_url_retry=%d "
1660 "info->num_used_proxies=%d opt_num_proxies_=%d",
1661 name_.c_str(), info->id(), typ.c_str(),
1662 ✗ static_cast<int>(info->probe_hosts()), host_chain,
1663 ✗ num_used_hosts, host_chain ? host_chain->size() : -1,
1664 static_cast<int>(same_url_retry),
1665 ✗ info->num_used_proxies(), opt_num_proxies_);
1666 ✗ info->SetNumUsedProxies(1);
1667 ✗ RebalanceProxiesUnlocked(
1668 "download failed - failover indefinitely");
1669 ✗ try_again = !Interrupted(fqrn_, info);
1670 } else {
1671 ✗ try_again = false;
1672 }
1673 }
1674 } // Make a proxy failure a host failure
1675 } // Proxy failure assumed
1676 } // end !sharding
1677 713 }
1678
1679
2/2
✓ Branch 0 taken 370 times.
✓ Branch 1 taken 5684 times.
 6054 if (try_again) {
1680
4/6
✗ Branch 0 not taken.
✓ Branch 1 taken 370 times.
✓ Branch 2 taken 135 times.
✓ Branch 3 taken 235 times.
✓ Branch 7 taken 370 times.
✗ Branch 8 not taken.
 740 LogCvmfs(kLogDownload, kLogDebug,
1681 "(manager '%s' - id %" PRId64 ") "
1682 "Trying again on same curl handle, %s"
1683 "error code %d%s",
1684 name_.c_str(), info->id(), same_url_retry ? "same url, " : "",
1685 740 info->error_code(), info->nocache() ? ", no cache" : "");
1686 // Reset internal state and destination. In parallel-decompress mode the
1687 // sink and zstream are owned by the caller thread (it pops and decompresses
1688 // the queued chunks). Resetting them here would race the caller and, worse,
1689 // leave the bytes of this failed attempt in the tube to be decompressed
1690 // into the output. Defer the reset by enqueuing an ordered marker; the
1691 // caller discards this attempt's bytes when it pops it (see below).
1692 370 const bool defer_reset = info->IsValidDataTube();
1693
1/2
✓ Branch 0 taken 370 times.
✗ Branch 1 not taken.
 370 if (!defer_reset) {
1694
4/8
✓ Branch 1 taken 370 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 370 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 370 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 370 times.
 370 if (info->sink() != NULL && info->sink()->Reset() != 0) {
1695 ✗ info->SetErrorCode(kFailLocalIO);
1696 ✗ goto verify_and_finalize_stop;
1697 }
1698 }
1699
6/8
✓ Branch 1 taken 36 times.
✓ Branch 2 taken 334 times.
✓ Branch 5 taken 36 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 36 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 36 times.
✓ Branch 10 taken 334 times.
 370 if (info->interrupt_cue() && info->interrupt_cue()->IsCanceled()) {
1700 36 info->SetErrorCode(kFailCanceled);
1701 36 goto verify_and_finalize_stop;
1702 }
1703
1704 // The hash context is updated on this (MainDownload) thread in
1705 // CallbackCurlData(), so it is reset here regardless of the decompress
1706 // mode.
1707
2/2
✓ Branch 1 taken 220 times.
✓ Branch 2 taken 114 times.
 334 if (info->expected_hash()) {
1708
1/2
✓ Branch 2 taken 220 times.
✗ Branch 3 not taken.
 220 shash::Init(info->hash_context());
1709 }
1710
5/6
✓ Branch 1 taken 220 times.
✓ Branch 2 taken 114 times.
✓ Branch 3 taken 220 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 220 times.
✓ Branch 6 taken 114 times.
 334 if (info->compressed() && !defer_reset) {
1711
1/2
✓ Branch 2 taken 220 times.
✗ Branch 3 not taken.
 220 zlib::DecompressInit(info->GetZstreamPtr());
1712 }
1713
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 334 times.
 334 if (defer_reset) {
1714 ✗ info->GetDataTubePtr()->EnqueueBack(new DataTubeElement(kActionReset));
1715 }
1716
1717
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 334 times.
 334 if (sharding_policy_.UseCount() > 0) { // sharding policy
1718 ✗ ReleaseCredential(info);
1719 ✗ SetUrlOptions(info);
1720 } else { // no sharding policy
1721
1/2
✓ Branch 1 taken 334 times.
✗ Branch 2 not taken.
 334 SetRegularCache(info);
1722
1723 // Failure handling
1724 334 bool switch_proxy = false;
1725 334 bool switch_host = false;
1726
2/4
✓ Branch 1 taken 91 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 243 times.
 334 switch (info->error_code()) {
1727 91 case kFailBadData:
1728
1/2
✓ Branch 1 taken 91 times.
✗ Branch 2 not taken.
 91 SetNocache(info);
1729 91 break;
1730 ✗ case kFailProxyResolve:
1731 case kFailProxyHttp:
1732 ✗ switch_proxy = true;
1733 ✗ break;
1734 ✗ case kFailHostResolve:
1735 case kFailHostHttp:
1736 case kFailHostAfterProxy:
1737 ✗ switch_host = true;
1738 ✗ break;
1739 243 default:
1740
2/2
✓ Branch 2 taken 36 times.
✓ Branch 3 taken 207 times.
 243 if (IsProxyTransferError(info->error_code())) {
1741
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 36 times.
 36 if (same_url_retry) {
1742 ✗ Backoff(info);
1743 } else {
1744 36 switch_proxy = true;
1745 }
1746
1/2
✓ Branch 2 taken 207 times.
✗ Branch 3 not taken.
 207 } else if (IsHostTransferError(info->error_code())) {
1747
2/2
✓ Branch 0 taken 135 times.
✓ Branch 1 taken 72 times.
 207 if (same_url_retry) {
1748
1/2
✓ Branch 1 taken 135 times.
✗ Branch 2 not taken.
 135 Backoff(info);
1749 } else {
1750 72 switch_host = true;
1751 }
1752 } else {
1753 // No other errors expected when retrying
1754 ✗ PANIC(NULL);
1755 }
1756 }
1757
2/2
✓ Branch 0 taken 36 times.
✓ Branch 1 taken 298 times.
 334 if (switch_proxy) {
1758
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 ReleaseCredential(info);
1759
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 SwitchProxy(info);
1760 36 info->SetNumUsedProxies(info->num_used_proxies() + 1);
1761
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 SetUrlOptions(info);
1762 }
1763
2/2
✓ Branch 0 taken 72 times.
✓ Branch 1 taken 262 times.
 334 if (switch_host) {
1764
1/2
✓ Branch 1 taken 72 times.
✗ Branch 2 not taken.
 72 ReleaseCredential(info);
1765
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 72 times.
 72 if (was_metalink) {
1766 ✗ SwitchMetalink(info);
1767 ✗ info->SetNumUsedMetalinks(num_used_hosts + 1);
1768 } else {
1769
1/2
✓ Branch 1 taken 72 times.
✗ Branch 2 not taken.
 72 SwitchHost(info);
1770 72 info->SetNumUsedHosts(num_used_hosts + 1);
1771 }
1772
1/2
✓ Branch 1 taken 72 times.
✗ Branch 2 not taken.
 72 SetUrlOptions(info);
1773 }
1774 } // end !sharding
1775
1776
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 334 times.
 334 if (failover_indefinitely_) {
1777 // try again, breaking if there's a cvmfs reload happening and we are in a
1778 // proxy failover. This will EIO the call application.
1779 ✗ return !Interrupted(fqrn_, info);
1780 }
1781 334 return true; // try again
1782 }
1783
1784 5684 verify_and_finalize_stop:
1785 // Finalize, flush destination file.
1786
1/2
✓ Branch 1 taken 5720 times.
✗ Branch 2 not taken.
 5720 ReleaseCredential(info);
1787
1788 // In parallel-decompress mode (data_tube_ valid) the caller has not yet
1789 // pushed bytes through the sink / zstream — those operations happen on
1790 // the caller thread when it pops the queued chunks. Flushing the sink
1791 // and tearing down the zstream here would race the caller's decompress
1792 // and corrupt the output. Defer both to the caller.
1793 5720 const bool defer_finalize = info->IsValidDataTube();
1794
1/2
✓ Branch 0 taken 5720 times.
✗ Branch 1 not taken.
 5720 if (!defer_finalize) {
1795
4/8
✓ Branch 1 taken 5720 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 5720 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 5720 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 5720 times.
 5720 if (info->sink() != NULL && info->sink()->Flush() != 0) {
1796 ✗ info->SetErrorCode(kFailLocalIO);
1797 }
1798
2/2
✓ Branch 1 taken 3533 times.
✓ Branch 2 taken 2187 times.
 5720 if (info->compressed())
1799
1/2
✓ Branch 2 taken 3533 times.
✗ Branch 3 not taken.
 3533 zlib::DecompressFini(info->GetZstreamPtr());
1800 }
1801
1802
1/2
✓ Branch 1 taken 5720 times.
✗ Branch 2 not taken.
 5720 if (info->headers()) {
1803
1/2
✓ Branch 2 taken 5720 times.
✗ Branch 3 not taken.
 5720 header_lists_->PutList(info->headers());
1804 5720 info->SetHeaders(NULL);
1805 }
1806
1807 5720 return false; // stop transfer and return to Fetch()
1808 6054 }
1809
1810 4674 DownloadManager::~DownloadManager() {
1811 // cleaned up fini
1812
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4674 times.
 4674 if (sharding_policy_.UseCount() > 0) {
1813 ✗ sharding_policy_.Reset();
1814 }
1815
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4674 times.
 4674 if (health_check_.UseCount() > 0) {
1816 ✗ if (health_check_.Unique()) {
1817 ✗ LogCvmfs(kLogDownload, kLogDebug,
1818 "(manager '%s') Stopping healthcheck thread", name_.c_str());
1819 ✗ health_check_->StopHealthcheck();
1820 }
1821 ✗ health_check_.Reset();
1822 }
1823
1824
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4674 times.
 9348 if (multi_threaded_.load() == 1) {
1825 // Shutdown I/O thread
1826 ✗ pipe_terminate_->Write(kPipeTerminateSignal);
1827 ✗ pthread_join(thread_download_, NULL);
1828 // All handles are removed from the multi stack
1829 ✗ pipe_terminate_.reset();
1830 ✗ pipe_jobs_.reset();
1831 }
1832
1833 9348 for (set<CURL *>::iterator i = pool_handles_idle_->begin(),
1834 4674 iEnd = pool_handles_idle_->end();
1835
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4674 times.
 4674 i != iEnd;
1836 ✗ ++i) {
1837 ✗ curl_easy_cleanup(*i);
1838 }
1839
1840
1/2
✓ Branch 0 taken 4674 times.
✗ Branch 1 not taken.
 4674 delete pool_handles_idle_;
1841
1/2
✓ Branch 0 taken 4674 times.
✗ Branch 1 not taken.
 4674 delete pool_handles_inuse_;
1842 4674 curl_multi_cleanup(curl_multi_);
1843
1844
1/2
✓ Branch 0 taken 4674 times.
✗ Branch 1 not taken.
 4674 delete header_lists_;
1845
1/2
✓ Branch 0 taken 4674 times.
✗ Branch 1 not taken.
 4674 if (user_agent_)
1846 4674 free(user_agent_);
1847
1848
1/2
✓ Branch 0 taken 4674 times.
✗ Branch 1 not taken.
 4674 delete counters_;
1849
2/2
✓ Branch 0 taken 2033 times.
✓ Branch 1 taken 2641 times.
 4674 delete opt_host_.chain;
1850
2/2
✓ Branch 0 taken 2033 times.
✓ Branch 1 taken 2641 times.
 4674 delete opt_host_chain_rtt_;
1851
2/2
✓ Branch 0 taken 1752 times.
✓ Branch 1 taken 2922 times.
 4674 delete opt_proxy_groups_;
1852
1853 4674 curl_global_cleanup();
1854
1/2
✓ Branch 0 taken 4674 times.
✗ Branch 1 not taken.
 4674 delete resolver_;
1855
1856 // old destructor
1857 4674 pthread_mutex_destroy(lock_options_);
1858 4674 pthread_mutex_destroy(lock_synchronous_mode_);
1859 4674 free(lock_options_);
1860 4674 free(lock_synchronous_mode_);
1861 4674 }
1862
1863 4675 void DownloadManager::InitHeaders() {
1864 // User-Agent
1865
1/2
✓ Branch 2 taken 4675 times.
✗ Branch 3 not taken.
 4675 string cernvm_id = "User-Agent: cvmfs ";
1866 #ifdef CVMFS_LIBCVMFS
1867
1/2
✓ Branch 1 taken 4675 times.
✗ Branch 2 not taken.
 4675 cernvm_id += "libcvmfs ";
1868 #else
1869 ✗ cernvm_id += "Fuse ";
1870 #endif
1871
2/4
✓ Branch 2 taken 4675 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 4675 times.
✗ Branch 6 not taken.
 4675 cernvm_id += string(CVMFS_VERSION);
1872
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4675 times.
 4675 if (getenv("CERNVM_UUID") != NULL) {
1873 cernvm_id += " "
1874 ✗ + sanitizer::InputSanitizer("az AZ 09 -")
1875 ✗ .Filter(getenv("CERNVM_UUID"));
1876 }
1877 4675 user_agent_ = strdup(cernvm_id.c_str());
1878
1879
1/2
✓ Branch 1 taken 4675 times.
✗ Branch 2 not taken.
 4675 header_lists_ = new HeaderLists();
1880
1881
1/2
✓ Branch 1 taken 4675 times.
✗ Branch 2 not taken.
 4675 default_headers_ = header_lists_->GetList("Connection: Keep-Alive");
1882
1/2
✓ Branch 1 taken 4675 times.
✗ Branch 2 not taken.
 4675 header_lists_->AppendHeader(default_headers_, "Pragma:");
1883
1/2
✓ Branch 1 taken 4675 times.
✗ Branch 2 not taken.
 4675 header_lists_->AppendHeader(default_headers_, user_agent_);
1884 4675 }
1885
1886 4675 DownloadManager::DownloadManager(const unsigned max_pool_handles,
1887 const perf::StatisticsTemplate &statistics,
1888 4675 const std::string &name)
1889 4675 : prng_(Prng())
1890 4675 , pool_handles_idle_(new set<CURL *>)
1891 4675 , pool_handles_inuse_(new set<CURL *>)
1892 4675 , pool_max_handles_(max_pool_handles)
1893 4675 , pipe_terminate_(nullptr)
1894 4675 , pipe_jobs_(nullptr)
1895 4675 , watch_fds_(nullptr)
1896 4675 , watch_fds_size_(0)
1897 4675 , watch_fds_inuse_(0)
1898 4675 , watch_fds_max_(4 * max_pool_handles)
1899 4675 , opt_timeout_proxy_(5)
1900 4675 , opt_timeout_direct_(10)
1901 4675 , opt_low_speed_limit_(1024)
1902 4675 , opt_max_retries_(0)
1903 4675 , opt_backoff_init_ms_(0)
1904 4675 , opt_backoff_max_ms_(0)
1905 4675 , enable_info_header_(false)
1906 4675 , opt_ipv4_only_(false)
1907 4675 , follow_redirects_(false)
1908 4675 , ignore_signature_failures_(false)
1909 4675 , enable_http_tracing_(false)
1910 4675 , opt_metalink_(NULL, 0, 0, 0)
1911 4675 , opt_metalink_timestamp_link_(0)
1912 4675 , opt_host_(NULL, 0, 0, 0)
1913 4675 , opt_host_chain_rtt_(NULL)
1914 4675 , opt_proxy_groups_(NULL)
1915 4675 , opt_proxy_groups_current_(0)
1916 4675 , opt_proxy_groups_current_burned_(0)
1917 4675 , opt_proxy_groups_fallback_(0)
1918 4675 , opt_num_proxies_(0)
1919 4675 , opt_proxy_shard_(false)
1920 4675 , failover_indefinitely_(false)
1921
1/2
✓ Branch 1 taken 4675 times.
✗ Branch 2 not taken.
 4675 , name_(name)
1922 4675 , opt_ip_preference_(dns::kIpPreferSystem)
1923 4675 , opt_timestamp_backup_proxies_(0)
1924 4675 , opt_timestamp_failover_proxies_(0)
1925 4675 , opt_proxy_groups_reset_after_(0)
1926 4675 , credentials_attachment_(NULL)
1927
4/8
✓ Branch 13 taken 4675 times.
✗ Branch 14 not taken.
✓ Branch 16 taken 4675 times.
✗ Branch 17 not taken.
✓ Branch 19 taken 4675 times.
✗ Branch 20 not taken.
✓ Branch 23 taken 4675 times.
✗ Branch 24 not taken.
 14025 , counters_(new Counters(statistics)) {
1928 4675 multi_threaded_.store(0);
1929
1930 4675 lock_options_ = reinterpret_cast<pthread_mutex_t *>(
1931 4675 smalloc(sizeof(pthread_mutex_t)));
1932 4675 int retval = pthread_mutex_init(lock_options_, NULL);
1933
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4675 times.
 4675 assert(retval == 0);
1934 4675 lock_synchronous_mode_ = reinterpret_cast<pthread_mutex_t *>(
1935 4675 smalloc(sizeof(pthread_mutex_t)));
1936 4675 retval = pthread_mutex_init(lock_synchronous_mode_, NULL);
1937
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4675 times.
 4675 assert(retval == 0);
1938
1939
1/2
✓ Branch 1 taken 4675 times.
✗ Branch 2 not taken.
 4675 retval = curl_global_init(CURL_GLOBAL_ALL);
1940
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4675 times.
 4675 assert(retval == CURLE_OK);
1941
1942
1/2
✓ Branch 1 taken 4675 times.
✗ Branch 2 not taken.
 4675 InitHeaders();
1943
1944
1/2
✓ Branch 1 taken 4675 times.
✗ Branch 2 not taken.
 4675 curl_multi_ = curl_multi_init();
1945
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4675 times.
 4675 assert(curl_multi_ != NULL);
1946
1/2
✓ Branch 1 taken 4675 times.
✗ Branch 2 not taken.
 4675 curl_multi_setopt(curl_multi_, CURLMOPT_SOCKETFUNCTION, CallbackCurlSocket);
1947
1/2
✓ Branch 1 taken 4675 times.
✗ Branch 2 not taken.
 4675 curl_multi_setopt(curl_multi_, CURLMOPT_SOCKETDATA,
1948 static_cast<void *>(this));
1949
1/2
✓ Branch 1 taken 4675 times.
✗ Branch 2 not taken.
 4675 curl_multi_setopt(curl_multi_, CURLMOPT_MAXCONNECTS, watch_fds_max_);
1950
1/2
✓ Branch 1 taken 4675 times.
✗ Branch 2 not taken.
 4675 curl_multi_setopt(curl_multi_, CURLMOPT_MAX_TOTAL_CONNECTIONS,
1951 pool_max_handles_);
1952
1953 4675 prng_.InitLocaltime();
1954
1955 // Name resolving
1956 4675 if ((getenv("CVMFS_IPV4_ONLY") != NULL)
1957
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 4675 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 4675 times.
 4675 && (strlen(getenv("CVMFS_IPV4_ONLY")) > 0)) {
1958 ✗ opt_ipv4_only_ = true;
1959 }
1960
1/2
✓ Branch 1 taken 4675 times.
✗ Branch 2 not taken.
 4675 resolver_ = dns::NormalResolver::Create(opt_ipv4_only_, kDnsDefaultRetries,
1961 kDnsDefaultTimeoutMs);
1962
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4675 times.
 4675 assert(resolver_);
1963 4675 }
1964
1965 /**
1966 * Spawns the I/O worker thread and switches the module in multi-threaded mode.
1967 * No way back except Fini(); Init();
1968 */
1969 ✗ void DownloadManager::Spawn() {
1970 ✗ pipe_terminate_ = std::unique_ptr<Pipe<kPipeThreadTerminator> >(
1971 ✗ new Pipe<kPipeThreadTerminator>());
1972 ✗ pipe_jobs_ = std::unique_ptr<Pipe<kPipeDownloadJobs> >(
1973 ✗ new Pipe<kPipeDownloadJobs>());
1974
1975 ✗ const int retval = pthread_create(&thread_download_, NULL, MainDownload,
1976 static_cast<void *>(this));
1977 ✗ assert(retval == 0);
1978
1979 ✗ multi_threaded_.fetch_add(1);
1980
1981 ✗ if (health_check_.UseCount() > 0) {
1982 ✗ LogCvmfs(kLogDownload, kLogDebug,
1983 "(manager '%s') Starting healthcheck thread", name_.c_str());
1984 ✗ health_check_->StartHealthcheck();
1985 }
1986 }
1987
1988
1989 /**
1990 * Downloads data from an insecure outside channel (currently HTTP or file).
1991 */
1992 5720 Failures DownloadManager::Fetch(JobInfo *info) {
1993
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5720 times.
 5720 assert(info != NULL);
1994
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5720 times.
 5720 assert(info->url() != NULL);
1995
1996 Failures result;
1997
1/2
✓ Branch 1 taken 5720 times.
✗ Branch 2 not taken.
 5720 result = PrepareDownloadDestination(info);
1998
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5720 times.
 5720 if (result != kFailOk)
1999 ✗ return result;
2000
2001
2/2
✓ Branch 1 taken 3552 times.
✓ Branch 2 taken 2168 times.
 5720 if (info->expected_hash()) {
2002 3552 const shash::Algorithms algorithm = info->expected_hash()->algorithm;
2003 3552 info->GetHashContextPtr()->algorithm = algorithm;
2004
1/2
✓ Branch 1 taken 3552 times.
✗ Branch 2 not taken.
 3552 info->GetHashContextPtr()->size = shash::GetContextSize(algorithm);
2005 3552 info->GetHashContextPtr()->buffer = alloca(info->hash_context().size);
2006 }
2007
2008 // Clear out any saved Links in case this will be a metalink request and
2009 // previous non-metalink requests sent Link headers
2010
2/4
✓ Branch 2 taken 5720 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 5720 times.
✗ Branch 6 not taken.
 5720 info->SetLink("");
2011
2012 // Prepare cvmfs-info: header, allocate string on the stack
2013 5720 info->SetInfoHeader(NULL);
2014
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5720 times.
 5720 if (enable_info_header_) {
2015 const string header_info = info->GetInfoHeaderContents(
2016 ✗ info_header_template_);
2017 ✗ if (header_info != "") {
2018 ✗ const char * const header_name = "cvmfs-info: ";
2019 ✗ const size_t header_name_len = strlen(header_name);
2020 ✗ const size_t info_len = header_info.length();
2021 ✗ char *buf = static_cast<char *>(alloca(header_name_len + info_len + 1));
2022 ✗ memcpy(buf, header_name, header_name_len);
2023 ✗ memcpy(buf + header_name_len, header_info.c_str(), info_len);
2024 ✗ buf[header_name_len + info_len] = '\0';
2025 ✗ info->SetInfoHeader(buf);
2026 }
2027 }
2028
2029
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5720 times.
 5720 if (enable_http_tracing_) {
2030 ✗ const std::string str_pid = "X-CVMFS-PID: " + StringifyInt(info->pid());
2031 ✗ const std::string str_gid = "X-CVMFS-GID: " + StringifyUint(info->gid());
2032 ✗ const std::string str_uid = "X-CVMFS-UID: " + StringifyUint(info->uid());
2033
2034 // will be auto freed at the end of this function Fetch(JobInfo *info)
2035 ✗ info->SetTracingHeaderPid(static_cast<char *>(alloca(str_pid.size() + 1)));
2036 ✗ info->SetTracingHeaderGid(static_cast<char *>(alloca(str_gid.size() + 1)));
2037 ✗ info->SetTracingHeaderUid(static_cast<char *>(alloca(str_uid.size() + 1)));
2038
2039 ✗ memcpy(info->tracing_header_pid(), str_pid.c_str(), str_pid.size() + 1);
2040 ✗ memcpy(info->tracing_header_gid(), str_gid.c_str(), str_gid.size() + 1);
2041 ✗ memcpy(info->tracing_header_uid(), str_uid.c_str(), str_uid.size() + 1);
2042 }
2043
2044
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5720 times.
 11440 if (multi_threaded_.load() == 1) {
2045 ✗ if (!info->IsValidPipeJobResults()) {
2046 ✗ info->CreatePipeJobResults();
2047 }
2048 ✗ if (!info->IsValidDataTube()) {
2049 ✗ info->CreateDataTube();
2050 }
2051
2052 // LogCvmfs(kLogDownload, kLogDebug, "send job to thread, pipe %d %d",
2053 // info->wait_at[0], info->wait_at[1]);
2054 ✗ pipe_jobs_->Write<JobInfo *>(info);
2055
2056 // Decompress / copy chunks on this caller thread (in parallel across
2057 // concurrent Fetch() callers) instead of on the single MainDownload
2058 // thread. Track decompression errors locally and apply them at the end
2059 // — VerifyAndFinalize already ran by the time kActionStop arrives.
2060 ✗ Failures decompress_err = kFailOk;
2061 do {
2062 ✗ DataTubeElement *ele = info->GetDataTubePtr()->PopFront();
2063
2064 ✗ if (ele->action == kActionStop) {
2065 ✗ delete ele;
2066 ✗ break;
2067 }
2068
2069 ✗ if (ele->action == kActionReset) {
2070 // The chunks popped so far belonged to a download attempt that was
2071 // superseded by a retry / host fail-over (VerifyAndFinalize enqueued
2072 // this marker). Discard their — possibly corrupt — decompressed output
2073 // and start fresh so the next attempt's bytes are processed cleanly.
2074 ✗ if (info->compressed()) {
2075 ✗ zlib::DecompressFini(info->GetZstreamPtr());
2076 ✗ zlib::DecompressInit(info->GetZstreamPtr());
2077 }
2078 ✗ if (info->sink() != NULL && info->sink()->Reset() != 0) {
2079 ✗ decompress_err = kFailLocalIO;
2080 } else {
2081 ✗ decompress_err = kFailOk;
2082 }
2083 ✗ delete ele;
2084 ✗ continue;
2085 }
2086
2087 ✗ if (decompress_err == kFailOk && ele->size > 0) {
2088 ✗ if (info->compressed()) {
2089 ✗ const zlib::StreamStates retval = zlib::DecompressZStream2Sink(
2090 ✗ ele->data, static_cast<int64_t>(ele->size), info->GetZstreamPtr(),
2091 info->sink());
2092 ✗ if (retval == zlib::kStreamDataError) {
2093 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
2094 "(id %" PRId64 ") failed to decompress %s", info->id(),
2095 info->url()->c_str());
2096 ✗ decompress_err = kFailBadData;
2097 ✗ } else if (retval == zlib::kStreamIOError) {
2098 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
2099 "(id %" PRId64 ") decompressing %s, local IO error",
2100 info->id(), info->url()->c_str());
2101 ✗ decompress_err = kFailLocalIO;
2102 }
2103 } else {
2104 ✗ const int64_t written = info->sink()->Write(ele->data, ele->size);
2105 ✗ if (written < 0 || static_cast<uint64_t>(written) != ele->size) {
2106 ✗ LogCvmfs(kLogDownload, kLogDebug,
2107 "(id %" PRId64 ") sink write failed for %s (%ld of %zu)",
2108 info->id(), info->url()->c_str(),
2109 static_cast<long>(written), ele->size);
2110 ✗ decompress_err = kFailLocalIO;
2111 }
2112 }
2113 }
2114 ✗ delete ele;
2115 ✗ } while (true);
2116
2117 ✗ info->GetPipeJobResultPtr()->Read<download::Failures>(&result);
2118
2119 // Caller-side finalize (deferred from VerifyAndFinalize). Must run after
2120 // all kActionDecompress chunks have been popped and decompressed.
2121 ✗ if (result == kFailOk && decompress_err == kFailOk) {
2122 ✗ if (info->sink() != NULL && info->sink()->Flush() != 0) {
2123 ✗ decompress_err = kFailLocalIO;
2124 }
2125 }
2126 ✗ if (info->compressed()) {
2127 ✗ zlib::DecompressFini(info->GetZstreamPtr());
2128 }
2129 ✗ if (result == kFailOk && decompress_err != kFailOk) {
2130 ✗ result = decompress_err;
2131 ✗ info->SetErrorCode(decompress_err);
2132 ✗ if (info->sink() != NULL) {
2133 ✗ info->sink()->Purge();
2134 }
2135 }
2136 // LogCvmfs(kLogDownload, kLogDebug, "got result %d", result);
2137 } else {
2138 5720 const MutexLockGuard l(lock_synchronous_mode_);
2139
1/2
✓ Branch 1 taken 5720 times.
✗ Branch 2 not taken.
 5720 CURL *handle = AcquireCurlHandle();
2140
1/2
✓ Branch 1 taken 5720 times.
✗ Branch 2 not taken.
 5720 InitializeRequest(info, handle);
2141
1/2
✓ Branch 1 taken 5720 times.
✗ Branch 2 not taken.
 5720 SetUrlOptions(info);
2142 // curl_easy_setopt(handle, CURLOPT_VERBOSE, 1);
2143 int retval;
2144 do {
2145
1/2
✓ Branch 1 taken 6054 times.
✗ Branch 2 not taken.
 6054 retval = curl_easy_perform(handle);
2146 6054 perf::Inc(counters_->n_requests);
2147 double elapsed;
2148
1/2
✓ Branch 1 taken 6054 times.
✗ Branch 2 not taken.
 6054 if (curl_easy_getinfo(handle, CURLINFO_TOTAL_TIME, &elapsed)
2149
1/2
✓ Branch 0 taken 6054 times.
✗ Branch 1 not taken.
 6054 == CURLE_OK) {
2150 6054 perf::Xadd(counters_->sz_transfer_time,
2151 6054 static_cast<int64_t>(elapsed * 1000));
2152 }
2153
3/4
✓ Branch 1 taken 6054 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 334 times.
✓ Branch 4 taken 5720 times.
 6054 } while (VerifyAndFinalize(retval, info));
2154 5720 result = info->error_code();
2155 // Prevent the handle from being added to the idle list, which
2156 // avoids that it is mistakenly picked up by the multi handle later
2157 // in multi-threaded context. This, in turn, would fail to wait for
2158 // for resolver threads that meanwhile vanished due to a fork()
2159 // in Daemonize()
2160
1/2
✓ Branch 2 taken 5720 times.
✗ Branch 3 not taken.
 5720 ReleaseCurlHandle(info->curl_handle(), false /* allow_reuse */);
2161 5720 }
2162
2163
2/2
✓ Branch 0 taken 379 times.
✓ Branch 1 taken 5341 times.
 5720 if (result != kFailOk) {
2164
1/2
✓ Branch 4 taken 379 times.
✗ Branch 5 not taken.
 379 LogCvmfs(kLogDownload, kLogDebug,
2165 "(manager '%s' - id %" PRId64 ") "
2166 "download failed (error %d - %s)",
2167 name_.c_str(), info->id(), result, Code2Ascii(result));
2168
2169
1/2
✓ Branch 1 taken 379 times.
✗ Branch 2 not taken.
 379 if (info->sink() != NULL) {
2170
1/2
✓ Branch 2 taken 379 times.
✗ Branch 3 not taken.
 379 info->sink()->Purge();
2171 }
2172 }
2173
2174 5720 return result;
2175 }
2176
2177
2178 /**
2179 * Used by the client to connect the authz session manager to the download
2180 * manager.
2181 */
2182 1411 void DownloadManager::SetCredentialsAttachment(CredentialsAttachment *ca) {
2183 1411 const MutexLockGuard m(lock_options_);
2184 1411 credentials_attachment_ = ca;
2185 1411 }
2186
2187 /**
2188 * Gets the DNS sever.
2189 */
2190 ✗ std::string DownloadManager::GetDnsServer() const { return opt_dns_server_; }
2191
2192 /**
2193 * Sets a DNS server. Only for testing as it cannot be reverted to the system
2194 * default.
2195 */
2196 ✗ void DownloadManager::SetDnsServer(const string &address) {
2197 ✗ if (!address.empty()) {
2198 ✗ const MutexLockGuard m(lock_options_);
2199 ✗ opt_dns_server_ = address;
2200 ✗ assert(!opt_dns_server_.empty());
2201
2202 ✗ vector<string> servers;
2203 ✗ servers.push_back(address);
2204 ✗ const bool retval = resolver_->SetResolvers(servers);
2205 ✗ assert(retval);
2206 }
2207 ✗ LogCvmfs(kLogDownload, kLogSyslog, "(manager '%s') set nameserver to %s",
2208 name_.c_str(), address.c_str());
2209 }
2210
2211
2212 /**
2213 * Sets the DNS query timeout parameters.
2214 */
2215 2305 void DownloadManager::SetDnsParameters(const unsigned retries,
2216 const unsigned timeout_ms) {
2217 2305 const MutexLockGuard m(lock_options_);
2218 2305 if ((resolver_->retries() == retries)
2219
3/6
✓ Branch 0 taken 2305 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 2305 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 2305 times.
✗ Branch 6 not taken.
 2305 && (resolver_->timeout_ms() == timeout_ms)) {
2220 2305 return;
2221 }
2222 ✗ delete resolver_;
2223 ✗ resolver_ = NULL;
2224 ✗ resolver_ = dns::NormalResolver::Create(opt_ipv4_only_, retries, timeout_ms);
2225 ✗ assert(resolver_);
2226
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 2305 times.
 2305 }
2227
2228
2229 2305 void DownloadManager::SetDnsTtlLimits(const unsigned min_seconds,
2230 const unsigned max_seconds) {
2231 2305 const MutexLockGuard m(lock_options_);
2232 2305 resolver_->set_min_ttl(min_seconds);
2233 2305 resolver_->set_max_ttl(max_seconds);
2234 2305 }
2235
2236
2237 ✗ void DownloadManager::SetIpPreference(dns::IpPreference preference) {
2238 ✗ const MutexLockGuard m(lock_options_);
2239 ✗ opt_ip_preference_ = preference;
2240 }
2241
2242
2243 /**
2244 * Sets two timeout values for proxied and for direct connections, respectively.
2245 * The timeout counts for all sorts of connection phases,
2246 * DNS, HTTP connect, etc.
2247 */
2248 2873 void DownloadManager::SetTimeout(const unsigned seconds_proxy,
2249 const unsigned seconds_direct) {
2250 2873 const MutexLockGuard m(lock_options_);
2251 2873 opt_timeout_proxy_ = seconds_proxy;
2252 2873 opt_timeout_direct_ = seconds_direct;
2253 2873 }
2254
2255
2256 /**
2257 * Sets contains the average transfer speed in bytes per second that the
2258 * transfer should be below during CURLOPT_LOW_SPEED_TIME seconds for libcurl to
2259 * consider it to be too slow and abort. Only effective for new connections.
2260 */
2261 ✗ void DownloadManager::SetLowSpeedLimit(const unsigned low_speed_limit) {
2262 ✗ const MutexLockGuard m(lock_options_);
2263 ✗ opt_low_speed_limit_ = low_speed_limit;
2264 }
2265
2266
2267 /**
2268 * Receives the currently active timeout values.
2269 */
2270 822 void DownloadManager::GetTimeout(unsigned *seconds_proxy,
2271 unsigned *seconds_direct) {
2272 822 const MutexLockGuard m(lock_options_);
2273 822 *seconds_proxy = opt_timeout_proxy_;
2274 822 *seconds_direct = opt_timeout_direct_;
2275 822 }
2276
2277
2278 /**
2279 * Parses a list of ';'-separated hosts for the metalink chain. The empty
2280 * string removes the metalink list.
2281 */
2282 ✗ void DownloadManager::SetMetalinkChain(const string &metalink_list) {
2283 ✗ SetMetalinkChain(SplitString(metalink_list, ';'));
2284 }
2285
2286
2287 ✗ void DownloadManager::SetMetalinkChain(
2288 const std::vector<std::string> &metalink_list) {
2289 ✗ const MutexLockGuard m(lock_options_);
2290 ✗ opt_metalink_.timestamp_backup = 0;
2291 ✗ delete opt_metalink_.chain;
2292 ✗ opt_metalink_.current = 0;
2293
2294 ✗ if (metalink_list.empty()) {
2295 ✗ opt_metalink_.chain = NULL;
2296 ✗ return;
2297 }
2298
2299 ✗ opt_metalink_.chain = new vector<string>(metalink_list);
2300 }
2301
2302
2303 /**
2304 * Retrieves the currently set chain of metalink hosts and the currently
2305 * used metalink host.
2306 */
2307 ✗ void DownloadManager::GetMetalinkInfo(vector<string> *metalink_chain,
2308 unsigned *current_metalink) {
2309 ✗ const MutexLockGuard m(lock_options_);
2310 ✗ if (opt_metalink_.chain) {
2311 ✗ if (current_metalink) {
2312 ✗ *current_metalink = opt_metalink_.current;
2313 }
2314 ✗ if (metalink_chain) {
2315 ✗ *metalink_chain = *opt_metalink_.chain;
2316 }
2317 }
2318 }
2319
2320
2321 /**
2322 * Parses a list of ';'-separated hosts for the host chain. The empty string
2323 * removes the host list.
2324 */
2325 2052 void DownloadManager::SetHostChain(const string &host_list) {
2326
1/2
✓ Branch 2 taken 2052 times.
✗ Branch 3 not taken.
 2052 SetHostChain(SplitString(host_list, ';'));
2327 2052 }
2328
2329
2330 2126 void DownloadManager::SetHostChain(const std::vector<std::string> &host_list) {
2331 2126 const MutexLockGuard m(lock_options_);
2332 2126 opt_host_.timestamp_backup = 0;
2333
2/2
✓ Branch 0 taken 915 times.
✓ Branch 1 taken 1211 times.
 2126 delete opt_host_.chain;
2334
2/2
✓ Branch 0 taken 915 times.
✓ Branch 1 taken 1211 times.
 2126 delete opt_host_chain_rtt_;
2335 2126 opt_host_.current = 0;
2336
2337
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 2126 times.
 2126 if (host_list.empty()) {
2338 ✗ opt_host_.chain = NULL;
2339 ✗ opt_host_chain_rtt_ = NULL;
2340 ✗ return;
2341 }
2342
2343
2/4
✓ Branch 1 taken 2126 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2126 times.
✗ Branch 5 not taken.
 2126 opt_host_.chain = new vector<string>(host_list);
2344 4252 opt_host_chain_rtt_ = new vector<int>(opt_host_.chain->size(),
2345
2/4
✓ Branch 1 taken 2126 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2126 times.
✗ Branch 5 not taken.
 2126 kProbeUnprobed);
2346 // LogCvmfs(kLogDownload, kLogSyslog, "using host %s",
2347 // (*opt_host_.chain)[0].c_str());
2348
1/2
✓ Branch 1 taken 2126 times.
✗ Branch 2 not taken.
 2126 }
2349
2350
2351 /**
2352 * Retrieves the currently set chain of hosts, their round trip times, and the
2353 * currently used host.
2354 */
2355 518 void DownloadManager::GetHostInfo(vector<string> *host_chain, vector<int> *rtt,
2356 unsigned *current_host) {
2357 518 const MutexLockGuard m(lock_options_);
2358
1/2
✓ Branch 0 taken 518 times.
✗ Branch 1 not taken.
 518 if (opt_host_.chain) {
2359
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 518 times.
 518 if (current_host) {
2360 ✗ *current_host = opt_host_.current;
2361 }
2362
1/2
✓ Branch 0 taken 518 times.
✗ Branch 1 not taken.
 518 if (host_chain) {
2363
1/2
✓ Branch 1 taken 518 times.
✗ Branch 2 not taken.
 518 *host_chain = *opt_host_.chain;
2364 }
2365
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 518 times.
 518 if (rtt) {
2366 ✗ *rtt = *opt_host_chain_rtt_;
2367 }
2368 }
2369 518 }
2370
2371
2372 /**
2373 * Jumps to the next proxy in the ring of forward proxy servers.
2374 * Selects one randomly from a load-balancing group.
2375 *
2376 * Allow for the fact that the proxy may have already been failed by
2377 * another transfer, or that the proxy may no longer be part of the
2378 * current load-balancing group.
2379 */
2380 36 void DownloadManager::SwitchProxy(JobInfo *info) {
2381 36 const MutexLockGuard m(lock_options_);
2382
2383
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 36 times.
 36 if (!opt_proxy_groups_) {
2384 ✗ return;
2385 }
2386
2387 // Fail any matching proxies within the current load-balancing group
2388 36 vector<ProxyInfo> *group = current_proxy_group();
2389 36 const unsigned group_size = group->size();
2390 36 unsigned failed = 0;
2391
2/2
✓ Branch 0 taken 36 times.
✓ Branch 1 taken 36 times.
 72 for (unsigned i = 0; i < group_size - opt_proxy_groups_current_burned_; ++i) {
2392
5/14
✓ Branch 0 taken 36 times.
✗ Branch 1 not taken.
✓ Branch 4 taken 36 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 36 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 36 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✓ Branch 12 taken 36 times.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
 36 if (info && (info->proxy() == (*group)[i].url)) {
2393 // Move to list of failed proxies
2394 36 opt_proxy_groups_current_burned_++;
2395
1/2
✓ Branch 2 taken 36 times.
✗ Branch 3 not taken.
 36 swap((*group)[i],
2396 36 (*group)[group_size - opt_proxy_groups_current_burned_]);
2397 36 perf::Inc(counters_->n_proxy_failover);
2398 36 failed++;
2399 }
2400 }
2401
2402 // Do nothing more unless at least one proxy was marked as failed
2403
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 36 times.
 36 if (!failed)
2404 ✗ return;
2405
2406 // If all proxies from the current load-balancing group are burned, switch to
2407 // another group
2408
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 if (opt_proxy_groups_current_burned_ == group->size()) {
2409 36 opt_proxy_groups_current_burned_ = 0;
2410
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 if (opt_proxy_groups_->size() > 1) {
2411 72 opt_proxy_groups_current_ = (opt_proxy_groups_current_ + 1)
2412 36 % opt_proxy_groups_->size();
2413 // Remember the timestamp of switching to backup proxies
2414
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 36 times.
 36 if (opt_proxy_groups_reset_after_ > 0) {
2415 ✗ if (opt_proxy_groups_current_ > 0) {
2416 ✗ if (opt_timestamp_backup_proxies_ == 0)
2417 ✗ opt_timestamp_backup_proxies_ = time(NULL);
2418 // LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2419 // "switched to (another) backup proxy group");
2420 } else {
2421 ✗ opt_timestamp_backup_proxies_ = 0;
2422 // LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2423 // "switched back to primary proxy group");
2424 }
2425 ✗ opt_timestamp_failover_proxies_ = 0;
2426 }
2427 }
2428 } else {
2429 // Record failover time
2430 ✗ if (opt_proxy_groups_reset_after_ > 0) {
2431 ✗ if (opt_timestamp_failover_proxies_ == 0)
2432 ✗ opt_timestamp_failover_proxies_ = time(NULL);
2433 }
2434 }
2435
2436
2/4
✓ Branch 2 taken 36 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 36 times.
✗ Branch 6 not taken.
 36 UpdateProxiesUnlocked("failed proxy");
2437
1/2
✓ Branch 2 taken 36 times.
✗ Branch 3 not taken.
 36 LogCvmfs(kLogDownload, kLogDebug,
2438 "(manager '%s' - id %" PRId64 ") "
2439 "%lu proxies remain in group",
2440 name_.c_str(), info->id(),
2441 36 current_proxy_group()->size() - opt_proxy_groups_current_burned_);
2442
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 }
2443
2444
2445 /**
2446 * Switches to the next host in the chain. If jobinfo is set, switch only if
2447 * the current host is identical to the one used by jobinfo, otherwise another
2448 * transfer has already done the switch.
2449 */
2450 91 void DownloadManager::SwitchHostInfo(const std::string &typ,
2451 HostInfo &info,
2452 JobInfo *jobinfo) {
2453 91 const MutexLockGuard m(lock_options_);
2454
2455
5/6
✓ Branch 0 taken 91 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 19 times.
✓ Branch 4 taken 72 times.
✓ Branch 5 taken 19 times.
✓ Branch 6 taken 72 times.
 91 if (!info.chain || (info.chain->size() == 1)) {
2456 19 return;
2457 }
2458
2459
1/2
✓ Branch 0 taken 72 times.
✗ Branch 1 not taken.
 72 if (jobinfo) {
2460 int lastused;
2461
1/2
✓ Branch 1 taken 72 times.
✗ Branch 2 not taken.
 72 if (typ == "host") {
2462 72 lastused = jobinfo->current_host_chain_index();
2463 } else {
2464 ✗ lastused = jobinfo->current_metalink_chain_index();
2465 }
2466
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 72 times.
 72 if (lastused != info.current) {
2467 ✗ LogCvmfs(kLogDownload, kLogDebug,
2468 "(manager '%s' - id %" PRId64 ")"
2469 "don't switch %s, "
2470 "last used %s: %s, current %s: %s",
2471 name_.c_str(), jobinfo->id(), typ.c_str(), typ.c_str(),
2472 ✗ (*info.chain)[lastused].c_str(), typ.c_str(),
2473 ✗ (*info.chain)[info.current].c_str());
2474 ✗ return;
2475 }
2476 }
2477
2478
1/2
✓ Branch 2 taken 72 times.
✗ Branch 3 not taken.
 72 string reason = "manually triggered";
2479
2/4
✓ Branch 1 taken 72 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 72 times.
✗ Branch 5 not taken.
 72 string info_id = "(manager '" + name_ + "'";
2480
1/2
✓ Branch 0 taken 72 times.
✗ Branch 1 not taken.
 72 if (jobinfo) {
2481
1/2
✓ Branch 3 taken 72 times.
✗ Branch 4 not taken.
 72 reason = download::Code2Ascii(jobinfo->error_code());
2482
3/6
✓ Branch 2 taken 72 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 72 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 72 times.
✗ Branch 9 not taken.
 72 info_id += " - id " + StringifyInt(jobinfo->id());
2483 }
2484
1/2
✓ Branch 1 taken 72 times.
✗ Branch 2 not taken.
 72 info_id += ")";
2485
2486
1/2
✓ Branch 2 taken 72 times.
✗ Branch 3 not taken.
 72 const std::string old_host = (*info.chain)[info.current];
2487 72 info.current = (info.current + 1) % static_cast<int>(info.chain->size());
2488
1/2
✓ Branch 1 taken 72 times.
✗ Branch 2 not taken.
 72 if (typ == "host") {
2489 72 perf::Inc(counters_->n_host_failover);
2490 } else {
2491 ✗ perf::Inc(counters_->n_metalink_failover);
2492 }
2493
1/3
✓ Branch 6 taken 72 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
 144 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2494 "%s switching %s from %s to %s (%s)", info_id.c_str(), typ.c_str(),
2495 72 old_host.c_str(), (*info.chain)[info.current].c_str(),
2496 reason.c_str());
2497
2498 // Remember the timestamp of switching to backup host
2499
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 72 times.
 72 if (info.reset_after > 0) {
2500 ✗ if (info.current != 0) {
2501 ✗ if (info.timestamp_backup == 0)
2502 ✗ info.timestamp_backup = time(NULL);
2503 } else {
2504 ✗ info.timestamp_backup = 0;
2505 }
2506 }
2507
2/2
✓ Branch 4 taken 72 times.
✓ Branch 5 taken 19 times.
 91 }
2508
2509 91 void DownloadManager::SwitchHost(JobInfo *info) {
2510
2/4
✓ Branch 2 taken 91 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 91 times.
✗ Branch 6 not taken.
 91 SwitchHostInfo("host", opt_host_, info);
2511 91 }
2512
2513 19 void DownloadManager::SwitchHost() { SwitchHost(NULL); }
2514
2515
2516 ✗ void DownloadManager::SwitchMetalink(JobInfo *info) {
2517 ✗ SwitchHostInfo("metalink", opt_metalink_, info);
2518 }
2519
2520
2521 ✗ void DownloadManager::SwitchMetalink() { SwitchMetalink(NULL); }
2522
2523 2471 bool DownloadManager::CheckMetalinkChain(time_t now) {
2524 2471 return (opt_metalink_.chain
2525
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2471 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 2471 && ((opt_metalink_timestamp_link_ == 0)
2526 ✗ || (static_cast<int64_t>((now == 0) ? time(NULL) : now)
2527 ✗ > static_cast<int64_t>(opt_metalink_timestamp_link_
2528
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 2471 + opt_metalink_.reset_after))));
2529 }
2530
2531
2532 /**
2533 * Orders the hostlist according to RTT of downloading .cvmfschecksum.
2534 * Sets the current host to the best-responsive host.
2535 * If you change the host list in between by SetHostChain(), it will be
2536 * overwritten by this function.
2537 */
2538 ✗ void DownloadManager::ProbeHosts() {
2539 ✗ vector<string> host_chain;
2540 ✗ vector<int> host_rtt;
2541 unsigned current_host;
2542
2543 ✗ GetHostInfo(&host_chain, &host_rtt, &current_host);
2544
2545 // Stopwatch, two times to fill caches first
2546 unsigned i, retries;
2547 ✗ string url;
2548
2549 ✗ cvmfs::MemSink memsink;
2550 ✗ JobInfo info(&url, false, false, NULL, &memsink);
2551 ✗ for (retries = 0; retries < 2; ++retries) {
2552 ✗ for (i = 0; i < host_chain.size(); ++i) {
2553 ✗ url = host_chain[i] + "/.cvmfspublished";
2554
2555 struct timeval tv_start, tv_end;
2556 ✗ gettimeofday(&tv_start, NULL);
2557 ✗ const Failures result = Fetch(&info);
2558 ✗ gettimeofday(&tv_end, NULL);
2559 ✗ memsink.Reset();
2560 ✗ if (result == kFailOk) {
2561 ✗ host_rtt[i] = static_cast<int>(DiffTimeSeconds(tv_start, tv_end)
2562 ✗ * 1000);
2563 ✗ LogCvmfs(kLogDownload, kLogDebug,
2564 "(manager '%s' - id %" PRId64 ") "
2565 "probing host %s had %dms rtt",
2566 ✗ name_.c_str(), info.id(), url.c_str(), host_rtt[i]);
2567 } else {
2568 ✗ LogCvmfs(kLogDownload, kLogDebug,
2569 "(manager '%s' - id %" PRId64 ") "
2570 "error while probing host %s: %d %s",
2571 name_.c_str(), info.id(), url.c_str(), result,
2572 Code2Ascii(result));
2573 ✗ host_rtt[i] = INT_MAX;
2574 }
2575 }
2576 }
2577
2578 ✗ SortTeam(&host_rtt, &host_chain);
2579 ✗ for (i = 0; i < host_chain.size(); ++i) {
2580 ✗ if (host_rtt[i] == INT_MAX)
2581 ✗ host_rtt[i] = kProbeDown;
2582 }
2583
2584 ✗ const MutexLockGuard m(lock_options_);
2585 ✗ delete opt_host_.chain;
2586 ✗ delete opt_host_chain_rtt_;
2587 ✗ opt_host_.chain = new vector<string>(host_chain);
2588 ✗ opt_host_chain_rtt_ = new vector<int>(host_rtt);
2589 ✗ opt_host_.current = 0;
2590 }
2591
2592 ✗ bool DownloadManager::GeoSortServers(std::vector<std::string> *servers,
2593 std::vector<uint64_t> *output_order) {
2594 ✗ if (!servers) {
2595 ✗ return false;
2596 }
2597 ✗ if (servers->size() == 1) {
2598 ✗ if (output_order) {
2599 ✗ output_order->clear();
2600 ✗ output_order->push_back(0);
2601 }
2602 ✗ return true;
2603 }
2604
2605 ✗ std::vector<std::string> host_chain;
2606 ✗ GetHostInfo(&host_chain, NULL, NULL);
2607
2608 ✗ std::vector<std::string> server_dns_names;
2609 ✗ server_dns_names.reserve(servers->size());
2610 ✗ for (unsigned i = 0; i < servers->size(); ++i) {
2611 ✗ const std::string host = dns::ExtractHost((*servers)[i]);
2612 ✗ server_dns_names.push_back(host.empty() ? (*servers)[i] : host);
2613 }
2614 ✗ const std::string host_list = JoinStrings(server_dns_names, ",");
2615
2616 ✗ vector<string> host_chain_shuffled;
2617 {
2618 // Protect against concurrent access to prng_
2619 ✗ const MutexLockGuard m(lock_options_);
2620 // Determine random hosts for the Geo-API query
2621 ✗ host_chain_shuffled = Shuffle(host_chain, &prng_);
2622 }
2623 // Request ordered list via Geo-API
2624 ✗ bool success = false;
2625 ✗ const unsigned max_attempts = std::min(host_chain_shuffled.size(), size_t(3));
2626 ✗ vector<uint64_t> geo_order(servers->size());
2627 ✗ for (unsigned i = 0; i < max_attempts; ++i) {
2628 ✗ const string url = host_chain_shuffled[i] + "/api/v1.0/geo/@proxy@/"
2629 ✗ + host_list;
2630 ✗ LogCvmfs(kLogDownload, kLogDebug,
2631 "(manager '%s') requesting ordered server list from %s",
2632 name_.c_str(), url.c_str());
2633 ✗ cvmfs::MemSink memsink;
2634 ✗ JobInfo info(&url, false, false, NULL, &memsink);
2635 ✗ const Failures result = Fetch(&info);
2636 ✗ if (result == kFailOk) {
2637 ✗ const string order(reinterpret_cast<char *>(memsink.data()),
2638 ✗ memsink.pos());
2639 ✗ memsink.Reset();
2640 ✗ const bool retval = ValidateGeoReply(order, servers->size(), &geo_order);
2641 ✗ if (!retval) {
2642 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2643 "(manager '%s') retrieved invalid GeoAPI reply from %s [%s]",
2644 name_.c_str(), url.c_str(), order.c_str());
2645 } else {
2646 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslog,
2647 "(manager '%s') "
2648 "geographic order of servers retrieved from %s",
2649 name_.c_str(),
2650 ✗ dns::ExtractHost(host_chain_shuffled[i]).c_str());
2651 // remove new line at end of "order"
2652 ✗ LogCvmfs(kLogDownload, kLogDebug, "order is %s",
2653 ✗ Trim(order, true /* trim_newline */).c_str());
2654 ✗ success = true;
2655 ✗ break;
2656 }
2657 ✗ } else {
2658 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2659 "(manager '%s') GeoAPI request for %s failed with error %d [%s]",
2660 name_.c_str(), url.c_str(), result, Code2Ascii(result));
2661 }
2662 }
2663 ✗ if (!success) {
2664 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2665 "(manager '%s') "
2666 "failed to retrieve geographic order from stratum 1 servers",
2667 name_.c_str());
2668 ✗ return false;
2669 }
2670
2671 ✗ if (output_order) {
2672 ✗ output_order->swap(geo_order);
2673 } else {
2674 ✗ std::vector<std::string> sorted_servers;
2675 ✗ sorted_servers.reserve(geo_order.size());
2676 ✗ for (unsigned i = 0; i < geo_order.size(); ++i) {
2677 ✗ const uint64_t orderval = geo_order[i];
2678 ✗ sorted_servers.push_back((*servers)[orderval]);
2679 }
2680 ✗ servers->swap(sorted_servers);
2681 }
2682 ✗ return true;
2683 }
2684
2685
2686 /**
2687 * Uses the Geo-API of Stratum 1s to let any of them order the list of servers
2688 * and fallback proxies (if any).
2689 * Tries at most three random Stratum 1s before giving up.
2690 * If you change the host list in between by SetHostChain() or the fallback
2691 * proxy list by SetProxyChain(), they will be overwritten by this function.
2692 */
2693 ✗ bool DownloadManager::ProbeGeo() {
2694 ✗ vector<string> host_chain;
2695 ✗ vector<int> host_rtt;
2696 unsigned current_host;
2697 ✗ vector<vector<ProxyInfo> > proxy_chain;
2698 unsigned fallback_group;
2699
2700 ✗ GetHostInfo(&host_chain, &host_rtt, &current_host);
2701 ✗ GetProxyInfo(&proxy_chain, NULL, &fallback_group);
2702 ✗ if ((host_chain.size() < 2) && ((proxy_chain.size() - fallback_group) < 2))
2703 ✗ return true;
2704
2705 ✗ vector<string> host_names;
2706 ✗ for (unsigned i = 0; i < host_chain.size(); ++i)
2707 ✗ host_names.push_back(dns::ExtractHost(host_chain[i]));
2708 ✗ SortTeam(&host_names, &host_chain);
2709 ✗ const unsigned last_geo_host = host_names.size();
2710
2711 ✗ if ((fallback_group == 0) && (last_geo_host > 1)) {
2712 // There are no non-fallback proxies, which means that the client
2713 // will always use the fallback proxies. Add a keyword separator
2714 // between the hosts and fallback proxies so the geosorting service
2715 // will know to sort the hosts based on the distance from the
2716 // closest fallback proxy rather than the distance from the client.
2717 ✗ host_names.push_back("+PXYSEP+");
2718 }
2719
2720 // Add fallback proxy names to the end of the host list
2721 ✗ const unsigned first_geo_fallback = host_names.size();
2722 ✗ for (unsigned i = fallback_group; i < proxy_chain.size(); ++i) {
2723 // We only take the first fallback proxy name from every group under the
2724 // assumption that load-balanced servers are at the same location
2725 ✗ host_names.push_back(proxy_chain[i][0].host.name());
2726 }
2727
2728 ✗ std::vector<uint64_t> geo_order;
2729 ✗ const bool success = GeoSortServers(&host_names, &geo_order);
2730 ✗ if (!success) {
2731 // GeoSortServers already logged a failure message.
2732 ✗ return false;
2733 }
2734
2735 // Re-install host chain and proxy chain
2736 ✗ const MutexLockGuard m(lock_options_);
2737 ✗ delete opt_host_.chain;
2738 ✗ opt_num_proxies_ = 0;
2739 ✗ opt_host_.chain = new vector<string>(host_chain.size());
2740
2741 // It's possible that opt_proxy_groups_fallback_ might have changed while
2742 // the lock wasn't held
2743 vector<vector<ProxyInfo> > *proxy_groups = new vector<vector<ProxyInfo> >(
2744 ✗ opt_proxy_groups_fallback_ + proxy_chain.size() - fallback_group);
2745 // First copy the non-fallback part of the current proxy chain
2746 ✗ for (unsigned i = 0; i < opt_proxy_groups_fallback_; ++i) {
2747 ✗ (*proxy_groups)[i] = (*opt_proxy_groups_)[i];
2748 ✗ opt_num_proxies_ += (*opt_proxy_groups_)[i].size();
2749 }
2750
2751 // Copy the host chain and fallback proxies by geo order. Array indices
2752 // in geo_order that are smaller than last_geo_host refer to a stratum 1,
2753 // and those indices greater than or equal to first_geo_fallback refer to
2754 // a fallback proxy.
2755 ✗ unsigned hosti = 0;
2756 ✗ unsigned proxyi = opt_proxy_groups_fallback_;
2757 ✗ for (unsigned i = 0; i < geo_order.size(); ++i) {
2758 ✗ const uint64_t orderval = geo_order[i];
2759 ✗ if (orderval < static_cast<uint64_t>(last_geo_host)) {
2760 // LogCvmfs(kLogCvmfs, kLogSyslog, "this is orderval %u at host index
2761 // %u", orderval, hosti);
2762 ✗ (*opt_host_.chain)[hosti++] = host_chain[orderval];
2763 ✗ } else if (orderval >= static_cast<uint64_t>(first_geo_fallback)) {
2764 // LogCvmfs(kLogCvmfs, kLogSyslog,
2765 // "this is orderval %u at proxy index %u, using proxy_chain index %u",
2766 // orderval, proxyi, fallback_group + orderval - first_geo_fallback);
2767 ✗ (*proxy_groups)[proxyi] = proxy_chain[fallback_group + orderval
2768 ✗ - first_geo_fallback];
2769 ✗ opt_num_proxies_ += (*proxy_groups)[proxyi].size();
2770 ✗ proxyi++;
2771 }
2772 }
2773
2774 ✗ opt_proxy_map_.clear();
2775 ✗ delete opt_proxy_groups_;
2776 ✗ opt_proxy_groups_ = proxy_groups;
2777 // In pathological cases, opt_proxy_groups_current_ can be larger now when
2778 // proxies changed in-between.
2779 ✗ if (opt_proxy_groups_current_ > opt_proxy_groups_->size()) {
2780 ✗ if (opt_proxy_groups_->size() == 0) {
2781 ✗ opt_proxy_groups_current_ = 0;
2782 } else {
2783 ✗ opt_proxy_groups_current_ = opt_proxy_groups_->size() - 1;
2784 }
2785 ✗ opt_proxy_groups_current_burned_ = 0;
2786 }
2787
2788 ✗ UpdateProxiesUnlocked("geosort");
2789
2790 ✗ delete opt_host_chain_rtt_;
2791 ✗ opt_host_chain_rtt_ = new vector<int>(host_chain.size(), kProbeGeo);
2792 ✗ opt_host_.current = 0;
2793
2794 ✗ return true;
2795 }
2796
2797
2798 /**
2799 * Validates a string of the form "1,4,2,3" representing in which order the
2800 * the expected_size number of hosts should be put for optimal geographic
2801 * proximity. Returns false if the reply_order string is invalid, otherwise
2802 * fills in the reply_vals array with zero-based order indexes (e.g.
2803 * [0,3,1,2]) and returns true.
2804 */
2805 504 bool DownloadManager::ValidateGeoReply(const string &reply_order,
2806 const unsigned expected_size,
2807 vector<uint64_t> *reply_vals) {
2808
2/2
✓ Branch 1 taken 36 times.
✓ Branch 2 taken 468 times.
 504 if (reply_order.empty())
2809 36 return false;
2810
2/4
✓ Branch 2 taken 468 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 468 times.
✗ Branch 6 not taken.
 936 const sanitizer::InputSanitizer sanitizer("09 , \n");
2811
3/4
✓ Branch 1 taken 468 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 36 times.
✓ Branch 4 taken 432 times.
 468 if (!sanitizer.IsValid(reply_order))
2812 36 return false;
2813
2/4
✓ Branch 2 taken 432 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 432 times.
✗ Branch 6 not taken.
 864 const sanitizer::InputSanitizer strip_newline("09 ,");
2814
1/2
✓ Branch 1 taken 432 times.
✗ Branch 2 not taken.
 432 vector<string> reply_strings = SplitString(strip_newline.Filter(reply_order),
2815
1/2
✓ Branch 1 taken 432 times.
✗ Branch 2 not taken.
 432 ',');
2816 432 vector<uint64_t> tmp_vals;
2817
2/2
✓ Branch 1 taken 828 times.
✓ Branch 2 taken 360 times.
 1188 for (unsigned i = 0; i < reply_strings.size(); ++i) {
2818
2/2
✓ Branch 2 taken 72 times.
✓ Branch 3 taken 756 times.
 828 if (reply_strings[i].empty())
2819 72 return false;
2820
2/4
✓ Branch 2 taken 756 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 756 times.
✗ Branch 6 not taken.
 756 tmp_vals.push_back(String2Uint64(reply_strings[i]));
2821 }
2822
2/2
✓ Branch 1 taken 180 times.
✓ Branch 2 taken 180 times.
 360 if (tmp_vals.size() != expected_size)
2823 180 return false;
2824
2825 // Check if tmp_vals contains the number 1..n
2826
1/2
✓ Branch 3 taken 180 times.
✗ Branch 4 not taken.
 180 set<uint64_t> const coverage(tmp_vals.begin(), tmp_vals.end());
2827
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 180 times.
 180 if (coverage.size() != tmp_vals.size())
2828 ✗ return false;
2829
5/6
✓ Branch 2 taken 144 times.
✓ Branch 3 taken 36 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 144 times.
✓ Branch 9 taken 36 times.
✓ Branch 10 taken 144 times.
 180 if ((*coverage.begin() != 1) || (*coverage.rbegin() != coverage.size()))
2830 36 return false;
2831
2832
2/2
✓ Branch 0 taken 324 times.
✓ Branch 1 taken 144 times.
 468 for (unsigned i = 0; i < expected_size; ++i) {
2833 324 (*reply_vals)[i] = tmp_vals[i] - 1;
2834 }
2835 144 return true;
2836 468 }
2837
2838
2839 /**
2840 * Removes DIRECT from a list of ';' and '|' separated proxies.
2841 * \return true if DIRECT was present, false otherwise
2842 */
2843 2112 bool DownloadManager::StripDirect(const string &proxy_list,
2844 string *cleaned_list) {
2845
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2112 times.
 2112 assert(cleaned_list);
2846
2/2
✓ Branch 1 taken 1788 times.
✓ Branch 2 taken 324 times.
 2112 if (proxy_list == "") {
2847
1/2
✓ Branch 1 taken 1788 times.
✗ Branch 2 not taken.
 1788 *cleaned_list = "";
2848 1788 return false;
2849 }
2850 324 bool result = false;
2851
2852
1/2
✓ Branch 1 taken 324 times.
✗ Branch 2 not taken.
 324 vector<string> proxy_groups = SplitString(proxy_list, ';');
2853 324 vector<string> cleaned_groups;
2854
2/2
✓ Branch 1 taken 792 times.
✓ Branch 2 taken 324 times.
 1116 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2855
1/2
✓ Branch 2 taken 792 times.
✗ Branch 3 not taken.
 792 vector<string> group = SplitString(proxy_groups[i], '|');
2856 792 vector<string> cleaned;
2857
2/2
✓ Branch 1 taken 1332 times.
✓ Branch 2 taken 792 times.
 2124 for (unsigned j = 0; j < group.size(); ++j) {
2858
6/6
✓ Branch 2 taken 972 times.
✓ Branch 3 taken 360 times.
✓ Branch 6 taken 468 times.
✓ Branch 7 taken 504 times.
✓ Branch 8 taken 828 times.
✓ Branch 9 taken 504 times.
 1332 if ((group[j] == "DIRECT") || (group[j] == "")) {
2859 828 result = true;
2860 } else {
2861
1/2
✓ Branch 2 taken 504 times.
✗ Branch 3 not taken.
 504 cleaned.push_back(group[j]);
2862 }
2863 }
2864
2/2
✓ Branch 1 taken 252 times.
✓ Branch 2 taken 540 times.
 792 if (!cleaned.empty())
2865
3/6
✓ Branch 2 taken 252 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 252 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 252 times.
✗ Branch 9 not taken.
 252 cleaned_groups.push_back(JoinStrings(cleaned, "|"));
2866 792 }
2867
2868
2/4
✓ Branch 2 taken 324 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 324 times.
✗ Branch 6 not taken.
 324 *cleaned_list = JoinStrings(cleaned_groups, ";");
2869 324 return result;
2870 324 }
2871
2872
2873 /**
2874 * Parses a list of ';'- and '|'-separated proxy servers and fallback proxy
2875 * servers for the proxy groups.
2876 * The empty string for both removes the proxy chain.
2877 * The set_mode parameter can be used to set either proxies (leaving fallback
2878 * proxies unchanged) or fallback proxies (leaving regular proxies unchanged)
2879 * or both.
2880 */
2881 1752 void DownloadManager::SetProxyChain(const string &proxy_list,
2882 const string &fallback_proxy_list,
2883 const ProxySetModes set_mode) {
2884 1752 const MutexLockGuard m(lock_options_);
2885
2886 1752 opt_timestamp_backup_proxies_ = 0;
2887 1752 opt_timestamp_failover_proxies_ = 0;
2888
1/2
✓ Branch 1 taken 1752 times.
✗ Branch 2 not taken.
 1752 string set_proxy_list = opt_proxy_list_;
2889
1/2
✓ Branch 1 taken 1752 times.
✗ Branch 2 not taken.
 1752 string set_proxy_fallback_list = opt_proxy_fallback_list_;
2890 bool contains_direct;
2891
3/4
✓ Branch 0 taken 1752 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1644 times.
✓ Branch 3 taken 108 times.
 1752 if ((set_mode == kSetProxyFallback) || (set_mode == kSetProxyBoth)) {
2892
1/2
✓ Branch 1 taken 1644 times.
✗ Branch 2 not taken.
 1644 opt_proxy_fallback_list_ = fallback_proxy_list;
2893 }
2894
3/4
✓ Branch 0 taken 1644 times.
✓ Branch 1 taken 108 times.
✓ Branch 2 taken 1644 times.
✗ Branch 3 not taken.
 1752 if ((set_mode == kSetProxyRegular) || (set_mode == kSetProxyBoth)) {
2895
1/2
✓ Branch 1 taken 1752 times.
✗ Branch 2 not taken.
 1752 opt_proxy_list_ = proxy_list;
2896 }
2897
1/2
✓ Branch 1 taken 1752 times.
✗ Branch 2 not taken.
 1752 contains_direct = StripDirect(opt_proxy_fallback_list_,
2898 &set_proxy_fallback_list);
2899
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1752 times.
 1752 if (contains_direct) {
2900 ✗ LogCvmfs(kLogDownload, kLogSyslogWarn | kLogDebug,
2901 "(manager '%s') fallback proxies do not support DIRECT, removing",
2902 name_.c_str());
2903 }
2904
1/2
✓ Branch 1 taken 1752 times.
✗ Branch 2 not taken.
 1752 if (set_proxy_fallback_list == "") {
2905
1/2
✓ Branch 1 taken 1752 times.
✗ Branch 2 not taken.
 1752 set_proxy_list = opt_proxy_list_;
2906 } else {
2907 ✗ const bool contains_direct = StripDirect(opt_proxy_list_, &set_proxy_list);
2908 ✗ if (contains_direct) {
2909 ✗ LogCvmfs(kLogDownload, kLogSyslog | kLogDebug,
2910 "(manager '%s') skipping DIRECT proxy to use fallback proxy",
2911 name_.c_str());
2912 }
2913 }
2914
2915 // From this point on, use set_proxy_list and set_fallback_proxy_list as
2916 // effective proxy lists!
2917
2918 1752 opt_proxy_map_.clear();
2919
2/2
✓ Branch 0 taken 822 times.
✓ Branch 1 taken 930 times.
 1752 delete opt_proxy_groups_;
2920
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 1752 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 1752 times.
 1752 if ((set_proxy_list == "") && (set_proxy_fallback_list == "")) {
2921 ✗ opt_proxy_groups_ = NULL;
2922 ✗ opt_proxy_groups_current_ = 0;
2923 ✗ opt_proxy_groups_current_burned_ = 0;
2924 ✗ opt_proxy_groups_fallback_ = 0;
2925 ✗ opt_num_proxies_ = 0;
2926 ✗ return;
2927 }
2928
2929 // Determine number of regular proxy groups (== first fallback proxy group)
2930 1752 opt_proxy_groups_fallback_ = 0;
2931
1/2
✓ Branch 1 taken 1752 times.
✗ Branch 2 not taken.
 1752 if (set_proxy_list != "") {
2932
1/2
✓ Branch 1 taken 1752 times.
✗ Branch 2 not taken.
 1752 opt_proxy_groups_fallback_ = SplitString(set_proxy_list, ';').size();
2933 }
2934
1/2
✓ Branch 2 taken 1752 times.
✗ Branch 3 not taken.
 1752 LogCvmfs(kLogDownload, kLogDebug,
2935 "(manager '%s') "
2936 "first fallback proxy group %u",
2937 name_.c_str(), opt_proxy_groups_fallback_);
2938
2939 // Concatenate regular proxies and fallback proxies, both of which can be
2940 // empty.
2941
1/2
✓ Branch 1 taken 1752 times.
✗ Branch 2 not taken.
 1752 string all_proxy_list = set_proxy_list;
2942
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1752 times.
 1752 if (set_proxy_fallback_list != "") {
2943 ✗ if (all_proxy_list != "")
2944 ✗ all_proxy_list += ";";
2945 ✗ all_proxy_list += set_proxy_fallback_list;
2946 }
2947
1/2
✓ Branch 3 taken 1752 times.
✗ Branch 4 not taken.
 1752 LogCvmfs(kLogDownload, kLogDebug, "(manager '%s') full proxy list %s",
2948 name_.c_str(), all_proxy_list.c_str());
2949
2950 // Resolve server names in provided urls
2951 1752 vector<string> hostnames; // All encountered hostnames
2952 1752 vector<string> proxy_groups;
2953
1/2
✓ Branch 1 taken 1752 times.
✗ Branch 2 not taken.
 1752 if (all_proxy_list != "")
2954
1/2
✓ Branch 1 taken 1752 times.
✗ Branch 2 not taken.
 1752 proxy_groups = SplitString(all_proxy_list, ';');
2955
2/2
✓ Branch 1 taken 1788 times.
✓ Branch 2 taken 1752 times.
 3540 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2956
1/2
✓ Branch 2 taken 1788 times.
✗ Branch 3 not taken.
 1788 vector<string> this_group = SplitString(proxy_groups[i], '|');
2957
2/2
✓ Branch 1 taken 1788 times.
✓ Branch 2 taken 1788 times.
 3576 for (unsigned j = 0; j < this_group.size(); ++j) {
2958
1/2
✓ Branch 2 taken 1788 times.
✗ Branch 3 not taken.
 1788 this_group[j] = dns::AddDefaultScheme(this_group[j]);
2959 // Note: DIRECT strings will be "extracted" to an empty string.
2960
1/2
✓ Branch 2 taken 1788 times.
✗ Branch 3 not taken.
 1788 const string hostname = dns::ExtractHost(this_group[j]);
2961 // Save the hostname. Leave empty (DIRECT) names so indexes will
2962 // match later.
2963
1/2
✓ Branch 1 taken 1788 times.
✗ Branch 2 not taken.
 1788 hostnames.push_back(hostname);
2964 1788 }
2965 1788 }
2966 1752 vector<dns::Host> hosts;
2967
1/2
✓ Branch 3 taken 1752 times.
✗ Branch 4 not taken.
 1752 LogCvmfs(kLogDownload, kLogDebug,
2968 "(manager '%s') "
2969 "resolving %lu proxy addresses",
2970 name_.c_str(), hostnames.size());
2971
1/2
✓ Branch 1 taken 1752 times.
✗ Branch 2 not taken.
 1752 resolver_->ResolveMany(hostnames, &hosts);
2972
2973 // Construct opt_proxy_groups_: traverse proxy list in same order and expand
2974 // names to resolved IP addresses.
2975
1/2
✓ Branch 1 taken 1752 times.
✗ Branch 2 not taken.
 1752 opt_proxy_groups_ = new vector<vector<ProxyInfo> >();
2976 1752 opt_num_proxies_ = 0;
2977 1752 unsigned num_proxy = 0; // Combined i, j counter
2978
2/2
✓ Branch 1 taken 1788 times.
✓ Branch 2 taken 1752 times.
 3540 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2979
1/2
✓ Branch 2 taken 1788 times.
✗ Branch 3 not taken.
 1788 vector<string> this_group = SplitString(proxy_groups[i], '|');
2980 // Construct ProxyInfo objects from proxy string and DNS resolver result for
2981 // every proxy in this_group. One URL can result in multiple ProxyInfo
2982 // objects, one for each IP address.
2983 1788 vector<ProxyInfo> infos;
2984
2/2
✓ Branch 1 taken 1788 times.
✓ Branch 2 taken 1788 times.
 3576 for (unsigned j = 0; j < this_group.size(); ++j, ++num_proxy) {
2985
1/2
✓ Branch 2 taken 1788 times.
✗ Branch 3 not taken.
 1788 this_group[j] = dns::AddDefaultScheme(this_group[j]);
2986
2/2
✓ Branch 2 taken 1644 times.
✓ Branch 3 taken 144 times.
 1788 if (this_group[j] == "DIRECT") {
2987
3/6
✓ Branch 2 taken 1644 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1644 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1644 times.
✗ Branch 9 not taken.
 1644 infos.push_back(ProxyInfo("DIRECT"));
2988 1644 continue;
2989 }
2990
2991
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 144 times.
 144 if (hosts[num_proxy].status() != dns::kFailOk) {
2992 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2993 "(manager '%s') "
2994 "failed to resolve IP addresses for %s (%d - %s)",
2995 ✗ name_.c_str(), hosts[num_proxy].name().c_str(),
2996 ✗ hosts[num_proxy].status(),
2997 ✗ dns::Code2Ascii(hosts[num_proxy].status()));
2998 const dns::Host failed_host = dns::Host::ExtendDeadline(
2999 ✗ hosts[num_proxy], resolver_->min_ttl());
3000 ✗ infos.push_back(ProxyInfo(failed_host, this_group[j]));
3001 ✗ continue;
3002 }
3003
3004 // IPv4 addresses have precedence
3005 144 set<string> const best_addresses = hosts[num_proxy].ViewBestAddresses(
3006
2/4
✓ Branch 1 taken 144 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 144 times.
✗ Branch 5 not taken.
 144 opt_ip_preference_);
3007 144 set<string>::const_iterator iter_ips = best_addresses.begin();
3008
2/2
✓ Branch 3 taken 144 times.
✓ Branch 4 taken 144 times.
 288 for (; iter_ips != best_addresses.end(); ++iter_ips) {
3009
1/2
✓ Branch 3 taken 144 times.
✗ Branch 4 not taken.
 144 const string url_ip = dns::RewriteUrl(this_group[j], *iter_ips);
3010
2/4
✓ Branch 2 taken 144 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 144 times.
✗ Branch 6 not taken.
 144 infos.push_back(ProxyInfo(hosts[num_proxy], url_ip));
3011
3012
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 144 times.
 144 if (sharding_policy_.UseCount() > 0) {
3013 ✗ sharding_policy_->AddProxy(url_ip);
3014 }
3015 144 }
3016 144 }
3017
1/2
✓ Branch 1 taken 1788 times.
✗ Branch 2 not taken.
 1788 opt_proxy_groups_->push_back(infos);
3018 1788 opt_num_proxies_ += infos.size();
3019 1788 }
3020
1/2
✓ Branch 2 taken 1752 times.
✗ Branch 3 not taken.
 1752 LogCvmfs(kLogDownload, kLogDebug,
3021 "(manager '%s') installed %u proxies in %lu load-balance groups",
3022 1752 name_.c_str(), opt_num_proxies_, opt_proxy_groups_->size());
3023 1752 opt_proxy_groups_current_ = 0;
3024 1752 opt_proxy_groups_current_burned_ = 0;
3025
3026 // Select random start proxy from the first group.
3027
1/2
✓ Branch 1 taken 1752 times.
✗ Branch 2 not taken.
 1752 if (opt_proxy_groups_->size() > 0) {
3028 // Select random start proxy from the first group.
3029
2/4
✓ Branch 2 taken 1752 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1752 times.
✗ Branch 6 not taken.
 1752 UpdateProxiesUnlocked("set random start proxy from the first proxy group");
3030 }
3031
3/6
✓ Branch 5 taken 1752 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1752 times.
✗ Branch 9 not taken.
✓ Branch 11 taken 1752 times.
✗ Branch 12 not taken.
 1752 }
3032
3033
3034 /**
3035 * Retrieves the proxy chain, optionally the currently active load-balancing
3036 * group, and optionally the index of the first fallback proxy group.
3037 * If there are no fallback proxies, the index will equal the size of
3038 * the proxy chain.
3039 */
3040 ✗ void DownloadManager::GetProxyInfo(vector<vector<ProxyInfo> > *proxy_chain,
3041 unsigned *current_group,
3042 unsigned *fallback_group) {
3043 ✗ assert(proxy_chain != NULL);
3044 ✗ const MutexLockGuard m(lock_options_);
3045
3046 ✗ if (!opt_proxy_groups_) {
3047 ✗ const vector<vector<ProxyInfo> > empty_chain;
3048 ✗ *proxy_chain = empty_chain;
3049 ✗ if (current_group != NULL)
3050 ✗ *current_group = 0;
3051 ✗ if (fallback_group != NULL)
3052 ✗ *fallback_group = 0;
3053 ✗ return;
3054 }
3055
3056 ✗ *proxy_chain = *opt_proxy_groups_;
3057 ✗ if (current_group != NULL)
3058 ✗ *current_group = opt_proxy_groups_current_;
3059 ✗ if (fallback_group != NULL)
3060 ✗ *fallback_group = opt_proxy_groups_fallback_;
3061 }
3062
3063 ✗ string DownloadManager::GetProxyList() { return opt_proxy_list_; }
3064
3065 ✗ string DownloadManager::GetFallbackProxyList() {
3066 ✗ return opt_proxy_fallback_list_;
3067 }
3068
3069 /**
3070 * Choose proxy
3071 */
3072 5828 DownloadManager::ProxyInfo *DownloadManager::ChooseProxyUnlocked(
3073 const shash::Any *hash) {
3074
2/2
✓ Branch 0 taken 3797 times.
✓ Branch 1 taken 2031 times.
 5828 if (!opt_proxy_groups_)
3075 3797 return NULL;
3076
3077
2/2
✓ Branch 0 taken 907 times.
✓ Branch 1 taken 1124 times.
 2031 const uint32_t key = (hash ? hash->Partial32() : 0);
3078
1/2
✓ Branch 1 taken 2031 times.
✗ Branch 2 not taken.
 2031 const map<uint32_t, ProxyInfo *>::iterator it = opt_proxy_map_.lower_bound(
3079 key);
3080 2031 ProxyInfo *proxy = it->second;
3081
3082 2031 return proxy;
3083 }
3084
3085 /**
3086 * Update currently selected proxy
3087 */
3088 2610 void DownloadManager::UpdateProxiesUnlocked(const string &reason) {
3089
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2610 times.
 2610 if (!opt_proxy_groups_)
3090 ✗ return;
3091
3092 // Identify number of non-burned proxies within the current group
3093 2610 vector<ProxyInfo> *group = current_proxy_group();
3094 2610 const unsigned num_alive = (group->size() - opt_proxy_groups_current_burned_);
3095
2/4
✓ Branch 2 taken 2610 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2610 times.
✗ Branch 6 not taken.
 5220 const string old_proxy = JoinStrings(opt_proxies_, "|");
3096
3097 // Rebuild proxy map and URL list
3098 2610 opt_proxy_map_.clear();
3099 2610 opt_proxies_.clear();
3100 2610 const uint32_t max_key = 0xffffffffUL;
3101
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2610 times.
 2610 if (opt_proxy_shard_) {
3102 // Build a consistent map with multiple entries for each proxy
3103 ✗ for (unsigned i = 0; i < num_alive; ++i) {
3104 ✗ ProxyInfo *proxy = &(*group)[i];
3105 ✗ shash::Any proxy_hash(shash::kSha1);
3106 ✗ HashString(proxy->url, &proxy_hash);
3107 ✗ Prng prng;
3108 ✗ prng.InitSeed(proxy_hash.Partial32());
3109 ✗ for (unsigned j = 0; j < kProxyMapScale; ++j) {
3110 ✗ const std::pair<uint32_t, ProxyInfo *> entry(prng.Next(max_key), proxy);
3111 ✗ opt_proxy_map_.insert(entry);
3112 }
3113 ✗ const std::string proxy_name = proxy->host.name().empty()
3114 ? ""
3115 ✗ : " (" + proxy->host.name() + ")";
3116 ✗ opt_proxies_.push_back(proxy->url + proxy_name);
3117 }
3118 // Ensure lower_bound() finds a value for all keys
3119 ✗ ProxyInfo *first_proxy = opt_proxy_map_.begin()->second;
3120 ✗ const std::pair<uint32_t, ProxyInfo *> last_entry(max_key, first_proxy);
3121 ✗ opt_proxy_map_.insert(last_entry);
3122 } else {
3123 // Build a map with a single entry for one randomly selected proxy
3124 2610 const unsigned select = prng_.Next(num_alive);
3125 2610 ProxyInfo *proxy = &(*group)[select];
3126 2610 const std::pair<uint32_t, ProxyInfo *> entry(max_key, proxy);
3127
1/2
✓ Branch 1 taken 2610 times.
✗ Branch 2 not taken.
 2610 opt_proxy_map_.insert(entry);
3128 2610 const std::string proxy_name = proxy->host.name().empty()
3129 ? ""
3130
9/18
✓ Branch 0 taken 2466 times.
✓ Branch 1 taken 144 times.
✓ Branch 4 taken 2466 times.
✗ Branch 5 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 144 times.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✓ Branch 11 taken 144 times.
✗ Branch 12 not taken.
✓ Branch 13 taken 144 times.
✓ Branch 14 taken 2466 times.
✓ Branch 15 taken 2466 times.
✓ Branch 16 taken 144 times.
✗ Branch 17 not taken.
✗ Branch 18 not taken.
✗ Branch 19 not taken.
✗ Branch 20 not taken.
 2754 : " (" + proxy->host.name() + ")";
3131
2/4
✓ Branch 1 taken 2610 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2610 times.
✗ Branch 5 not taken.
 2610 opt_proxies_.push_back(proxy->url + proxy_name);
3132 2610 }
3133
1/2
✓ Branch 3 taken 2610 times.
✗ Branch 4 not taken.
 2610 sort(opt_proxies_.begin(), opt_proxies_.end());
3134
3135 // Report any change in proxy usage
3136
2/4
✓ Branch 2 taken 2610 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2610 times.
✗ Branch 6 not taken.
 5220 const string new_proxy = JoinStrings(opt_proxies_, "|");
3137 const string curr_host = "Current host: "
3138 2610 + (opt_host_.chain
3139
6/12
✓ Branch 0 taken 2466 times.
✓ Branch 1 taken 144 times.
✓ Branch 4 taken 2466 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 144 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 144 times.
✓ Branch 11 taken 2466 times.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
 5220 ? (*opt_host_.chain)[opt_host_.current]
3140
1/2
✓ Branch 1 taken 2610 times.
✗ Branch 2 not taken.
 2610 : "");
3141
2/2
✓ Branch 1 taken 1788 times.
✓ Branch 2 taken 822 times.
 2610 if (new_proxy != old_proxy) {
3142
4/9
✗ Branch 2 not taken.
✓ Branch 3 taken 1788 times.
✓ Branch 4 taken 1752 times.
✓ Branch 5 taken 36 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 1788 times.
✗ Branch 9 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
 5400 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
3143 "(manager '%s') switching proxy from %s to %s. Reason: %s [%s]",
3144 1824 name_.c_str(), (old_proxy.empty() ? "(none)" : old_proxy.c_str()),
3145 3576 (new_proxy.empty() ? "(none)" : new_proxy.c_str()), reason.c_str(),
3146 curr_host.c_str());
3147 }
3148 2610 }
3149
3150 /**
3151 * Enable proxy sharding
3152 */
3153 ✗ void DownloadManager::ShardProxies() {
3154 ✗ opt_proxy_shard_ = true;
3155 ✗ RebalanceProxiesUnlocked("enable sharding");
3156 }
3157
3158 /**
3159 * Selects a new random proxy in the current load-balancing group. Resets the
3160 * "burned" counter.
3161 */
3162 ✗ void DownloadManager::RebalanceProxiesUnlocked(const string &reason) {
3163 ✗ if (!opt_proxy_groups_)
3164 ✗ return;
3165
3166 ✗ opt_timestamp_failover_proxies_ = 0;
3167 ✗ opt_proxy_groups_current_burned_ = 0;
3168 ✗ UpdateProxiesUnlocked(reason);
3169 }
3170
3171
3172 ✗ void DownloadManager::RebalanceProxies() {
3173 ✗ const MutexLockGuard m(lock_options_);
3174 ✗ RebalanceProxiesUnlocked("rebalance invoked manually");
3175 }
3176
3177
3178 /**
3179 * Switches to the next load-balancing group of proxy servers.
3180 */
3181 ✗ void DownloadManager::SwitchProxyGroup() {
3182 ✗ const MutexLockGuard m(lock_options_);
3183
3184 ✗ if (!opt_proxy_groups_ || (opt_proxy_groups_->size() < 2)) {
3185 ✗ return;
3186 }
3187
3188 ✗ opt_proxy_groups_current_ = (opt_proxy_groups_current_ + 1)
3189 ✗ % opt_proxy_groups_->size();
3190 ✗ opt_timestamp_backup_proxies_ = time(NULL);
3191
3192 const std::string msg = "switch to proxy group "
3193 ✗ + StringifyUint(opt_proxy_groups_current_);
3194 ✗ RebalanceProxiesUnlocked(msg);
3195 }
3196
3197
3198 ✗ void DownloadManager::SetProxyGroupResetDelay(const unsigned seconds) {
3199 ✗ const MutexLockGuard m(lock_options_);
3200 ✗ opt_proxy_groups_reset_after_ = seconds;
3201 ✗ if (opt_proxy_groups_reset_after_ == 0) {
3202 ✗ opt_timestamp_backup_proxies_ = 0;
3203 ✗ opt_timestamp_failover_proxies_ = 0;
3204 }
3205 }
3206
3207
3208 ✗ void DownloadManager::SetMetalinkResetDelay(const unsigned seconds) {
3209 ✗ const MutexLockGuard m(lock_options_);
3210 ✗ opt_metalink_.reset_after = seconds;
3211 ✗ if (opt_metalink_.reset_after == 0)
3212 ✗ opt_metalink_.timestamp_backup = 0;
3213 }
3214
3215
3216 ✗ void DownloadManager::SetHostResetDelay(const unsigned seconds) {
3217 ✗ const MutexLockGuard m(lock_options_);
3218 ✗ opt_host_.reset_after = seconds;
3219 ✗ if (opt_host_.reset_after == 0)
3220 ✗ opt_host_.timestamp_backup = 0;
3221 }
3222
3223
3224 2051 void DownloadManager::SetRetryParameters(const unsigned max_retries,
3225 const unsigned backoff_init_ms,
3226 const unsigned backoff_max_ms) {
3227 2051 const MutexLockGuard m(lock_options_);
3228 2051 opt_max_retries_ = max_retries;
3229 2051 opt_backoff_init_ms_ = backoff_init_ms;
3230 2051 opt_backoff_max_ms_ = backoff_max_ms;
3231 2051 }
3232
3233
3234 894 void DownloadManager::SetMaxIpaddrPerProxy(unsigned limit) {
3235 894 const MutexLockGuard m(lock_options_);
3236 894 resolver_->set_throttle(limit);
3237 894 }
3238
3239
3240 1411 void DownloadManager::SetProxyTemplates(const std::string &direct,
3241 const std::string &forced) {
3242 1411 const MutexLockGuard m(lock_options_);
3243
1/2
✓ Branch 1 taken 1411 times.
✗ Branch 2 not taken.
 1411 proxy_template_direct_ = direct;
3244
1/2
✓ Branch 1 taken 1411 times.
✗ Branch 2 not taken.
 1411 proxy_template_forced_ = forced;
3245 1411 }
3246
3247
3248 ✗ void DownloadManager::EnableInfoHeader() { enable_info_header_ = true; }
3249
3250
3251 748 void DownloadManager::EnableRedirects() { follow_redirects_ = true; }
3252
3253 ✗ void DownloadManager::EnableIgnoreSignatureFailures() {
3254 ✗ ignore_signature_failures_ = true;
3255 }
3256
3257 ✗ void DownloadManager::EnableHTTPTracing() { enable_http_tracing_ = true; }
3258
3259 ✗ void DownloadManager::AddHTTPTracingHeader(const std::string &header) {
3260 ✗ http_tracing_headers_.push_back(header);
3261 }
3262
3263 640 void DownloadManager::UseSystemCertificatePath() {
3264 640 ssl_certificate_store_.UseSystemCertificatePath();
3265 640 }
3266
3267 ✗ bool DownloadManager::SetShardingPolicy(const ShardingPolicySelector type) {
3268 ✗ const bool success = false;
3269 ✗ switch (type) {
3270 default:
3271 ✗ LogCvmfs(
3272 kLogDownload, kLogDebug | kLogSyslogErr,
3273 "(manager '%s') "
3274 "Proposed sharding policy does not exist. Falling back to default",
3275 name_.c_str());
3276 }
3277 ✗ return success;
3278 }
3279
3280 ✗ void DownloadManager::SetFailoverIndefinitely() {
3281 ✗ failover_indefinitely_ = true;
3282 }
3283
3284 /**
3285 * Creates a copy of the existing download manager. Must only be called in
3286 * single-threaded stage because it calls curl_global_init().
3287 */
3288 894 DownloadManager *DownloadManager::Clone(
3289 const perf::StatisticsTemplate &statistics,
3290 const std::string &cloned_name) {
3291 DownloadManager *clone = new DownloadManager(pool_max_handles_, statistics,
3292
1/2
✓ Branch 2 taken 894 times.
✗ Branch 3 not taken.
 894 cloned_name);
3293
3294 894 clone->SetDnsParameters(resolver_->retries(), resolver_->timeout_ms());
3295 894 clone->SetDnsTtlLimits(resolver_->min_ttl(), resolver_->max_ttl());
3296 894 clone->SetMaxIpaddrPerProxy(resolver_->throttle());
3297
3298
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 894 times.
 894 if (!opt_dns_server_.empty())
3299 ✗ clone->SetDnsServer(opt_dns_server_);
3300 894 clone->opt_timeout_proxy_ = opt_timeout_proxy_;
3301 894 clone->opt_timeout_direct_ = opt_timeout_direct_;
3302 894 clone->opt_low_speed_limit_ = opt_low_speed_limit_;
3303 894 clone->opt_max_retries_ = opt_max_retries_;
3304 894 clone->opt_backoff_init_ms_ = opt_backoff_init_ms_;
3305 894 clone->opt_backoff_max_ms_ = opt_backoff_max_ms_;
3306 894 clone->enable_info_header_ = enable_info_header_;
3307 894 clone->enable_http_tracing_ = enable_http_tracing_;
3308 894 clone->http_tracing_headers_ = http_tracing_headers_;
3309 894 clone->follow_redirects_ = follow_redirects_;
3310 894 clone->ignore_signature_failures_ = ignore_signature_failures_;
3311
2/2
✓ Branch 0 taken 822 times.
✓ Branch 1 taken 72 times.
 894 if (opt_host_.chain) {
3312
1/2
✓ Branch 2 taken 822 times.
✗ Branch 3 not taken.
 822 clone->opt_host_.chain = new vector<string>(*opt_host_.chain);
3313
1/2
✓ Branch 2 taken 822 times.
✗ Branch 3 not taken.
 822 clone->opt_host_chain_rtt_ = new vector<int>(*opt_host_chain_rtt_);
3314 }
3315
3316 894 CloneProxyConfig(clone);
3317 894 clone->opt_ip_preference_ = opt_ip_preference_;
3318 894 clone->proxy_template_direct_ = proxy_template_direct_;
3319 894 clone->proxy_template_forced_ = proxy_template_forced_;
3320 894 clone->opt_proxy_groups_reset_after_ = opt_proxy_groups_reset_after_;
3321 894 clone->opt_metalink_.reset_after = opt_metalink_.reset_after;
3322 894 clone->opt_host_.reset_after = opt_host_.reset_after;
3323 894 clone->credentials_attachment_ = credentials_attachment_;
3324 894 clone->ssl_certificate_store_ = ssl_certificate_store_;
3325
3326 894 clone->health_check_ = health_check_;
3327 894 clone->sharding_policy_ = sharding_policy_;
3328 894 clone->failover_indefinitely_ = failover_indefinitely_;
3329 894 clone->fqrn_ = fqrn_;
3330
3331 894 return clone;
3332 }
3333
3334
3335 894 void DownloadManager::CloneProxyConfig(DownloadManager *clone) {
3336 894 clone->opt_proxy_groups_current_ = opt_proxy_groups_current_;
3337 894 clone->opt_proxy_groups_current_burned_ = opt_proxy_groups_current_burned_;
3338 894 clone->opt_proxy_groups_fallback_ = opt_proxy_groups_fallback_;
3339 894 clone->opt_num_proxies_ = opt_num_proxies_;
3340 894 clone->opt_proxy_shard_ = opt_proxy_shard_;
3341 894 clone->opt_proxy_list_ = opt_proxy_list_;
3342 894 clone->opt_proxy_fallback_list_ = opt_proxy_fallback_list_;
3343
2/2
✓ Branch 0 taken 72 times.
✓ Branch 1 taken 822 times.
 894 if (opt_proxy_groups_ == NULL)
3344 72 return;
3345
3346
1/2
✓ Branch 2 taken 822 times.
✗ Branch 3 not taken.
 822 clone->opt_proxy_groups_ = new vector<vector<ProxyInfo> >(*opt_proxy_groups_);
3347
2/4
✓ Branch 2 taken 822 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 822 times.
✗ Branch 6 not taken.
 822 clone->UpdateProxiesUnlocked("cloned");
3348 }
3349
3350 } // namespace download
3351