GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/network/download.cc
Date: 2025-11-02 02:35:35
Exec Total Coverage
Lines: 890 1747 50.9%
Branches: 697 2244 31.1%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * The download module provides an interface for fetching files via HTTP
5 * and file. It is internally using libcurl and the asynchronous DNS resolver
6 * c-ares. The JobInfo struct describes a single file/url to download and
7 * keeps the state during the several phases of downloading.
8 *
9 * The module starts in single-threaded mode and can be switched to multi-
10 * threaded mode by Spawn(). In multi-threaded mode, the Fetch() function still
11 * blocks but there is a separate I/O thread using asynchronous I/O, which
12 * maintains all concurrent connections simultaneously. As there might be more
13 * than 1024 file descriptors for the CernVM-FS process, the I/O thread uses
14 * poll and the libcurl multi socket interface.
15 *
16 * While downloading, files can be decompressed and the secure hash can be
17 * calculated on the fly.
18 *
19 * The module also implements failure handling. If corrupted data has been
20 * downloaded, the transfer is restarted using HTTP "no-cache" pragma.
21 * A "host chain" can be configured. When a host fails, there is automatic
22 * fail-over to the next host in the chain until all hosts are probed.
23 * Similarly a chain of proxy sets can be configured. Inside a proxy set,
24 * proxies are selected randomly (load-balancing set).
25 */
26
27 // TODO(jblomer): MS for time summing
28 // NOLINTNEXTLINE
29 #define __STDC_FORMAT_MACROS
30
31
32 #include "download.h"
33
34 #include <alloca.h>
35 #include <errno.h>
36 #include <inttypes.h>
37 #include <poll.h>
38 #include <pthread.h>
39 #include <signal.h>
40 #include <stdint.h>
41 #include <sys/time.h>
42 #include <unistd.h>
43
44 #include <algorithm>
45 #include <cassert>
46 #include <cstdio>
47 #include <cstdlib>
48 #include <cstring>
49 #include <map>
50 #include <set>
51 #include <utility>
52
53 #include "compression/compression.h"
54 #include "crypto/hash.h"
55 #include "duplex_curl.h"
56 #include "interrupt.h"
57 #include "sanitizer.h"
58 #include "ssl.h"
59 #include "util/algorithm.h"
60 #include "util/atomic.h"
61 #include "util/concurrency.h"
62 #include "util/exception.h"
63 #include "util/logging.h"
64 #include "util/posix.h"
65 #include "util/prng.h"
66 #include "util/smalloc.h"
67 #include "util/string.h"
68
69 using namespace std; // NOLINT
70
71 namespace download {
72
73 /**
74 * Returns the status if an interrupt happened for a given repository.
75 *
76 * Used only in case CVMFS_FAILOVER_INDEFINITELY (failover_indefinitely_) is set
77 * where failed downloads are retried indefinitely, unless an interrupt occurred
78 *
79 * @note If you use this functionality you need to change the source code of
80 * e.g. cvmfs_config reload to create a sentinel file. See comment below.
81 *
82 * @return true if an interrupt occurred
83 * false otherwise
84 */
85 ✗ bool Interrupted(const std::string &fqrn, JobInfo *info) {
86 ✗ if (info->allow_failure()) {
87 ✗ return true;
88 }
89
90 ✗ if (!fqrn.empty()) {
91 // it is up to the user the create this sentinel file ("pause_file") if
92 // CVMFS_FAILOVER_INDEFINITELY is used. It must be created during
93 // "cvmfs_config reload" and "cvmfs_config reload $fqrn"
94 ✗ const std::string pause_file = std::string("/var/run/cvmfs/interrupt.")
95 ✗ + fqrn;
96
97 ✗ LogCvmfs(kLogDownload, kLogDebug,
98 "(id %" PRId64 ") Interrupted(): checking for existence of %s",
99 info->id(), pause_file.c_str());
100 ✗ if (FileExists(pause_file)) {
101 ✗ LogCvmfs(kLogDownload, kLogDebug,
102 "(id %" PRId64 ") Interrupt marker found - "
103 "Interrupting current download, this will EIO outstanding IO.",
104 info->id());
105 ✗ if (0 != unlink(pause_file.c_str())) {
106 ✗ LogCvmfs(kLogDownload, kLogDebug,
107 "(id %" PRId64 ") Couldn't delete interrupt marker: errno=%d",
108 ✗ info->id(), errno);
109 }
110 ✗ return true;
111 }
112 }
113 ✗ return false;
114 }
115
116 6717 static Failures PrepareDownloadDestination(JobInfo *info) {
117
3/6
✓ Branch 1 taken 6717 times.
✗ Branch 2 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 6717 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 6717 times.
 6717 if (info->sink() != NULL && !info->sink()->IsValid()) {
118 ✗ cvmfs::PathSink *psink = dynamic_cast<cvmfs::PathSink *>(info->sink());
119 ✗ if (psink != NULL) {
120 ✗ LogCvmfs(kLogDownload, kLogDebug,
121 "(id %" PRId64 ") Failed to open path %s: %s (errno=%d).",
122 ✗ info->id(), psink->path().c_str(), strerror(errno), errno);
123 ✗ return kFailLocalIO;
124 } else {
125 ✗ LogCvmfs(kLogDownload, kLogDebug,
126 "(id %" PRId64 ") "
127 "Failed to create a valid sink: \n %s",
128 ✗ info->id(), info->sink()->Describe().c_str());
129 ✗ return kFailOther;
130 }
131 }
132
133 6717 return kFailOk;
134 }
135
136
137 /**
138 * Called by curl for every HTTP header. Not called for file:// transfers.
139 */
140 20120 static size_t CallbackCurlHeader(void *ptr, size_t size, size_t nmemb,
141 void *info_link) {
142 20120 const size_t num_bytes = size * nmemb;
143
1/2
✓ Branch 2 taken 20120 times.
✗ Branch 3 not taken.
 20120 const string header_line(static_cast<const char *>(ptr), num_bytes);
144 20120 JobInfo *info = static_cast<JobInfo *>(info_link);
145
146 // LogCvmfs(kLogDownload, kLogDebug, "REMOVE-ME: Header callback with %s",
147 // header_line.c_str());
148
149 // Check http status codes
150
4/6
✓ Branch 2 taken 20120 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 20120 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 588 times.
✓ Branch 10 taken 19532 times.
 20120 if (HasPrefix(header_line, "HTTP/1.", false)) {
151
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 588 times.
 588 if (header_line.length() < 10) {
152 ✗ return 0;
153 }
154
155 unsigned i;
156
5/6
✓ Branch 1 taken 1176 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 588 times.
✓ Branch 5 taken 588 times.
✓ Branch 6 taken 588 times.
✓ Branch 7 taken 588 times.
 1176 for (i = 8; (i < header_line.length()) && (header_line[i] == ' '); ++i) {
157 }
158
159 // Code is initialized to -1
160
1/2
✓ Branch 1 taken 588 times.
✗ Branch 2 not taken.
 588 if (header_line.length() > i + 2) {
161 588 info->SetHttpCode(DownloadManager::ParseHttpCode(&header_line[i]));
162 }
163
164
2/2
✓ Branch 1 taken 441 times.
✓ Branch 2 taken 147 times.
 588 if ((info->http_code() / 100) == 2) {
165 441 return num_bytes;
166
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 147 } else if ((info->http_code() == 301) || (info->http_code() == 302)
167
2/8
✗ Branch 0 not taken.
✓ Branch 1 taken 147 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 147 times.
✗ Branch 9 not taken.
 147 || (info->http_code() == 303) || (info->http_code() == 307)) {
168
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 147 times.
 147 if (!info->follow_redirects()) {
169 ✗ LogCvmfs(kLogDownload, kLogDebug,
170 "(id %" PRId64 ") redirect support not enabled: %s",
171 info->id(), header_line.c_str());
172 ✗ info->SetErrorCode(kFailHostHttp);
173 ✗ return 0;
174 }
175
1/2
✓ Branch 3 taken 147 times.
✗ Branch 4 not taken.
 147 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") http redirect: %s",
176 info->id(), header_line.c_str());
177 // libcurl will handle this because of CURLOPT_FOLLOWLOCATION
178 147 return num_bytes;
179 } else {
180 ✗ LogCvmfs(kLogDownload, kLogDebug,
181 "(id %" PRId64 ") http status error code: %s [%d]", info->id(),
182 header_line.c_str(), info->http_code());
183 ✗ if (((info->http_code() / 100) == 5) || (info->http_code() == 400)
184 ✗ || (info->http_code() == 404)) {
185 // 5XX returned by host
186 // 400: error from the GeoAPI module
187 // 404: the stratum 1 does not have the newest files
188 ✗ info->SetErrorCode(kFailHostHttp);
189 ✗ } else if (info->http_code() == 429) {
190 // 429: rate throttling (we ignore the backoff hint for the time being)
191 ✗ info->SetErrorCode(kFailHostConnection);
192 } else {
193 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostHttp
194 : kFailProxyHttp);
195 }
196 ✗ return 0;
197 }
198 }
199
200 // If needed: allocate space in sink
201
3/4
✓ Branch 2 taken 19532 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 7625 times.
✓ Branch 5 taken 11907 times.
 19532 if (info->sink() != NULL && info->sink()->RequiresReserve()
202
11/20
✓ Branch 1 taken 19532 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 7625 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 7625 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 2362 times.
✓ Branch 11 taken 5263 times.
✓ Branch 12 taken 7625 times.
✓ Branch 13 taken 11907 times.
✗ Branch 14 not taken.
✓ Branch 15 taken 7625 times.
✓ Branch 16 taken 11907 times.
✓ Branch 18 taken 2362 times.
✓ Branch 19 taken 17170 times.
✗ Branch 20 not taken.
✗ Branch 21 not taken.
✗ Branch 23 not taken.
✗ Branch 24 not taken.
 39064 && HasPrefix(header_line, "CONTENT-LENGTH:", true)) {
203 2362 char *tmp = reinterpret_cast<char *>(alloca(num_bytes + 1));
204 2362 uint64_t length = 0;
205 2362 sscanf(header_line.c_str(), "%s %" PRIu64, tmp, &length);
206
1/2
✓ Branch 0 taken 2362 times.
✗ Branch 1 not taken.
 2362 if (length > 0) {
207
2/4
✓ Branch 2 taken 2362 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 2362 times.
 2362 if (!info->sink()->Reserve(length)) {
208 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
209 "(id %" PRId64 ") "
210 "resource %s too large to store in memory (%" PRIu64 ")",
211 info->id(), info->url()->c_str(), length);
212 ✗ info->SetErrorCode(kFailTooBig);
213 ✗ return 0;
214 }
215 } else {
216 // Empty resource
217 ✗ info->sink()->Reserve(0);
218 }
219
4/6
✓ Branch 2 taken 17170 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 17170 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 147 times.
✓ Branch 10 taken 17023 times.
 17170 } else if (HasPrefix(header_line, "LOCATION:", true)) {
220 // This comes along with redirects
221
1/2
✓ Branch 3 taken 147 times.
✗ Branch 4 not taken.
 147 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") %s", info->id(),
222 header_line.c_str());
223
3/6
✓ Branch 2 taken 17023 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 17023 times.
✗ Branch 6 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 17023 times.
 17023 } else if (HasPrefix(header_line, "LINK:", true)) {
224 // This is metalink info
225 ✗ LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") %s", info->id(),
226 header_line.c_str());
227 ✗ std::string link = info->link();
228 ✗ if (link.size() != 0) {
229 // multiple LINK headers are allowed
230 ✗ link = link + ", " + header_line.substr(5);
231 } else {
232 ✗ link = header_line.substr(5);
233 }
234 ✗ info->SetLink(link);
235
3/6
✓ Branch 3 taken 17023 times.
✗ Branch 4 not taken.
✓ Branch 6 taken 17023 times.
✗ Branch 7 not taken.
✗ Branch 10 not taken.
✓ Branch 11 taken 17023 times.
 17023 } else if (HasPrefix(header_line, "X-SQUID-ERROR:", true)) {
236 // Reinterpret host error as proxy error
237 ✗ if (info->error_code() == kFailHostHttp) {
238 ✗ info->SetErrorCode(kFailProxyHttp);
239 }
240
3/6
✓ Branch 2 taken 17023 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 17023 times.
✗ Branch 6 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 17023 times.
 17023 } else if (HasPrefix(header_line, "PROXY-STATUS:", true)) {
241 // Reinterpret host error as proxy error if applicable
242 ✗ if ((info->error_code() == kFailHostHttp)
243 ✗ && (header_line.find("error=") != string::npos)) {
244 ✗ info->SetErrorCode(kFailProxyHttp);
245 }
246 }
247
248 19532 return num_bytes;
249 20120 }
250
251
252 /**
253 * Called by curl for every received data chunk.
254 */
255 8179 static size_t CallbackCurlData(void *ptr, size_t size, size_t nmemb,
256 void *info_link) {
257 8179 const size_t num_bytes = size * nmemb;
258 8179 JobInfo *info = static_cast<JobInfo *>(info_link);
259
260 // TODO(heretherebedragons) remove if no error comes up
261 // as this means only jobinfo data request (and not header only)
262 // come here
263
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 8179 times.
 8179 assert(info->sink() != NULL);
264
265 // LogCvmfs(kLogDownload, kLogDebug, "Data callback, %d bytes", num_bytes);
266
267
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 8179 times.
 8179 if (num_bytes == 0)
268 ✗ return 0;
269
270
2/2
✓ Branch 1 taken 4331 times.
✓ Branch 2 taken 3848 times.
 8179 if (info->expected_hash()) {
271 4331 shash::Update(reinterpret_cast<unsigned char *>(ptr), num_bytes,
272 info->hash_context());
273 }
274
275
2/2
✓ Branch 1 taken 4282 times.
✓ Branch 2 taken 3897 times.
 8179 if (info->compressed()) {
276 4282 const zlib::StreamStates retval = zlib::DecompressZStream2Sink(
277 ptr, static_cast<int64_t>(num_bytes), info->GetZstreamPtr(),
278 info->sink());
279
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4282 times.
 4282 if (retval == zlib::kStreamDataError) {
280 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
281 "(id %" PRId64 ") failed to decompress %s", info->id(),
282 info->url()->c_str());
283 ✗ info->SetErrorCode(kFailBadData);
284 ✗ return 0;
285
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4282 times.
 4282 } else if (retval == zlib::kStreamIOError) {
286 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
287 "(id %" PRId64 ") decompressing %s, local IO error", info->id(),
288 info->url()->c_str());
289 ✗ info->SetErrorCode(kFailLocalIO);
290 ✗ return 0;
291 }
292 } else {
293 3897 const int64_t written = info->sink()->Write(ptr, num_bytes);
294
2/4
✓ Branch 0 taken 3897 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 3897 times.
 3897 if (written < 0 || static_cast<uint64_t>(written) != num_bytes) {
295 ✗ LogCvmfs(kLogDownload, kLogDebug,
296 "(id %" PRId64 ") "
297 "Failed to perform write of %zu bytes to sink %s with errno %ld",
298 ✗ info->id(), num_bytes, info->sink()->Describe().c_str(),
299 written);
300 }
301 }
302
303 8179 return num_bytes;
304 }
305
306 #ifdef DEBUGMSG
307 5346 static int CallbackCurlDebug(CURL *handle,
308 curl_infotype type,
309 char *data,
310 size_t size,
311 void * /* clientp */) {
312 JobInfo *info;
313
1/2
✓ Branch 1 taken 5346 times.
✗ Branch 2 not taken.
 5346 curl_easy_getinfo(handle, CURLINFO_PRIVATE, &info);
314
315
3/6
✓ Branch 2 taken 5346 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 5346 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 5346 times.
✗ Branch 9 not taken.
 10692 std::string prefix = "(id " + StringifyInt(info->id()) + ") ";
316
4/8
✓ Branch 0 taken 691 times.
✓ Branch 1 taken 2303 times.
✓ Branch 2 taken 588 times.
✓ Branch 3 taken 1764 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
 5346 switch (type) {
317 691 case CURLINFO_TEXT:
318
1/2
✓ Branch 1 taken 691 times.
✗ Branch 2 not taken.
 691 prefix += "{info} ";
319 691 break;
320 2303 case CURLINFO_HEADER_IN:
321
1/2
✓ Branch 1 taken 2303 times.
✗ Branch 2 not taken.
 2303 prefix += "{header/recv} ";
322 2303 break;
323 588 case CURLINFO_HEADER_OUT:
324
1/2
✓ Branch 1 taken 588 times.
✗ Branch 2 not taken.
 588 prefix += "{header/sent} ";
325 588 break;
326 1764 case CURLINFO_DATA_IN:
327
2/2
✓ Branch 0 taken 196 times.
✓ Branch 1 taken 1568 times.
 1764 if (size < 50) {
328
1/2
✓ Branch 1 taken 196 times.
✗ Branch 2 not taken.
 196 prefix += "{data/recv} ";
329 196 break;
330 } else {
331
1/2
✓ Branch 2 taken 1568 times.
✗ Branch 3 not taken.
 1568 LogCvmfs(kLogCurl, kLogDebug, "%s{data/recv} <snip>", prefix.c_str());
332 1568 return 0;
333 }
334 ✗ case CURLINFO_DATA_OUT:
335 ✗ if (size < 50) {
336 ✗ prefix += "{data/sent} ";
337 ✗ break;
338 } else {
339 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{data/sent} <snip>", prefix.c_str());
340 ✗ return 0;
341 }
342 ✗ case CURLINFO_SSL_DATA_IN:
343 ✗ if (size < 50) {
344 ✗ prefix += "{ssldata/recv} ";
345 ✗ break;
346 } else {
347 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{ssldata/recv} <snip>",
348 prefix.c_str());
349 ✗ return 0;
350 }
351 ✗ case CURLINFO_SSL_DATA_OUT:
352 ✗ if (size < 50) {
353 ✗ prefix += "{ssldata/sent} ";
354 ✗ break;
355 } else {
356 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{ssldata/sent} <snip>",
357 prefix.c_str());
358 ✗ return 0;
359 }
360 ✗ default:
361 // just log the message
362 ✗ break;
363 }
364
365 3778 bool valid_char = true;
366
1/2
✓ Branch 2 taken 3778 times.
✗ Branch 3 not taken.
 3778 std::string msg(data, size);
367
2/2
✓ Branch 1 taken 183256 times.
✓ Branch 2 taken 3778 times.
 187034 for (size_t i = 0; i < msg.length(); ++i) {
368
2/4
✓ Branch 1 taken 183256 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 183256 times.
 183256 if (msg[i] == '\0') {
369 ✗ msg[i] = '~';
370 }
371
372 // verify that char is a valid printable char
373
3/6
✓ Branch 1 taken 183256 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 170511 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 170511 times.
 353767 if ((msg[i] < ' ' || msg[i] > '~')
374
6/8
✓ Branch 0 taken 170511 times.
✓ Branch 1 taken 12745 times.
✓ Branch 3 taken 12745 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 6027 times.
✓ Branch 6 taken 6718 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 183256 times.
 359794 && (msg[i] != 10 /*line feed*/
375
2/4
✓ Branch 1 taken 6027 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 6027 times.
 6027 && msg[i] != 13 /*carriage return*/)) {
376 ✗ valid_char = false;
377 }
378 }
379
380
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3778 times.
 3778 if (!valid_char) {
381 ✗ msg = "<Non-plaintext sequence>";
382 }
383
384
1/2
✓ Branch 3 taken 3778 times.
✗ Branch 4 not taken.
 3778 LogCvmfs(kLogCurl, kLogDebug, "%s%s", prefix.c_str(),
385
1/2
✓ Branch 1 taken 3778 times.
✗ Branch 2 not taken.
 7556 Trim(msg, true /* trim_newline */).c_str());
386 3778 return 0;
387 5346 }
388 #endif
389
390 //------------------------------------------------------------------------------
391
392
393 const int DownloadManager::kProbeUnprobed = -1;
394 const int DownloadManager::kProbeDown = -2;
395 const int DownloadManager::kProbeGeo = -3;
396
397 563168 bool DownloadManager::EscapeUrlChar(unsigned char input, char output[3]) {
398
8/8
✓ Branch 0 taken 502962 times.
✓ Branch 1 taken 60206 times.
✓ Branch 2 taken 390436 times.
✓ Branch 3 taken 112526 times.
✓ Branch 4 taken 382922 times.
✓ Branch 5 taken 67720 times.
✓ Branch 6 taken 350314 times.
✓ Branch 7 taken 32608 times.
 563168 if (((input >= '0') && (input <= '9')) || ((input >= 'A') && (input <= 'Z'))
399
8/8
✓ Branch 0 taken 329899 times.
✓ Branch 1 taken 88135 times.
✓ Branch 2 taken 441 times.
✓ Branch 3 taken 329458 times.
✓ Branch 4 taken 40028 times.
✓ Branch 5 taken 48548 times.
✓ Branch 6 taken 32514 times.
✓ Branch 7 taken 7514 times.
 418034 || ((input >= 'a') && (input <= 'z')) || (input == '/') || (input == ':')
400
6/8
✓ Branch 0 taken 21025 times.
✓ Branch 1 taken 11489 times.
✓ Branch 2 taken 21025 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 21025 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 20856 times.
✓ Branch 7 taken 169 times.
 32514 || (input == '.') || (input == '@') || (input == '+') || (input == '-')
401
5/8
✓ Branch 0 taken 441 times.
✓ Branch 1 taken 20415 times.
✓ Branch 2 taken 441 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 441 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 441 times.
✗ Branch 7 not taken.
 20856 || (input == '_') || (input == '~') || (input == '[') || (input == ']')
402
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 441 times.
 441 || (input == ',')) {
403 562727 output[0] = static_cast<char>(input);
404 562727 return false;
405 }
406
407 441 output[0] = '%';
408 882 output[1] = static_cast<char>((input / 16)
409
2/2
✓ Branch 0 taken 98 times.
✓ Branch 1 taken 343 times.
 441 + ((input / 16 <= 9) ? '0' : 'A' - 10));
410 882 output[2] = static_cast<char>((input % 16)
411
2/2
✓ Branch 0 taken 196 times.
✓ Branch 1 taken 245 times.
 441 + ((input % 16 <= 9) ? '0' : 'A' - 10));
412 441 return true;
413 }
414
415
416 /**
417 * Escape special chars from the URL, except for ':' and '/',
418 * which should keep their meaning.
419 */
420 6913 string DownloadManager::EscapeUrl(const int64_t jobinfo_id, const string &url) {
421 6913 string escaped;
422
1/2
✓ Branch 2 taken 6913 times.
✗ Branch 3 not taken.
 6913 escaped.reserve(url.length());
423
424 char escaped_char[3];
425
2/2
✓ Branch 1 taken 563168 times.
✓ Branch 2 taken 6913 times.
 570081 for (unsigned i = 0, s = url.length(); i < s; ++i) {
426
2/2
✓ Branch 2 taken 441 times.
✓ Branch 3 taken 562727 times.
 563168 if (EscapeUrlChar(url[i], escaped_char)) {
427
1/2
✓ Branch 1 taken 441 times.
✗ Branch 2 not taken.
 441 escaped.append(escaped_char, 3);
428 } else {
429
1/2
✓ Branch 1 taken 562727 times.
✗ Branch 2 not taken.
 562727 escaped.push_back(escaped_char[0]);
430 }
431 }
432
1/2
✓ Branch 3 taken 6913 times.
✗ Branch 4 not taken.
 6913 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") escaped %s to %s",
433 jobinfo_id, url.c_str(), escaped.c_str());
434
435 13826 return escaped;
436 }
437
438 /**
439 * escaped array needs to be sufficiently large. Its size is calculated by
440 * passing NULL to EscapeHeader.
441 */
442 ✗ unsigned DownloadManager::EscapeHeader(const string &header,
443 char *escaped_buf,
444 size_t buf_size) {
445 ✗ unsigned esc_pos = 0;
446 char escaped_char[3];
447 ✗ for (unsigned i = 0, s = header.size(); i < s; ++i) {
448 ✗ if (EscapeUrlChar(header[i], escaped_char)) {
449 ✗ for (unsigned j = 0; j < 3; ++j) {
450 ✗ if (escaped_buf) {
451 ✗ if (esc_pos >= buf_size)
452 ✗ return esc_pos;
453 ✗ escaped_buf[esc_pos] = escaped_char[j];
454 }
455 ✗ esc_pos++;
456 }
457 } else {
458 ✗ if (escaped_buf) {
459 ✗ if (esc_pos >= buf_size)
460 ✗ return esc_pos;
461 ✗ escaped_buf[esc_pos] = escaped_char[0];
462 }
463 ✗ esc_pos++;
464 }
465 }
466
467 ✗ return esc_pos;
468 }
469
470 /**
471 * -1 of digits is not a valid Http return code
472 */
473 833 int DownloadManager::ParseHttpCode(const char digits[3]) {
474 833 int result = 0;
475 833 int factor = 100;
476
2/2
✓ Branch 0 taken 2499 times.
✓ Branch 1 taken 784 times.
 3283 for (int i = 0; i < 3; ++i) {
477
3/4
✓ Branch 0 taken 2499 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 49 times.
✓ Branch 3 taken 2450 times.
 2499 if ((digits[i] < '0') || (digits[i] > '9'))
478 49 return -1;
479 2450 result += (digits[i] - '0') * factor;
480 2450 factor /= 10;
481 }
482 784 return result;
483 }
484
485
486 /**
487 * Called when new curl sockets arrive or existing curl sockets depart.
488 */
489 ✗ int DownloadManager::CallbackCurlSocket(CURL * /* easy */,
490 curl_socket_t s,
491 int action,
492 void *userp,
493 void * /* socketp */) {
494 // LogCvmfs(kLogDownload, kLogDebug, "CallbackCurlSocket called with easy "
495 // "handle %p, socket %d, action %d", easy, s, action);
496 ✗ DownloadManager *download_mgr = static_cast<DownloadManager *>(userp);
497 ✗ if (action == CURL_POLL_NONE)
498 ✗ return 0;
499
500 // Find s in watch_fds_
501 unsigned index;
502
503 // TODO(heretherebedragons) why start at index = 0 and not 2?
504 // fd[0] and fd[1] are fixed?
505 ✗ for (index = 0; index < download_mgr->watch_fds_inuse_; ++index) {
506 ✗ if (download_mgr->watch_fds_[index].fd == s)
507 ✗ break;
508 }
509 // Or create newly
510 ✗ if (index == download_mgr->watch_fds_inuse_) {
511 // Extend array if necessary
512 ✗ if (download_mgr->watch_fds_inuse_ == download_mgr->watch_fds_size_) {
513 ✗ assert(download_mgr->watch_fds_size_ > 0);
514 ✗ download_mgr->watch_fds_size_ *= 2;
515 ✗ download_mgr->watch_fds_ = static_cast<struct pollfd *>(
516 ✗ srealloc(download_mgr->watch_fds_,
517 ✗ download_mgr->watch_fds_size_ * sizeof(struct pollfd)));
518 }
519 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].fd = s;
520 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].events = 0;
521 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].revents = 0;
522 ✗ download_mgr->watch_fds_inuse_++;
523 }
524
525 ✗ switch (action) {
526 ✗ case CURL_POLL_IN:
527 ✗ download_mgr->watch_fds_[index].events = POLLIN | POLLPRI;
528 ✗ break;
529 ✗ case CURL_POLL_OUT:
530 ✗ download_mgr->watch_fds_[index].events = POLLOUT | POLLWRBAND;
531 ✗ break;
532 ✗ case CURL_POLL_INOUT:
533 ✗ download_mgr->watch_fds_[index].events = POLLIN | POLLPRI | POLLOUT
534 | POLLWRBAND;
535 ✗ break;
536 ✗ case CURL_POLL_REMOVE:
537 ✗ if (index < download_mgr->watch_fds_inuse_ - 1) {
538 download_mgr
539 ✗ ->watch_fds_[index] = download_mgr->watch_fds_
540 ✗ [download_mgr->watch_fds_inuse_ - 1];
541 }
542 ✗ download_mgr->watch_fds_inuse_--;
543 // Shrink array if necessary
544 ✗ if ((download_mgr->watch_fds_inuse_ > download_mgr->watch_fds_max_)
545 ✗ && (download_mgr->watch_fds_inuse_
546 ✗ < download_mgr->watch_fds_size_ / 2)) {
547 ✗ download_mgr->watch_fds_size_ /= 2;
548 // LogCvmfs(kLogDownload, kLogDebug, "shrinking watch_fds_ (%d)",
549 // watch_fds_size_);
550 ✗ download_mgr->watch_fds_ = static_cast<struct pollfd *>(
551 ✗ srealloc(download_mgr->watch_fds_,
552 ✗ download_mgr->watch_fds_size_ * sizeof(struct pollfd)));
553 // LogCvmfs(kLogDownload, kLogDebug, "shrinking watch_fds_ done",
554 // watch_fds_size_);
555 }
556 ✗ break;
557 ✗ default:
558 ✗ break;
559 }
560
561 ✗ return 0;
562 }
563
564
565 /**
566 * Worker thread event loop. Waits on new JobInfo structs on a pipe.
567 */
568 ✗ void *DownloadManager::MainDownload(void *data) {
569 ✗ DownloadManager *download_mgr = static_cast<DownloadManager *>(data);
570 ✗ LogCvmfs(kLogDownload, kLogDebug,
571 "download I/O thread of DownloadManager '%s' started",
572 download_mgr->name_.c_str());
573
574 ✗ const int kIdxPipeTerminate = 0;
575 ✗ const int kIdxPipeJobs = 1;
576
577 ✗ download_mgr->watch_fds_ = static_cast<struct pollfd *>(
578 ✗ smalloc(2 * sizeof(struct pollfd)));
579 ✗ download_mgr->watch_fds_size_ = 2;
580 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].fd = download_mgr->pipe_terminate_
581 ✗ ->GetReadFd();
582 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].events = POLLIN | POLLPRI;
583 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].revents = 0;
584 ✗ download_mgr->watch_fds_[kIdxPipeJobs].fd = download_mgr->pipe_jobs_
585 ✗ ->GetReadFd();
586 ✗ download_mgr->watch_fds_[kIdxPipeJobs].events = POLLIN | POLLPRI;
587 ✗ download_mgr->watch_fds_[kIdxPipeJobs].revents = 0;
588 ✗ download_mgr->watch_fds_inuse_ = 2;
589
590 ✗ int still_running = 0;
591 struct timeval timeval_start, timeval_stop;
592 ✗ gettimeofday(&timeval_start, NULL);
593 while (true) {
594 int timeout;
595 ✗ if (still_running) {
596 /* NOTE: The following might degrade the performance for many small files
597 * use case. TODO(jblomer): look into it.
598 // Specify a timeout for polling in ms; this allows us to return
599 // to libcurl once a second so it can look for internal operations
600 // which timed out. libcurl has a more elaborate mechanism
601 // (CURLMOPT_TIMERFUNCTION) that would inform us of the next potential
602 // timeout. TODO(bbockelm) we should switch to that in the future.
603 timeout = 100;
604 */
605 ✗ timeout = 1;
606 } else {
607 ✗ timeout = -1;
608 ✗ gettimeofday(&timeval_stop, NULL);
609 const int64_t delta = static_cast<int64_t>(
610 ✗ 1000 * DiffTimeSeconds(timeval_start, timeval_stop));
611 ✗ perf::Xadd(download_mgr->counters_->sz_transfer_time, delta);
612 }
613 ✗ const int retval = poll(download_mgr->watch_fds_,
614 ✗ download_mgr->watch_fds_inuse_, timeout);
615 ✗ if (retval < 0) {
616 ✗ continue;
617 }
618
619 // Handle timeout
620 ✗ if (retval == 0) {
621 ✗ curl_multi_socket_action(download_mgr->curl_multi_, CURL_SOCKET_TIMEOUT,
622 0, &still_running);
623 }
624
625 // Terminate I/O thread
626 ✗ if (download_mgr->watch_fds_[kIdxPipeTerminate].revents)
627 ✗ break;
628
629 // New job arrives
630 ✗ if (download_mgr->watch_fds_[kIdxPipeJobs].revents) {
631 ✗ download_mgr->watch_fds_[kIdxPipeJobs].revents = 0;
632 JobInfo *info;
633 ✗ download_mgr->pipe_jobs_->Read<JobInfo *>(&info);
634 ✗ if (!still_running) {
635 ✗ gettimeofday(&timeval_start, NULL);
636 }
637 ✗ CURL *handle = download_mgr->AcquireCurlHandle();
638 ✗ download_mgr->InitializeRequest(info, handle);
639 ✗ download_mgr->SetUrlOptions(info);
640 ✗ curl_multi_add_handle(download_mgr->curl_multi_, handle);
641 ✗ curl_multi_socket_action(download_mgr->curl_multi_, CURL_SOCKET_TIMEOUT,
642 0, &still_running);
643 }
644
645 // Activity on curl sockets
646 // Within this loop the curl_multi_socket_action() may cause socket(s)
647 // to be removed from watch_fds_. If a socket is removed it is replaced
648 // by the socket at the end of the array and the inuse count is decreased.
649 // Therefore loop over the array in reverse order.
650 ✗ for (int64_t i = download_mgr->watch_fds_inuse_ - 1; i >= 2; --i) {
651 ✗ if (i >= download_mgr->watch_fds_inuse_) {
652 ✗ continue;
653 }
654 ✗ if (download_mgr->watch_fds_[i].revents) {
655 ✗ int ev_bitmask = 0;
656 ✗ if (download_mgr->watch_fds_[i].revents & (POLLIN | POLLPRI))
657 ✗ ev_bitmask |= CURL_CSELECT_IN;
658 ✗ if (download_mgr->watch_fds_[i].revents & (POLLOUT | POLLWRBAND))
659 ✗ ev_bitmask |= CURL_CSELECT_OUT;
660 ✗ if (download_mgr->watch_fds_[i].revents
661 ✗ & (POLLERR | POLLHUP | POLLNVAL)) {
662 ✗ ev_bitmask |= CURL_CSELECT_ERR;
663 }
664 ✗ download_mgr->watch_fds_[i].revents = 0;
665
666 ✗ curl_multi_socket_action(download_mgr->curl_multi_,
667 ✗ download_mgr->watch_fds_[i].fd,
668 ev_bitmask,
669 &still_running);
670 }
671 }
672
673 // Check if transfers are completed
674 CURLMsg *curl_msg;
675 int msgs_in_queue;
676 ✗ while ((curl_msg = curl_multi_info_read(download_mgr->curl_multi_,
677 &msgs_in_queue))) {
678 ✗ if (curl_msg->msg == CURLMSG_DONE) {
679 ✗ perf::Inc(download_mgr->counters_->n_requests);
680 JobInfo *info;
681 ✗ CURL *easy_handle = curl_msg->easy_handle;
682 ✗ const int curl_error = curl_msg->data.result;
683 ✗ curl_easy_getinfo(easy_handle, CURLINFO_PRIVATE, &info);
684
685 int64_t redir_count;
686 ✗ curl_easy_getinfo(easy_handle, CURLINFO_REDIRECT_COUNT, &redir_count);
687 ✗ LogCvmfs(kLogDownload, kLogDebug,
688 "(manager '%s' - id %" PRId64 ") "
689 "Number of CURL redirects %" PRId64,
690 download_mgr->name_.c_str(), info->id(), redir_count);
691
692 ✗ curl_multi_remove_handle(download_mgr->curl_multi_, easy_handle);
693 ✗ if (download_mgr->VerifyAndFinalize(curl_error, info)) {
694 ✗ curl_multi_add_handle(download_mgr->curl_multi_, easy_handle);
695 ✗ curl_multi_socket_action(download_mgr->curl_multi_,
696 CURL_SOCKET_TIMEOUT,
697 0,
698 &still_running);
699 } else {
700 // Return easy handle into pool and write result back
701 ✗ download_mgr->ReleaseCurlHandle(easy_handle);
702
703 ✗ DataTubeElement *ele = new DataTubeElement(kActionStop);
704 ✗ info->GetDataTubePtr()->EnqueueBack(ele);
705 ✗ info->GetPipeJobResultPtr()->Write<download::Failures>(
706 ✗ info->error_code());
707 }
708 }
709 }
710 }
711
712 ✗ for (set<CURL *>::iterator i = download_mgr->pool_handles_inuse_->begin(),
713 ✗ iEnd = download_mgr->pool_handles_inuse_->end();
714 ✗ i != iEnd;
715 ✗ ++i) {
716 ✗ curl_multi_remove_handle(download_mgr->curl_multi_, *i);
717 ✗ curl_easy_cleanup(*i);
718 }
719 ✗ download_mgr->pool_handles_inuse_->clear();
720 ✗ free(download_mgr->watch_fds_);
721
722 ✗ LogCvmfs(kLogDownload, kLogDebug,
723 "download I/O thread of DownloadManager '%s' terminated",
724 download_mgr->name_.c_str());
725 ✗ return NULL;
726 }
727
728
729 //------------------------------------------------------------------------------
730
731
732 5642 HeaderLists::~HeaderLists() {
733
2/2
✓ Branch 1 taken 5679 times.
✓ Branch 2 taken 5642 times.
 11321 for (unsigned i = 0; i < blocks_.size(); ++i) {
734
1/2
✓ Branch 1 taken 5679 times.
✗ Branch 2 not taken.
 5679 delete[] blocks_[i];
735 }
736 5642 blocks_.clear();
737 5642 }
738
739
740 31489 curl_slist *HeaderLists::GetList(const char *header) { return Get(header); }
741
742
743 6717 curl_slist *HeaderLists::DuplicateList(curl_slist *slist) {
744
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6717 times.
 6717 assert(slist);
745 6717 curl_slist *copy = GetList(slist->data);
746 6717 copy->next = slist->next;
747 6717 curl_slist *prev = copy;
748 6717 slist = slist->next;
749
2/2
✓ Branch 0 taken 13434 times.
✓ Branch 1 taken 6717 times.
 20151 while (slist) {
750 13434 curl_slist *new_link = Get(slist->data);
751 13434 new_link->next = slist->next;
752 13434 prev->next = new_link;
753 13434 prev = new_link;
754 13434 slist = slist->next;
755 }
756 6717 return copy;
757 }
758
759
760 11738 void HeaderLists::AppendHeader(curl_slist *slist, const char *header) {
761
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11738 times.
 11738 assert(slist);
762 11738 curl_slist *new_link = Get(header);
763 11738 new_link->next = NULL;
764
765
2/2
✓ Branch 0 taken 6736 times.
✓ Branch 1 taken 11738 times.
 18474 while (slist->next)
766 6736 slist = slist->next;
767 11738 slist->next = new_link;
768 11738 }
769
770
771 /**
772 * Ensures that a certain header string is _not_ part of slist on return.
773 * Note that if the first header element matches, the returned slist points
774 * to a different value.
775 */
776 185 void HeaderLists::CutHeader(const char *header, curl_slist **slist) {
777
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 185 times.
 185 assert(slist);
778 curl_slist head;
779 185 head.next = *slist;
780 185 curl_slist *prev = &head;
781 185 curl_slist *rover = *slist;
782
2/2
✓ Branch 0 taken 333 times.
✓ Branch 1 taken 185 times.
 518 while (rover) {
783
2/2
✓ Branch 0 taken 148 times.
✓ Branch 1 taken 185 times.
 333 if (strcmp(rover->data, header) == 0) {
784 148 prev->next = rover->next;
785 148 Put(rover);
786 148 rover = prev;
787 }
788 333 prev = rover;
789 333 rover = rover->next;
790 }
791 185 *slist = head.next;
792 185 }
793
794
795 16337 void HeaderLists::PutList(curl_slist *slist) {
796
2/2
✓ Branch 0 taken 30297 times.
✓ Branch 1 taken 16337 times.
 46634 while (slist) {
797 30297 curl_slist *next = slist->next;
798 30297 Put(slist);
799 30297 slist = next;
800 }
801 16337 }
802
803
804 148 string HeaderLists::Print(curl_slist *slist) {
805 148 string verbose;
806
2/2
✓ Branch 0 taken 296 times.
✓ Branch 1 taken 148 times.
 444 while (slist) {
807
3/6
✓ Branch 2 taken 296 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 296 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 296 times.
✗ Branch 9 not taken.
 296 verbose += string(slist->data) + "\n";
808 296 slist = slist->next;
809 }
810 148 return verbose;
811 }
812
813
814 56661 curl_slist *HeaderLists::Get(const char *header) {
815
2/2
✓ Branch 1 taken 50981 times.
✓ Branch 2 taken 5717 times.
 56698 for (unsigned i = 0; i < blocks_.size(); ++i) {
816
2/2
✓ Branch 0 taken 2585423 times.
✓ Branch 1 taken 37 times.
 2585460 for (unsigned j = 0; j < kBlockSize; ++j) {
817
2/2
✓ Branch 2 taken 50944 times.
✓ Branch 3 taken 2534479 times.
 2585423 if (!IsUsed(&(blocks_[i][j]))) {
818 50944 blocks_[i][j].data = const_cast<char *>(header);
819 50944 return &(blocks_[i][j]);
820 }
821 }
822 }
823
824 // All used, new block
825 5717 AddBlock();
826 5717 blocks_[blocks_.size() - 1][0].data = const_cast<char *>(header);
827 5717 return &(blocks_[blocks_.size() - 1][0]);
828 }
829
830
831 1493997 void HeaderLists::Put(curl_slist *slist) {
832 1493997 slist->data = NULL;
833 1493997 slist->next = NULL;
834 1493997 }
835
836
837 5717 void HeaderLists::AddBlock() {
838
1/2
✓ Branch 1 taken 5717 times.
✗ Branch 2 not taken.
 5717 curl_slist *new_block = new curl_slist[kBlockSize];
839
2/2
✓ Branch 0 taken 1463552 times.
✓ Branch 1 taken 5717 times.
 1469269 for (unsigned i = 0; i < kBlockSize; ++i) {
840 1463552 Put(&new_block[i]);
841 }
842
1/2
✓ Branch 1 taken 5717 times.
✗ Branch 2 not taken.
 5717 blocks_.push_back(new_block);
843 5717 }
844
845
846 //------------------------------------------------------------------------------
847
848
849 ✗ string DownloadManager::ProxyInfo::Print() {
850 ✗ if (url == "DIRECT")
851 ✗ return url;
852
853 ✗ string result = url;
854 ✗ const int remaining = static_cast<int>(host.deadline())
855 ✗ - static_cast<int>(time(NULL));
856 ✗ string expinfo = (remaining >= 0) ? "+" : "";
857 ✗ if (abs(remaining) >= 3600) {
858 ✗ expinfo += StringifyInt(remaining / 3600) + "h";
859 ✗ } else if (abs(remaining) >= 60) {
860 ✗ expinfo += StringifyInt(remaining / 60) + "m";
861 } else {
862 ✗ expinfo += StringifyInt(remaining) + "s";
863 }
864 ✗ if (host.status() == dns::kFailOk) {
865 ✗ result += " (" + host.name() + ", " + expinfo + ")";
866 } else {
867 ✗ result += " (:unresolved:, " + expinfo + ")";
868 }
869 ✗ return result;
870 }
871
872
873 /**
874 * Gets an idle CURL handle from the pool. Creates a new one and adds it to
875 * the pool if necessary.
876 */
877 6717 CURL *DownloadManager::AcquireCurlHandle() {
878 CURL *handle;
879
880
2/2
✓ Branch 1 taken 2907 times.
✓ Branch 2 taken 3810 times.
 6717 if (pool_handles_idle_->empty()) {
881 // Create a new handle
882
1/2
✓ Branch 1 taken 2907 times.
✗ Branch 2 not taken.
 2907 handle = curl_easy_init();
883
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2907 times.
 2907 assert(handle != NULL);
884
885
1/2
✓ Branch 1 taken 2907 times.
✗ Branch 2 not taken.
 2907 curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1);
886 // curl_easy_setopt(curl_default, CURLOPT_FAILONERROR, 1);
887
1/2
✓ Branch 1 taken 2907 times.
✗ Branch 2 not taken.
 2907 curl_easy_setopt(handle, CURLOPT_HEADERFUNCTION, CallbackCurlHeader);
888
1/2
✓ Branch 1 taken 2907 times.
✗ Branch 2 not taken.
 2907 curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, CallbackCurlData);
889 } else {
890 3810 handle = *(pool_handles_idle_->begin());
891
1/2
✓ Branch 2 taken 3810 times.
✗ Branch 3 not taken.
 3810 pool_handles_idle_->erase(pool_handles_idle_->begin());
892 }
893
894
1/2
✓ Branch 1 taken 6717 times.
✗ Branch 2 not taken.
 6717 pool_handles_inuse_->insert(handle);
895
896 6717 return handle;
897 }
898
899
900 6717 void DownloadManager::ReleaseCurlHandle(CURL *handle) {
901
1/2
✓ Branch 1 taken 6717 times.
✗ Branch 2 not taken.
 6717 const set<CURL *>::iterator elem = pool_handles_inuse_->find(handle);
902
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 6717 times.
 6717 assert(elem != pool_handles_inuse_->end());
903
904
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6717 times.
 6717 if (pool_handles_idle_->size() > pool_max_handles_) {
905 ✗ curl_easy_cleanup(*elem);
906 } else {
907
1/2
✓ Branch 2 taken 6717 times.
✗ Branch 3 not taken.
 6717 pool_handles_idle_->insert(*elem);
908 }
909
910
1/2
✓ Branch 1 taken 6717 times.
✗ Branch 2 not taken.
 6717 pool_handles_inuse_->erase(elem);
911 6717 }
912
913
914 /**
915 * HTTP request options: set the URL and other options such as timeout and
916 * proxy.
917 */
918 6717 void DownloadManager::InitializeRequest(JobInfo *info, CURL *handle) {
919 // Initialize internal download state
920 6717 info->SetCurlHandle(handle);
921 6717 info->SetErrorCode(kFailOk);
922 6717 info->SetHttpCode(-1);
923 6717 info->SetFollowRedirects(follow_redirects_);
924 6717 info->SetNumUsedProxies(1);
925 6717 info->SetNumUsedMetalinks(1);
926 6717 info->SetNumUsedHosts(1);
927 6717 info->SetNumRetries(0);
928 6717 info->SetBackoffMs(0);
929 6717 info->SetHeaders(header_lists_->DuplicateList(default_headers_));
930
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6717 times.
 6717 if (info->info_header()) {
931 ✗ header_lists_->AppendHeader(info->headers(), info->info_header());
932 }
933
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6717 times.
 6717 if (enable_http_tracing_) {
934 ✗ for (unsigned int i = 0; i < http_tracing_headers_.size(); i++) {
935 ✗ header_lists_->AppendHeader(info->headers(),
936 ✗ (http_tracing_headers_)[i].c_str());
937 }
938
939 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_pid());
940 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_gid());
941 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_uid());
942
943 ✗ LogCvmfs(kLogDownload, kLogDebug,
944 "(manager '%s' - id %" PRId64 ") "
945 "CURL Header for URL: %s is:\n %s",
946 name_.c_str(), info->id(), info->url()->c_str(),
947 ✗ header_lists_->Print(info->headers()).c_str());
948 }
949
950
2/2
✓ Branch 1 taken 84 times.
✓ Branch 2 taken 6633 times.
 6717 if (info->force_nocache()) {
951 84 SetNocache(info);
952 } else {
953 6633 info->SetNocache(false);
954 }
955
2/2
✓ Branch 1 taken 4384 times.
✓ Branch 2 taken 2333 times.
 6717 if (info->compressed()) {
956 4384 zlib::DecompressInit(info->GetZstreamPtr());
957 }
958
2/2
✓ Branch 1 taken 4433 times.
✓ Branch 2 taken 2284 times.
 6717 if (info->expected_hash()) {
959
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4433 times.
 4433 assert(info->hash_context().buffer != NULL);
960 4433 shash::Init(info->hash_context());
961 }
962
963
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 6717 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 6717 times.
 6717 if ((info->range_offset() != -1) && (info->range_size())) {
964 char byte_range_array[100];
965 ✗ const int64_t range_lower = static_cast<int64_t>(info->range_offset());
966 ✗ const int64_t range_upper = static_cast<int64_t>(info->range_offset()
967 ✗ + info->range_size() - 1);
968 ✗ if (snprintf(byte_range_array, sizeof(byte_range_array),
969 "%" PRId64 "-%" PRId64, range_lower, range_upper)
970 ✗ == 100) {
971 ✗ PANIC(NULL); // Should be impossible given limits on offset size.
972 }
973 ✗ curl_easy_setopt(handle, CURLOPT_RANGE, byte_range_array);
974 } else {
975 6717 curl_easy_setopt(handle, CURLOPT_RANGE, NULL);
976 }
977
978 // Set curl parameters
979 6717 curl_easy_setopt(handle, CURLOPT_PRIVATE, static_cast<void *>(info));
980 6717 curl_easy_setopt(handle, CURLOPT_WRITEHEADER, static_cast<void *>(info));
981 6717 curl_easy_setopt(handle, CURLOPT_WRITEDATA, static_cast<void *>(info));
982 6717 curl_easy_setopt(handle, CURLOPT_HTTPHEADER, info->headers());
983
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6717 times.
 6717 if (info->head_request()) {
984 ✗ curl_easy_setopt(handle, CURLOPT_NOBODY, 1);
985 } else {
986 6717 curl_easy_setopt(handle, CURLOPT_HTTPGET, 1);
987 }
988
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6717 times.
 6717 if (opt_ipv4_only_) {
989 ✗ curl_easy_setopt(handle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
990 }
991
2/2
✓ Branch 0 taken 1924 times.
✓ Branch 1 taken 4793 times.
 6717 if (follow_redirects_) {
992 1924 curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1);
993 1924 curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 4);
994 }
995 #ifdef DEBUGMSG
996 6717 curl_easy_setopt(handle, CURLOPT_VERBOSE, 1);
997 6717 curl_easy_setopt(handle, CURLOPT_DEBUGFUNCTION, CallbackCurlDebug);
998 #endif
999 6717 }
1000
1001 13728 void DownloadManager::CheckHostInfoReset(const std::string &typ,
1002 HostInfo &info,
1003 JobInfo *jobinfo,
1004 time_t &now) {
1005
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 13728 times.
 13728 if (info.timestamp_backup > 0) {
1006 ✗ if (now == 0)
1007 ✗ now = time(NULL);
1008 ✗ if (static_cast<int64_t>(now)
1009 ✗ > static_cast<int64_t>(info.timestamp_backup + info.reset_after)) {
1010 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1011 "(manager %s - id %" PRId64 ") "
1012 "switching %s from %s to %s (reset %s)",
1013 name_.c_str(), jobinfo->id(), typ.c_str(),
1014 ✗ (*info.chain)[info.current].c_str(), (*info.chain)[0].c_str(),
1015 typ.c_str());
1016 ✗ info.current = 0;
1017 ✗ info.timestamp_backup = 0;
1018 }
1019 }
1020 13728 }
1021
1022
1023 /**
1024 * Sets the URL specific options such as host to use and timeout. It might also
1025 * set an error code, in which case the further processing should react on.
1026 */
1027 6864 void DownloadManager::SetUrlOptions(JobInfo *info) {
1028 6864 CURL *curl_handle = info->curl_handle();
1029 6864 string url_prefix;
1030 6864 time_t now = 0;
1031
1032 6864 const MutexLockGuard m(lock_options_);
1033
1034 // sharding policy
1035
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6864 times.
 6864 if (sharding_policy_.UseCount() > 0) {
1036 ✗ if (info->proxy() != "") {
1037 // proxy already set, so this is a failover event
1038 ✗ perf::Inc(counters_->n_proxy_failover);
1039 }
1040 ✗ info->SetProxy(sharding_policy_->GetNextProxy(
1041 ✗ info->url(), info->proxy(),
1042 ✗ info->range_offset() == -1 ? 0 : info->range_offset()));
1043
1044 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, info->proxy().c_str());
1045 } else { // no sharding policy
1046 // Check if proxy group needs to be reset from backup to primary
1047
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6864 times.
 6864 if (opt_timestamp_backup_proxies_ > 0) {
1048 ✗ now = time(NULL);
1049 ✗ if (static_cast<int64_t>(now) > static_cast<int64_t>(
1050 ✗ opt_timestamp_backup_proxies_ + opt_proxy_groups_reset_after_)) {
1051 ✗ opt_proxy_groups_current_ = 0;
1052 ✗ opt_timestamp_backup_proxies_ = 0;
1053 ✗ RebalanceProxiesUnlocked("Reset proxy group from backup to primary");
1054 }
1055 }
1056 // Check if load-balanced proxies within the group need to be reset
1057
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6864 times.
 6864 if (opt_timestamp_failover_proxies_ > 0) {
1058 ✗ if (now == 0)
1059 ✗ now = time(NULL);
1060 ✗ if (static_cast<int64_t>(now)
1061 ✗ > static_cast<int64_t>(opt_timestamp_failover_proxies_
1062 ✗ + opt_proxy_groups_reset_after_)) {
1063 ✗ RebalanceProxiesUnlocked(
1064 "Reset load-balanced proxies within the active group");
1065 }
1066 }
1067
1068
1/2
✓ Branch 2 taken 6864 times.
✗ Branch 3 not taken.
 6864 ProxyInfo *proxy = ChooseProxyUnlocked(info->expected_hash());
1069
6/6
✓ Branch 0 taken 2461 times.
✓ Branch 1 taken 4403 times.
✓ Branch 3 taken 2265 times.
✓ Branch 4 taken 196 times.
✓ Branch 5 taken 6668 times.
✓ Branch 6 taken 196 times.
 6864 if (!proxy || (proxy->url == "DIRECT")) {
1070
2/4
✓ Branch 2 taken 6668 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 6668 times.
✗ Branch 6 not taken.
 6668 info->SetProxy("DIRECT");
1071
1/2
✓ Branch 2 taken 6668 times.
✗ Branch 3 not taken.
 6668 curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "");
1072 } else {
1073 // Note: inside ValidateProxyIpsUnlocked() we may change the proxy data
1074 // structure, so we must not pass proxy->... (== current_proxy())
1075 // parameters directly
1076
1/2
✓ Branch 1 taken 196 times.
✗ Branch 2 not taken.
 196 const std::string purl = proxy->url;
1077
1/2
✓ Branch 1 taken 196 times.
✗ Branch 2 not taken.
 196 const dns::Host phost = proxy->host;
1078
1/2
✓ Branch 1 taken 196 times.
✗ Branch 2 not taken.
 196 const bool changed = ValidateProxyIpsUnlocked(purl, phost);
1079 // Current proxy may have changed
1080
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 196 times.
 196 if (changed) {
1081 ✗ proxy = ChooseProxyUnlocked(info->expected_hash());
1082 }
1083
1/2
✓ Branch 1 taken 196 times.
✗ Branch 2 not taken.
 196 info->SetProxy(proxy->url);
1084
1/2
✓ Branch 1 taken 196 times.
✗ Branch 2 not taken.
 196 if (proxy->host.status() == dns::kFailOk) {
1085
2/4
✓ Branch 1 taken 196 times.
✗ Branch 2 not taken.
✓ Branch 6 taken 196 times.
✗ Branch 7 not taken.
 196 curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY,
1086 info->proxy().c_str());
1087 } else {
1088 // We know it can't work, don't even try to download
1089 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "0.0.0.0");
1090 }
1091 196 }
1092 } // end !sharding
1093
1094 // Check if metalink and host chains need to be reset
1095
2/4
✓ Branch 2 taken 6864 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 6864 times.
✗ Branch 6 not taken.
 6864 CheckHostInfoReset("metalink", opt_metalink_, info, now);
1096
2/4
✓ Branch 2 taken 6864 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 6864 times.
✗ Branch 6 not taken.
 6864 CheckHostInfoReset("host", opt_host_, info, now);
1097
1098
1/2
✓ Branch 1 taken 6864 times.
✗ Branch 2 not taken.
 6864 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_LIMIT, opt_low_speed_limit_);
1099
3/6
✓ Branch 1 taken 6864 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 196 times.
✓ Branch 6 taken 6668 times.
 6864 if (info->proxy() != "DIRECT") {
1100
1/2
✓ Branch 1 taken 196 times.
✗ Branch 2 not taken.
 196 curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT, opt_timeout_proxy_);
1101
1/2
✓ Branch 1 taken 196 times.
✗ Branch 2 not taken.
 196 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_TIME, opt_timeout_proxy_);
1102 } else {
1103
1/2
✓ Branch 1 taken 6668 times.
✗ Branch 2 not taken.
 6668 curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT, opt_timeout_direct_);
1104
1/2
✓ Branch 1 taken 6668 times.
✗ Branch 2 not taken.
 6668 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_TIME, opt_timeout_direct_);
1105 }
1106
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6864 times.
 6864 if (!opt_dns_server_.empty())
1107 ✗ curl_easy_setopt(curl_handle, CURLOPT_DNS_SERVERS, opt_dns_server_.c_str());
1108
1109
2/2
✓ Branch 1 taken 3316 times.
✓ Branch 2 taken 3548 times.
 6864 if (info->probe_hosts()) {
1110
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 3316 times.
 3316 if (CheckMetalinkChain(now)) {
1111 ✗ url_prefix = (*opt_metalink_.chain)[opt_metalink_.current];
1112 ✗ info->SetCurrentMetalinkChainIndex(opt_metalink_.current);
1113 ✗ LogCvmfs(kLogDownload, kLogDebug,
1114 "(manager %s - id %" PRId64 ") "
1115 "reading from metalink %d",
1116 name_.c_str(), info->id(), opt_metalink_.current);
1117
1/2
✓ Branch 0 taken 3316 times.
✗ Branch 1 not taken.
 3316 } else if (opt_host_.chain) {
1118
1/2
✓ Branch 2 taken 3316 times.
✗ Branch 3 not taken.
 3316 url_prefix = (*opt_host_.chain)[opt_host_.current];
1119 3316 info->SetCurrentHostChainIndex(opt_host_.current);
1120
1/2
✓ Branch 3 taken 3316 times.
✗ Branch 4 not taken.
 3316 LogCvmfs(kLogDownload, kLogDebug,
1121 "(manager %s - id %" PRId64 ") "
1122 "reading from host %d",
1123 name_.c_str(), info->id(), opt_host_.current);
1124 }
1125 }
1126
1127
1/2
✓ Branch 2 taken 6864 times.
✗ Branch 3 not taken.
 6864 string url = url_prefix + *(info->url());
1128
1129
1/2
✓ Branch 1 taken 6864 times.
✗ Branch 2 not taken.
 6864 curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, 1L);
1130
2/6
✓ Branch 1 taken 6864 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 6864 times.
 6864 if (url.substr(0, 5) == "https") {
1131 ✗ const bool rvb = ssl_certificate_store_.ApplySslCertificatePath(
1132 curl_handle);
1133 ✗ if (!rvb) {
1134 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1135 "(manager %s - id %" PRId64 ") "
1136 "Failed to set SSL certificate path %s",
1137 name_.c_str(), info->id(),
1138 ✗ ssl_certificate_store_.GetCaPath().c_str());
1139 }
1140 ✗ if (info->pid() != -1) {
1141 ✗ if (credentials_attachment_ == NULL) {
1142 ✗ LogCvmfs(kLogDownload, kLogDebug,
1143 "(manager %s - id %" PRId64 ") "
1144 "uses secure downloads but no credentials attachment set",
1145 name_.c_str(), info->id());
1146 } else {
1147 ✗ const bool retval = credentials_attachment_->ConfigureCurlHandle(
1148 curl_handle, info->pid(), info->GetCredDataPtr());
1149 ✗ if (!retval) {
1150 ✗ LogCvmfs(kLogDownload, kLogDebug,
1151 "(manager %s - id %" PRId64 ") "
1152 "failed attaching credentials",
1153 name_.c_str(), info->id());
1154 }
1155 }
1156 }
1157 // The download manager disables signal handling in the curl library;
1158 // as OpenSSL's implementation of TLS will generate a sigpipe in some
1159 // error paths, we must explicitly disable SIGPIPE here.
1160 // TODO(jblomer): it should be enough to do this once
1161 ✗ signal(SIGPIPE, SIG_IGN);
1162 }
1163
1164
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6864 times.
 6864 if (url.find("@proxy@") != string::npos) {
1165 // This is used in Geo-API requests (only), to replace a portion of the
1166 // URL with the current proxy name for the sake of caching the result.
1167 // Replace the @proxy@ either with a passed in "forced" template (which
1168 // is set from $CVMFS_PROXY_TEMPLATE) if there is one, or a "direct"
1169 // template (which is the uuid) if there's no proxy, or the name of the
1170 // proxy.
1171 ✗ string replacement;
1172 ✗ if (proxy_template_forced_ != "") {
1173 ✗ replacement = proxy_template_forced_;
1174 ✗ } else if (info->proxy() == "DIRECT") {
1175 ✗ replacement = proxy_template_direct_;
1176 } else {
1177 ✗ if (opt_proxy_groups_current_ >= opt_proxy_groups_fallback_) {
1178 // It doesn't make sense to use the fallback proxies in Geo-API requests
1179 // since the fallback proxies are supposed to get sorted, too.
1180 ✗ info->SetProxy("DIRECT");
1181 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "");
1182 ✗ replacement = proxy_template_direct_;
1183 } else {
1184 ✗ replacement = ChooseProxyUnlocked(info->expected_hash())->host.name();
1185 }
1186 }
1187 ✗ replacement = (replacement == "") ? proxy_template_direct_ : replacement;
1188 ✗ LogCvmfs(kLogDownload, kLogDebug,
1189 "(manager %s - id %" PRId64 ") "
1190 "replacing @proxy@ by %s",
1191 name_.c_str(), info->id(), replacement.c_str());
1192 ✗ url = ReplaceAll(url, "@proxy@", replacement);
1193 }
1194
1195 // TODO(heretherebedragons) before removing
1196 // static_cast<cvmfs::MemSink*>(info->sink)->size() == 0
1197 // and just always call info->sink->Reserve()
1198 // we should do a speed check
1199
3/4
✓ Branch 2 taken 6864 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2709 times.
✓ Branch 5 taken 4155 times.
 6864 if ((info->sink() != NULL) && info->sink()->RequiresReserve()
1200
1/2
✓ Branch 2 taken 2709 times.
✗ Branch 3 not taken.
 2709 && (static_cast<cvmfs::MemSink *>(info->sink())->size() == 0)
1201
11/20
✓ Branch 1 taken 6864 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 2709 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 2709 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 2085 times.
✓ Branch 11 taken 624 times.
✓ Branch 12 taken 2709 times.
✓ Branch 13 taken 4155 times.
✗ Branch 14 not taken.
✓ Branch 15 taken 2709 times.
✓ Branch 16 taken 4155 times.
✓ Branch 18 taken 2085 times.
✓ Branch 19 taken 4779 times.
✗ Branch 20 not taken.
✗ Branch 21 not taken.
✗ Branch 23 not taken.
✗ Branch 24 not taken.
 13728 && HasPrefix(url, "file://", false)) {
1202 platform_stat64 stat_buf;
1203 2085 const int retval = platform_stat(url.c_str(), &stat_buf);
1204
1/2
✓ Branch 0 taken 2085 times.
✗ Branch 1 not taken.
 2085 if (retval != 0) {
1205 // this is an error: file does not exist or out of memory
1206 // error is caught in other code section.
1207
1/2
✓ Branch 2 taken 2085 times.
✗ Branch 3 not taken.
 2085 info->sink()->Reserve(64ul * 1024ul);
1208 } else {
1209 ✗ info->sink()->Reserve(stat_buf.st_size);
1210 }
1211 }
1212
1213
2/4
✓ Branch 2 taken 6864 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 6864 times.
✗ Branch 7 not taken.
 6864 curl_easy_setopt(curl_handle, CURLOPT_URL,
1214 EscapeUrl(info->id(), url).c_str());
1215 6864 }
1216
1217
1218 /**
1219 * Checks if the name resolving information is still up to date. The host
1220 * object should be one from the current load-balance group. If the information
1221 * changed, gather new set of resolved IPs and, if different, exchange them in
1222 * the load-balance group on the fly. In the latter case, also rebalance the
1223 * proxies. The options mutex needs to be open.
1224 *
1225 * Returns true if proxies may have changed.
1226 */
1227 196 bool DownloadManager::ValidateProxyIpsUnlocked(const string &url,
1228 const dns::Host &host) {
1229
2/4
✓ Branch 1 taken 196 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 196 times.
✗ Branch 4 not taken.
 196 if (!host.IsExpired())
1230 196 return false;
1231 ✗ LogCvmfs(kLogDownload, kLogDebug, "(manager '%s') validate DNS entry for %s",
1232 ✗ name_.c_str(), host.name().c_str());
1233
1234 ✗ const unsigned group_idx = opt_proxy_groups_current_;
1235 ✗ dns::Host new_host = resolver_->Resolve(host.name());
1236
1237 ✗ bool update_only = true; // No changes to the list of IP addresses.
1238 ✗ if (new_host.status() != dns::kFailOk) {
1239 // Try again later in case resolving fails.
1240 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1241 "(manager '%s') failed to resolve IP addresses for %s (%d - %s)",
1242 ✗ name_.c_str(), host.name().c_str(), new_host.status(),
1243 dns::Code2Ascii(new_host.status()));
1244 ✗ new_host = dns::Host::ExtendDeadline(host, resolver_->min_ttl());
1245 ✗ } else if (!host.IsEquivalent(new_host)) {
1246 ✗ update_only = false;
1247 }
1248
1249 ✗ if (update_only) {
1250 ✗ for (unsigned i = 0; i < (*opt_proxy_groups_)[group_idx].size(); ++i) {
1251 ✗ if ((*opt_proxy_groups_)[group_idx][i].host.id() == host.id())
1252 ✗ (*opt_proxy_groups_)[group_idx][i].host = new_host;
1253 }
1254 ✗ return false;
1255 }
1256
1257 ✗ assert(new_host.status() == dns::kFailOk);
1258
1259 // Remove old host objects, insert new objects, and rebalance.
1260 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslog,
1261 "(manager '%s') DNS entries for proxy %s changed, adjusting",
1262 ✗ name_.c_str(), host.name().c_str());
1263 ✗ vector<ProxyInfo> *group = current_proxy_group();
1264 ✗ opt_num_proxies_ -= group->size();
1265 ✗ for (unsigned i = 0; i < group->size();) {
1266 ✗ if ((*group)[i].host.id() == host.id()) {
1267 ✗ group->erase(group->begin() + i);
1268 } else {
1269 ✗ i++;
1270 }
1271 }
1272 ✗ vector<ProxyInfo> new_infos;
1273 ✗ set<string> best_addresses = new_host.ViewBestAddresses(opt_ip_preference_);
1274 ✗ set<string>::const_iterator iter_ips = best_addresses.begin();
1275 ✗ for (; iter_ips != best_addresses.end(); ++iter_ips) {
1276 ✗ const string url_ip = dns::RewriteUrl(url, *iter_ips);
1277 ✗ new_infos.push_back(ProxyInfo(new_host, url_ip));
1278 }
1279 ✗ group->insert(group->end(), new_infos.begin(), new_infos.end());
1280 ✗ opt_num_proxies_ += new_infos.size();
1281
1282 ✗ const std::string msg = "DNS entries for proxy " + host.name() + " changed";
1283
1284 ✗ RebalanceProxiesUnlocked(msg);
1285 ✗ return true;
1286 }
1287
1288
1289 /**
1290 * Adds transfer time and downloaded bytes to the global counters.
1291 */
1292 7131 void DownloadManager::UpdateStatistics(CURL *handle) {
1293 double val;
1294 int retval;
1295 7131 int64_t sum = 0;
1296
1297
1/2
✓ Branch 1 taken 7131 times.
✗ Branch 2 not taken.
 7131 retval = curl_easy_getinfo(handle, CURLINFO_SIZE_DOWNLOAD, &val);
1298
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7131 times.
 7131 assert(retval == CURLE_OK);
1299 7131 sum += static_cast<int64_t>(val);
1300 /*retval = curl_easy_getinfo(handle, CURLINFO_HEADER_SIZE, &val);
1301 assert(retval == CURLE_OK);
1302 sum += static_cast<int64_t>(val);*/
1303 7131 perf::Xadd(counters_->sz_transferred_bytes, sum);
1304 7131 }
1305
1306
1307 /**
1308 * Retry if possible if not on no-cache and if not already done too often.
1309 */
1310 7131 bool DownloadManager::CanRetry(const JobInfo *info) {
1311 7131 const MutexLockGuard m(lock_options_);
1312 7131 const unsigned max_retries = opt_max_retries_;
1313
1314
2/2
✓ Branch 2 taken 2280 times.
✓ Branch 3 taken 4662 times.
 14073 return !(info->nocache()) && (info->num_retries() < max_retries)
1315
3/4
✓ Branch 0 taken 6942 times.
✓ Branch 1 taken 189 times.
✓ Branch 4 taken 2280 times.
✗ Branch 5 not taken.
 16353 && (IsProxyTransferError(info->error_code())
1316
2/2
✓ Branch 2 taken 162 times.
✓ Branch 3 taken 2118 times.
 2280 || IsHostTransferError(info->error_code()));
1317 7131 }
1318
1319 /**
1320 * Backoff for retry to introduce a jitter into a cluster of requesting
1321 * cvmfs nodes.
1322 * Retry only when HTTP caching is on.
1323 *
1324 * \return true if backoff has been performed, false otherwise
1325 */
1326 162 void DownloadManager::Backoff(JobInfo *info) {
1327 162 unsigned backoff_init_ms = 0;
1328 162 unsigned backoff_max_ms = 0;
1329 {
1330 162 const MutexLockGuard m(lock_options_);
1331 162 backoff_init_ms = opt_backoff_init_ms_;
1332 162 backoff_max_ms = opt_backoff_max_ms_;
1333 162 }
1334
1335 162 info->SetNumRetries(info->num_retries() + 1);
1336 162 perf::Inc(counters_->n_retries);
1337
2/2
✓ Branch 1 taken 66 times.
✓ Branch 2 taken 96 times.
 162 if (info->backoff_ms() == 0) {
1338 66 info->SetBackoffMs(prng_.Next(backoff_init_ms + 1)); // Must be != 0
1339 } else {
1340 96 info->SetBackoffMs(info->backoff_ms() * 2);
1341 }
1342
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 162 times.
 162 if (info->backoff_ms() > backoff_max_ms) {
1343 ✗ info->SetBackoffMs(backoff_max_ms);
1344 }
1345
1346 162 LogCvmfs(kLogDownload, kLogDebug,
1347 "(manager '%s' - id %" PRId64 ") backing off for %d ms",
1348 name_.c_str(), info->id(), info->backoff_ms());
1349 162 SafeSleepMs(info->backoff_ms());
1350 162 }
1351
1352 189 void DownloadManager::SetNocache(JobInfo *info) {
1353
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 189 times.
 189 if (info->nocache())
1354 ✗ return;
1355 189 header_lists_->AppendHeader(info->headers(), "Pragma: no-cache");
1356 189 header_lists_->AppendHeader(info->headers(), "Cache-Control: no-cache");
1357 189 curl_easy_setopt(info->curl_handle(), CURLOPT_HTTPHEADER, info->headers());
1358 189 info->SetNocache(true);
1359 }
1360
1361
1362 /**
1363 * Reverse operation of SetNocache. Makes sure that "no-cache" header
1364 * disappears from the list of headers to let proxies work normally.
1365 */
1366 414 void DownloadManager::SetRegularCache(JobInfo *info) {
1367
1/2
✓ Branch 1 taken 414 times.
✗ Branch 2 not taken.
 414 if (info->nocache() == false)
1368 414 return;
1369 ✗ header_lists_->CutHeader("Pragma: no-cache", info->GetHeadersPtr());
1370 ✗ header_lists_->CutHeader("Cache-Control: no-cache", info->GetHeadersPtr());
1371 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_HTTPHEADER, info->headers());
1372 ✗ info->SetNocache(false);
1373 }
1374
1375
1376 /**
1377 * Frees the storage associated with the authz attachment from the job
1378 */
1379 6864 void DownloadManager::ReleaseCredential(JobInfo *info) {
1380
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6864 times.
 6864 if (info->cred_data()) {
1381 ✗ assert(credentials_attachment_ != NULL); // Someone must have set it
1382 ✗ credentials_attachment_->ReleaseCurlHandle(info->curl_handle(),
1383 info->cred_data());
1384 ✗ info->SetCredData(NULL);
1385 }
1386 6864 }
1387
1388
1389 /* Sort links based on the "pri=" parameter */
1390 ✗ static bool sortlinks(const std::string &s1, const std::string &s2) {
1391 ✗ const size_t pos1 = s1.find("; pri=");
1392 ✗ const size_t pos2 = s2.find("; pri=");
1393 int pri1, pri2;
1394 ✗ if ((pos1 != std::string::npos) && (pos2 != std::string::npos)
1395 ✗ && (sscanf(s1.substr(pos1 + 6).c_str(), "%d", &pri1) == 1)
1396 ✗ && (sscanf(s2.substr(pos2 + 6).c_str(), "%d", &pri2) == 1)) {
1397 ✗ return pri1 < pri2;
1398 }
1399 ✗ return false;
1400 }
1401
1402 /**
1403 * Parses Link header and uses it to set a new host chain.
1404 * See rfc6249.
1405 */
1406 ✗ void DownloadManager::ProcessLink(JobInfo *info) {
1407 ✗ std::vector<std::string> links = SplitString(info->link(), ',');
1408 ✗ if (info->link().find("; pri=") != std::string::npos)
1409 ✗ std::sort(links.begin(), links.end(), sortlinks);
1410
1411 ✗ std::vector<std::string> host_list;
1412
1413 ✗ std::vector<std::string>::const_iterator il = links.begin();
1414 ✗ for (; il != links.end(); ++il) {
1415 ✗ const std::string &link = *il;
1416 ✗ if ((link.find("; rel=duplicate") == std::string::npos)
1417 ✗ && (link.find("; rel=\"duplicate\"") == std::string::npos)) {
1418 ✗ LogCvmfs(kLogDownload, kLogDebug,
1419 "skipping link '%s' because it does not contain rel=duplicate",
1420 link.c_str());
1421 ✗ continue;
1422 }
1423 // ignore depth= field since there's nothing useful we can do with it
1424
1425 ✗ size_t start = link.find('<');
1426 ✗ if (start == std::string::npos) {
1427 ✗ LogCvmfs(
1428 kLogDownload, kLogDebug,
1429 "skipping link '%s' because it does not have a left angle bracket",
1430 link.c_str());
1431 ✗ continue;
1432 }
1433
1434 ✗ start++;
1435 ✗ if ((link.substr(start, 7) != "http://")
1436 ✗ && (link.substr(start, 8) != "https://")) {
1437 ✗ LogCvmfs(kLogDownload, kLogDebug,
1438 "skipping link '%s' of unrecognized url protocol", link.c_str());
1439 ✗ continue;
1440 }
1441
1442 ✗ size_t end = link.find('/', start + 8);
1443 ✗ if (end == std::string::npos)
1444 ✗ end = link.find('>');
1445 ✗ if (end == std::string::npos) {
1446 ✗ LogCvmfs(kLogDownload, kLogDebug,
1447 "skipping link '%s' because no slash in url and no right angle "
1448 "bracket",
1449 link.c_str());
1450 ✗ continue;
1451 }
1452 ✗ const std::string host = link.substr(start, end - start);
1453 ✗ LogCvmfs(kLogDownload, kLogDebug, "adding linked host '%s'", host.c_str());
1454 ✗ host_list.push_back(host);
1455 }
1456
1457 ✗ if (host_list.size() > 0) {
1458 ✗ SetHostChain(host_list);
1459 ✗ opt_metalink_timestamp_link_ = time(NULL);
1460 }
1461 }
1462
1463
1464 /**
1465 * Checks the result of a curl download and implements the failure logic, such
1466 * as changing the proxy server. Takes care of cleanup.
1467 *
1468 * \return true if another download should be performed, false otherwise
1469 */
1470 7131 bool DownloadManager::VerifyAndFinalize(const int curl_error, JobInfo *info) {
1471
1/2
✓ Branch 6 taken 7131 times.
✗ Branch 7 not taken.
 7131 LogCvmfs(kLogDownload, kLogDebug,
1472 "(manager '%s' - id %" PRId64 ") "
1473 "Verify downloaded url %s, proxy %s (curl error %d)",
1474 name_.c_str(), info->id(), info->url()->c_str(),
1475
1/2
✓ Branch 1 taken 7131 times.
✗ Branch 2 not taken.
 14262 info->proxy().c_str(), curl_error);
1476
1/2
✓ Branch 2 taken 7131 times.
✗ Branch 3 not taken.
 7131 UpdateStatistics(info->curl_handle());
1477
1478 bool was_metalink;
1479 7131 std::string typ;
1480
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 7131 times.
 7131 if (info->current_metalink_chain_index() >= 0) {
1481 ✗ was_metalink = true;
1482 ✗ typ = "metalink";
1483 ✗ if (info->link() != "") {
1484 // process Link header whether or not the redirected URL got an error
1485 ✗ ProcessLink(info);
1486 }
1487 } else {
1488 7131 was_metalink = false;
1489
1/2
✓ Branch 1 taken 7131 times.
✗ Branch 2 not taken.
 7131 typ = "host";
1490 }
1491
1492
1493 // Verification and error classification
1494
3/14
✓ Branch 0 taken 6380 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 60 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 691 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
 7131 switch (curl_error) {
1495 6380 case CURLE_OK:
1496 // Verify content hash
1497
2/2
✓ Branch 1 taken 4247 times.
✓ Branch 2 taken 2133 times.
 6380 if (info->expected_hash()) {
1498
1/2
✓ Branch 1 taken 4247 times.
✗ Branch 2 not taken.
 4247 shash::Any match_hash;
1499
1/2
✓ Branch 2 taken 4247 times.
✗ Branch 3 not taken.
 4247 shash::Final(info->hash_context(), &match_hash);
1500
3/4
✓ Branch 2 taken 4247 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 210 times.
✓ Branch 5 taken 4037 times.
 4247 if (match_hash != *(info->expected_hash())) {
1501
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 210 times.
 210 if (ignore_signature_failures_) {
1502 ✗ LogCvmfs(
1503 kLogDownload, kLogDebug | kLogSyslogErr,
1504 "(manager '%s' - id %" PRId64 ") "
1505 "ignoring failed hash verification of %s (expected %s, got %s)",
1506 name_.c_str(), info->id(), info->url()->c_str(),
1507 ✗ info->expected_hash()->ToString().c_str(),
1508 ✗ match_hash.ToString().c_str());
1509 } else {
1510
1/2
✓ Branch 7 taken 210 times.
✗ Branch 8 not taken.
 420 LogCvmfs(kLogDownload, kLogDebug,
1511 "(manager '%s' - id %" PRId64 ") "
1512 "hash verification of %s failed (expected %s, got %s)",
1513 name_.c_str(), info->id(), info->url()->c_str(),
1514
1/2
✓ Branch 2 taken 210 times.
✗ Branch 3 not taken.
 420 info->expected_hash()->ToString().c_str(),
1515
1/2
✓ Branch 1 taken 210 times.
✗ Branch 2 not taken.
 420 match_hash.ToString().c_str());
1516 210 info->SetErrorCode(kFailBadData);
1517 210 break;
1518 }
1519 }
1520 }
1521
1522 6170 info->SetErrorCode(kFailOk);
1523 6170 break;
1524 ✗ case CURLE_UNSUPPORTED_PROTOCOL:
1525 ✗ info->SetErrorCode(kFailUnsupportedProtocol);
1526 ✗ break;
1527 60 case CURLE_URL_MALFORMAT:
1528 60 info->SetErrorCode(kFailBadUrl);
1529 60 break;
1530 ✗ case CURLE_COULDNT_RESOLVE_PROXY:
1531 ✗ info->SetErrorCode(kFailProxyResolve);
1532 ✗ break;
1533 ✗ case CURLE_COULDNT_RESOLVE_HOST:
1534 ✗ info->SetErrorCode(kFailHostResolve);
1535 ✗ break;
1536 ✗ case CURLE_OPERATION_TIMEDOUT:
1537 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostTooSlow
1538 : kFailProxyTooSlow);
1539 ✗ break;
1540 ✗ case CURLE_PARTIAL_FILE:
1541 case CURLE_GOT_NOTHING:
1542 case CURLE_RECV_ERROR:
1543 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostShortTransfer
1544 : kFailProxyShortTransfer);
1545 ✗ break;
1546 691 case CURLE_FILE_COULDNT_READ_FILE:
1547 case CURLE_COULDNT_CONNECT:
1548
3/6
✓ Branch 1 taken 691 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 49 times.
✓ Branch 6 taken 642 times.
 691 if (info->proxy() != "DIRECT") {
1549 // This is a guess. Fail-over can still change to switching host
1550 49 info->SetErrorCode(kFailProxyConnection);
1551 } else {
1552 642 info->SetErrorCode(kFailHostConnection);
1553 }
1554 691 break;
1555 ✗ case CURLE_TOO_MANY_REDIRECTS:
1556 ✗ info->SetErrorCode(kFailHostConnection);
1557 ✗ break;
1558 ✗ case CURLE_SSL_CACERT_BADFILE:
1559 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
1560 "(manager '%s' -id %" PRId64 ") "
1561 "Failed to load certificate bundle. "
1562 "X509_CERT_BUNDLE might point to the wrong location.",
1563 name_.c_str(), info->id());
1564 ✗ info->SetErrorCode(kFailHostConnection);
1565 ✗ break;
1566 // As of curl 7.62.0, CURLE_SSL_CACERT is the same as
1567 // CURLE_PEER_FAILED_VERIFICATION
1568 ✗ case CURLE_PEER_FAILED_VERIFICATION:
1569 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
1570 "(manager '%s' - id %" PRId64 ") "
1571 "invalid SSL certificate of remote host. "
1572 "X509_CERT_DIR and/or X509_CERT_BUNDLE might point to the wrong "
1573 "location.",
1574 name_.c_str(), info->id());
1575 ✗ info->SetErrorCode(kFailHostConnection);
1576 ✗ break;
1577 ✗ case CURLE_ABORTED_BY_CALLBACK:
1578 case CURLE_WRITE_ERROR:
1579 // Error set by callback
1580 ✗ break;
1581 ✗ case CURLE_SEND_ERROR:
1582 // The curl error CURLE_SEND_ERROR can be seen when a cache is misbehaving
1583 // and closing connections before the http request send is completed.
1584 // Handle this error, treating it as a short transfer error.
1585 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostShortTransfer
1586 : kFailProxyShortTransfer);
1587 ✗ break;
1588 ✗ default:
1589 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
1590 "(manager '%s' - id %" PRId64 ") "
1591 "unexpected curl error (%d) while trying to fetch %s",
1592 name_.c_str(), info->id(), curl_error, info->url()->c_str());
1593 ✗ info->SetErrorCode(kFailOther);
1594 ✗ break;
1595 }
1596
1597 std::vector<std::string> *host_chain;
1598 unsigned char num_used_hosts;
1599
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7131 times.
 7131 if (was_metalink) {
1600 ✗ host_chain = opt_metalink_.chain;
1601 ✗ num_used_hosts = info->num_used_metalinks();
1602 } else {
1603 7131 host_chain = opt_host_.chain;
1604 7131 num_used_hosts = info->num_used_hosts();
1605 }
1606
1607 // Determination if download should be repeated
1608 7131 bool try_again = false;
1609
1/2
✓ Branch 1 taken 7131 times.
✗ Branch 2 not taken.
 7131 bool same_url_retry = CanRetry(info);
1610
2/2
✓ Branch 1 taken 961 times.
✓ Branch 2 taken 6170 times.
 7131 if (info->error_code() != kFailOk) {
1611 961 const MutexLockGuard m(lock_options_);
1612
2/2
✓ Branch 1 taken 210 times.
✓ Branch 2 taken 751 times.
 961 if (info->error_code() == kFailBadData) {
1613
2/2
✓ Branch 1 taken 105 times.
✓ Branch 2 taken 105 times.
 210 if (!info->nocache()) {
1614 105 try_again = true;
1615 } else {
1616 // Make it a host failure
1617
1/2
✓ Branch 4 taken 105 times.
✗ Branch 5 not taken.
 105 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1618 "(manager '%s' - id %" PRId64 ") "
1619 "data corruption with no-cache header, try another %s",
1620 name_.c_str(), info->id(), typ.c_str());
1621
1622 105 info->SetErrorCode(kFailHostHttp);
1623 }
1624 }
1625 961 if (same_url_retry
1626
5/6
✓ Branch 0 taken 799 times.
✓ Branch 1 taken 162 times.
✓ Branch 3 taken 799 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 309 times.
✓ Branch 6 taken 652 times.
 1760 || (((info->error_code() == kFailHostResolve)
1627
2/2
✓ Branch 2 taken 319 times.
✓ Branch 3 taken 480 times.
 799 || IsHostTransferError(info->error_code())
1628
2/2
✓ Branch 1 taken 105 times.
✓ Branch 2 taken 214 times.
 319 || (info->error_code() == kFailHostHttp))
1629
3/4
✓ Branch 1 taken 415 times.
✓ Branch 2 taken 170 times.
✓ Branch 3 taken 415 times.
✗ Branch 4 not taken.
 585 && info->probe_hosts() && host_chain
1630
2/2
✓ Branch 1 taken 147 times.
✓ Branch 2 taken 268 times.
 415 && (num_used_hosts < host_chain->size()))) {
1631 309 try_again = true;
1632 }
1633 961 if (same_url_retry
1634
5/6
✓ Branch 0 taken 799 times.
✓ Branch 1 taken 162 times.
✓ Branch 3 taken 799 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 211 times.
✓ Branch 6 taken 750 times.
 1760 || (((info->error_code() == kFailProxyResolve)
1635
2/2
✓ Branch 2 taken 750 times.
✓ Branch 3 taken 49 times.
 799 || IsProxyTransferError(info->error_code())
1636
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 750 times.
 750 || (info->error_code() == kFailProxyHttp)))) {
1637
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 211 times.
 211 if (sharding_policy_.UseCount() > 0) { // sharding policy
1638 ✗ try_again = true;
1639 ✗ same_url_retry = false;
1640 } else { // no sharding
1641 211 try_again = true;
1642 // If all proxies failed, do a next round with the next host
1643
4/6
✓ Branch 0 taken 49 times.
✓ Branch 1 taken 162 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 49 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 211 times.
 211 if (!same_url_retry && (info->num_used_proxies() >= opt_num_proxies_)) {
1644 // Check if this can be made a host fail-over
1645 ✗ if (info->probe_hosts() && host_chain
1646 ✗ && (num_used_hosts < host_chain->size())) {
1647 // reset proxy group if not already performed by other handle
1648 ✗ if (opt_proxy_groups_) {
1649 ✗ if ((opt_proxy_groups_current_ > 0)
1650 ✗ || (opt_proxy_groups_current_burned_ > 0)) {
1651 ✗ opt_proxy_groups_current_ = 0;
1652 ✗ opt_timestamp_backup_proxies_ = 0;
1653 ✗ const std::string msg = "reset proxies for " + typ
1654 ✗ + " failover";
1655 ✗ RebalanceProxiesUnlocked(msg);
1656 }
1657 }
1658
1659 // Make it a host failure
1660 ✗ LogCvmfs(kLogDownload, kLogDebug,
1661 "(manager '%s' - id %" PRId64 ") make it a %s failure",
1662 name_.c_str(), info->id(), typ.c_str());
1663 ✗ info->SetNumUsedProxies(1);
1664 ✗ info->SetErrorCode(kFailHostAfterProxy);
1665 } else {
1666 ✗ if (failover_indefinitely_) {
1667 // Instead of giving up, reset the num_used_proxies counter,
1668 // switch proxy and try again
1669 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1670 "(manager '%s' - id %" PRId64 ") "
1671 "VerifyAndFinalize() would fail the download here. "
1672 "Instead switch proxy and retry download. "
1673 "typ=%s "
1674 "info->probe_hosts=%d host_chain=%p num_used_hosts=%d "
1675 "host_chain->size()=%lu same_url_retry=%d "
1676 "info->num_used_proxies=%d opt_num_proxies_=%d",
1677 name_.c_str(), info->id(), typ.c_str(),
1678 ✗ static_cast<int>(info->probe_hosts()), host_chain,
1679 ✗ num_used_hosts, host_chain ? host_chain->size() : -1,
1680 static_cast<int>(same_url_retry),
1681 ✗ info->num_used_proxies(), opt_num_proxies_);
1682 ✗ info->SetNumUsedProxies(1);
1683 ✗ RebalanceProxiesUnlocked(
1684 "download failed - failover indefinitely");
1685 ✗ try_again = !Interrupted(fqrn_, info);
1686 } else {
1687 ✗ try_again = false;
1688 }
1689 }
1690 } // Make a proxy failure a host failure
1691 } // Proxy failure assumed
1692 } // end !sharding
1693 961 }
1694
1695
2/2
✓ Branch 0 taken 463 times.
✓ Branch 1 taken 6668 times.
 7131 if (try_again) {
1696
1/2
✓ Branch 3 taken 463 times.
✗ Branch 4 not taken.
 463 LogCvmfs(kLogDownload, kLogDebug,
1697 "(manager '%s' - id %" PRId64 ") "
1698 "Trying again on same curl handle, same url: %d, "
1699 "error code %d no-cache %d",
1700 463 name_.c_str(), info->id(), same_url_retry, info->error_code(),
1701 463 info->nocache());
1702 // Reset internal state and destination
1703
4/8
✓ Branch 1 taken 463 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 463 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 463 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 463 times.
 463 if (info->sink() != NULL && info->sink()->Reset() != 0) {
1704 ✗ info->SetErrorCode(kFailLocalIO);
1705 ✗ goto verify_and_finalize_stop;
1706 }
1707
6/8
✓ Branch 1 taken 49 times.
✓ Branch 2 taken 414 times.
✓ Branch 5 taken 49 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 49 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 49 times.
✓ Branch 10 taken 414 times.
 463 if (info->interrupt_cue() && info->interrupt_cue()->IsCanceled()) {
1708 49 info->SetErrorCode(kFailCanceled);
1709 49 goto verify_and_finalize_stop;
1710 }
1711
1712
2/2
✓ Branch 1 taken 195 times.
✓ Branch 2 taken 219 times.
 414 if (info->expected_hash()) {
1713
1/2
✓ Branch 2 taken 195 times.
✗ Branch 3 not taken.
 195 shash::Init(info->hash_context());
1714 }
1715
2/2
✓ Branch 1 taken 195 times.
✓ Branch 2 taken 219 times.
 414 if (info->compressed()) {
1716
1/2
✓ Branch 2 taken 195 times.
✗ Branch 3 not taken.
 195 zlib::DecompressInit(info->GetZstreamPtr());
1717 }
1718
1719
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 414 times.
 414 if (sharding_policy_.UseCount() > 0) { // sharding policy
1720 ✗ ReleaseCredential(info);
1721 ✗ SetUrlOptions(info);
1722 } else { // no sharding policy
1723
1/2
✓ Branch 1 taken 414 times.
✗ Branch 2 not taken.
 414 SetRegularCache(info);
1724
1725 // Failure handling
1726 414 bool switch_proxy = false;
1727 414 bool switch_host = false;
1728
2/4
✓ Branch 1 taken 105 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 309 times.
 414 switch (info->error_code()) {
1729 105 case kFailBadData:
1730
1/2
✓ Branch 1 taken 105 times.
✗ Branch 2 not taken.
 105 SetNocache(info);
1731 105 break;
1732 ✗ case kFailProxyResolve:
1733 case kFailProxyHttp:
1734 ✗ switch_proxy = true;
1735 ✗ break;
1736 ✗ case kFailHostResolve:
1737 case kFailHostHttp:
1738 case kFailHostAfterProxy:
1739 ✗ switch_host = true;
1740 ✗ break;
1741 309 default:
1742
2/2
✓ Branch 2 taken 49 times.
✓ Branch 3 taken 260 times.
 309 if (IsProxyTransferError(info->error_code())) {
1743
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
 49 if (same_url_retry) {
1744 ✗ Backoff(info);
1745 } else {
1746 49 switch_proxy = true;
1747 }
1748
1/2
✓ Branch 2 taken 260 times.
✗ Branch 3 not taken.
 260 } else if (IsHostTransferError(info->error_code())) {
1749
2/2
✓ Branch 0 taken 162 times.
✓ Branch 1 taken 98 times.
 260 if (same_url_retry) {
1750
1/2
✓ Branch 1 taken 162 times.
✗ Branch 2 not taken.
 162 Backoff(info);
1751 } else {
1752 98 switch_host = true;
1753 }
1754 } else {
1755 // No other errors expected when retrying
1756 ✗ PANIC(NULL);
1757 }
1758 }
1759
2/2
✓ Branch 0 taken 49 times.
✓ Branch 1 taken 365 times.
 414 if (switch_proxy) {
1760
1/2
✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
 49 ReleaseCredential(info);
1761
1/2
✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
 49 SwitchProxy(info);
1762 49 info->SetNumUsedProxies(info->num_used_proxies() + 1);
1763
1/2
✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
 49 SetUrlOptions(info);
1764 }
1765
2/2
✓ Branch 0 taken 98 times.
✓ Branch 1 taken 316 times.
 414 if (switch_host) {
1766
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 ReleaseCredential(info);
1767
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 98 times.
 98 if (was_metalink) {
1768 ✗ SwitchMetalink(info);
1769 ✗ info->SetNumUsedMetalinks(num_used_hosts + 1);
1770 } else {
1771
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 SwitchHost(info);
1772 98 info->SetNumUsedHosts(num_used_hosts + 1);
1773 }
1774
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 SetUrlOptions(info);
1775 }
1776 } // end !sharding
1777
1778
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 414 times.
 414 if (failover_indefinitely_) {
1779 // try again, breaking if there's a cvmfs reload happening and we are in a
1780 // proxy failover. This will EIO the call application.
1781 ✗ return !Interrupted(fqrn_, info);
1782 }
1783 414 return true; // try again
1784 }
1785
1786 6668 verify_and_finalize_stop:
1787 // Finalize, flush destination file
1788
1/2
✓ Branch 1 taken 6717 times.
✗ Branch 2 not taken.
 6717 ReleaseCredential(info);
1789
4/8
✓ Branch 1 taken 6717 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 6717 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 6717 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 6717 times.
 6717 if (info->sink() != NULL && info->sink()->Flush() != 0) {
1790 ✗ info->SetErrorCode(kFailLocalIO);
1791 }
1792
1793
2/2
✓ Branch 1 taken 4384 times.
✓ Branch 2 taken 2333 times.
 6717 if (info->compressed())
1794
1/2
✓ Branch 2 taken 4384 times.
✗ Branch 3 not taken.
 4384 zlib::DecompressFini(info->GetZstreamPtr());
1795
1796
1/2
✓ Branch 1 taken 6717 times.
✗ Branch 2 not taken.
 6717 if (info->headers()) {
1797
1/2
✓ Branch 2 taken 6717 times.
✗ Branch 3 not taken.
 6717 header_lists_->PutList(info->headers());
1798 6717 info->SetHeaders(NULL);
1799 }
1800
1801 6717 return false; // stop transfer and return to Fetch()
1802 7131 }
1803
1804 5568 DownloadManager::~DownloadManager() {
1805 // cleaned up fini
1806
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5568 times.
 5568 if (sharding_policy_.UseCount() > 0) {
1807 ✗ sharding_policy_.Reset();
1808 }
1809
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5568 times.
 5568 if (health_check_.UseCount() > 0) {
1810 ✗ if (health_check_.Unique()) {
1811 ✗ LogCvmfs(kLogDownload, kLogDebug,
1812 "(manager '%s') Stopping healthcheck thread", name_.c_str());
1813 ✗ health_check_->StopHealthcheck();
1814 }
1815 ✗ health_check_.Reset();
1816 }
1817
1818
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5568 times.
 5568 if (atomic_xadd32(&multi_threaded_, 0) == 1) {
1819 // Shutdown I/O thread
1820 ✗ pipe_terminate_->Write(kPipeTerminateSignal);
1821 ✗ pthread_join(thread_download_, NULL);
1822 // All handles are removed from the multi stack
1823 ✗ pipe_terminate_.Destroy();
1824 ✗ pipe_jobs_.Destroy();
1825 }
1826
1827 11136 for (set<CURL *>::iterator i = pool_handles_idle_->begin(),
1828 5568 iEnd = pool_handles_idle_->end();
1829
2/2
✓ Branch 1 taken 2906 times.
✓ Branch 2 taken 5568 times.
 8474 i != iEnd;
1830 2906 ++i) {
1831 2906 curl_easy_cleanup(*i);
1832 }
1833
1834
1/2
✓ Branch 0 taken 5568 times.
✗ Branch 1 not taken.
 5568 delete pool_handles_idle_;
1835
1/2
✓ Branch 0 taken 5568 times.
✗ Branch 1 not taken.
 5568 delete pool_handles_inuse_;
1836 5568 curl_multi_cleanup(curl_multi_);
1837
1838
1/2
✓ Branch 0 taken 5568 times.
✗ Branch 1 not taken.
 5568 delete header_lists_;
1839
1/2
✓ Branch 0 taken 5568 times.
✗ Branch 1 not taken.
 5568 if (user_agent_)
1840 5568 free(user_agent_);
1841
1842
1/2
✓ Branch 0 taken 5568 times.
✗ Branch 1 not taken.
 5568 delete counters_;
1843
2/2
✓ Branch 0 taken 2689 times.
✓ Branch 1 taken 2879 times.
 5568 delete opt_host_.chain;
1844
2/2
✓ Branch 0 taken 2689 times.
✓ Branch 1 taken 2879 times.
 5568 delete opt_host_chain_rtt_;
1845
2/2
✓ Branch 0 taken 2185 times.
✓ Branch 1 taken 3383 times.
 5568 delete opt_proxy_groups_;
1846
1847 5568 curl_global_cleanup();
1848
1/2
✓ Branch 0 taken 5568 times.
✗ Branch 1 not taken.
 5568 delete resolver_;
1849
1850 // old destructor
1851 5568 pthread_mutex_destroy(lock_options_);
1852 5568 pthread_mutex_destroy(lock_synchronous_mode_);
1853 5568 free(lock_options_);
1854 5568 free(lock_synchronous_mode_);
1855 5568 }
1856
1857 5569 void DownloadManager::InitHeaders() {
1858 // User-Agent
1859
1/2
✓ Branch 2 taken 5569 times.
✗ Branch 3 not taken.
 5569 string cernvm_id = "User-Agent: cvmfs ";
1860 #ifdef CVMFS_LIBCVMFS
1861
1/2
✓ Branch 1 taken 5569 times.
✗ Branch 2 not taken.
 5569 cernvm_id += "libcvmfs ";
1862 #else
1863 ✗ cernvm_id += "Fuse ";
1864 #endif
1865
2/4
✓ Branch 2 taken 5569 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 5569 times.
✗ Branch 6 not taken.
 5569 cernvm_id += string(CVMFS_VERSION);
1866
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5569 times.
 5569 if (getenv("CERNVM_UUID") != NULL) {
1867 cernvm_id += " "
1868 ✗ + sanitizer::InputSanitizer("az AZ 09 -")
1869 ✗ .Filter(getenv("CERNVM_UUID"));
1870 }
1871 5569 user_agent_ = strdup(cernvm_id.c_str());
1872
1873
1/2
✓ Branch 1 taken 5569 times.
✗ Branch 2 not taken.
 5569 header_lists_ = new HeaderLists();
1874
1875
1/2
✓ Branch 1 taken 5569 times.
✗ Branch 2 not taken.
 5569 default_headers_ = header_lists_->GetList("Connection: Keep-Alive");
1876
1/2
✓ Branch 1 taken 5569 times.
✗ Branch 2 not taken.
 5569 header_lists_->AppendHeader(default_headers_, "Pragma:");
1877
1/2
✓ Branch 1 taken 5569 times.
✗ Branch 2 not taken.
 5569 header_lists_->AppendHeader(default_headers_, user_agent_);
1878 5569 }
1879
1880 5569 DownloadManager::DownloadManager(const unsigned max_pool_handles,
1881 const perf::StatisticsTemplate &statistics,
1882 5569 const std::string &name)
1883 5569 : prng_(Prng())
1884 5569 , pool_handles_idle_(new set<CURL *>)
1885 5569 , pool_handles_inuse_(new set<CURL *>)
1886 5569 , pool_max_handles_(max_pool_handles)
1887 5569 , pipe_terminate_(NULL)
1888
1/2
✓ Branch 1 taken 5569 times.
✗ Branch 2 not taken.
 5569 , pipe_jobs_(NULL)
1889 5569 , watch_fds_(NULL)
1890 5569 , watch_fds_size_(0)
1891 5569 , watch_fds_inuse_(0)
1892 5569 , watch_fds_max_(4 * max_pool_handles)
1893 5569 , opt_timeout_proxy_(5)
1894 5569 , opt_timeout_direct_(10)
1895 5569 , opt_low_speed_limit_(1024)
1896 5569 , opt_max_retries_(0)
1897 5569 , opt_backoff_init_ms_(0)
1898 5569 , opt_backoff_max_ms_(0)
1899 5569 , enable_info_header_(false)
1900 5569 , opt_ipv4_only_(false)
1901 5569 , follow_redirects_(false)
1902 5569 , ignore_signature_failures_(false)
1903 5569 , enable_http_tracing_(false)
1904 5569 , opt_metalink_(NULL, 0, 0, 0)
1905 5569 , opt_metalink_timestamp_link_(0)
1906 5569 , opt_host_(NULL, 0, 0, 0)
1907 5569 , opt_host_chain_rtt_(NULL)
1908 5569 , opt_proxy_groups_(NULL)
1909 5569 , opt_proxy_groups_current_(0)
1910 5569 , opt_proxy_groups_current_burned_(0)
1911 5569 , opt_proxy_groups_fallback_(0)
1912 5569 , opt_num_proxies_(0)
1913 5569 , opt_proxy_shard_(false)
1914 5569 , failover_indefinitely_(false)
1915
1/2
✓ Branch 1 taken 5569 times.
✗ Branch 2 not taken.
 5569 , name_(name)
1916 5569 , opt_ip_preference_(dns::kIpPreferSystem)
1917 5569 , opt_timestamp_backup_proxies_(0)
1918 5569 , opt_timestamp_failover_proxies_(0)
1919 5569 , opt_proxy_groups_reset_after_(0)
1920 5569 , credentials_attachment_(NULL)
1921
4/8
✓ Branch 12 taken 5569 times.
✗ Branch 13 not taken.
✓ Branch 15 taken 5569 times.
✗ Branch 16 not taken.
✓ Branch 18 taken 5569 times.
✗ Branch 19 not taken.
✓ Branch 22 taken 5569 times.
✗ Branch 23 not taken.
 16707 , counters_(new Counters(statistics)) {
1922 5569 atomic_init32(&multi_threaded_);
1923
1924 5569 lock_options_ = reinterpret_cast<pthread_mutex_t *>(
1925 5569 smalloc(sizeof(pthread_mutex_t)));
1926 5569 int retval = pthread_mutex_init(lock_options_, NULL);
1927
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5569 times.
 5569 assert(retval == 0);
1928 5569 lock_synchronous_mode_ = reinterpret_cast<pthread_mutex_t *>(
1929 5569 smalloc(sizeof(pthread_mutex_t)));
1930 5569 retval = pthread_mutex_init(lock_synchronous_mode_, NULL);
1931
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5569 times.
 5569 assert(retval == 0);
1932
1933
1/2
✓ Branch 1 taken 5569 times.
✗ Branch 2 not taken.
 5569 retval = curl_global_init(CURL_GLOBAL_ALL);
1934
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5569 times.
 5569 assert(retval == CURLE_OK);
1935
1936
1/2
✓ Branch 1 taken 5569 times.
✗ Branch 2 not taken.
 5569 InitHeaders();
1937
1938
1/2
✓ Branch 1 taken 5569 times.
✗ Branch 2 not taken.
 5569 curl_multi_ = curl_multi_init();
1939
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5569 times.
 5569 assert(curl_multi_ != NULL);
1940
1/2
✓ Branch 1 taken 5569 times.
✗ Branch 2 not taken.
 5569 curl_multi_setopt(curl_multi_, CURLMOPT_SOCKETFUNCTION, CallbackCurlSocket);
1941
1/2
✓ Branch 1 taken 5569 times.
✗ Branch 2 not taken.
 5569 curl_multi_setopt(curl_multi_, CURLMOPT_SOCKETDATA,
1942 static_cast<void *>(this));
1943
1/2
✓ Branch 1 taken 5569 times.
✗ Branch 2 not taken.
 5569 curl_multi_setopt(curl_multi_, CURLMOPT_MAXCONNECTS, watch_fds_max_);
1944
1/2
✓ Branch 1 taken 5569 times.
✗ Branch 2 not taken.
 5569 curl_multi_setopt(curl_multi_, CURLMOPT_MAX_TOTAL_CONNECTIONS,
1945 pool_max_handles_);
1946
1947 5569 prng_.InitLocaltime();
1948
1949 // Name resolving
1950 5569 if ((getenv("CVMFS_IPV4_ONLY") != NULL)
1951
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 5569 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 5569 times.
 5569 && (strlen(getenv("CVMFS_IPV4_ONLY")) > 0)) {
1952 ✗ opt_ipv4_only_ = true;
1953 }
1954
1/2
✓ Branch 1 taken 5569 times.
✗ Branch 2 not taken.
 5569 resolver_ = dns::NormalResolver::Create(opt_ipv4_only_, kDnsDefaultRetries,
1955 kDnsDefaultTimeoutMs);
1956
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5569 times.
 5569 assert(resolver_);
1957 5569 }
1958
1959 /**
1960 * Spawns the I/O worker thread and switches the module in multi-threaded mode.
1961 * No way back except Fini(); Init();
1962 */
1963 ✗ void DownloadManager::Spawn() {
1964 ✗ pipe_terminate_ = new Pipe<kPipeThreadTerminator>();
1965 ✗ pipe_jobs_ = new Pipe<kPipeDownloadJobs>();
1966
1967 ✗ const int retval = pthread_create(&thread_download_, NULL, MainDownload,
1968 static_cast<void *>(this));
1969 ✗ assert(retval == 0);
1970
1971 ✗ atomic_inc32(&multi_threaded_);
1972
1973 ✗ if (health_check_.UseCount() > 0) {
1974 ✗ LogCvmfs(kLogDownload, kLogDebug,
1975 "(manager '%s') Starting healthcheck thread", name_.c_str());
1976 ✗ health_check_->StartHealthcheck();
1977 }
1978 }
1979
1980
1981 /**
1982 * Downloads data from an insecure outside channel (currently HTTP or file).
1983 */
1984 6717 Failures DownloadManager::Fetch(JobInfo *info) {
1985
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6717 times.
 6717 assert(info != NULL);
1986
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6717 times.
 6717 assert(info->url() != NULL);
1987
1988 Failures result;
1989
1/2
✓ Branch 1 taken 6717 times.
✗ Branch 2 not taken.
 6717 result = PrepareDownloadDestination(info);
1990
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6717 times.
 6717 if (result != kFailOk)
1991 ✗ return result;
1992
1993
2/2
✓ Branch 1 taken 4433 times.
✓ Branch 2 taken 2284 times.
 6717 if (info->expected_hash()) {
1994 4433 const shash::Algorithms algorithm = info->expected_hash()->algorithm;
1995 4433 info->GetHashContextPtr()->algorithm = algorithm;
1996
1/2
✓ Branch 1 taken 4433 times.
✗ Branch 2 not taken.
 4433 info->GetHashContextPtr()->size = shash::GetContextSize(algorithm);
1997 4433 info->GetHashContextPtr()->buffer = alloca(info->hash_context().size);
1998 }
1999
2000 // In case JobInfo object is being reused
2001
2/4
✓ Branch 2 taken 6717 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 6717 times.
✗ Branch 6 not taken.
 6717 info->SetLink("");
2002
2003 // Prepare cvmfs-info: header, allocate string on the stack
2004 6717 info->SetInfoHeader(NULL);
2005
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 6717 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 6717 times.
 6717 if (enable_info_header_ && info->extra_info()) {
2006 ✗ const char *header_name = "cvmfs-info: ";
2007 ✗ const size_t header_name_len = strlen(header_name);
2008 const unsigned header_size = 1 + header_name_len
2009 ✗ + EscapeHeader(*(info->extra_info()), NULL, 0);
2010 ✗ info->SetInfoHeader(static_cast<char *>(alloca(header_size)));
2011 ✗ memcpy(info->info_header(), header_name, header_name_len);
2012 ✗ EscapeHeader(*(info->extra_info()), info->info_header() + header_name_len,
2013 ✗ header_size - header_name_len);
2014 ✗ info->info_header()[header_size - 1] = '\0';
2015 }
2016
2017
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6717 times.
 6717 if (enable_http_tracing_) {
2018 ✗ const std::string str_pid = "X-CVMFS-PID: " + StringifyInt(info->pid());
2019 ✗ const std::string str_gid = "X-CVMFS-GID: " + StringifyUint(info->gid());
2020 ✗ const std::string str_uid = "X-CVMFS-UID: " + StringifyUint(info->uid());
2021
2022 // will be auto freed at the end of this function Fetch(JobInfo *info)
2023 ✗ info->SetTracingHeaderPid(static_cast<char *>(alloca(str_pid.size() + 1)));
2024 ✗ info->SetTracingHeaderGid(static_cast<char *>(alloca(str_gid.size() + 1)));
2025 ✗ info->SetTracingHeaderUid(static_cast<char *>(alloca(str_uid.size() + 1)));
2026
2027 ✗ memcpy(info->tracing_header_pid(), str_pid.c_str(), str_pid.size() + 1);
2028 ✗ memcpy(info->tracing_header_gid(), str_gid.c_str(), str_gid.size() + 1);
2029 ✗ memcpy(info->tracing_header_uid(), str_uid.c_str(), str_uid.size() + 1);
2030 }
2031
2032
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6717 times.
 6717 if (atomic_xadd32(&multi_threaded_, 0) == 1) {
2033 ✗ if (!info->IsValidPipeJobResults()) {
2034 ✗ info->CreatePipeJobResults();
2035 }
2036 ✗ if (!info->IsValidDataTube()) {
2037 ✗ info->CreateDataTube();
2038 }
2039
2040 // LogCvmfs(kLogDownload, kLogDebug, "send job to thread, pipe %d %d",
2041 // info->wait_at[0], info->wait_at[1]);
2042 ✗ pipe_jobs_->Write<JobInfo *>(info);
2043
2044 do {
2045 ✗ DataTubeElement *ele = info->GetDataTubePtr()->PopFront();
2046
2047 ✗ if (ele->action == kActionStop) {
2048 ✗ delete ele;
2049 ✗ break;
2050 }
2051 // TODO(heretherebedragons) add compression
2052 ✗ } while (true);
2053
2054 ✗ info->GetPipeJobResultPtr()->Read<download::Failures>(&result);
2055 // LogCvmfs(kLogDownload, kLogDebug, "got result %d", result);
2056 } else {
2057 6717 const MutexLockGuard l(lock_synchronous_mode_);
2058
1/2
✓ Branch 1 taken 6717 times.
✗ Branch 2 not taken.
 6717 CURL *handle = AcquireCurlHandle();
2059
1/2
✓ Branch 1 taken 6717 times.
✗ Branch 2 not taken.
 6717 InitializeRequest(info, handle);
2060
1/2
✓ Branch 1 taken 6717 times.
✗ Branch 2 not taken.
 6717 SetUrlOptions(info);
2061 // curl_easy_setopt(handle, CURLOPT_VERBOSE, 1);
2062 int retval;
2063 do {
2064
1/2
✓ Branch 1 taken 7131 times.
✗ Branch 2 not taken.
 7131 retval = curl_easy_perform(handle);
2065 7131 perf::Inc(counters_->n_requests);
2066 double elapsed;
2067
1/2
✓ Branch 1 taken 7131 times.
✗ Branch 2 not taken.
 7131 if (curl_easy_getinfo(handle, CURLINFO_TOTAL_TIME, &elapsed)
2068
1/2
✓ Branch 0 taken 7131 times.
✗ Branch 1 not taken.
 7131 == CURLE_OK) {
2069 7131 perf::Xadd(counters_->sz_transfer_time,
2070 7131 static_cast<int64_t>(elapsed * 1000));
2071 }
2072
3/4
✓ Branch 1 taken 7131 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 414 times.
✓ Branch 4 taken 6717 times.
 7131 } while (VerifyAndFinalize(retval, info));
2073 6717 result = info->error_code();
2074
1/2
✓ Branch 2 taken 6717 times.
✗ Branch 3 not taken.
 6717 ReleaseCurlHandle(info->curl_handle());
2075 6717 }
2076
2077
2/2
✓ Branch 0 taken 547 times.
✓ Branch 1 taken 6170 times.
 6717 if (result != kFailOk) {
2078
1/2
✓ Branch 4 taken 547 times.
✗ Branch 5 not taken.
 547 LogCvmfs(kLogDownload, kLogDebug,
2079 "(manager '%s' - id %" PRId64 ") "
2080 "download failed (error %d - %s)",
2081 name_.c_str(), info->id(), result, Code2Ascii(result));
2082
2083
1/2
✓ Branch 1 taken 547 times.
✗ Branch 2 not taken.
 547 if (info->sink() != NULL) {
2084
1/2
✓ Branch 2 taken 547 times.
✗ Branch 3 not taken.
 547 info->sink()->Purge();
2085 }
2086 }
2087
2088 6717 return result;
2089 }
2090
2091
2092 /**
2093 * Used by the client to connect the authz session manager to the download
2094 * manager.
2095 */
2096 1231 void DownloadManager::SetCredentialsAttachment(CredentialsAttachment *ca) {
2097 1231 const MutexLockGuard m(lock_options_);
2098 1231 credentials_attachment_ = ca;
2099 1231 }
2100
2101 /**
2102 * Gets the DNS sever.
2103 */
2104 ✗ std::string DownloadManager::GetDnsServer() const { return opt_dns_server_; }
2105
2106 /**
2107 * Sets a DNS server. Only for testing as it cannot be reverted to the system
2108 * default.
2109 */
2110 ✗ void DownloadManager::SetDnsServer(const string &address) {
2111 ✗ if (!address.empty()) {
2112 ✗ const MutexLockGuard m(lock_options_);
2113 ✗ opt_dns_server_ = address;
2114 ✗ assert(!opt_dns_server_.empty());
2115
2116 ✗ vector<string> servers;
2117 ✗ servers.push_back(address);
2118 ✗ const bool retval = resolver_->SetResolvers(servers);
2119 ✗ assert(retval);
2120 }
2121 ✗ LogCvmfs(kLogDownload, kLogSyslog, "(manager '%s') set nameserver to %s",
2122 name_.c_str(), address.c_str());
2123 }
2124
2125
2126 /**
2127 * Sets the DNS query timeout parameters.
2128 */
2129 2348 void DownloadManager::SetDnsParameters(const unsigned retries,
2130 const unsigned timeout_ms) {
2131 2348 const MutexLockGuard m(lock_options_);
2132 2348 if ((resolver_->retries() == retries)
2133
3/6
✓ Branch 0 taken 2348 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 2348 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 2348 times.
✗ Branch 6 not taken.
 2348 && (resolver_->timeout_ms() == timeout_ms)) {
2134 2348 return;
2135 }
2136 ✗ delete resolver_;
2137 ✗ resolver_ = NULL;
2138 ✗ resolver_ = dns::NormalResolver::Create(opt_ipv4_only_, retries, timeout_ms);
2139 ✗ assert(resolver_);
2140
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 2348 times.
 2348 }
2141
2142
2143 2348 void DownloadManager::SetDnsTtlLimits(const unsigned min_seconds,
2144 const unsigned max_seconds) {
2145 2348 const MutexLockGuard m(lock_options_);
2146 2348 resolver_->set_min_ttl(min_seconds);
2147 2348 resolver_->set_max_ttl(max_seconds);
2148 2348 }
2149
2150
2151 ✗ void DownloadManager::SetIpPreference(dns::IpPreference preference) {
2152 ✗ const MutexLockGuard m(lock_options_);
2153 ✗ opt_ip_preference_ = preference;
2154 }
2155
2156
2157 /**
2158 * Sets two timeout values for proxied and for direct connections, respectively.
2159 * The timeout counts for all sorts of connection phases,
2160 * DNS, HTTP connect, etc.
2161 */
2162 3080 void DownloadManager::SetTimeout(const unsigned seconds_proxy,
2163 const unsigned seconds_direct) {
2164 3080 const MutexLockGuard m(lock_options_);
2165 3080 opt_timeout_proxy_ = seconds_proxy;
2166 3080 opt_timeout_direct_ = seconds_direct;
2167 3080 }
2168
2169
2170 /**
2171 * Sets contains the average transfer speed in bytes per second that the
2172 * transfer should be below during CURLOPT_LOW_SPEED_TIME seconds for libcurl to
2173 * consider it to be too slow and abort. Only effective for new connections.
2174 */
2175 ✗ void DownloadManager::SetLowSpeedLimit(const unsigned low_speed_limit) {
2176 ✗ const MutexLockGuard m(lock_options_);
2177 ✗ opt_low_speed_limit_ = low_speed_limit;
2178 }
2179
2180
2181 /**
2182 * Receives the currently active timeout values.
2183 */
2184 1019 void DownloadManager::GetTimeout(unsigned *seconds_proxy,
2185 unsigned *seconds_direct) {
2186 1019 const MutexLockGuard m(lock_options_);
2187 1019 *seconds_proxy = opt_timeout_proxy_;
2188 1019 *seconds_direct = opt_timeout_direct_;
2189 1019 }
2190
2191
2192 /**
2193 * Parses a list of ';'-separated hosts for the metalink chain. The empty
2194 * string removes the metalink list.
2195 */
2196 ✗ void DownloadManager::SetMetalinkChain(const string &metalink_list) {
2197 ✗ SetMetalinkChain(SplitString(metalink_list, ';'));
2198 }
2199
2200
2201 ✗ void DownloadManager::SetMetalinkChain(
2202 const std::vector<std::string> &metalink_list) {
2203 ✗ const MutexLockGuard m(lock_options_);
2204 ✗ opt_metalink_.timestamp_backup = 0;
2205 ✗ delete opt_metalink_.chain;
2206 ✗ opt_metalink_.current = 0;
2207
2208 ✗ if (metalink_list.empty()) {
2209 ✗ opt_metalink_.chain = NULL;
2210 ✗ return;
2211 }
2212
2213 ✗ opt_metalink_.chain = new vector<string>(metalink_list);
2214 }
2215
2216
2217 /**
2218 * Retrieves the currently set chain of metalink hosts and the currently
2219 * used metalink host.
2220 */
2221 ✗ void DownloadManager::GetMetalinkInfo(vector<string> *metalink_chain,
2222 unsigned *current_metalink) {
2223 ✗ const MutexLockGuard m(lock_options_);
2224 ✗ if (opt_metalink_.chain) {
2225 ✗ if (current_metalink) {
2226 ✗ *current_metalink = opt_metalink_.current;
2227 }
2228 ✗ if (metalink_chain) {
2229 ✗ *metalink_chain = *opt_metalink_.chain;
2230 }
2231 }
2232 }
2233
2234
2235 /**
2236 * Parses a list of ';'-separated hosts for the host chain. The empty string
2237 * removes the host list.
2238 */
2239 2719 void DownloadManager::SetHostChain(const string &host_list) {
2240
1/2
✓ Branch 2 taken 2719 times.
✗ Branch 3 not taken.
 2719 SetHostChain(SplitString(host_list, ';'));
2241 2719 }
2242
2243
2244 2783 void DownloadManager::SetHostChain(const std::vector<std::string> &host_list) {
2245 2783 const MutexLockGuard m(lock_options_);
2246 2783 opt_host_.timestamp_backup = 0;
2247
2/2
✓ Branch 0 taken 1113 times.
✓ Branch 1 taken 1670 times.
 2783 delete opt_host_.chain;
2248
2/2
✓ Branch 0 taken 1113 times.
✓ Branch 1 taken 1670 times.
 2783 delete opt_host_chain_rtt_;
2249 2783 opt_host_.current = 0;
2250
2251
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 2783 times.
 2783 if (host_list.empty()) {
2252 ✗ opt_host_.chain = NULL;
2253 ✗ opt_host_chain_rtt_ = NULL;
2254 ✗ return;
2255 }
2256
2257
2/4
✓ Branch 1 taken 2783 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2783 times.
✗ Branch 5 not taken.
 2783 opt_host_.chain = new vector<string>(host_list);
2258 5566 opt_host_chain_rtt_ = new vector<int>(opt_host_.chain->size(),
2259
2/4
✓ Branch 1 taken 2783 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2783 times.
✗ Branch 5 not taken.
 2783 kProbeUnprobed);
2260 // LogCvmfs(kLogDownload, kLogSyslog, "using host %s",
2261 // (*opt_host_.chain)[0].c_str());
2262
1/2
✓ Branch 1 taken 2783 times.
✗ Branch 2 not taken.
 2783 }
2263
2264
2265 /**
2266 * Retrieves the currently set chain of hosts, their round trip times, and the
2267 * currently used host.
2268 */
2269 448 void DownloadManager::GetHostInfo(vector<string> *host_chain, vector<int> *rtt,
2270 unsigned *current_host) {
2271 448 const MutexLockGuard m(lock_options_);
2272
1/2
✓ Branch 0 taken 448 times.
✗ Branch 1 not taken.
 448 if (opt_host_.chain) {
2273
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 448 times.
 448 if (current_host) {
2274 ✗ *current_host = opt_host_.current;
2275 }
2276
1/2
✓ Branch 0 taken 448 times.
✗ Branch 1 not taken.
 448 if (host_chain) {
2277
1/2
✓ Branch 1 taken 448 times.
✗ Branch 2 not taken.
 448 *host_chain = *opt_host_.chain;
2278 }
2279
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 448 times.
 448 if (rtt) {
2280 ✗ *rtt = *opt_host_chain_rtt_;
2281 }
2282 }
2283 448 }
2284
2285
2286 /**
2287 * Jumps to the next proxy in the ring of forward proxy servers.
2288 * Selects one randomly from a load-balancing group.
2289 *
2290 * Allow for the fact that the proxy may have already been failed by
2291 * another transfer, or that the proxy may no longer be part of the
2292 * current load-balancing group.
2293 */
2294 49 void DownloadManager::SwitchProxy(JobInfo *info) {
2295 49 const MutexLockGuard m(lock_options_);
2296
2297
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
 49 if (!opt_proxy_groups_) {
2298 ✗ return;
2299 }
2300
2301 // Fail any matching proxies within the current load-balancing group
2302 49 vector<ProxyInfo> *group = current_proxy_group();
2303 49 const unsigned group_size = group->size();
2304 49 unsigned failed = 0;
2305
2/2
✓ Branch 0 taken 49 times.
✓ Branch 1 taken 49 times.
 98 for (unsigned i = 0; i < group_size - opt_proxy_groups_current_burned_; ++i) {
2306
5/14
✓ Branch 0 taken 49 times.
✗ Branch 1 not taken.
✓ Branch 4 taken 49 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 49 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 49 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✓ Branch 12 taken 49 times.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
 49 if (info && (info->proxy() == (*group)[i].url)) {
2307 // Move to list of failed proxies
2308 49 opt_proxy_groups_current_burned_++;
2309
1/2
✓ Branch 2 taken 49 times.
✗ Branch 3 not taken.
 49 swap((*group)[i],
2310 49 (*group)[group_size - opt_proxy_groups_current_burned_]);
2311 49 perf::Inc(counters_->n_proxy_failover);
2312 49 failed++;
2313 }
2314 }
2315
2316 // Do nothing more unless at least one proxy was marked as failed
2317
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
 49 if (!failed)
2318 ✗ return;
2319
2320 // If all proxies from the current load-balancing group are burned, switch to
2321 // another group
2322
1/2
✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
 49 if (opt_proxy_groups_current_burned_ == group->size()) {
2323 49 opt_proxy_groups_current_burned_ = 0;
2324
1/2
✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
 49 if (opt_proxy_groups_->size() > 1) {
2325 98 opt_proxy_groups_current_ = (opt_proxy_groups_current_ + 1)
2326 49 % opt_proxy_groups_->size();
2327 // Remember the timestamp of switching to backup proxies
2328
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
 49 if (opt_proxy_groups_reset_after_ > 0) {
2329 ✗ if (opt_proxy_groups_current_ > 0) {
2330 ✗ if (opt_timestamp_backup_proxies_ == 0)
2331 ✗ opt_timestamp_backup_proxies_ = time(NULL);
2332 // LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2333 // "switched to (another) backup proxy group");
2334 } else {
2335 ✗ opt_timestamp_backup_proxies_ = 0;
2336 // LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2337 // "switched back to primary proxy group");
2338 }
2339 ✗ opt_timestamp_failover_proxies_ = 0;
2340 }
2341 }
2342 } else {
2343 // Record failover time
2344 ✗ if (opt_proxy_groups_reset_after_ > 0) {
2345 ✗ if (opt_timestamp_failover_proxies_ == 0)
2346 ✗ opt_timestamp_failover_proxies_ = time(NULL);
2347 }
2348 }
2349
2350
2/4
✓ Branch 2 taken 49 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 49 times.
✗ Branch 6 not taken.
 49 UpdateProxiesUnlocked("failed proxy");
2351
1/2
✓ Branch 2 taken 49 times.
✗ Branch 3 not taken.
 49 LogCvmfs(kLogDownload, kLogDebug,
2352 "(manager '%s' - id %" PRId64 ") "
2353 "%lu proxies remain in group",
2354 name_.c_str(), info->id(),
2355 49 current_proxy_group()->size() - opt_proxy_groups_current_burned_);
2356
1/2
✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
 49 }
2357
2358
2359 /**
2360 * Switches to the next host in the chain. If jobinfo is set, switch only if
2361 * the current host is identical to the one used by jobinfo, otherwise another
2362 * transfer has already done the switch.
2363 */
2364 128 void DownloadManager::SwitchHostInfo(const std::string &typ,
2365 HostInfo &info,
2366 JobInfo *jobinfo) {
2367 128 const MutexLockGuard m(lock_options_);
2368
2369
5/6
✓ Branch 0 taken 128 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 30 times.
✓ Branch 4 taken 98 times.
✓ Branch 5 taken 30 times.
✓ Branch 6 taken 98 times.
 128 if (!info.chain || (info.chain->size() == 1)) {
2370 30 return;
2371 }
2372
2373
1/2
✓ Branch 0 taken 98 times.
✗ Branch 1 not taken.
 98 if (jobinfo) {
2374 int lastused;
2375
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 if (typ == "host") {
2376 98 lastused = jobinfo->current_host_chain_index();
2377 } else {
2378 ✗ lastused = jobinfo->current_metalink_chain_index();
2379 }
2380
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 98 times.
 98 if (lastused != info.current) {
2381 ✗ LogCvmfs(kLogDownload, kLogDebug,
2382 "(manager '%s' - id %" PRId64 ")"
2383 "don't switch %s, "
2384 "last used %s: %s, current %s: %s",
2385 name_.c_str(), jobinfo->id(), typ.c_str(), typ.c_str(),
2386 ✗ (*info.chain)[lastused].c_str(), typ.c_str(),
2387 ✗ (*info.chain)[info.current].c_str());
2388 ✗ return;
2389 }
2390 }
2391
2392
1/2
✓ Branch 2 taken 98 times.
✗ Branch 3 not taken.
 98 string reason = "manually triggered";
2393
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 string info_id = "(manager " + name_;
2394
1/2
✓ Branch 0 taken 98 times.
✗ Branch 1 not taken.
 98 if (jobinfo) {
2395
1/2
✓ Branch 3 taken 98 times.
✗ Branch 4 not taken.
 98 reason = download::Code2Ascii(jobinfo->error_code());
2396
2/4
✓ Branch 2 taken 98 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 98 times.
✗ Branch 6 not taken.
 98 info_id = " - id " + StringifyInt(jobinfo->id());
2397 }
2398
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 info_id += ")";
2399
2400
1/2
✓ Branch 2 taken 98 times.
✗ Branch 3 not taken.
 98 const std::string old_host = (*info.chain)[info.current];
2401 98 info.current = (info.current + 1) % static_cast<int>(info.chain->size());
2402
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 if (typ == "host") {
2403 98 perf::Inc(counters_->n_host_failover);
2404 } else {
2405 ✗ perf::Inc(counters_->n_metalink_failover);
2406 }
2407
1/3
✓ Branch 6 taken 98 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
 196 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2408 "%s switching %s from %s to %s (%s)", info_id.c_str(), typ.c_str(),
2409 98 old_host.c_str(), (*info.chain)[info.current].c_str(),
2410 reason.c_str());
2411
2412 // Remember the timestamp of switching to backup host
2413
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 98 times.
 98 if (info.reset_after > 0) {
2414 ✗ if (info.current != 0) {
2415 ✗ if (info.timestamp_backup == 0)
2416 ✗ info.timestamp_backup = time(NULL);
2417 } else {
2418 ✗ info.timestamp_backup = 0;
2419 }
2420 }
2421
2/2
✓ Branch 4 taken 98 times.
✓ Branch 5 taken 30 times.
 128 }
2422
2423 128 void DownloadManager::SwitchHost(JobInfo *info) {
2424
2/4
✓ Branch 2 taken 128 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 128 times.
✗ Branch 6 not taken.
 128 SwitchHostInfo("host", opt_host_, info);
2425 128 }
2426
2427 30 void DownloadManager::SwitchHost() { SwitchHost(NULL); }
2428
2429
2430 ✗ void DownloadManager::SwitchMetalink(JobInfo *info) {
2431 ✗ SwitchHostInfo("metalink", opt_metalink_, info);
2432 }
2433
2434
2435 ✗ void DownloadManager::SwitchMetalink() { SwitchMetalink(NULL); }
2436
2437 3316 bool DownloadManager::CheckMetalinkChain(time_t now) {
2438 3316 return (opt_metalink_.chain
2439
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 3316 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 3316 && ((opt_metalink_timestamp_link_ == 0)
2440 ✗ || (static_cast<int64_t>((now == 0) ? time(NULL) : now)
2441 ✗ > static_cast<int64_t>(opt_metalink_timestamp_link_
2442
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 3316 + opt_metalink_.reset_after))));
2443 }
2444
2445
2446 /**
2447 * Orders the hostlist according to RTT of downloading .cvmfschecksum.
2448 * Sets the current host to the best-responsive host.
2449 * If you change the host list in between by SetHostChain(), it will be
2450 * overwritten by this function.
2451 */
2452 ✗ void DownloadManager::ProbeHosts() {
2453 ✗ vector<string> host_chain;
2454 ✗ vector<int> host_rtt;
2455 unsigned current_host;
2456
2457 ✗ GetHostInfo(&host_chain, &host_rtt, &current_host);
2458
2459 // Stopwatch, two times to fill caches first
2460 unsigned i, retries;
2461 ✗ string url;
2462
2463 ✗ cvmfs::MemSink memsink;
2464 ✗ JobInfo info(&url, false, false, NULL, &memsink);
2465 ✗ for (retries = 0; retries < 2; ++retries) {
2466 ✗ for (i = 0; i < host_chain.size(); ++i) {
2467 ✗ url = host_chain[i] + "/.cvmfspublished";
2468
2469 struct timeval tv_start, tv_end;
2470 ✗ gettimeofday(&tv_start, NULL);
2471 ✗ const Failures result = Fetch(&info);
2472 ✗ gettimeofday(&tv_end, NULL);
2473 ✗ memsink.Reset();
2474 ✗ if (result == kFailOk) {
2475 ✗ host_rtt[i] = static_cast<int>(DiffTimeSeconds(tv_start, tv_end)
2476 ✗ * 1000);
2477 ✗ LogCvmfs(kLogDownload, kLogDebug,
2478 "(manager '%s' - id %" PRId64 ") "
2479 "probing host %s had %dms rtt",
2480 ✗ name_.c_str(), info.id(), url.c_str(), host_rtt[i]);
2481 } else {
2482 ✗ LogCvmfs(kLogDownload, kLogDebug,
2483 "(manager '%s' - id %" PRId64 ") "
2484 "error while probing host %s: %d %s",
2485 name_.c_str(), info.id(), url.c_str(), result,
2486 Code2Ascii(result));
2487 ✗ host_rtt[i] = INT_MAX;
2488 }
2489 }
2490 }
2491
2492 ✗ SortTeam(&host_rtt, &host_chain);
2493 ✗ for (i = 0; i < host_chain.size(); ++i) {
2494 ✗ if (host_rtt[i] == INT_MAX)
2495 ✗ host_rtt[i] = kProbeDown;
2496 }
2497
2498 ✗ const MutexLockGuard m(lock_options_);
2499 ✗ delete opt_host_.chain;
2500 ✗ delete opt_host_chain_rtt_;
2501 ✗ opt_host_.chain = new vector<string>(host_chain);
2502 ✗ opt_host_chain_rtt_ = new vector<int>(host_rtt);
2503 ✗ opt_host_.current = 0;
2504 }
2505
2506 ✗ bool DownloadManager::GeoSortServers(std::vector<std::string> *servers,
2507 std::vector<uint64_t> *output_order) {
2508 ✗ if (!servers) {
2509 ✗ return false;
2510 }
2511 ✗ if (servers->size() == 1) {
2512 ✗ if (output_order) {
2513 ✗ output_order->clear();
2514 ✗ output_order->push_back(0);
2515 }
2516 ✗ return true;
2517 }
2518
2519 ✗ std::vector<std::string> host_chain;
2520 ✗ GetHostInfo(&host_chain, NULL, NULL);
2521
2522 ✗ std::vector<std::string> server_dns_names;
2523 ✗ server_dns_names.reserve(servers->size());
2524 ✗ for (unsigned i = 0; i < servers->size(); ++i) {
2525 ✗ const std::string host = dns::ExtractHost((*servers)[i]);
2526 ✗ server_dns_names.push_back(host.empty() ? (*servers)[i] : host);
2527 }
2528 ✗ const std::string host_list = JoinStrings(server_dns_names, ",");
2529
2530 ✗ vector<string> host_chain_shuffled;
2531 {
2532 // Protect against concurrent access to prng_
2533 ✗ const MutexLockGuard m(lock_options_);
2534 // Determine random hosts for the Geo-API query
2535 ✗ host_chain_shuffled = Shuffle(host_chain, &prng_);
2536 }
2537 // Request ordered list via Geo-API
2538 ✗ bool success = false;
2539 ✗ const unsigned max_attempts = std::min(host_chain_shuffled.size(), size_t(3));
2540 ✗ vector<uint64_t> geo_order(servers->size());
2541 ✗ for (unsigned i = 0; i < max_attempts; ++i) {
2542 ✗ const string url = host_chain_shuffled[i] + "/api/v1.0/geo/@proxy@/"
2543 ✗ + host_list;
2544 ✗ LogCvmfs(kLogDownload, kLogDebug,
2545 "(manager '%s') requesting ordered server list from %s",
2546 name_.c_str(), url.c_str());
2547 ✗ cvmfs::MemSink memsink;
2548 ✗ JobInfo info(&url, false, false, NULL, &memsink);
2549 ✗ const Failures result = Fetch(&info);
2550 ✗ if (result == kFailOk) {
2551 ✗ const string order(reinterpret_cast<char *>(memsink.data()),
2552 ✗ memsink.pos());
2553 ✗ memsink.Reset();
2554 ✗ const bool retval = ValidateGeoReply(order, servers->size(), &geo_order);
2555 ✗ if (!retval) {
2556 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2557 "(manager '%s') retrieved invalid GeoAPI reply from %s [%s]",
2558 name_.c_str(), url.c_str(), order.c_str());
2559 } else {
2560 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslog,
2561 "(manager '%s') "
2562 "geographic order of servers retrieved from %s",
2563 name_.c_str(),
2564 ✗ dns::ExtractHost(host_chain_shuffled[i]).c_str());
2565 // remove new line at end of "order"
2566 ✗ LogCvmfs(kLogDownload, kLogDebug, "order is %s",
2567 ✗ Trim(order, true /* trim_newline */).c_str());
2568 ✗ success = true;
2569 ✗ break;
2570 }
2571 ✗ } else {
2572 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2573 "(manager '%s') GeoAPI request for %s failed with error %d [%s]",
2574 name_.c_str(), url.c_str(), result, Code2Ascii(result));
2575 }
2576 }
2577 ✗ if (!success) {
2578 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2579 "(manager '%s') "
2580 "failed to retrieve geographic order from stratum 1 servers",
2581 name_.c_str());
2582 ✗ return false;
2583 }
2584
2585 ✗ if (output_order) {
2586 ✗ output_order->swap(geo_order);
2587 } else {
2588 ✗ std::vector<std::string> sorted_servers;
2589 ✗ sorted_servers.reserve(geo_order.size());
2590 ✗ for (unsigned i = 0; i < geo_order.size(); ++i) {
2591 ✗ const uint64_t orderval = geo_order[i];
2592 ✗ sorted_servers.push_back((*servers)[orderval]);
2593 }
2594 ✗ servers->swap(sorted_servers);
2595 }
2596 ✗ return true;
2597 }
2598
2599
2600 /**
2601 * Uses the Geo-API of Stratum 1s to let any of them order the list of servers
2602 * and fallback proxies (if any).
2603 * Tries at most three random Stratum 1s before giving up.
2604 * If you change the host list in between by SetHostChain() or the fallback
2605 * proxy list by SetProxyChain(), they will be overwritten by this function.
2606 */
2607 ✗ bool DownloadManager::ProbeGeo() {
2608 ✗ vector<string> host_chain;
2609 ✗ vector<int> host_rtt;
2610 unsigned current_host;
2611 ✗ vector<vector<ProxyInfo> > proxy_chain;
2612 unsigned fallback_group;
2613
2614 ✗ GetHostInfo(&host_chain, &host_rtt, &current_host);
2615 ✗ GetProxyInfo(&proxy_chain, NULL, &fallback_group);
2616 ✗ if ((host_chain.size() < 2) && ((proxy_chain.size() - fallback_group) < 2))
2617 ✗ return true;
2618
2619 ✗ vector<string> host_names;
2620 ✗ for (unsigned i = 0; i < host_chain.size(); ++i)
2621 ✗ host_names.push_back(dns::ExtractHost(host_chain[i]));
2622 ✗ SortTeam(&host_names, &host_chain);
2623 ✗ const unsigned last_geo_host = host_names.size();
2624
2625 ✗ if ((fallback_group == 0) && (last_geo_host > 1)) {
2626 // There are no non-fallback proxies, which means that the client
2627 // will always use the fallback proxies. Add a keyword separator
2628 // between the hosts and fallback proxies so the geosorting service
2629 // will know to sort the hosts based on the distance from the
2630 // closest fallback proxy rather than the distance from the client.
2631 ✗ host_names.push_back("+PXYSEP+");
2632 }
2633
2634 // Add fallback proxy names to the end of the host list
2635 ✗ const unsigned first_geo_fallback = host_names.size();
2636 ✗ for (unsigned i = fallback_group; i < proxy_chain.size(); ++i) {
2637 // We only take the first fallback proxy name from every group under the
2638 // assumption that load-balanced servers are at the same location
2639 ✗ host_names.push_back(proxy_chain[i][0].host.name());
2640 }
2641
2642 ✗ std::vector<uint64_t> geo_order;
2643 ✗ const bool success = GeoSortServers(&host_names, &geo_order);
2644 ✗ if (!success) {
2645 // GeoSortServers already logged a failure message.
2646 ✗ return false;
2647 }
2648
2649 // Re-install host chain and proxy chain
2650 ✗ const MutexLockGuard m(lock_options_);
2651 ✗ delete opt_host_.chain;
2652 ✗ opt_num_proxies_ = 0;
2653 ✗ opt_host_.chain = new vector<string>(host_chain.size());
2654
2655 // It's possible that opt_proxy_groups_fallback_ might have changed while
2656 // the lock wasn't held
2657 vector<vector<ProxyInfo> > *proxy_groups = new vector<vector<ProxyInfo> >(
2658 ✗ opt_proxy_groups_fallback_ + proxy_chain.size() - fallback_group);
2659 // First copy the non-fallback part of the current proxy chain
2660 ✗ for (unsigned i = 0; i < opt_proxy_groups_fallback_; ++i) {
2661 ✗ (*proxy_groups)[i] = (*opt_proxy_groups_)[i];
2662 ✗ opt_num_proxies_ += (*opt_proxy_groups_)[i].size();
2663 }
2664
2665 // Copy the host chain and fallback proxies by geo order. Array indices
2666 // in geo_order that are smaller than last_geo_host refer to a stratum 1,
2667 // and those indices greater than or equal to first_geo_fallback refer to
2668 // a fallback proxy.
2669 ✗ unsigned hosti = 0;
2670 ✗ unsigned proxyi = opt_proxy_groups_fallback_;
2671 ✗ for (unsigned i = 0; i < geo_order.size(); ++i) {
2672 ✗ const uint64_t orderval = geo_order[i];
2673 ✗ if (orderval < static_cast<uint64_t>(last_geo_host)) {
2674 // LogCvmfs(kLogCvmfs, kLogSyslog, "this is orderval %u at host index
2675 // %u", orderval, hosti);
2676 ✗ (*opt_host_.chain)[hosti++] = host_chain[orderval];
2677 ✗ } else if (orderval >= static_cast<uint64_t>(first_geo_fallback)) {
2678 // LogCvmfs(kLogCvmfs, kLogSyslog,
2679 // "this is orderval %u at proxy index %u, using proxy_chain index %u",
2680 // orderval, proxyi, fallback_group + orderval - first_geo_fallback);
2681 ✗ (*proxy_groups)[proxyi] = proxy_chain[fallback_group + orderval
2682 ✗ - first_geo_fallback];
2683 ✗ opt_num_proxies_ += (*proxy_groups)[proxyi].size();
2684 ✗ proxyi++;
2685 }
2686 }
2687
2688 ✗ opt_proxy_map_.clear();
2689 ✗ delete opt_proxy_groups_;
2690 ✗ opt_proxy_groups_ = proxy_groups;
2691 // In pathological cases, opt_proxy_groups_current_ can be larger now when
2692 // proxies changed in-between.
2693 ✗ if (opt_proxy_groups_current_ > opt_proxy_groups_->size()) {
2694 ✗ if (opt_proxy_groups_->size() == 0) {
2695 ✗ opt_proxy_groups_current_ = 0;
2696 } else {
2697 ✗ opt_proxy_groups_current_ = opt_proxy_groups_->size() - 1;
2698 }
2699 ✗ opt_proxy_groups_current_burned_ = 0;
2700 }
2701
2702 ✗ UpdateProxiesUnlocked("geosort");
2703
2704 ✗ delete opt_host_chain_rtt_;
2705 ✗ opt_host_chain_rtt_ = new vector<int>(host_chain.size(), kProbeGeo);
2706 ✗ opt_host_.current = 0;
2707
2708 ✗ return true;
2709 }
2710
2711
2712 /**
2713 * Validates a string of the form "1,4,2,3" representing in which order the
2714 * the expected_size number of hosts should be put for optimal geographic
2715 * proximity. Returns false if the reply_order string is invalid, otherwise
2716 * fills in the reply_vals array with zero-based order indexes (e.g.
2717 * [0,3,1,2]) and returns true.
2718 */
2719 686 bool DownloadManager::ValidateGeoReply(const string &reply_order,
2720 const unsigned expected_size,
2721 vector<uint64_t> *reply_vals) {
2722
2/2
✓ Branch 1 taken 49 times.
✓ Branch 2 taken 637 times.
 686 if (reply_order.empty())
2723 49 return false;
2724
2/4
✓ Branch 2 taken 637 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 637 times.
✗ Branch 6 not taken.
 1274 const sanitizer::InputSanitizer sanitizer("09 , \n");
2725
3/4
✓ Branch 1 taken 637 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 49 times.
✓ Branch 4 taken 588 times.
 637 if (!sanitizer.IsValid(reply_order))
2726 49 return false;
2727
2/4
✓ Branch 2 taken 588 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 588 times.
✗ Branch 6 not taken.
 1176 const sanitizer::InputSanitizer strip_newline("09 ,");
2728
1/2
✓ Branch 1 taken 588 times.
✗ Branch 2 not taken.
 588 vector<string> reply_strings = SplitString(strip_newline.Filter(reply_order),
2729
1/2
✓ Branch 1 taken 588 times.
✗ Branch 2 not taken.
 588 ',');
2730 588 vector<uint64_t> tmp_vals;
2731
2/2
✓ Branch 1 taken 1127 times.
✓ Branch 2 taken 490 times.
 1617 for (unsigned i = 0; i < reply_strings.size(); ++i) {
2732
2/2
✓ Branch 2 taken 98 times.
✓ Branch 3 taken 1029 times.
 1127 if (reply_strings[i].empty())
2733 98 return false;
2734
2/4
✓ Branch 2 taken 1029 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1029 times.
✗ Branch 6 not taken.
 1029 tmp_vals.push_back(String2Uint64(reply_strings[i]));
2735 }
2736
2/2
✓ Branch 1 taken 245 times.
✓ Branch 2 taken 245 times.
 490 if (tmp_vals.size() != expected_size)
2737 245 return false;
2738
2739 // Check if tmp_vals contains the number 1..n
2740
1/2
✓ Branch 3 taken 245 times.
✗ Branch 4 not taken.
 245 set<uint64_t> coverage(tmp_vals.begin(), tmp_vals.end());
2741
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 245 times.
 245 if (coverage.size() != tmp_vals.size())
2742 ✗ return false;
2743
5/6
✓ Branch 2 taken 196 times.
✓ Branch 3 taken 49 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 196 times.
✓ Branch 9 taken 49 times.
✓ Branch 10 taken 196 times.
 245 if ((*coverage.begin() != 1) || (*coverage.rbegin() != coverage.size()))
2744 49 return false;
2745
2746
2/2
✓ Branch 0 taken 441 times.
✓ Branch 1 taken 196 times.
 637 for (unsigned i = 0; i < expected_size; ++i) {
2747 441 (*reply_vals)[i] = tmp_vals[i] - 1;
2748 }
2749 196 return true;
2750 637 }
2751
2752
2753 /**
2754 * Removes DIRECT from a list of ';' and '|' separated proxies.
2755 * \return true if DIRECT was present, false otherwise
2756 */
2757 2675 bool DownloadManager::StripDirect(const string &proxy_list,
2758 string *cleaned_list) {
2759
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2675 times.
 2675 assert(cleaned_list);
2760
2/2
✓ Branch 1 taken 2234 times.
✓ Branch 2 taken 441 times.
 2675 if (proxy_list == "") {
2761
1/2
✓ Branch 1 taken 2234 times.
✗ Branch 2 not taken.
 2234 *cleaned_list = "";
2762 2234 return false;
2763 }
2764 441 bool result = false;
2765
2766
1/2
✓ Branch 1 taken 441 times.
✗ Branch 2 not taken.
 441 vector<string> proxy_groups = SplitString(proxy_list, ';');
2767 441 vector<string> cleaned_groups;
2768
2/2
✓ Branch 1 taken 1078 times.
✓ Branch 2 taken 441 times.
 1519 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2769
1/2
✓ Branch 2 taken 1078 times.
✗ Branch 3 not taken.
 1078 vector<string> group = SplitString(proxy_groups[i], '|');
2770 1078 vector<string> cleaned;
2771
2/2
✓ Branch 1 taken 1813 times.
✓ Branch 2 taken 1078 times.
 2891 for (unsigned j = 0; j < group.size(); ++j) {
2772
6/6
✓ Branch 2 taken 1323 times.
✓ Branch 3 taken 490 times.
✓ Branch 6 taken 637 times.
✓ Branch 7 taken 686 times.
✓ Branch 8 taken 1127 times.
✓ Branch 9 taken 686 times.
 1813 if ((group[j] == "DIRECT") || (group[j] == "")) {
2773 1127 result = true;
2774 } else {
2775
1/2
✓ Branch 2 taken 686 times.
✗ Branch 3 not taken.
 686 cleaned.push_back(group[j]);
2776 }
2777 }
2778
2/2
✓ Branch 1 taken 343 times.
✓ Branch 2 taken 735 times.
 1078 if (!cleaned.empty())
2779
3/6
✓ Branch 2 taken 343 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 343 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 343 times.
✗ Branch 9 not taken.
 343 cleaned_groups.push_back(JoinStrings(cleaned, "|"));
2780 1078 }
2781
2782
2/4
✓ Branch 2 taken 441 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 441 times.
✗ Branch 6 not taken.
 441 *cleaned_list = JoinStrings(cleaned_groups, ";");
2783 441 return result;
2784 441 }
2785
2786
2787 /**
2788 * Parses a list of ';'- and '|'-separated proxy servers and fallback proxy
2789 * servers for the proxy groups.
2790 * The empty string for both removes the proxy chain.
2791 * The set_mode parameter can be used to set either proxies (leaving fallback
2792 * proxies unchanged) or fallback proxies (leaving regular proxies unchanged)
2793 * or both.
2794 */
2795 2185 void DownloadManager::SetProxyChain(const string &proxy_list,
2796 const string &fallback_proxy_list,
2797 const ProxySetModes set_mode) {
2798 2185 const MutexLockGuard m(lock_options_);
2799
2800 2185 opt_timestamp_backup_proxies_ = 0;
2801 2185 opt_timestamp_failover_proxies_ = 0;
2802
1/2
✓ Branch 1 taken 2185 times.
✗ Branch 2 not taken.
 2185 string set_proxy_list = opt_proxy_list_;
2803
1/2
✓ Branch 1 taken 2185 times.
✗ Branch 2 not taken.
 2185 string set_proxy_fallback_list = opt_proxy_fallback_list_;
2804 bool contains_direct;
2805
3/4
✓ Branch 0 taken 2185 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2038 times.
✓ Branch 3 taken 147 times.
 2185 if ((set_mode == kSetProxyFallback) || (set_mode == kSetProxyBoth)) {
2806
1/2
✓ Branch 1 taken 2038 times.
✗ Branch 2 not taken.
 2038 opt_proxy_fallback_list_ = fallback_proxy_list;
2807 }
2808
3/4
✓ Branch 0 taken 2038 times.
✓ Branch 1 taken 147 times.
✓ Branch 2 taken 2038 times.
✗ Branch 3 not taken.
 2185 if ((set_mode == kSetProxyRegular) || (set_mode == kSetProxyBoth)) {
2809
1/2
✓ Branch 1 taken 2185 times.
✗ Branch 2 not taken.
 2185 opt_proxy_list_ = proxy_list;
2810 }
2811
1/2
✓ Branch 1 taken 2185 times.
✗ Branch 2 not taken.
 2185 contains_direct = StripDirect(opt_proxy_fallback_list_,
2812 &set_proxy_fallback_list);
2813
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2185 times.
 2185 if (contains_direct) {
2814 ✗ LogCvmfs(kLogDownload, kLogSyslogWarn | kLogDebug,
2815 "(manager '%s') fallback proxies do not support DIRECT, removing",
2816 name_.c_str());
2817 }
2818
1/2
✓ Branch 1 taken 2185 times.
✗ Branch 2 not taken.
 2185 if (set_proxy_fallback_list == "") {
2819
1/2
✓ Branch 1 taken 2185 times.
✗ Branch 2 not taken.
 2185 set_proxy_list = opt_proxy_list_;
2820 } else {
2821 ✗ const bool contains_direct = StripDirect(opt_proxy_list_, &set_proxy_list);
2822 ✗ if (contains_direct) {
2823 ✗ LogCvmfs(kLogDownload, kLogSyslog | kLogDebug,
2824 "(manager '%s') skipping DIRECT proxy to use fallback proxy",
2825 name_.c_str());
2826 }
2827 }
2828
2829 // From this point on, use set_proxy_list and set_fallback_proxy_list as
2830 // effective proxy lists!
2831
2832 2185 opt_proxy_map_.clear();
2833
2/2
✓ Branch 0 taken 1019 times.
✓ Branch 1 taken 1166 times.
 2185 delete opt_proxy_groups_;
2834
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 2185 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 2185 times.
 2185 if ((set_proxy_list == "") && (set_proxy_fallback_list == "")) {
2835 ✗ opt_proxy_groups_ = NULL;
2836 ✗ opt_proxy_groups_current_ = 0;
2837 ✗ opt_proxy_groups_current_burned_ = 0;
2838 ✗ opt_proxy_groups_fallback_ = 0;
2839 ✗ opt_num_proxies_ = 0;
2840 ✗ return;
2841 }
2842
2843 // Determine number of regular proxy groups (== first fallback proxy group)
2844 2185 opt_proxy_groups_fallback_ = 0;
2845
1/2
✓ Branch 1 taken 2185 times.
✗ Branch 2 not taken.
 2185 if (set_proxy_list != "") {
2846
1/2
✓ Branch 1 taken 2185 times.
✗ Branch 2 not taken.
 2185 opt_proxy_groups_fallback_ = SplitString(set_proxy_list, ';').size();
2847 }
2848
1/2
✓ Branch 2 taken 2185 times.
✗ Branch 3 not taken.
 2185 LogCvmfs(kLogDownload, kLogDebug,
2849 "(manager '%s') "
2850 "first fallback proxy group %u",
2851 name_.c_str(), opt_proxy_groups_fallback_);
2852
2853 // Concatenate regular proxies and fallback proxies, both of which can be
2854 // empty.
2855
1/2
✓ Branch 1 taken 2185 times.
✗ Branch 2 not taken.
 2185 string all_proxy_list = set_proxy_list;
2856
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 2185 times.
 2185 if (set_proxy_fallback_list != "") {
2857 ✗ if (all_proxy_list != "")
2858 ✗ all_proxy_list += ";";
2859 ✗ all_proxy_list += set_proxy_fallback_list;
2860 }
2861
1/2
✓ Branch 3 taken 2185 times.
✗ Branch 4 not taken.
 2185 LogCvmfs(kLogDownload, kLogDebug, "(manager '%s') full proxy list %s",
2862 name_.c_str(), all_proxy_list.c_str());
2863
2864 // Resolve server names in provided urls
2865 2185 vector<string> hostnames; // All encountered hostnames
2866 2185 vector<string> proxy_groups;
2867
1/2
✓ Branch 1 taken 2185 times.
✗ Branch 2 not taken.
 2185 if (all_proxy_list != "")
2868
1/2
✓ Branch 1 taken 2185 times.
✗ Branch 2 not taken.
 2185 proxy_groups = SplitString(all_proxy_list, ';');
2869
2/2
✓ Branch 1 taken 2234 times.
✓ Branch 2 taken 2185 times.
 4419 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2870
1/2
✓ Branch 2 taken 2234 times.
✗ Branch 3 not taken.
 2234 vector<string> this_group = SplitString(proxy_groups[i], '|');
2871
2/2
✓ Branch 1 taken 2234 times.
✓ Branch 2 taken 2234 times.
 4468 for (unsigned j = 0; j < this_group.size(); ++j) {
2872
1/2
✓ Branch 2 taken 2234 times.
✗ Branch 3 not taken.
 2234 this_group[j] = dns::AddDefaultScheme(this_group[j]);
2873 // Note: DIRECT strings will be "extracted" to an empty string.
2874
1/2
✓ Branch 2 taken 2234 times.
✗ Branch 3 not taken.
 2234 const string hostname = dns::ExtractHost(this_group[j]);
2875 // Save the hostname. Leave empty (DIRECT) names so indexes will
2876 // match later.
2877
1/2
✓ Branch 1 taken 2234 times.
✗ Branch 2 not taken.
 2234 hostnames.push_back(hostname);
2878 2234 }
2879 2234 }
2880 2185 vector<dns::Host> hosts;
2881
1/2
✓ Branch 3 taken 2185 times.
✗ Branch 4 not taken.
 2185 LogCvmfs(kLogDownload, kLogDebug,
2882 "(manager '%s') "
2883 "resolving %lu proxy addresses",
2884 name_.c_str(), hostnames.size());
2885
1/2
✓ Branch 1 taken 2185 times.
✗ Branch 2 not taken.
 2185 resolver_->ResolveMany(hostnames, &hosts);
2886
2887 // Construct opt_proxy_groups_: traverse proxy list in same order and expand
2888 // names to resolved IP addresses.
2889
1/2
✓ Branch 1 taken 2185 times.
✗ Branch 2 not taken.
 2185 opt_proxy_groups_ = new vector<vector<ProxyInfo> >();
2890 2185 opt_num_proxies_ = 0;
2891 2185 unsigned num_proxy = 0; // Combined i, j counter
2892
2/2
✓ Branch 1 taken 2234 times.
✓ Branch 2 taken 2185 times.
 4419 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2893
1/2
✓ Branch 2 taken 2234 times.
✗ Branch 3 not taken.
 2234 vector<string> this_group = SplitString(proxy_groups[i], '|');
2894 // Construct ProxyInfo objects from proxy string and DNS resolver result for
2895 // every proxy in this_group. One URL can result in multiple ProxyInfo
2896 // objects, one for each IP address.
2897 2234 vector<ProxyInfo> infos;
2898
2/2
✓ Branch 1 taken 2234 times.
✓ Branch 2 taken 2234 times.
 4468 for (unsigned j = 0; j < this_group.size(); ++j, ++num_proxy) {
2899
1/2
✓ Branch 2 taken 2234 times.
✗ Branch 3 not taken.
 2234 this_group[j] = dns::AddDefaultScheme(this_group[j]);
2900
2/2
✓ Branch 2 taken 2038 times.
✓ Branch 3 taken 196 times.
 2234 if (this_group[j] == "DIRECT") {
2901
3/6
✓ Branch 2 taken 2038 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2038 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 2038 times.
✗ Branch 9 not taken.
 2038 infos.push_back(ProxyInfo("DIRECT"));
2902 2038 continue;
2903 }
2904
2905
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 196 times.
 196 if (hosts[num_proxy].status() != dns::kFailOk) {
2906 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2907 "(manager '%s') "
2908 "failed to resolve IP addresses for %s (%d - %s)",
2909 ✗ name_.c_str(), hosts[num_proxy].name().c_str(),
2910 ✗ hosts[num_proxy].status(),
2911 ✗ dns::Code2Ascii(hosts[num_proxy].status()));
2912 const dns::Host failed_host = dns::Host::ExtendDeadline(
2913 ✗ hosts[num_proxy], resolver_->min_ttl());
2914 ✗ infos.push_back(ProxyInfo(failed_host, this_group[j]));
2915 ✗ continue;
2916 }
2917
2918 // IPv4 addresses have precedence
2919 196 set<string> best_addresses = hosts[num_proxy].ViewBestAddresses(
2920
2/4
✓ Branch 1 taken 196 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 196 times.
✗ Branch 5 not taken.
 196 opt_ip_preference_);
2921 196 set<string>::const_iterator iter_ips = best_addresses.begin();
2922
2/2
✓ Branch 3 taken 196 times.
✓ Branch 4 taken 196 times.
 392 for (; iter_ips != best_addresses.end(); ++iter_ips) {
2923
1/2
✓ Branch 3 taken 196 times.
✗ Branch 4 not taken.
 196 const string url_ip = dns::RewriteUrl(this_group[j], *iter_ips);
2924
2/4
✓ Branch 2 taken 196 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 196 times.
✗ Branch 6 not taken.
 196 infos.push_back(ProxyInfo(hosts[num_proxy], url_ip));
2925
2926
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 196 times.
 196 if (sharding_policy_.UseCount() > 0) {
2927 ✗ sharding_policy_->AddProxy(url_ip);
2928 }
2929 196 }
2930 196 }
2931
1/2
✓ Branch 1 taken 2234 times.
✗ Branch 2 not taken.
 2234 opt_proxy_groups_->push_back(infos);
2932 2234 opt_num_proxies_ += infos.size();
2933 2234 }
2934
1/2
✓ Branch 2 taken 2185 times.
✗ Branch 3 not taken.
 2185 LogCvmfs(kLogDownload, kLogDebug,
2935 "(manager '%s') installed %u proxies in %lu load-balance groups",
2936 2185 name_.c_str(), opt_num_proxies_, opt_proxy_groups_->size());
2937 2185 opt_proxy_groups_current_ = 0;
2938 2185 opt_proxy_groups_current_burned_ = 0;
2939
2940 // Select random start proxy from the first group.
2941
1/2
✓ Branch 1 taken 2185 times.
✗ Branch 2 not taken.
 2185 if (opt_proxy_groups_->size() > 0) {
2942 // Select random start proxy from the first group.
2943
2/4
✓ Branch 2 taken 2185 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2185 times.
✗ Branch 6 not taken.
 2185 UpdateProxiesUnlocked("set random start proxy from the first proxy group");
2944 }
2945
3/6
✓ Branch 5 taken 2185 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 2185 times.
✗ Branch 9 not taken.
✓ Branch 11 taken 2185 times.
✗ Branch 12 not taken.
 2185 }
2946
2947
2948 /**
2949 * Retrieves the proxy chain, optionally the currently active load-balancing
2950 * group, and optionally the index of the first fallback proxy group.
2951 * If there are no fallback proxies, the index will equal the size of
2952 * the proxy chain.
2953 */
2954 ✗ void DownloadManager::GetProxyInfo(vector<vector<ProxyInfo> > *proxy_chain,
2955 unsigned *current_group,
2956 unsigned *fallback_group) {
2957 ✗ assert(proxy_chain != NULL);
2958 ✗ const MutexLockGuard m(lock_options_);
2959
2960 ✗ if (!opt_proxy_groups_) {
2961 ✗ const vector<vector<ProxyInfo> > empty_chain;
2962 ✗ *proxy_chain = empty_chain;
2963 ✗ if (current_group != NULL)
2964 ✗ *current_group = 0;
2965 ✗ if (fallback_group != NULL)
2966 ✗ *fallback_group = 0;
2967 ✗ return;
2968 }
2969
2970 ✗ *proxy_chain = *opt_proxy_groups_;
2971 ✗ if (current_group != NULL)
2972 ✗ *current_group = opt_proxy_groups_current_;
2973 ✗ if (fallback_group != NULL)
2974 ✗ *fallback_group = opt_proxy_groups_fallback_;
2975 }
2976
2977 ✗ string DownloadManager::GetProxyList() { return opt_proxy_list_; }
2978
2979 ✗ string DownloadManager::GetFallbackProxyList() {
2980 ✗ return opt_proxy_fallback_list_;
2981 }
2982
2983 /**
2984 * Choose proxy
2985 */
2986 6864 DownloadManager::ProxyInfo *DownloadManager::ChooseProxyUnlocked(
2987 const shash::Any *hash) {
2988
2/2
✓ Branch 0 taken 4403 times.
✓ Branch 1 taken 2461 times.
 6864 if (!opt_proxy_groups_)
2989 4403 return NULL;
2990
2991
2/2
✓ Branch 0 taken 1241 times.
✓ Branch 1 taken 1220 times.
 2461 const uint32_t key = (hash ? hash->Partial32() : 0);
2992
1/2
✓ Branch 1 taken 2461 times.
✗ Branch 2 not taken.
 2461 const map<uint32_t, ProxyInfo *>::iterator it = opt_proxy_map_.lower_bound(
2993 key);
2994 2461 ProxyInfo *proxy = it->second;
2995
2996 2461 return proxy;
2997 }
2998
2999 /**
3000 * Update currently selected proxy
3001 */
3002 3253 void DownloadManager::UpdateProxiesUnlocked(const string &reason) {
3003
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3253 times.
 3253 if (!opt_proxy_groups_)
3004 ✗ return;
3005
3006 // Identify number of non-burned proxies within the current group
3007 3253 vector<ProxyInfo> *group = current_proxy_group();
3008 3253 const unsigned num_alive = (group->size() - opt_proxy_groups_current_burned_);
3009
2/4
✓ Branch 2 taken 3253 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 3253 times.
✗ Branch 6 not taken.
 6506 const string old_proxy = JoinStrings(opt_proxies_, "|");
3010
3011 // Rebuild proxy map and URL list
3012 3253 opt_proxy_map_.clear();
3013 3253 opt_proxies_.clear();
3014 3253 const uint32_t max_key = 0xffffffffUL;
3015
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3253 times.
 3253 if (opt_proxy_shard_) {
3016 // Build a consistent map with multiple entries for each proxy
3017 ✗ for (unsigned i = 0; i < num_alive; ++i) {
3018 ✗ ProxyInfo *proxy = &(*group)[i];
3019 ✗ shash::Any proxy_hash(shash::kSha1);
3020 ✗ HashString(proxy->url, &proxy_hash);
3021 ✗ Prng prng;
3022 ✗ prng.InitSeed(proxy_hash.Partial32());
3023 ✗ for (unsigned j = 0; j < kProxyMapScale; ++j) {
3024 ✗ const std::pair<uint32_t, ProxyInfo *> entry(prng.Next(max_key), proxy);
3025 ✗ opt_proxy_map_.insert(entry);
3026 }
3027 ✗ const std::string proxy_name = proxy->host.name().empty()
3028 ? ""
3029 ✗ : " (" + proxy->host.name() + ")";
3030 ✗ opt_proxies_.push_back(proxy->url + proxy_name);
3031 }
3032 // Ensure lower_bound() finds a value for all keys
3033 ✗ ProxyInfo *first_proxy = opt_proxy_map_.begin()->second;
3034 ✗ const std::pair<uint32_t, ProxyInfo *> last_entry(max_key, first_proxy);
3035 ✗ opt_proxy_map_.insert(last_entry);
3036 } else {
3037 // Build a map with a single entry for one randomly selected proxy
3038 3253 const unsigned select = prng_.Next(num_alive);
3039 3253 ProxyInfo *proxy = &(*group)[select];
3040 3253 const std::pair<uint32_t, ProxyInfo *> entry(max_key, proxy);
3041
1/2
✓ Branch 1 taken 3253 times.
✗ Branch 2 not taken.
 3253 opt_proxy_map_.insert(entry);
3042 3253 const std::string proxy_name = proxy->host.name().empty()
3043 ? ""
3044
9/18
✓ Branch 0 taken 3057 times.
✓ Branch 1 taken 196 times.
✓ Branch 4 taken 3057 times.
✗ Branch 5 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 196 times.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✓ Branch 11 taken 196 times.
✗ Branch 12 not taken.
✓ Branch 13 taken 196 times.
✓ Branch 14 taken 3057 times.
✓ Branch 15 taken 3057 times.
✓ Branch 16 taken 196 times.
✗ Branch 17 not taken.
✗ Branch 18 not taken.
✗ Branch 19 not taken.
✗ Branch 20 not taken.
 3449 : " (" + proxy->host.name() + ")";
3045
2/4
✓ Branch 1 taken 3253 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3253 times.
✗ Branch 5 not taken.
 3253 opt_proxies_.push_back(proxy->url + proxy_name);
3046 3253 }
3047
1/2
✓ Branch 3 taken 3253 times.
✗ Branch 4 not taken.
 3253 sort(opt_proxies_.begin(), opt_proxies_.end());
3048
3049 // Report any change in proxy usage
3050
2/4
✓ Branch 2 taken 3253 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 3253 times.
✗ Branch 6 not taken.
 6506 const string new_proxy = JoinStrings(opt_proxies_, "|");
3051 const string curr_host = "Current host: "
3052 3253 + (opt_host_.chain
3053
6/12
✓ Branch 0 taken 3057 times.
✓ Branch 1 taken 196 times.
✓ Branch 4 taken 3057 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 196 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 196 times.
✓ Branch 11 taken 3057 times.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
 6506 ? (*opt_host_.chain)[opt_host_.current]
3054
1/2
✓ Branch 1 taken 3253 times.
✗ Branch 2 not taken.
 3253 : "");
3055
2/2
✓ Branch 1 taken 2234 times.
✓ Branch 2 taken 1019 times.
 3253 if (new_proxy != old_proxy) {
3056
4/9
✗ Branch 2 not taken.
✓ Branch 3 taken 2234 times.
✓ Branch 4 taken 2185 times.
✓ Branch 5 taken 49 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 2234 times.
✗ Branch 9 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
 6751 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
3057 "(manager '%s') switching proxy from %s to %s. Reason: %s [%s]",
3058 2283 name_.c_str(), (old_proxy.empty() ? "(none)" : old_proxy.c_str()),
3059 4468 (new_proxy.empty() ? "(none)" : new_proxy.c_str()), reason.c_str(),
3060 curr_host.c_str());
3061 }
3062 3253 }
3063
3064 /**
3065 * Enable proxy sharding
3066 */
3067 ✗ void DownloadManager::ShardProxies() {
3068 ✗ opt_proxy_shard_ = true;
3069 ✗ RebalanceProxiesUnlocked("enable sharding");
3070 }
3071
3072 /**
3073 * Selects a new random proxy in the current load-balancing group. Resets the
3074 * "burned" counter.
3075 */
3076 ✗ void DownloadManager::RebalanceProxiesUnlocked(const string &reason) {
3077 ✗ if (!opt_proxy_groups_)
3078 ✗ return;
3079
3080 ✗ opt_timestamp_failover_proxies_ = 0;
3081 ✗ opt_proxy_groups_current_burned_ = 0;
3082 ✗ UpdateProxiesUnlocked(reason);
3083 }
3084
3085
3086 ✗ void DownloadManager::RebalanceProxies() {
3087 ✗ const MutexLockGuard m(lock_options_);
3088 ✗ RebalanceProxiesUnlocked("rebalance invoked manually");
3089 }
3090
3091
3092 /**
3093 * Switches to the next load-balancing group of proxy servers.
3094 */
3095 ✗ void DownloadManager::SwitchProxyGroup() {
3096 ✗ const MutexLockGuard m(lock_options_);
3097
3098 ✗ if (!opt_proxy_groups_ || (opt_proxy_groups_->size() < 2)) {
3099 ✗ return;
3100 }
3101
3102 ✗ opt_proxy_groups_current_ = (opt_proxy_groups_current_ + 1)
3103 ✗ % opt_proxy_groups_->size();
3104 ✗ opt_timestamp_backup_proxies_ = time(NULL);
3105
3106 const std::string msg = "switch to proxy group "
3107 ✗ + StringifyUint(opt_proxy_groups_current_);
3108 ✗ RebalanceProxiesUnlocked(msg);
3109 }
3110
3111
3112 ✗ void DownloadManager::SetProxyGroupResetDelay(const unsigned seconds) {
3113 ✗ const MutexLockGuard m(lock_options_);
3114 ✗ opt_proxy_groups_reset_after_ = seconds;
3115 ✗ if (opt_proxy_groups_reset_after_ == 0) {
3116 ✗ opt_timestamp_backup_proxies_ = 0;
3117 ✗ opt_timestamp_failover_proxies_ = 0;
3118 }
3119 }
3120
3121
3122 ✗ void DownloadManager::SetMetalinkResetDelay(const unsigned seconds) {
3123 ✗ const MutexLockGuard m(lock_options_);
3124 ✗ opt_metalink_.reset_after = seconds;
3125 ✗ if (opt_metalink_.reset_after == 0)
3126 ✗ opt_metalink_.timestamp_backup = 0;
3127 }
3128
3129
3130 ✗ void DownloadManager::SetHostResetDelay(const unsigned seconds) {
3131 ✗ const MutexLockGuard m(lock_options_);
3132 ✗ opt_host_.reset_after = seconds;
3133 ✗ if (opt_host_.reset_after == 0)
3134 ✗ opt_host_.timestamp_backup = 0;
3135 }
3136
3137
3138 2061 void DownloadManager::SetRetryParameters(const unsigned max_retries,
3139 const unsigned backoff_init_ms,
3140 const unsigned backoff_max_ms) {
3141 2061 const MutexLockGuard m(lock_options_);
3142 2061 opt_max_retries_ = max_retries;
3143 2061 opt_backoff_init_ms_ = backoff_init_ms;
3144 2061 opt_backoff_max_ms_ = backoff_max_ms;
3145 2061 }
3146
3147
3148 1117 void DownloadManager::SetMaxIpaddrPerProxy(unsigned limit) {
3149 1117 const MutexLockGuard m(lock_options_);
3150 1117 resolver_->set_throttle(limit);
3151 1117 }
3152
3153
3154 1231 void DownloadManager::SetProxyTemplates(const std::string &direct,
3155 const std::string &forced) {
3156 1231 const MutexLockGuard m(lock_options_);
3157
1/2
✓ Branch 1 taken 1231 times.
✗ Branch 2 not taken.
 1231 proxy_template_direct_ = direct;
3158
1/2
✓ Branch 1 taken 1231 times.
✗ Branch 2 not taken.
 1231 proxy_template_forced_ = forced;
3159 1231 }
3160
3161
3162 ✗ void DownloadManager::EnableInfoHeader() { enable_info_header_ = true; }
3163
3164
3165 977 void DownloadManager::EnableRedirects() { follow_redirects_ = true; }
3166
3167 ✗ void DownloadManager::EnableIgnoreSignatureFailures() {
3168 ✗ ignore_signature_failures_ = true;
3169 }
3170
3171 ✗ void DownloadManager::EnableHTTPTracing() { enable_http_tracing_ = true; }
3172
3173 ✗ void DownloadManager::AddHTTPTracingHeader(const std::string &header) {
3174 ✗ http_tracing_headers_.push_back(header);
3175 }
3176
3177 830 void DownloadManager::UseSystemCertificatePath() {
3178 830 ssl_certificate_store_.UseSystemCertificatePath();
3179 830 }
3180
3181 ✗ bool DownloadManager::SetShardingPolicy(const ShardingPolicySelector type) {
3182 ✗ const bool success = false;
3183 ✗ switch (type) {
3184 default:
3185 ✗ LogCvmfs(
3186 kLogDownload, kLogDebug | kLogSyslogErr,
3187 "(manager '%s') "
3188 "Proposed sharding policy does not exist. Falling back to default",
3189 name_.c_str());
3190 }
3191 ✗ return success;
3192 }
3193
3194 ✗ void DownloadManager::SetFailoverIndefinitely() {
3195 ✗ failover_indefinitely_ = true;
3196 }
3197
3198 /**
3199 * Creates a copy of the existing download manager. Must only be called in
3200 * single-threaded stage because it calls curl_global_init().
3201 */
3202 1117 DownloadManager *DownloadManager::Clone(
3203 const perf::StatisticsTemplate &statistics,
3204 const std::string &cloned_name) {
3205 DownloadManager *clone = new DownloadManager(pool_max_handles_, statistics,
3206
1/2
✓ Branch 2 taken 1117 times.
✗ Branch 3 not taken.
 1117 cloned_name);
3207
3208 1117 clone->SetDnsParameters(resolver_->retries(), resolver_->timeout_ms());
3209 1117 clone->SetDnsTtlLimits(resolver_->min_ttl(), resolver_->max_ttl());
3210 1117 clone->SetMaxIpaddrPerProxy(resolver_->throttle());
3211
3212
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1117 times.
 1117 if (!opt_dns_server_.empty())
3213 ✗ clone->SetDnsServer(opt_dns_server_);
3214 1117 clone->opt_timeout_proxy_ = opt_timeout_proxy_;
3215 1117 clone->opt_timeout_direct_ = opt_timeout_direct_;
3216 1117 clone->opt_low_speed_limit_ = opt_low_speed_limit_;
3217 1117 clone->opt_max_retries_ = opt_max_retries_;
3218 1117 clone->opt_backoff_init_ms_ = opt_backoff_init_ms_;
3219 1117 clone->opt_backoff_max_ms_ = opt_backoff_max_ms_;
3220 1117 clone->enable_info_header_ = enable_info_header_;
3221 1117 clone->enable_http_tracing_ = enable_http_tracing_;
3222 1117 clone->http_tracing_headers_ = http_tracing_headers_;
3223 1117 clone->follow_redirects_ = follow_redirects_;
3224 1117 clone->ignore_signature_failures_ = ignore_signature_failures_;
3225
2/2
✓ Branch 0 taken 1019 times.
✓ Branch 1 taken 98 times.
 1117 if (opt_host_.chain) {
3226
1/2
✓ Branch 2 taken 1019 times.
✗ Branch 3 not taken.
 1019 clone->opt_host_.chain = new vector<string>(*opt_host_.chain);
3227
1/2
✓ Branch 2 taken 1019 times.
✗ Branch 3 not taken.
 1019 clone->opt_host_chain_rtt_ = new vector<int>(*opt_host_chain_rtt_);
3228 }
3229
3230 1117 CloneProxyConfig(clone);
3231 1117 clone->opt_ip_preference_ = opt_ip_preference_;
3232 1117 clone->proxy_template_direct_ = proxy_template_direct_;
3233 1117 clone->proxy_template_forced_ = proxy_template_forced_;
3234 1117 clone->opt_proxy_groups_reset_after_ = opt_proxy_groups_reset_after_;
3235 1117 clone->opt_metalink_.reset_after = opt_metalink_.reset_after;
3236 1117 clone->opt_host_.reset_after = opt_host_.reset_after;
3237 1117 clone->credentials_attachment_ = credentials_attachment_;
3238 1117 clone->ssl_certificate_store_ = ssl_certificate_store_;
3239
3240 1117 clone->health_check_ = health_check_;
3241 1117 clone->sharding_policy_ = sharding_policy_;
3242 1117 clone->failover_indefinitely_ = failover_indefinitely_;
3243 1117 clone->fqrn_ = fqrn_;
3244
3245 1117 return clone;
3246 }
3247
3248
3249 1117 void DownloadManager::CloneProxyConfig(DownloadManager *clone) {
3250 1117 clone->opt_proxy_groups_current_ = opt_proxy_groups_current_;
3251 1117 clone->opt_proxy_groups_current_burned_ = opt_proxy_groups_current_burned_;
3252 1117 clone->opt_proxy_groups_fallback_ = opt_proxy_groups_fallback_;
3253 1117 clone->opt_num_proxies_ = opt_num_proxies_;
3254 1117 clone->opt_proxy_shard_ = opt_proxy_shard_;
3255 1117 clone->opt_proxy_list_ = opt_proxy_list_;
3256 1117 clone->opt_proxy_fallback_list_ = opt_proxy_fallback_list_;
3257
2/2
✓ Branch 0 taken 98 times.
✓ Branch 1 taken 1019 times.
 1117 if (opt_proxy_groups_ == NULL)
3258 98 return;
3259
3260
1/2
✓ Branch 2 taken 1019 times.
✗ Branch 3 not taken.
 1019 clone->opt_proxy_groups_ = new vector<vector<ProxyInfo> >(*opt_proxy_groups_);
3261
2/4
✓ Branch 2 taken 1019 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1019 times.
✗ Branch 6 not taken.
 1019 clone->UpdateProxiesUnlocked("cloned");
3262 }
3263
3264 } // namespace download
3265