GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/network/download.cc
Date: 2026-06-14 02:36:34
Exec Total Coverage
Lines: 873 1719 50.8%
Branches: 664 2198 30.2%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * The download module provides an interface for fetching files via HTTP
5 * and file. It is internally using libcurl and the asynchronous DNS resolver
6 * c-ares. The JobInfo struct describes a single file/url to download and
7 * keeps the state during the several phases of downloading.
8 *
9 * The module starts in single-threaded mode and can be switched to multi-
10 * threaded mode by Spawn(). In multi-threaded mode, the Fetch() function still
11 * blocks but there is a separate I/O thread using asynchronous I/O, which
12 * maintains all concurrent connections simultaneously. As there might be more
13 * than 1024 file descriptors for the CernVM-FS process, the I/O thread uses
14 * poll and the libcurl multi socket interface.
15 *
16 * While downloading, files can be decompressed and the secure hash can be
17 * calculated on the fly.
18 *
19 * The module also implements failure handling. If corrupted data has been
20 * downloaded, the transfer is restarted using HTTP "no-cache" pragma.
21 * A "host chain" can be configured. When a host fails, there is automatic
22 * fail-over to the next host in the chain until all hosts are probed.
23 * Similarly a chain of proxy sets can be configured. Inside a proxy set,
24 * proxies are selected randomly (load-balancing set).
25 */
26
27 // TODO(jblomer): MS for time summing
28
29 #include "download.h"
30
31 #include <alloca.h>
32 #include <errno.h>
33 #include <inttypes.h>
34 #include <poll.h>
35 #include <pthread.h>
36 #include <signal.h>
37 #include <stdint.h>
38 #include <sys/time.h>
39 #include <unistd.h>
40
41 #include <algorithm>
42 #include <cassert>
43 #include <cstdio>
44 #include <cstdlib>
45 #include <cstring>
46 #include <map>
47 #include <set>
48 #include <utility>
49
50 #include "compression/compression.h"
51 #include "crypto/hash.h"
52 #include "duplex_curl.h" // IWYU pragma: keep
53 #include "interrupt.h"
54 #include "network/sink_mem.h"
55 #include "network/sink_path.h"
56 #include "sanitizer.h"
57 #include "ssl.h"
58 #include "util/algorithm.h"
59 #include "util/atomic.h"
60 #include "util/exception.h"
61 #include "util/logging.h"
62 #include "util/posix.h"
63 #include "util/prng.h"
64 #include "util/smalloc.h"
65 #include "util/string.h"
66
67 using namespace std; // NOLINT
68
69 namespace download {
70
71 /**
72 * Returns the status if an interrupt happened for a given repository.
73 *
74 * Used only in case CVMFS_FAILOVER_INDEFINITELY (failover_indefinitely_) is set
75 * where failed downloads are retried indefinitely, unless an interrupt occurred
76 *
77 * @note If you use this functionality you need to change the source code of
78 * e.g. cvmfs_config reload to create a sentinel file. See comment below.
79 *
80 * @return true if an interrupt occurred
81 * false otherwise
82 */
83 ✗ bool Interrupted(const std::string &fqrn, JobInfo *info) {
84 ✗ if (info->allow_failure()) {
85 ✗ return true;
86 }
87
88 ✗ if (!fqrn.empty()) {
89 // it is up to the user the create this sentinel file ("pause_file") if
90 // CVMFS_FAILOVER_INDEFINITELY is used. It must be created during
91 // "cvmfs_config reload" and "cvmfs_config reload $fqrn"
92 ✗ const std::string pause_file = std::string("/var/run/cvmfs/interrupt.")
93 ✗ + fqrn;
94
95 ✗ LogCvmfs(kLogDownload, kLogDebug,
96 "(id %" PRId64 ") Interrupted(): checking for existence of %s",
97 info->id(), pause_file.c_str());
98 ✗ if (FileExists(pause_file)) {
99 ✗ LogCvmfs(kLogDownload, kLogDebug,
100 "(id %" PRId64 ") Interrupt marker found - "
101 "Interrupting current download, this will EIO outstanding IO.",
102 info->id());
103 ✗ if (0 != unlink(pause_file.c_str())) {
104 ✗ LogCvmfs(kLogDownload, kLogDebug,
105 "(id %" PRId64 ") Couldn't delete interrupt marker: errno=%d",
106 ✗ info->id(), errno);
107 }
108 ✗ return true;
109 }
110 }
111 ✗ return false;
112 }
113
114 4913 static Failures PrepareDownloadDestination(JobInfo *info) {
115
3/6
✓ Branch 1 taken 4913 times.
✗ Branch 2 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 4913 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 4913 times.
 4913 if (info->sink() != NULL && !info->sink()->IsValid()) {
116 ✗ cvmfs::PathSink *psink = dynamic_cast<cvmfs::PathSink *>(info->sink());
117 ✗ if (psink != NULL) {
118 ✗ LogCvmfs(kLogDownload, kLogDebug,
119 "(id %" PRId64 ") Failed to open path %s: %s (errno=%d).",
120 ✗ info->id(), psink->path().c_str(), strerror(errno), errno);
121 ✗ return kFailLocalIO;
122 } else {
123 ✗ LogCvmfs(kLogDownload, kLogDebug,
124 "(id %" PRId64 ") "
125 "Failed to create a valid sink: \n %s",
126 ✗ info->id(), info->sink()->Describe().c_str());
127 ✗ return kFailOther;
128 }
129 }
130
131 4913 return kFailOk;
132 }
133
134
135 /**
136 * Called by curl for every HTTP header. Not called for file:// transfers.
137 */
138 14942 static size_t CallbackCurlHeader(void *ptr, size_t size, size_t nmemb,
139 void *info_link) {
140 14942 const size_t num_bytes = size * nmemb;
141
1/2
✓ Branch 2 taken 14942 times.
✗ Branch 3 not taken.
 14942 const string header_line(static_cast<const char *>(ptr), num_bytes);
142 14942 JobInfo *info = static_cast<JobInfo *>(info_link);
143
144 // LogCvmfs(kLogDownload, kLogDebug, "REMOVE-ME: Header callback with %s",
145 // header_line.c_str());
146
147 // Check http status codes
148
4/6
✓ Branch 2 taken 14942 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 14942 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 588 times.
✓ Branch 10 taken 14354 times.
 14942 if (HasPrefix(header_line, "HTTP/1.", false)) {
149
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 588 times.
 588 if (header_line.length() < 10) {
150 ✗ return 0;
151 }
152
153 unsigned i;
154
5/6
✓ Branch 1 taken 1176 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 588 times.
✓ Branch 5 taken 588 times.
✓ Branch 6 taken 588 times.
✓ Branch 7 taken 588 times.
 1176 for (i = 8; (i < header_line.length()) && (header_line[i] == ' '); ++i) {
155 }
156
157 // Code is initialized to -1
158
1/2
✓ Branch 1 taken 588 times.
✗ Branch 2 not taken.
 588 if (header_line.length() > i + 2) {
159 588 info->SetHttpCode(DownloadManager::ParseHttpCode(&header_line[i]));
160 }
161
162
2/2
✓ Branch 1 taken 441 times.
✓ Branch 2 taken 147 times.
 588 if ((info->http_code() / 100) == 2) {
163 441 return num_bytes;
164
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 147 } else if ((info->http_code() == 301) || (info->http_code() == 302)
165
2/8
✗ Branch 0 not taken.
✓ Branch 1 taken 147 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 147 times.
✗ Branch 9 not taken.
 147 || (info->http_code() == 303) || (info->http_code() == 307)) {
166
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 147 times.
 147 if (!info->follow_redirects()) {
167 ✗ LogCvmfs(kLogDownload, kLogDebug,
168 "(id %" PRId64 ") redirect support not enabled: %s",
169 info->id(), header_line.c_str());
170 ✗ info->SetErrorCode(kFailHostHttp);
171 ✗ return 0;
172 }
173
1/2
✓ Branch 3 taken 147 times.
✗ Branch 4 not taken.
 147 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") http redirect: %s",
174 info->id(), header_line.c_str());
175 // libcurl will handle this because of CURLOPT_FOLLOWLOCATION
176 147 return num_bytes;
177 } else {
178 ✗ LogCvmfs(kLogDownload, kLogDebug,
179 "(id %" PRId64 ") http status error code: %s [%d]", info->id(),
180 header_line.c_str(), info->http_code());
181 ✗ if (((info->http_code() / 100) == 5) || (info->http_code() == 400)
182 ✗ || (info->http_code() == 404)) {
183 // 5XX returned by host
184 // 400: error from the GeoAPI module
185 // 404: the stratum 1 does not have the newest files
186 ✗ info->SetErrorCode(kFailHostHttp);
187 ✗ } else if (info->http_code() == 429) {
188 // 429: rate throttling (we ignore the backoff hint for the time being)
189 ✗ info->SetErrorCode(kFailHostConnection);
190 } else {
191 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostHttp
192 : kFailProxyHttp);
193 }
194 ✗ return 0;
195 }
196 }
197
198 // If needed: allocate space in sink
199
3/4
✓ Branch 2 taken 14354 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 5087 times.
✓ Branch 5 taken 9267 times.
 14354 if (info->sink() != NULL && info->sink()->RequiresReserve()
200
11/20
✓ Branch 1 taken 14354 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 5087 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 5087 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 1516 times.
✓ Branch 11 taken 3571 times.
✓ Branch 12 taken 5087 times.
✓ Branch 13 taken 9267 times.
✗ Branch 14 not taken.
✓ Branch 15 taken 5087 times.
✓ Branch 16 taken 9267 times.
✓ Branch 18 taken 1516 times.
✓ Branch 19 taken 12838 times.
✗ Branch 20 not taken.
✗ Branch 21 not taken.
✗ Branch 23 not taken.
✗ Branch 24 not taken.
 28708 && HasPrefix(header_line, "CONTENT-LENGTH:", true)) {
201 1516 char *tmp = reinterpret_cast<char *>(alloca(num_bytes + 1));
202 1516 uint64_t length = 0;
203 1516 sscanf(header_line.c_str(), "%s %" PRIu64, tmp, &length);
204
1/2
✓ Branch 0 taken 1516 times.
✗ Branch 1 not taken.
 1516 if (length > 0) {
205
2/4
✓ Branch 2 taken 1516 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 1516 times.
 1516 if (!info->sink()->Reserve(length)) {
206 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
207 "(id %" PRId64 ") "
208 "resource %s too large to store in memory (%" PRIu64 ")",
209 info->id(), info->url()->c_str(), length);
210 ✗ info->SetErrorCode(kFailTooBig);
211 ✗ return 0;
212 }
213 } else {
214 // Empty resource
215 ✗ info->sink()->Reserve(0);
216 }
217
4/6
✓ Branch 2 taken 12838 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 12838 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 147 times.
✓ Branch 10 taken 12691 times.
 12838 } else if (HasPrefix(header_line, "LOCATION:", true)) {
218 // This comes along with redirects
219
1/2
✓ Branch 3 taken 147 times.
✗ Branch 4 not taken.
 147 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") %s", info->id(),
220 header_line.c_str());
221
3/6
✓ Branch 2 taken 12691 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 12691 times.
✗ Branch 6 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 12691 times.
 12691 } else if (HasPrefix(header_line, "LINK:", true)) {
222 // This is metalink info
223 ✗ LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") %s", info->id(),
224 header_line.c_str());
225 ✗ std::string link = info->link();
226 ✗ if (link.size() != 0) {
227 // multiple LINK headers are allowed
228 ✗ link = link + ", " + header_line.substr(5);
229 } else {
230 ✗ link = header_line.substr(5);
231 }
232 ✗ info->SetLink(link);
233
3/6
✓ Branch 3 taken 12691 times.
✗ Branch 4 not taken.
✓ Branch 6 taken 12691 times.
✗ Branch 7 not taken.
✗ Branch 10 not taken.
✓ Branch 11 taken 12691 times.
 12691 } else if (HasPrefix(header_line, "X-SQUID-ERROR:", true)) {
234 // Reinterpret host error as proxy error
235 ✗ if (info->error_code() == kFailHostHttp) {
236 ✗ info->SetErrorCode(kFailProxyHttp);
237 }
238
3/6
✓ Branch 2 taken 12691 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 12691 times.
✗ Branch 6 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 12691 times.
 12691 } else if (HasPrefix(header_line, "PROXY-STATUS:", true)) {
239 // Reinterpret host error as proxy error if applicable
240 ✗ if ((info->error_code() == kFailHostHttp)
241 ✗ && (header_line.find("error=") != string::npos)) {
242 ✗ info->SetErrorCode(kFailProxyHttp);
243 }
244 }
245
246 14354 return num_bytes;
247 14942 }
248
249
250 /**
251 * Called by curl for every received data chunk.
252 */
253 6397 static size_t CallbackCurlData(void *ptr, size_t size, size_t nmemb,
254 void *info_link) {
255 6397 const size_t num_bytes = size * nmemb;
256 6397 JobInfo *info = static_cast<JobInfo *>(info_link);
257
258 // TODO(heretherebedragons) remove if no error comes up
259 // as this means only jobinfo data request (and not header only)
260 // come here
261
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6397 times.
 6397 assert(info->sink() != NULL);
262
263 // LogCvmfs(kLogDownload, kLogDebug, "Data callback, %d bytes", num_bytes);
264
265
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6397 times.
 6397 if (num_bytes == 0)
266 ✗ return 0;
267
268
2/2
✓ Branch 1 taken 3097 times.
✓ Branch 2 taken 3300 times.
 6397 if (info->expected_hash()) {
269 3097 shash::Update(reinterpret_cast<unsigned char *>(ptr), num_bytes,
270 info->hash_context());
271 }
272
273
2/2
✓ Branch 1 taken 3085 times.
✓ Branch 2 taken 3312 times.
 6397 if (info->compressed()) {
274 3085 const zlib::StreamStates retval = zlib::DecompressZStream2Sink(
275 ptr, static_cast<int64_t>(num_bytes), info->GetZstreamPtr(),
276 info->sink());
277
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3085 times.
 3085 if (retval == zlib::kStreamDataError) {
278 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
279 "(id %" PRId64 ") failed to decompress %s", info->id(),
280 info->url()->c_str());
281 ✗ info->SetErrorCode(kFailBadData);
282 ✗ return 0;
283
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3085 times.
 3085 } else if (retval == zlib::kStreamIOError) {
284 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
285 "(id %" PRId64 ") decompressing %s, local IO error", info->id(),
286 info->url()->c_str());
287 ✗ info->SetErrorCode(kFailLocalIO);
288 ✗ return 0;
289 }
290 } else {
291 3312 const int64_t written = info->sink()->Write(ptr, num_bytes);
292
2/4
✓ Branch 0 taken 3312 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 3312 times.
 3312 if (written < 0 || static_cast<uint64_t>(written) != num_bytes) {
293 ✗ LogCvmfs(kLogDownload, kLogDebug,
294 "(id %" PRId64 ") "
295 "Failed to perform write of %zu bytes to sink %s with errno %ld",
296 ✗ info->id(), num_bytes, info->sink()->Describe().c_str(),
297 written);
298 }
299 }
300
301 6397 return num_bytes;
302 }
303
304 #ifdef DEBUGMSG
305 5329 static int CallbackCurlDebug(CURL *handle,
306 curl_infotype type,
307 char *data,
308 size_t size,
309 void * /* clientp */) {
310 JobInfo *info;
311
1/2
✓ Branch 1 taken 5329 times.
✗ Branch 2 not taken.
 5329 curl_easy_getinfo(handle, CURLINFO_PRIVATE, &info);
312
313
3/6
✓ Branch 2 taken 5329 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 5329 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 5329 times.
✗ Branch 9 not taken.
 10658 std::string prefix = "(id " + StringifyInt(info->id()) + ") ";
314
4/8
✓ Branch 0 taken 674 times.
✓ Branch 1 taken 2303 times.
✓ Branch 2 taken 588 times.
✓ Branch 3 taken 1764 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
 5329 switch (type) {
315 674 case CURLINFO_TEXT:
316
1/2
✓ Branch 1 taken 674 times.
✗ Branch 2 not taken.
 674 prefix += "{info} ";
317 674 break;
318 2303 case CURLINFO_HEADER_IN:
319
1/2
✓ Branch 1 taken 2303 times.
✗ Branch 2 not taken.
 2303 prefix += "{header/recv} ";
320 2303 break;
321 588 case CURLINFO_HEADER_OUT:
322
1/2
✓ Branch 1 taken 588 times.
✗ Branch 2 not taken.
 588 prefix += "{header/sent} ";
323 588 break;
324 1764 case CURLINFO_DATA_IN:
325
2/2
✓ Branch 0 taken 196 times.
✓ Branch 1 taken 1568 times.
 1764 if (size < 50) {
326
1/2
✓ Branch 1 taken 196 times.
✗ Branch 2 not taken.
 196 prefix += "{data/recv} ";
327 196 break;
328 } else {
329
1/2
✓ Branch 2 taken 1568 times.
✗ Branch 3 not taken.
 1568 LogCvmfs(kLogCurl, kLogDebug, "%s{data/recv} <snip>", prefix.c_str());
330 1568 return 0;
331 }
332 ✗ case CURLINFO_DATA_OUT:
333 ✗ if (size < 50) {
334 ✗ prefix += "{data/sent} ";
335 ✗ break;
336 } else {
337 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{data/sent} <snip>", prefix.c_str());
338 ✗ return 0;
339 }
340 ✗ case CURLINFO_SSL_DATA_IN:
341 ✗ if (size < 50) {
342 ✗ prefix += "{ssldata/recv} ";
343 ✗ break;
344 } else {
345 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{ssldata/recv} <snip>",
346 prefix.c_str());
347 ✗ return 0;
348 }
349 ✗ case CURLINFO_SSL_DATA_OUT:
350 ✗ if (size < 50) {
351 ✗ prefix += "{ssldata/sent} ";
352 ✗ break;
353 } else {
354 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{ssldata/sent} <snip>",
355 prefix.c_str());
356 ✗ return 0;
357 }
358 ✗ default:
359 // just log the message
360 ✗ break;
361 }
362
363 3761 bool valid_char = true;
364
1/2
✓ Branch 2 taken 3761 times.
✗ Branch 3 not taken.
 3761 std::string msg(data, size);
365
2/2
✓ Branch 1 taken 181967 times.
✓ Branch 2 taken 3761 times.
 185728 for (size_t i = 0; i < msg.length(); ++i) {
366
2/4
✓ Branch 1 taken 181967 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 181967 times.
 181967 if (msg[i] == '\0') {
367 ✗ msg[i] = '~';
368 }
369
370 // verify that char is a valid printable char
371
3/6
✓ Branch 1 taken 181967 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 169239 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 169239 times.
 351206 if ((msg[i] < ' ' || msg[i] > '~')
372
6/8
✓ Branch 0 taken 169239 times.
✓ Branch 1 taken 12728 times.
✓ Branch 3 taken 12728 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 6027 times.
✓ Branch 6 taken 6701 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 181967 times.
 357233 && (msg[i] != 10 /*line feed*/
373
2/4
✓ Branch 1 taken 6027 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 6027 times.
 6027 && msg[i] != 13 /*carriage return*/)) {
374 ✗ valid_char = false;
375 }
376 }
377
378
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3761 times.
 3761 if (!valid_char) {
379 ✗ msg = "<Non-plaintext sequence>";
380 }
381
382
1/2
✓ Branch 3 taken 3761 times.
✗ Branch 4 not taken.
 3761 LogCvmfs(kLogCurl, kLogDebug, "%s%s", prefix.c_str(),
383
1/2
✓ Branch 1 taken 3761 times.
✗ Branch 2 not taken.
 7522 Trim(msg, true /* trim_newline */).c_str());
384 3761 return 0;
385 5329 }
386 #endif
387
388 //------------------------------------------------------------------------------
389
390
391 const int DownloadManager::kProbeUnprobed = -1;
392 const int DownloadManager::kProbeDown = -2;
393 const int DownloadManager::kProbeGeo = -3;
394
395 /**
396 * Escape special chars from the URL, except for ':' and '/',
397 * which should keep their meaning.
398 */
399 5109 string DownloadManager::EscapeUrl(const int64_t jobinfo_id, const string &url) {
400 5109 string escaped;
401
1/2
✓ Branch 2 taken 5109 times.
✗ Branch 3 not taken.
 5109 escaped.reserve(url.length());
402
403 char escaped_char[3];
404
2/2
✓ Branch 1 taken 435431 times.
✓ Branch 2 taken 5109 times.
 440540 for (unsigned i = 0, s = url.length(); i < s; ++i) {
405
3/4
✓ Branch 2 taken 435431 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 441 times.
✓ Branch 5 taken 434990 times.
 435431 if (JobInfo::EscapeUrlChar(url[i], escaped_char)) {
406
1/2
✓ Branch 1 taken 441 times.
✗ Branch 2 not taken.
 441 escaped.append(escaped_char, 3);
407 } else {
408
1/2
✓ Branch 1 taken 434990 times.
✗ Branch 2 not taken.
 434990 escaped.push_back(escaped_char[0]);
409 }
410 }
411
1/2
✓ Branch 3 taken 5109 times.
✗ Branch 4 not taken.
 5109 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") escaped %s to %s",
412 jobinfo_id, url.c_str(), escaped.c_str());
413
414 10218 return escaped;
415 }
416
417 /**
418 * -1 of digits is not a valid Http return code
419 */
420 833 int DownloadManager::ParseHttpCode(const char digits[3]) {
421 833 int result = 0;
422 833 int factor = 100;
423
2/2
✓ Branch 0 taken 2499 times.
✓ Branch 1 taken 784 times.
 3283 for (int i = 0; i < 3; ++i) {
424
3/4
✓ Branch 0 taken 2499 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 49 times.
✓ Branch 3 taken 2450 times.
 2499 if ((digits[i] < '0') || (digits[i] > '9'))
425 49 return -1;
426 2450 result += (digits[i] - '0') * factor;
427 2450 factor /= 10;
428 }
429 784 return result;
430 }
431
432
433 /**
434 * Called when new curl sockets arrive or existing curl sockets depart.
435 */
436 ✗ int DownloadManager::CallbackCurlSocket(CURL * /* easy */,
437 curl_socket_t s,
438 int action,
439 void *userp,
440 void * /* socketp */) {
441 // LogCvmfs(kLogDownload, kLogDebug, "CallbackCurlSocket called with easy "
442 // "handle %p, socket %d, action %d", easy, s, action);
443 ✗ DownloadManager *download_mgr = static_cast<DownloadManager *>(userp);
444 ✗ if (action == CURL_POLL_NONE)
445 ✗ return 0;
446
447 // Find s in watch_fds_
448 unsigned index;
449
450 // TODO(heretherebedragons) why start at index = 0 and not 2?
451 // fd[0] and fd[1] are fixed?
452 ✗ for (index = 0; index < download_mgr->watch_fds_inuse_; ++index) {
453 ✗ if (download_mgr->watch_fds_[index].fd == s)
454 ✗ break;
455 }
456 // Or create newly
457 ✗ if (index == download_mgr->watch_fds_inuse_) {
458 // Extend array if necessary
459 ✗ if (download_mgr->watch_fds_inuse_ == download_mgr->watch_fds_size_) {
460 ✗ assert(download_mgr->watch_fds_size_ > 0);
461 ✗ download_mgr->watch_fds_size_ *= 2;
462 ✗ download_mgr->watch_fds_ = static_cast<struct pollfd *>(
463 ✗ srealloc(download_mgr->watch_fds_,
464 ✗ download_mgr->watch_fds_size_ * sizeof(struct pollfd)));
465 }
466 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].fd = s;
467 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].events = 0;
468 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].revents = 0;
469 ✗ download_mgr->watch_fds_inuse_++;
470 }
471
472 ✗ switch (action) {
473 ✗ case CURL_POLL_IN:
474 ✗ download_mgr->watch_fds_[index].events = POLLIN | POLLPRI;
475 ✗ break;
476 ✗ case CURL_POLL_OUT:
477 ✗ download_mgr->watch_fds_[index].events = POLLOUT | POLLWRBAND;
478 ✗ break;
479 ✗ case CURL_POLL_INOUT:
480 ✗ download_mgr->watch_fds_[index].events = POLLIN | POLLPRI | POLLOUT
481 | POLLWRBAND;
482 ✗ break;
483 ✗ case CURL_POLL_REMOVE:
484 ✗ if (index < download_mgr->watch_fds_inuse_ - 1) {
485 download_mgr
486 ✗ ->watch_fds_[index] = download_mgr->watch_fds_
487 ✗ [download_mgr->watch_fds_inuse_ - 1];
488 }
489 ✗ download_mgr->watch_fds_inuse_--;
490 // Shrink array if necessary
491 ✗ if ((download_mgr->watch_fds_inuse_ > download_mgr->watch_fds_max_)
492 ✗ && (download_mgr->watch_fds_inuse_
493 ✗ < download_mgr->watch_fds_size_ / 2)) {
494 ✗ download_mgr->watch_fds_size_ /= 2;
495 // LogCvmfs(kLogDownload, kLogDebug, "shrinking watch_fds_ (%d)",
496 // watch_fds_size_);
497 ✗ download_mgr->watch_fds_ = static_cast<struct pollfd *>(
498 ✗ srealloc(download_mgr->watch_fds_,
499 ✗ download_mgr->watch_fds_size_ * sizeof(struct pollfd)));
500 // LogCvmfs(kLogDownload, kLogDebug, "shrinking watch_fds_ done",
501 // watch_fds_size_);
502 }
503 ✗ break;
504 ✗ default:
505 ✗ break;
506 }
507
508 ✗ return 0;
509 }
510
511
512 /**
513 * Worker thread event loop. Waits on new JobInfo structs on a pipe.
514 */
515 ✗ void *DownloadManager::MainDownload(void *data) {
516 ✗ DownloadManager *download_mgr = static_cast<DownloadManager *>(data);
517 ✗ LogCvmfs(kLogDownload, kLogDebug,
518 "download I/O thread of DownloadManager '%s' started",
519 download_mgr->name_.c_str());
520
521 ✗ const int kIdxPipeTerminate = 0;
522 ✗ const int kIdxPipeJobs = 1;
523
524 ✗ download_mgr->watch_fds_ = static_cast<struct pollfd *>(
525 ✗ smalloc(2 * sizeof(struct pollfd)));
526 ✗ download_mgr->watch_fds_size_ = 2;
527 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].fd = download_mgr->pipe_terminate_
528 ✗ ->GetReadFd();
529 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].events = POLLIN | POLLPRI;
530 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].revents = 0;
531 ✗ download_mgr->watch_fds_[kIdxPipeJobs].fd = download_mgr->pipe_jobs_
532 ✗ ->GetReadFd();
533 ✗ download_mgr->watch_fds_[kIdxPipeJobs].events = POLLIN | POLLPRI;
534 ✗ download_mgr->watch_fds_[kIdxPipeJobs].revents = 0;
535 ✗ download_mgr->watch_fds_inuse_ = 2;
536
537 ✗ int still_running = 0;
538 struct timeval timeval_start, timeval_stop;
539 ✗ gettimeofday(&timeval_start, NULL);
540 while (true) {
541 int timeout;
542 ✗ if (still_running) {
543 /* NOTE: The following might degrade the performance for many small files
544 * use case. TODO(jblomer): look into it.
545 // Specify a timeout for polling in ms; this allows us to return
546 // to libcurl once a second so it can look for internal operations
547 // which timed out. libcurl has a more elaborate mechanism
548 // (CURLMOPT_TIMERFUNCTION) that would inform us of the next potential
549 // timeout. TODO(bbockelm) we should switch to that in the future.
550 timeout = 100;
551 */
552 ✗ timeout = 1;
553 } else {
554 ✗ timeout = -1;
555 ✗ gettimeofday(&timeval_stop, NULL);
556 const int64_t delta = static_cast<int64_t>(
557 ✗ 1000 * DiffTimeSeconds(timeval_start, timeval_stop));
558 ✗ perf::Xadd(download_mgr->counters_->sz_transfer_time, delta);
559 }
560 ✗ const int retval = poll(download_mgr->watch_fds_,
561 ✗ download_mgr->watch_fds_inuse_, timeout);
562 ✗ if (retval < 0) {
563 ✗ continue;
564 }
565
566 // Handle timeout
567 ✗ if (retval == 0) {
568 ✗ curl_multi_socket_action(download_mgr->curl_multi_, CURL_SOCKET_TIMEOUT,
569 0, &still_running);
570 }
571
572 // Terminate I/O thread
573 ✗ if (download_mgr->watch_fds_[kIdxPipeTerminate].revents)
574 ✗ break;
575
576 // New job arrives
577 ✗ if (download_mgr->watch_fds_[kIdxPipeJobs].revents) {
578 ✗ download_mgr->watch_fds_[kIdxPipeJobs].revents = 0;
579 JobInfo *info;
580 ✗ download_mgr->pipe_jobs_->Read<JobInfo *>(&info);
581 ✗ if (!still_running) {
582 ✗ gettimeofday(&timeval_start, NULL);
583 }
584 ✗ CURL *handle = download_mgr->AcquireCurlHandle();
585 ✗ download_mgr->InitializeRequest(info, handle);
586 ✗ download_mgr->SetUrlOptions(info);
587 ✗ curl_multi_add_handle(download_mgr->curl_multi_, handle);
588 ✗ curl_multi_socket_action(download_mgr->curl_multi_, CURL_SOCKET_TIMEOUT,
589 0, &still_running);
590 }
591
592 // Activity on curl sockets
593 // Within this loop the curl_multi_socket_action() may cause socket(s)
594 // to be removed from watch_fds_. If a socket is removed it is replaced
595 // by the socket at the end of the array and the inuse count is decreased.
596 // Therefore loop over the array in reverse order.
597 ✗ for (int64_t i = download_mgr->watch_fds_inuse_ - 1; i >= 2; --i) {
598 ✗ if (i >= download_mgr->watch_fds_inuse_) {
599 ✗ continue;
600 }
601 ✗ if (download_mgr->watch_fds_[i].revents) {
602 ✗ int ev_bitmask = 0;
603 ✗ if (download_mgr->watch_fds_[i].revents & (POLLIN | POLLPRI))
604 ✗ ev_bitmask |= CURL_CSELECT_IN;
605 ✗ if (download_mgr->watch_fds_[i].revents & (POLLOUT | POLLWRBAND))
606 ✗ ev_bitmask |= CURL_CSELECT_OUT;
607 ✗ if (download_mgr->watch_fds_[i].revents
608 ✗ & (POLLERR | POLLHUP | POLLNVAL)) {
609 ✗ ev_bitmask |= CURL_CSELECT_ERR;
610 }
611 ✗ download_mgr->watch_fds_[i].revents = 0;
612
613 ✗ curl_multi_socket_action(download_mgr->curl_multi_,
614 ✗ download_mgr->watch_fds_[i].fd,
615 ev_bitmask,
616 &still_running);
617 }
618 }
619
620 // Check if transfers are completed
621 CURLMsg *curl_msg;
622 int msgs_in_queue;
623 ✗ while ((curl_msg = curl_multi_info_read(download_mgr->curl_multi_,
624 &msgs_in_queue))) {
625 ✗ if (curl_msg->msg == CURLMSG_DONE) {
626 ✗ perf::Inc(download_mgr->counters_->n_requests);
627 JobInfo *info;
628 ✗ CURL *easy_handle = curl_msg->easy_handle;
629 ✗ const int curl_error = curl_msg->data.result;
630 ✗ curl_easy_getinfo(easy_handle, CURLINFO_PRIVATE, &info);
631
632 int64_t redir_count;
633 ✗ curl_easy_getinfo(easy_handle, CURLINFO_REDIRECT_COUNT, &redir_count);
634 ✗ LogCvmfs(kLogDownload, kLogDebug,
635 "(manager '%s' - id %" PRId64 ") "
636 "Number of CURL redirects %" PRId64,
637 download_mgr->name_.c_str(), info->id(), redir_count);
638
639 ✗ curl_multi_remove_handle(download_mgr->curl_multi_, easy_handle);
640 ✗ if (download_mgr->VerifyAndFinalize(curl_error, info)) {
641 ✗ curl_multi_add_handle(download_mgr->curl_multi_, easy_handle);
642 ✗ curl_multi_socket_action(download_mgr->curl_multi_,
643 CURL_SOCKET_TIMEOUT,
644 0,
645 &still_running);
646 } else {
647 // Return easy handle into pool and write result back
648 ✗ download_mgr->ReleaseCurlHandle(easy_handle, true /* allow_reuse */);
649
650 ✗ DataTubeElement *ele = new DataTubeElement(kActionStop);
651 ✗ info->GetDataTubePtr()->EnqueueBack(ele);
652 ✗ info->GetPipeJobResultPtr()->Write<download::Failures>(
653 ✗ info->error_code());
654 }
655 }
656 }
657 }
658
659 ✗ for (set<CURL *>::iterator i = download_mgr->pool_handles_inuse_->begin(),
660 ✗ iEnd = download_mgr->pool_handles_inuse_->end();
661 ✗ i != iEnd;
662 ✗ ++i) {
663 ✗ curl_multi_remove_handle(download_mgr->curl_multi_, *i);
664 ✗ curl_easy_cleanup(*i);
665 }
666 ✗ download_mgr->pool_handles_inuse_->clear();
667 ✗ free(download_mgr->watch_fds_);
668
669 ✗ LogCvmfs(kLogDownload, kLogDebug,
670 "download I/O thread of DownloadManager '%s' terminated",
671 download_mgr->name_.c_str());
672 ✗ return NULL;
673 }
674
675
676 //------------------------------------------------------------------------------
677
678
679 3602 HeaderLists::~HeaderLists() {
680
2/2
✓ Branch 1 taken 3603 times.
✓ Branch 2 taken 3602 times.
 7205 for (unsigned i = 0; i < blocks_.size(); ++i) {
681
1/2
✓ Branch 1 taken 3603 times.
✗ Branch 2 not taken.
 3603 delete[] blocks_[i];
682 }
683 3602 blocks_.clear();
684 3602 }
685
686
687 9033 curl_slist *HeaderLists::GetList(const char *header) { return Get(header); }
688
689
690 4913 curl_slist *HeaderLists::DuplicateList(curl_slist *slist) {
691
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4913 times.
 4913 assert(slist);
692 4913 curl_slist *copy = GetList(slist->data);
693 4913 copy->next = slist->next;
694 4913 curl_slist *prev = copy;
695 4913 slist = slist->next;
696
2/2
✓ Branch 0 taken 9826 times.
✓ Branch 1 taken 4913 times.
 14739 while (slist) {
697 9826 curl_slist *new_link = Get(slist->data);
698 9826 new_link->next = slist->next;
699 9826 prev->next = new_link;
700 9826 prev = new_link;
701 9826 slist = slist->next;
702 }
703 4913 return copy;
704 }
705
706
707 7652 void HeaderLists::AppendHeader(curl_slist *slist, const char *header) {
708
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7652 times.
 7652 assert(slist);
709 7652 curl_slist *new_link = Get(header);
710 7652 new_link->next = NULL;
711
712
2/2
✓ Branch 0 taken 4717 times.
✓ Branch 1 taken 7652 times.
 12369 while (slist->next)
713 4717 slist = slist->next;
714 7652 slist->next = new_link;
715 7652 }
716
717
718 /**
719 * Ensures that a certain header string is _not_ part of slist on return.
720 * Note that if the first header element matches, the returned slist points
721 * to a different value.
722 */
723 5 void HeaderLists::CutHeader(const char *header, curl_slist **slist) {
724
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5 times.
 5 assert(slist);
725 curl_slist head;
726 5 head.next = *slist;
727 5 curl_slist *prev = &head;
728 5 curl_slist *rover = *slist;
729
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 5 times.
 14 while (rover) {
730
2/2
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 5 times.
 9 if (strcmp(rover->data, header) == 0) {
731 4 prev->next = rover->next;
732 4 Put(rover);
733 4 rover = prev;
734 }
735 9 prev = rover;
736 9 rover = rover->next;
737 }
738 5 *slist = head.next;
739 5 }
740
741
742 5173 void HeaderLists::PutList(curl_slist *slist) {
743
2/2
✓ Branch 0 taken 15447 times.
✓ Branch 1 taken 5173 times.
 20620 while (slist) {
744 15447 curl_slist *next = slist->next;
745 15447 Put(slist);
746 15447 slist = next;
747 }
748 5173 }
749
750
751 4 string HeaderLists::Print(curl_slist *slist) {
752 4 string verbose;
753
2/2
✓ Branch 0 taken 8 times.
✓ Branch 1 taken 4 times.
 12 while (slist) {
754
3/6
✓ Branch 2 taken 8 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 8 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 8 times.
✗ Branch 9 not taken.
 8 verbose += string(slist->data) + "\n";
755 8 slist = slist->next;
756 }
757 4 return verbose;
758 }
759
760
761 26511 curl_slist *HeaderLists::Get(const char *header) {
762
2/2
✓ Branch 1 taken 22907 times.
✓ Branch 2 taken 3605 times.
 26512 for (unsigned i = 0; i < blocks_.size(); ++i) {
763
2/2
✓ Branch 0 taken 161354 times.
✓ Branch 1 taken 1 times.
 161355 for (unsigned j = 0; j < kBlockSize; ++j) {
764
2/2
✓ Branch 2 taken 22906 times.
✓ Branch 3 taken 138448 times.
 161354 if (!IsUsed(&(blocks_[i][j]))) {
765 22906 blocks_[i][j].data = const_cast<char *>(header);
766 22906 return &(blocks_[i][j]);
767 }
768 }
769 }
770
771 // All used, new block
772 3605 AddBlock();
773 3605 blocks_[blocks_.size() - 1][0].data = const_cast<char *>(header);
774 3605 return &(blocks_[blocks_.size() - 1][0]);
775 }
776
777
778 938331 void HeaderLists::Put(curl_slist *slist) {
779 938331 slist->data = NULL;
780 938331 slist->next = NULL;
781 938331 }
782
783
784 3605 void HeaderLists::AddBlock() {
785
1/2
✓ Branch 1 taken 3605 times.
✗ Branch 2 not taken.
 3605 curl_slist *new_block = new curl_slist[kBlockSize];
786
2/2
✓ Branch 0 taken 922880 times.
✓ Branch 1 taken 3605 times.
 926485 for (unsigned i = 0; i < kBlockSize; ++i) {
787 922880 Put(&new_block[i]);
788 }
789
1/2
✓ Branch 1 taken 3605 times.
✗ Branch 2 not taken.
 3605 blocks_.push_back(new_block);
790 3605 }
791
792
793 //------------------------------------------------------------------------------
794
795
796 ✗ string DownloadManager::ProxyInfo::Print() {
797 ✗ if (url == "DIRECT")
798 ✗ return url;
799
800 ✗ string result = url;
801 ✗ const int remaining = static_cast<int>(host.deadline())
802 ✗ - static_cast<int>(time(NULL));
803 ✗ string expinfo = (remaining >= 0) ? "+" : "";
804 ✗ if (abs(remaining) >= 3600) {
805 ✗ expinfo += StringifyInt(remaining / 3600) + "h";
806 ✗ } else if (abs(remaining) >= 60) {
807 ✗ expinfo += StringifyInt(remaining / 60) + "m";
808 } else {
809 ✗ expinfo += StringifyInt(remaining) + "s";
810 }
811 ✗ if (host.status() == dns::kFailOk) {
812 ✗ result += " (" + host.name() + ", " + expinfo + ")";
813 } else {
814 ✗ result += " (:unresolved:, " + expinfo + ")";
815 }
816 ✗ return result;
817 }
818
819
820 /**
821 * Gets an idle CURL handle from the pool. Creates a new one and adds it to
822 * the pool if necessary.
823 */
824 4913 CURL *DownloadManager::AcquireCurlHandle() {
825 CURL *handle;
826
827
1/2
✓ Branch 1 taken 4913 times.
✗ Branch 2 not taken.
 4913 if (pool_handles_idle_->empty()) {
828 // Create a new handle
829
1/2
✓ Branch 1 taken 4913 times.
✗ Branch 2 not taken.
 4913 handle = curl_easy_init();
830
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4913 times.
 4913 assert(handle != NULL);
831
832
1/2
✓ Branch 1 taken 4913 times.
✗ Branch 2 not taken.
 4913 curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1);
833 // curl_easy_setopt(curl_default, CURLOPT_FAILONERROR, 1);
834
1/2
✓ Branch 1 taken 4913 times.
✗ Branch 2 not taken.
 4913 curl_easy_setopt(handle, CURLOPT_HEADERFUNCTION, CallbackCurlHeader);
835
1/2
✓ Branch 1 taken 4913 times.
✗ Branch 2 not taken.
 4913 curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, CallbackCurlData);
836 } else {
837 ✗ handle = *(pool_handles_idle_->begin());
838 ✗ pool_handles_idle_->erase(pool_handles_idle_->begin());
839 }
840
841
1/2
✓ Branch 1 taken 4913 times.
✗ Branch 2 not taken.
 4913 pool_handles_inuse_->insert(handle);
842
843 4913 return handle;
844 }
845
846
847 4913 void DownloadManager::ReleaseCurlHandle(CURL *handle, bool allow_reuse) {
848
1/2
✓ Branch 1 taken 4913 times.
✗ Branch 2 not taken.
 4913 const set<CURL *>::iterator elem = pool_handles_inuse_->find(handle);
849
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 4913 times.
 4913 assert(elem != pool_handles_inuse_->end());
850
851
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 4913 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 4913 times.
✗ Branch 6 not taken.
 4913 if (!allow_reuse || pool_handles_idle_->size() > pool_max_handles_) {
852
1/2
✓ Branch 2 taken 4913 times.
✗ Branch 3 not taken.
 4913 curl_easy_cleanup(*elem);
853 } else {
854 ✗ pool_handles_idle_->insert(*elem);
855 }
856
857
1/2
✓ Branch 1 taken 4913 times.
✗ Branch 2 not taken.
 4913 pool_handles_inuse_->erase(elem);
858 4913 }
859
860
861 /**
862 * HTTP request options: set the URL and other options such as timeout and
863 * proxy.
864 */
865 4913 void DownloadManager::InitializeRequest(JobInfo *info, CURL *handle) {
866 // Initialize internal download state
867 4913 info->SetCurlHandle(handle);
868 4913 info->SetErrorCode(kFailOk);
869 4913 info->SetHttpCode(-1);
870 4913 info->SetFollowRedirects(follow_redirects_);
871 4913 info->SetNumUsedProxies(1);
872 4913 info->SetNumUsedMetalinks(1);
873 4913 info->SetNumUsedHosts(1);
874 4913 info->SetNumRetries(0);
875 4913 info->SetBackoffMs(0);
876 4913 info->SetHeaders(header_lists_->DuplicateList(default_headers_));
877
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4913 times.
 4913 if (info->info_header()) {
878 ✗ header_lists_->AppendHeader(info->headers(), info->info_header());
879 }
880
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4913 times.
 4913 if (enable_http_tracing_) {
881 ✗ for (unsigned int i = 0; i < http_tracing_headers_.size(); i++) {
882 ✗ header_lists_->AppendHeader(info->headers(),
883 ✗ (http_tracing_headers_)[i].c_str());
884 }
885
886 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_pid());
887 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_gid());
888 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_uid());
889
890 ✗ LogCvmfs(kLogDownload, kLogDebug,
891 "(manager '%s' - id %" PRId64 ") "
892 "CURL Header for URL: %s is:\n %s",
893 name_.c_str(), info->id(), info->url()->c_str(),
894 ✗ header_lists_->Print(info->headers()).c_str());
895 }
896
897
2/2
✓ Branch 1 taken 126 times.
✓ Branch 2 taken 4787 times.
 4913 if (info->force_nocache()) {
898 126 SetNocache(info);
899 } else {
900 4787 info->SetNocache(false);
901 }
902
2/2
✓ Branch 1 taken 3173 times.
✓ Branch 2 taken 1740 times.
 4913 if (info->compressed()) {
903 3173 zlib::DecompressInit(info->GetZstreamPtr());
904 }
905
2/2
✓ Branch 1 taken 3185 times.
✓ Branch 2 taken 1728 times.
 4913 if (info->expected_hash()) {
906
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 3185 times.
 3185 assert(info->hash_context().buffer != NULL);
907 3185 shash::Init(info->hash_context());
908 }
909
910
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 4913 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 4913 times.
 4913 if ((info->range_offset() != -1) && (info->range_size())) {
911 char byte_range_array[100];
912 ✗ const int64_t range_lower = static_cast<int64_t>(info->range_offset());
913 ✗ const int64_t range_upper = static_cast<int64_t>(info->range_offset()
914 ✗ + info->range_size() - 1);
915 ✗ if (snprintf(byte_range_array, sizeof(byte_range_array),
916 "%" PRId64 "-%" PRId64, range_lower, range_upper)
917 ✗ == 100) {
918 ✗ PANIC(NULL); // Should be impossible given limits on offset size.
919 }
920 ✗ curl_easy_setopt(handle, CURLOPT_RANGE, byte_range_array);
921 } else {
922 4913 curl_easy_setopt(handle, CURLOPT_RANGE, NULL);
923 }
924
925 // Set curl parameters
926 4913 curl_easy_setopt(handle, CURLOPT_PRIVATE, static_cast<void *>(info));
927 4913 curl_easy_setopt(handle, CURLOPT_WRITEHEADER, static_cast<void *>(info));
928 4913 curl_easy_setopt(handle, CURLOPT_WRITEDATA, static_cast<void *>(info));
929 4913 curl_easy_setopt(handle, CURLOPT_HTTPHEADER, info->headers());
930
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4913 times.
 4913 if (info->head_request()) {
931 ✗ curl_easy_setopt(handle, CURLOPT_NOBODY, 1);
932 } else {
933 4913 curl_easy_setopt(handle, CURLOPT_HTTPGET, 1);
934 }
935
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4913 times.
 4913 if (opt_ipv4_only_) {
936 ✗ curl_easy_setopt(handle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
937 }
938
2/2
✓ Branch 0 taken 1709 times.
✓ Branch 1 taken 3204 times.
 4913 if (follow_redirects_) {
939 1709 curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1);
940 1709 curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 4);
941 }
942 #ifdef DEBUGMSG
943 4913 curl_easy_setopt(handle, CURLOPT_VERBOSE, 1);
944 4913 curl_easy_setopt(handle, CURLOPT_DEBUGFUNCTION, CallbackCurlDebug);
945 #endif
946 4913 }
947
948 10120 void DownloadManager::CheckHostInfoReset(const std::string &typ,
949 HostInfo &info,
950 JobInfo *jobinfo,
951 time_t &now) {
952
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 10120 times.
 10120 if (info.timestamp_backup > 0) {
953 ✗ if (now == 0)
954 ✗ now = time(NULL);
955 ✗ if (static_cast<int64_t>(now)
956 ✗ > static_cast<int64_t>(info.timestamp_backup + info.reset_after)) {
957 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
958 "(manager %s - id %" PRId64 ") "
959 "switching %s from %s to %s (reset %s)",
960 name_.c_str(), jobinfo->id(), typ.c_str(),
961 ✗ (*info.chain)[info.current].c_str(), (*info.chain)[0].c_str(),
962 typ.c_str());
963 ✗ info.current = 0;
964 ✗ info.timestamp_backup = 0;
965 }
966 }
967 10120 }
968
969
970 /**
971 * Sets the URL specific options such as host to use and timeout. It might also
972 * set an error code, in which case the further processing should react on.
973 */
974 5060 void DownloadManager::SetUrlOptions(JobInfo *info) {
975 5060 CURL *curl_handle = info->curl_handle();
976 5060 string url_prefix;
977 5060 time_t now = 0;
978
979 5060 const MutexLockGuard m(lock_options_);
980
981 // sharding policy
982
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5060 times.
 5060 if (sharding_policy_.UseCount() > 0) {
983 ✗ if (info->proxy() != "") {
984 // proxy already set, so this is a failover event
985 ✗ perf::Inc(counters_->n_proxy_failover);
986 }
987 ✗ info->SetProxy(sharding_policy_->GetNextProxy(
988 ✗ info->url(), info->proxy(),
989 ✗ info->range_offset() == -1 ? 0 : info->range_offset()));
990
991 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, info->proxy().c_str());
992 } else { // no sharding policy
993 // Check if proxy group needs to be reset from backup to primary
994
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5060 times.
 5060 if (opt_timestamp_backup_proxies_ > 0) {
995 ✗ now = time(NULL);
996 ✗ if (static_cast<int64_t>(now) > static_cast<int64_t>(
997 ✗ opt_timestamp_backup_proxies_ + opt_proxy_groups_reset_after_)) {
998 ✗ opt_proxy_groups_current_ = 0;
999 ✗ opt_timestamp_backup_proxies_ = 0;
1000 ✗ RebalanceProxiesUnlocked("Reset proxy group from backup to primary");
1001 }
1002 }
1003 // Check if load-balanced proxies within the group need to be reset
1004
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5060 times.
 5060 if (opt_timestamp_failover_proxies_ > 0) {
1005 ✗ if (now == 0)
1006 ✗ now = time(NULL);
1007 ✗ if (static_cast<int64_t>(now)
1008 ✗ > static_cast<int64_t>(opt_timestamp_failover_proxies_
1009 ✗ + opt_proxy_groups_reset_after_)) {
1010 ✗ RebalanceProxiesUnlocked(
1011 "Reset load-balanced proxies within the active group");
1012 }
1013 }
1014
1015
1/2
✓ Branch 2 taken 5060 times.
✗ Branch 3 not taken.
 5060 ProxyInfo *proxy = ChooseProxyUnlocked(info->expected_hash());
1016
6/6
✓ Branch 0 taken 989 times.
✓ Branch 1 taken 4071 times.
✓ Branch 3 taken 793 times.
✓ Branch 4 taken 196 times.
✓ Branch 5 taken 4864 times.
✓ Branch 6 taken 196 times.
 5060 if (!proxy || (proxy->url == "DIRECT")) {
1017
2/4
✓ Branch 2 taken 4864 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 4864 times.
✗ Branch 6 not taken.
 4864 info->SetProxy("DIRECT");
1018
1/2
✓ Branch 2 taken 4864 times.
✗ Branch 3 not taken.
 4864 curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "");
1019 } else {
1020 // Note: inside ValidateProxyIpsUnlocked() we may change the proxy data
1021 // structure, so we must not pass proxy->... (== current_proxy())
1022 // parameters directly
1023
1/2
✓ Branch 1 taken 196 times.
✗ Branch 2 not taken.
 196 const std::string purl = proxy->url;
1024
1/2
✓ Branch 1 taken 196 times.
✗ Branch 2 not taken.
 196 const dns::Host phost = proxy->host;
1025
1/2
✓ Branch 1 taken 196 times.
✗ Branch 2 not taken.
 196 const bool changed = ValidateProxyIpsUnlocked(purl, phost);
1026 // Current proxy may have changed
1027
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 196 times.
 196 if (changed) {
1028 ✗ proxy = ChooseProxyUnlocked(info->expected_hash());
1029 }
1030
1/2
✓ Branch 1 taken 196 times.
✗ Branch 2 not taken.
 196 info->SetProxy(proxy->url);
1031
1/2
✓ Branch 1 taken 196 times.
✗ Branch 2 not taken.
 196 if (proxy->host.status() == dns::kFailOk) {
1032
2/4
✓ Branch 1 taken 196 times.
✗ Branch 2 not taken.
✓ Branch 6 taken 196 times.
✗ Branch 7 not taken.
 196 curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY,
1033 info->proxy().c_str());
1034 } else {
1035 // We know it can't work, don't even try to download
1036 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "0.0.0.0");
1037 }
1038 196 }
1039 } // end !sharding
1040
1041 // Check if metalink and host chains need to be reset
1042
2/4
✓ Branch 2 taken 5060 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 5060 times.
✗ Branch 6 not taken.
 5060 CheckHostInfoReset("metalink", opt_metalink_, info, now);
1043
2/4
✓ Branch 2 taken 5060 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 5060 times.
✗ Branch 6 not taken.
 5060 CheckHostInfoReset("host", opt_host_, info, now);
1044
1045
1/2
✓ Branch 1 taken 5060 times.
✗ Branch 2 not taken.
 5060 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_LIMIT, opt_low_speed_limit_);
1046
3/6
✓ Branch 1 taken 5060 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 196 times.
✓ Branch 6 taken 4864 times.
 5060 if (info->proxy() != "DIRECT") {
1047
1/2
✓ Branch 1 taken 196 times.
✗ Branch 2 not taken.
 196 curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT, opt_timeout_proxy_);
1048
1/2
✓ Branch 1 taken 196 times.
✗ Branch 2 not taken.
 196 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_TIME, opt_timeout_proxy_);
1049 } else {
1050
1/2
✓ Branch 1 taken 4864 times.
✗ Branch 2 not taken.
 4864 curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT, opt_timeout_direct_);
1051
1/2
✓ Branch 1 taken 4864 times.
✗ Branch 2 not taken.
 4864 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_TIME, opt_timeout_direct_);
1052 }
1053
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5060 times.
 5060 if (!opt_dns_server_.empty())
1054 ✗ curl_easy_setopt(curl_handle, CURLOPT_DNS_SERVERS, opt_dns_server_.c_str());
1055
1056
2/2
✓ Branch 1 taken 1209 times.
✓ Branch 2 taken 3851 times.
 5060 if (info->probe_hosts()) {
1057
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1209 times.
 1209 if (CheckMetalinkChain(now)) {
1058 ✗ url_prefix = (*opt_metalink_.chain)[opt_metalink_.current];
1059 ✗ info->SetCurrentMetalinkChainIndex(opt_metalink_.current);
1060 ✗ LogCvmfs(kLogDownload, kLogDebug,
1061 "(manager %s - id %" PRId64 ") "
1062 "reading from metalink %d",
1063 name_.c_str(), info->id(), opt_metalink_.current);
1064
1/2
✓ Branch 0 taken 1209 times.
✗ Branch 1 not taken.
 1209 } else if (opt_host_.chain) {
1065
1/2
✓ Branch 2 taken 1209 times.
✗ Branch 3 not taken.
 1209 url_prefix = (*opt_host_.chain)[opt_host_.current];
1066 1209 info->SetCurrentHostChainIndex(opt_host_.current);
1067
1/2
✓ Branch 3 taken 1209 times.
✗ Branch 4 not taken.
 1209 LogCvmfs(kLogDownload, kLogDebug,
1068 "(manager %s - id %" PRId64 ") "
1069 "reading from host %d",
1070 name_.c_str(), info->id(), opt_host_.current);
1071 }
1072 }
1073
1074
1/2
✓ Branch 2 taken 5060 times.
✗ Branch 3 not taken.
 5060 string url = url_prefix + *(info->url());
1075
1076
1/2
✓ Branch 1 taken 5060 times.
✗ Branch 2 not taken.
 5060 curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, 1L);
1077
2/6
✓ Branch 1 taken 5060 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 5060 times.
 5060 if (url.substr(0, 5) == "https") {
1078 ✗ const bool rvb = ssl_certificate_store_.ApplySslCertificatePath(
1079 curl_handle);
1080 ✗ if (!rvb) {
1081 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1082 "(manager %s - id %" PRId64 ") "
1083 "Failed to set SSL certificate path %s",
1084 name_.c_str(), info->id(),
1085 ✗ ssl_certificate_store_.GetCaPath().c_str());
1086 }
1087 ✗ if (info->pid() != -1) {
1088 ✗ if (credentials_attachment_ == NULL) {
1089 ✗ LogCvmfs(kLogDownload, kLogDebug,
1090 "(manager %s - id %" PRId64 ") "
1091 "uses secure downloads but no credentials attachment set",
1092 name_.c_str(), info->id());
1093 } else {
1094 ✗ const bool retval = credentials_attachment_->ConfigureCurlHandle(
1095 curl_handle, info->pid(), info->GetCredDataPtr());
1096 ✗ if (!retval) {
1097 ✗ LogCvmfs(kLogDownload, kLogDebug,
1098 "(manager %s - id %" PRId64 ") "
1099 "failed attaching credentials",
1100 name_.c_str(), info->id());
1101 }
1102 }
1103 }
1104 // The download manager disables signal handling in the curl library;
1105 // as OpenSSL's implementation of TLS will generate a sigpipe in some
1106 // error paths, we must explicitly disable SIGPIPE here.
1107 // TODO(jblomer): it should be enough to do this once
1108 ✗ signal(SIGPIPE, SIG_IGN);
1109 }
1110
1111
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5060 times.
 5060 if (url.find("@proxy@") != string::npos) {
1112 // This is used in Geo-API requests (only), to replace a portion of the
1113 // URL with the current proxy name for the sake of caching the result.
1114 // Replace the @proxy@ either with a passed in "forced" template (which
1115 // is set from $CVMFS_PROXY_TEMPLATE) if there is one, or a "direct"
1116 // template (which is the uuid) if there's no proxy, or the name of the
1117 // proxy.
1118 ✗ string replacement;
1119 ✗ if (proxy_template_forced_ != "") {
1120 ✗ replacement = proxy_template_forced_;
1121 ✗ } else if (info->proxy() == "DIRECT") {
1122 ✗ replacement = proxy_template_direct_;
1123 } else {
1124 ✗ if (opt_proxy_groups_current_ >= opt_proxy_groups_fallback_) {
1125 // It doesn't make sense to use the fallback proxies in Geo-API requests
1126 // since the fallback proxies are supposed to get sorted, too.
1127 ✗ info->SetProxy("DIRECT");
1128 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "");
1129 ✗ replacement = proxy_template_direct_;
1130 } else {
1131 ✗ replacement = ChooseProxyUnlocked(info->expected_hash())->host.name();
1132 }
1133 }
1134 ✗ replacement = (replacement == "") ? proxy_template_direct_ : replacement;
1135 ✗ LogCvmfs(kLogDownload, kLogDebug,
1136 "(manager %s - id %" PRId64 ") "
1137 "replacing @proxy@ by %s",
1138 name_.c_str(), info->id(), replacement.c_str());
1139 ✗ url = ReplaceAll(url, "@proxy@", replacement);
1140 }
1141
1142 // TODO(heretherebedragons) before removing
1143 // static_cast<cvmfs::MemSink*>(info->sink)->size() == 0
1144 // and just always call info->sink->Reserve()
1145 // we should do a speed check
1146
3/4
✓ Branch 2 taken 5060 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1855 times.
✓ Branch 5 taken 3205 times.
 5060 if ((info->sink() != NULL) && info->sink()->RequiresReserve()
1147
1/2
✓ Branch 2 taken 1855 times.
✗ Branch 3 not taken.
 1855 && (static_cast<cvmfs::MemSink *>(info->sink())->size() == 0)
1148
11/20
✓ Branch 1 taken 5060 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 1855 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 1855 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 1226 times.
✓ Branch 11 taken 629 times.
✓ Branch 12 taken 1855 times.
✓ Branch 13 taken 3205 times.
✗ Branch 14 not taken.
✓ Branch 15 taken 1855 times.
✓ Branch 16 taken 3205 times.
✓ Branch 18 taken 1226 times.
✓ Branch 19 taken 3834 times.
✗ Branch 20 not taken.
✗ Branch 21 not taken.
✗ Branch 23 not taken.
✗ Branch 24 not taken.
 10120 && HasPrefix(url, "file://", false)) {
1149 platform_stat64 stat_buf;
1150 1226 const int retval = platform_stat(url.c_str(), &stat_buf);
1151
1/2
✓ Branch 0 taken 1226 times.
✗ Branch 1 not taken.
 1226 if (retval != 0) {
1152 // this is an error: file does not exist or out of memory
1153 // error is caught in other code section.
1154
1/2
✓ Branch 2 taken 1226 times.
✗ Branch 3 not taken.
 1226 info->sink()->Reserve(64ul * 1024ul);
1155 } else {
1156 ✗ info->sink()->Reserve(stat_buf.st_size);
1157 }
1158 }
1159
1160
2/4
✓ Branch 2 taken 5060 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 5060 times.
✗ Branch 7 not taken.
 5060 curl_easy_setopt(curl_handle, CURLOPT_URL,
1161 EscapeUrl(info->id(), url).c_str());
1162 5060 }
1163
1164
1165 /**
1166 * Checks if the name resolving information is still up to date. The host
1167 * object should be one from the current load-balance group. If the information
1168 * changed, gather new set of resolved IPs and, if different, exchange them in
1169 * the load-balance group on the fly. In the latter case, also rebalance the
1170 * proxies. The options mutex needs to be open.
1171 *
1172 * Returns true if proxies may have changed.
1173 */
1174 196 bool DownloadManager::ValidateProxyIpsUnlocked(const string &url,
1175 const dns::Host &host) {
1176
2/4
✓ Branch 1 taken 196 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 196 times.
✗ Branch 4 not taken.
 196 if (!host.IsExpired())
1177 196 return false;
1178 ✗ LogCvmfs(kLogDownload, kLogDebug, "(manager '%s') validate DNS entry for %s",
1179 ✗ name_.c_str(), host.name().c_str());
1180
1181 ✗ const unsigned group_idx = opt_proxy_groups_current_;
1182 ✗ dns::Host new_host = resolver_->Resolve(host.name());
1183
1184 ✗ bool update_only = true; // No changes to the list of IP addresses.
1185 ✗ if (new_host.status() != dns::kFailOk) {
1186 // Try again later in case resolving fails.
1187 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1188 "(manager '%s') failed to resolve IP addresses for %s (%d - %s)",
1189 ✗ name_.c_str(), host.name().c_str(), new_host.status(),
1190 dns::Code2Ascii(new_host.status()));
1191 ✗ new_host = dns::Host::ExtendDeadline(host, resolver_->min_ttl());
1192 ✗ } else if (!host.IsEquivalent(new_host)) {
1193 ✗ update_only = false;
1194 }
1195
1196 ✗ if (update_only) {
1197 ✗ for (unsigned i = 0; i < (*opt_proxy_groups_)[group_idx].size(); ++i) {
1198 ✗ if ((*opt_proxy_groups_)[group_idx][i].host.id() == host.id())
1199 ✗ (*opt_proxy_groups_)[group_idx][i].host = new_host;
1200 }
1201 ✗ return false;
1202 }
1203
1204 ✗ assert(new_host.status() == dns::kFailOk);
1205
1206 // Remove old host objects, insert new objects, and rebalance.
1207 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslog,
1208 "(manager '%s') DNS entries for proxy %s changed, adjusting",
1209 ✗ name_.c_str(), host.name().c_str());
1210 ✗ vector<ProxyInfo> *group = current_proxy_group();
1211 ✗ opt_num_proxies_ -= group->size();
1212 ✗ for (unsigned i = 0; i < group->size();) {
1213 ✗ if ((*group)[i].host.id() == host.id()) {
1214 ✗ group->erase(group->begin() + i);
1215 } else {
1216 ✗ i++;
1217 }
1218 }
1219 ✗ vector<ProxyInfo> new_infos;
1220 ✗ set<string> best_addresses = new_host.ViewBestAddresses(opt_ip_preference_);
1221 ✗ set<string>::const_iterator iter_ips = best_addresses.begin();
1222 ✗ for (; iter_ips != best_addresses.end(); ++iter_ips) {
1223 ✗ const string url_ip = dns::RewriteUrl(url, *iter_ips);
1224 ✗ new_infos.push_back(ProxyInfo(new_host, url_ip));
1225 }
1226 ✗ group->insert(group->end(), new_infos.begin(), new_infos.end());
1227 ✗ opt_num_proxies_ += new_infos.size();
1228
1229 ✗ const std::string msg = "DNS entries for proxy " + host.name() + " changed";
1230
1231 ✗ RebalanceProxiesUnlocked(msg);
1232 ✗ return true;
1233 }
1234
1235
1236 /**
1237 * Adds transfer time and downloaded bytes to the global counters.
1238 */
1239 5352 void DownloadManager::UpdateStatistics(CURL *handle) {
1240 double val;
1241 int retval;
1242 5352 int64_t sum = 0;
1243
1244
1/2
✓ Branch 1 taken 5352 times.
✗ Branch 2 not taken.
 5352 retval = curl_easy_getinfo(handle, CURLINFO_SIZE_DOWNLOAD, &val);
1245
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5352 times.
 5352 assert(retval == CURLE_OK);
1246 5352 sum += static_cast<int64_t>(val);
1247 /*retval = curl_easy_getinfo(handle, CURLINFO_HEADER_SIZE, &val);
1248 assert(retval == CURLE_OK);
1249 sum += static_cast<int64_t>(val);*/
1250 5352 perf::Xadd(counters_->sz_transferred_bytes, sum);
1251 5352 }
1252
1253
1254 /**
1255 * Retry if possible if not on no-cache and if not already done too often.
1256 */
1257 5352 bool DownloadManager::CanRetry(const JobInfo *info) {
1258 5352 const MutexLockGuard m(lock_options_);
1259 5352 const unsigned max_retries = opt_max_retries_;
1260
1261
2/2
✓ Branch 2 taken 2145 times.
✓ Branch 3 taken 2985 times.
 10482 return !(info->nocache()) && (info->num_retries() < max_retries)
1262
3/4
✓ Branch 0 taken 5130 times.
✓ Branch 1 taken 222 times.
✓ Branch 4 taken 2145 times.
✗ Branch 5 not taken.
 12627 && (IsProxyTransferError(info->error_code())
1263
2/2
✓ Branch 2 taken 196 times.
✓ Branch 3 taken 1949 times.
 2145 || IsHostTransferError(info->error_code()));
1264 5352 }
1265
1266 /**
1267 * Backoff for retry to introduce a jitter into a cluster of requesting
1268 * cvmfs nodes.
1269 * Retry only when HTTP caching is on.
1270 *
1271 * \return true if backoff has been performed, false otherwise
1272 */
1273 196 void DownloadManager::Backoff(JobInfo *info) {
1274 196 unsigned backoff_init_ms = 0;
1275 196 unsigned backoff_max_ms = 0;
1276 {
1277 196 const MutexLockGuard m(lock_options_);
1278 196 backoff_init_ms = opt_backoff_init_ms_;
1279 196 backoff_max_ms = opt_backoff_max_ms_;
1280 196 }
1281
1282 196 info->SetNumRetries(info->num_retries() + 1);
1283 196 perf::Inc(counters_->n_retries);
1284
2/2
✓ Branch 1 taken 79 times.
✓ Branch 2 taken 117 times.
 196 if (info->backoff_ms() == 0) {
1285 79 info->SetBackoffMs(prng_.Next(backoff_init_ms + 1)); // Must be != 0
1286 } else {
1287 117 info->SetBackoffMs(info->backoff_ms() * 2);
1288 }
1289
2/2
✓ Branch 1 taken 38 times.
✓ Branch 2 taken 158 times.
 196 if (info->backoff_ms() > backoff_max_ms) {
1290 38 info->SetBackoffMs(backoff_max_ms);
1291 }
1292
1293 196 LogCvmfs(kLogDownload, kLogDebug,
1294 "(manager '%s' - id %" PRId64 ") backing off for %d ms",
1295 name_.c_str(), info->id(), info->backoff_ms());
1296 196 SafeSleepMs(info->backoff_ms());
1297 196 }
1298
1299 222 void DownloadManager::SetNocache(JobInfo *info) {
1300
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 222 times.
 222 if (info->nocache())
1301 ✗ return;
1302 222 header_lists_->AppendHeader(info->headers(), "Pragma: no-cache");
1303 222 header_lists_->AppendHeader(info->headers(), "Cache-Control: no-cache");
1304 222 curl_easy_setopt(info->curl_handle(), CURLOPT_HTTPHEADER, info->headers());
1305 222 info->SetNocache(true);
1306 }
1307
1308
1309 /**
1310 * Reverse operation of SetNocache. Makes sure that "no-cache" header
1311 * disappears from the list of headers to let proxies work normally.
1312 */
1313 439 void DownloadManager::SetRegularCache(JobInfo *info) {
1314
1/2
✓ Branch 1 taken 439 times.
✗ Branch 2 not taken.
 439 if (info->nocache() == false)
1315 439 return;
1316 ✗ header_lists_->CutHeader("Pragma: no-cache", info->GetHeadersPtr());
1317 ✗ header_lists_->CutHeader("Cache-Control: no-cache", info->GetHeadersPtr());
1318 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_HTTPHEADER, info->headers());
1319 ✗ info->SetNocache(false);
1320 }
1321
1322
1323 /**
1324 * Frees the storage associated with the authz attachment from the job
1325 */
1326 5060 void DownloadManager::ReleaseCredential(JobInfo *info) {
1327
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5060 times.
 5060 if (info->cred_data()) {
1328 ✗ assert(credentials_attachment_ != NULL); // Someone must have set it
1329 ✗ credentials_attachment_->ReleaseCurlHandle(info->curl_handle(),
1330 info->cred_data());
1331 ✗ info->SetCredData(NULL);
1332 }
1333 5060 }
1334
1335
1336 /* Sort links based on the "pri=" parameter */
1337 ✗ static bool sortlinks(const std::string &s1, const std::string &s2) {
1338 ✗ const size_t pos1 = s1.find("; pri=");
1339 ✗ const size_t pos2 = s2.find("; pri=");
1340 int pri1, pri2;
1341 ✗ if ((pos1 != std::string::npos) && (pos2 != std::string::npos)
1342 ✗ && (sscanf(s1.substr(pos1 + 6).c_str(), "%d", &pri1) == 1)
1343 ✗ && (sscanf(s2.substr(pos2 + 6).c_str(), "%d", &pri2) == 1)) {
1344 ✗ return pri1 < pri2;
1345 }
1346 ✗ return false;
1347 }
1348
1349 /**
1350 * Parses Link header and uses it to set a new host chain.
1351 * See rfc6249.
1352 */
1353 ✗ void DownloadManager::ProcessLink(JobInfo *info) {
1354 ✗ std::vector<std::string> links = SplitString(info->link(), ',');
1355 ✗ if (info->link().find("; pri=") != std::string::npos)
1356 ✗ std::sort(links.begin(), links.end(), sortlinks);
1357
1358 ✗ std::vector<std::string> host_list;
1359
1360 ✗ std::vector<std::string>::const_iterator il = links.begin();
1361 ✗ for (; il != links.end(); ++il) {
1362 ✗ const std::string &link = *il;
1363 ✗ if ((link.find("; rel=duplicate") == std::string::npos)
1364 ✗ && (link.find("; rel=\"duplicate\"") == std::string::npos)) {
1365 ✗ LogCvmfs(kLogDownload, kLogDebug,
1366 "skipping link '%s' because it does not contain rel=duplicate",
1367 link.c_str());
1368 ✗ continue;
1369 }
1370 // ignore depth= field since there's nothing useful we can do with it
1371
1372 ✗ size_t start = link.find('<');
1373 ✗ if (start == std::string::npos) {
1374 ✗ LogCvmfs(
1375 kLogDownload, kLogDebug,
1376 "skipping link '%s' because it does not have a left angle bracket",
1377 link.c_str());
1378 ✗ continue;
1379 }
1380
1381 ✗ start++;
1382 ✗ if ((link.substr(start, 7) != "http://")
1383 ✗ && (link.substr(start, 8) != "https://")) {
1384 ✗ LogCvmfs(kLogDownload, kLogDebug,
1385 "skipping link '%s' of unrecognized url protocol", link.c_str());
1386 ✗ continue;
1387 }
1388
1389 ✗ size_t end = link.find('/', start + 8);
1390 ✗ if (end == std::string::npos)
1391 ✗ end = link.find('>');
1392 ✗ if (end == std::string::npos) {
1393 ✗ LogCvmfs(kLogDownload, kLogDebug,
1394 "skipping link '%s' because no slash in url and no right angle "
1395 "bracket",
1396 link.c_str());
1397 ✗ continue;
1398 }
1399 ✗ const std::string host = link.substr(start, end - start);
1400 ✗ LogCvmfs(kLogDownload, kLogDebug, "adding linked host '%s'", host.c_str());
1401 ✗ host_list.push_back(host);
1402 }
1403
1404 ✗ if (host_list.size() > 0) {
1405 ✗ SetHostChain(host_list);
1406 ✗ opt_metalink_timestamp_link_ = time(NULL);
1407 }
1408 }
1409
1410
1411 /**
1412 * Checks the result of a curl download and implements the failure logic, such
1413 * as changing the proxy server. Takes care of cleanup.
1414 *
1415 * \return true if another download should be performed, false otherwise
1416 */
1417 5352 bool DownloadManager::VerifyAndFinalize(const int curl_error, JobInfo *info) {
1418
1/2
✓ Branch 6 taken 5352 times.
✗ Branch 7 not taken.
 5352 LogCvmfs(kLogDownload, kLogDebug,
1419 "(manager '%s' - id %" PRId64 ") "
1420 "Verify downloaded url %s, proxy %s (curl error %d)",
1421 name_.c_str(), info->id(), info->url()->c_str(),
1422
1/2
✓ Branch 1 taken 5352 times.
✗ Branch 2 not taken.
 10704 info->proxy().c_str(), curl_error);
1423
1/2
✓ Branch 2 taken 5352 times.
✗ Branch 3 not taken.
 5352 UpdateStatistics(info->curl_handle());
1424
1425 bool was_metalink;
1426 5352 std::string typ;
1427
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5352 times.
 5352 if (info->current_metalink_chain_index() >= 0) {
1428 ✗ was_metalink = true;
1429 ✗ typ = "metalink";
1430 ✗ if (info->link() != "") {
1431 // process Link header whether or not the redirected URL got an error
1432 ✗ ProcessLink(info);
1433 }
1434 } else {
1435 5352 was_metalink = false;
1436
1/2
✓ Branch 1 taken 5352 times.
✗ Branch 2 not taken.
 5352 typ = "host";
1437 }
1438
1439
1440 // Verification and error classification
1441
3/14
✓ Branch 0 taken 4654 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 24 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 674 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
 5352 switch (curl_error) {
1442 4654 case CURLE_OK:
1443 // Verify content hash
1444
2/2
✓ Branch 1 taken 3069 times.
✓ Branch 2 taken 1585 times.
 4654 if (info->expected_hash()) {
1445
1/2
✓ Branch 1 taken 3069 times.
✗ Branch 2 not taken.
 3069 shash::Any match_hash;
1446
1/2
✓ Branch 2 taken 3069 times.
✗ Branch 3 not taken.
 3069 shash::Final(info->hash_context(), &match_hash);
1447
3/4
✓ Branch 2 taken 3069 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 192 times.
✓ Branch 5 taken 2877 times.
 3069 if (match_hash != *(info->expected_hash())) {
1448
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 192 times.
 192 if (ignore_signature_failures_) {
1449 ✗ LogCvmfs(
1450 kLogDownload, kLogDebug | kLogSyslogErr,
1451 "(manager '%s' - id %" PRId64 ") "
1452 "ignoring failed hash verification of %s (expected %s, got %s)",
1453 name_.c_str(), info->id(), info->url()->c_str(),
1454 ✗ info->expected_hash()->ToString().c_str(),
1455 ✗ match_hash.ToString().c_str());
1456 } else {
1457
1/2
✓ Branch 7 taken 192 times.
✗ Branch 8 not taken.
 384 LogCvmfs(kLogDownload, kLogDebug,
1458 "(manager '%s' - id %" PRId64 ") "
1459 "hash verification of %s failed (expected %s, got %s)",
1460 name_.c_str(), info->id(), info->url()->c_str(),
1461
1/2
✓ Branch 2 taken 192 times.
✗ Branch 3 not taken.
 384 info->expected_hash()->ToString().c_str(),
1462
1/2
✓ Branch 1 taken 192 times.
✗ Branch 2 not taken.
 384 match_hash.ToString().c_str());
1463 192 info->SetErrorCode(kFailBadData);
1464 192 break;
1465 }
1466 }
1467 }
1468
1469 4462 info->SetErrorCode(kFailOk);
1470 4462 break;
1471 ✗ case CURLE_UNSUPPORTED_PROTOCOL:
1472 ✗ info->SetErrorCode(kFailUnsupportedProtocol);
1473 ✗ break;
1474 24 case CURLE_URL_MALFORMAT:
1475 24 info->SetErrorCode(kFailBadUrl);
1476 24 break;
1477 ✗ case CURLE_COULDNT_RESOLVE_PROXY:
1478 ✗ info->SetErrorCode(kFailProxyResolve);
1479 ✗ break;
1480 ✗ case CURLE_COULDNT_RESOLVE_HOST:
1481 ✗ info->SetErrorCode(kFailHostResolve);
1482 ✗ break;
1483 ✗ case CURLE_OPERATION_TIMEDOUT:
1484 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostTooSlow
1485 : kFailProxyTooSlow);
1486 ✗ break;
1487 ✗ case CURLE_PARTIAL_FILE:
1488 case CURLE_GOT_NOTHING:
1489 case CURLE_RECV_ERROR:
1490 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostShortTransfer
1491 : kFailProxyShortTransfer);
1492 ✗ break;
1493 674 case CURLE_FILE_COULDNT_READ_FILE:
1494 case CURLE_COULDNT_CONNECT:
1495
3/6
✓ Branch 1 taken 674 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 49 times.
✓ Branch 6 taken 625 times.
 674 if (info->proxy() != "DIRECT") {
1496 // This is a guess. Fail-over can still change to switching host
1497 49 info->SetErrorCode(kFailProxyConnection);
1498 } else {
1499 625 info->SetErrorCode(kFailHostConnection);
1500 }
1501 674 break;
1502 ✗ case CURLE_TOO_MANY_REDIRECTS:
1503 ✗ info->SetErrorCode(kFailHostConnection);
1504 ✗ break;
1505 ✗ case CURLE_SSL_CACERT_BADFILE:
1506 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
1507 "(manager '%s' -id %" PRId64 ") "
1508 "Failed to load certificate bundle. "
1509 "X509_CERT_BUNDLE might point to the wrong location.",
1510 name_.c_str(), info->id());
1511 ✗ info->SetErrorCode(kFailHostConnection);
1512 ✗ break;
1513 // As of curl 7.62.0, CURLE_SSL_CACERT is the same as
1514 // CURLE_PEER_FAILED_VERIFICATION
1515 ✗ case CURLE_PEER_FAILED_VERIFICATION:
1516 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
1517 "(manager '%s' - id %" PRId64 ") "
1518 "invalid SSL certificate of remote host. "
1519 "X509_CERT_DIR and/or X509_CERT_BUNDLE might point to the wrong "
1520 "location.",
1521 name_.c_str(), info->id());
1522 ✗ info->SetErrorCode(kFailHostConnection);
1523 ✗ break;
1524 ✗ case CURLE_ABORTED_BY_CALLBACK:
1525 case CURLE_WRITE_ERROR:
1526 // Error set by callback
1527 ✗ break;
1528 ✗ case CURLE_SEND_ERROR:
1529 // The curl error CURLE_SEND_ERROR can be seen when a cache is misbehaving
1530 // and closing connections before the http request send is completed.
1531 // Handle this error, treating it as a short transfer error.
1532 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostShortTransfer
1533 : kFailProxyShortTransfer);
1534 ✗ break;
1535 ✗ default:
1536 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
1537 "(manager '%s' - id %" PRId64 ") "
1538 "unexpected curl error (%d) while trying to fetch %s",
1539 name_.c_str(), info->id(), curl_error, info->url()->c_str());
1540 ✗ info->SetErrorCode(kFailOther);
1541 ✗ break;
1542 }
1543
1544 std::vector<std::string> *host_chain;
1545 unsigned char num_used_hosts;
1546
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5352 times.
 5352 if (was_metalink) {
1547 ✗ host_chain = opt_metalink_.chain;
1548 ✗ num_used_hosts = info->num_used_metalinks();
1549 } else {
1550 5352 host_chain = opt_host_.chain;
1551 5352 num_used_hosts = info->num_used_hosts();
1552 }
1553
1554 // Determination if download should be repeated
1555 5352 bool try_again = false;
1556
1/2
✓ Branch 1 taken 5352 times.
✗ Branch 2 not taken.
 5352 bool same_url_retry = CanRetry(info);
1557
2/2
✓ Branch 1 taken 890 times.
✓ Branch 2 taken 4462 times.
 5352 if (info->error_code() != kFailOk) {
1558 890 const MutexLockGuard m(lock_options_);
1559
2/2
✓ Branch 1 taken 192 times.
✓ Branch 2 taken 698 times.
 890 if (info->error_code() == kFailBadData) {
1560
2/2
✓ Branch 1 taken 96 times.
✓ Branch 2 taken 96 times.
 192 if (!info->nocache()) {
1561 96 try_again = true;
1562 } else {
1563 // Make it a host failure
1564
1/2
✓ Branch 4 taken 96 times.
✗ Branch 5 not taken.
 96 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1565 "(manager '%s' - id %" PRId64 ") "
1566 "data corruption with no-cache header, try another %s",
1567 name_.c_str(), info->id(), typ.c_str());
1568
1569 96 info->SetErrorCode(kFailHostHttp);
1570 }
1571 }
1572 890 if (same_url_retry
1573
5/6
✓ Branch 0 taken 694 times.
✓ Branch 1 taken 196 times.
✓ Branch 3 taken 694 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 343 times.
✓ Branch 6 taken 547 times.
 1584 || (((info->error_code() == kFailHostResolve)
1574
2/2
✓ Branch 2 taken 265 times.
✓ Branch 3 taken 429 times.
 694 || IsHostTransferError(info->error_code())
1575
2/2
✓ Branch 1 taken 96 times.
✓ Branch 2 taken 169 times.
 265 || (info->error_code() == kFailHostHttp))
1576
3/4
✓ Branch 1 taken 277 times.
✓ Branch 2 taken 248 times.
✓ Branch 3 taken 277 times.
✗ Branch 4 not taken.
 525 && info->probe_hosts() && host_chain
1577
2/2
✓ Branch 1 taken 147 times.
✓ Branch 2 taken 130 times.
 277 && (num_used_hosts < host_chain->size()))) {
1578 343 try_again = true;
1579 }
1580 890 if (same_url_retry
1581
5/6
✓ Branch 0 taken 694 times.
✓ Branch 1 taken 196 times.
✓ Branch 3 taken 694 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 245 times.
✓ Branch 6 taken 645 times.
 1584 || (((info->error_code() == kFailProxyResolve)
1582
2/2
✓ Branch 2 taken 645 times.
✓ Branch 3 taken 49 times.
 694 || IsProxyTransferError(info->error_code())
1583
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 645 times.
 645 || (info->error_code() == kFailProxyHttp)))) {
1584
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 245 times.
 245 if (sharding_policy_.UseCount() > 0) { // sharding policy
1585 ✗ try_again = true;
1586 ✗ same_url_retry = false;
1587 } else { // no sharding
1588 245 try_again = true;
1589 // If all proxies failed, do a next round with the next host
1590
4/6
✓ Branch 0 taken 49 times.
✓ Branch 1 taken 196 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 49 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 245 times.
 245 if (!same_url_retry && (info->num_used_proxies() >= opt_num_proxies_)) {
1591 // Check if this can be made a host fail-over
1592 ✗ if (info->probe_hosts() && host_chain
1593 ✗ && (num_used_hosts < host_chain->size())) {
1594 // reset proxy group if not already performed by other handle
1595 ✗ if (opt_proxy_groups_) {
1596 ✗ if ((opt_proxy_groups_current_ > 0)
1597 ✗ || (opt_proxy_groups_current_burned_ > 0)) {
1598 ✗ opt_proxy_groups_current_ = 0;
1599 ✗ opt_timestamp_backup_proxies_ = 0;
1600 ✗ const std::string msg = "reset proxies for " + typ
1601 ✗ + " failover";
1602 ✗ RebalanceProxiesUnlocked(msg);
1603 }
1604 }
1605
1606 // Make it a host failure
1607 ✗ LogCvmfs(kLogDownload, kLogDebug,
1608 "(manager '%s' - id %" PRId64 ") make it a %s failure",
1609 name_.c_str(), info->id(), typ.c_str());
1610 ✗ info->SetNumUsedProxies(1);
1611 ✗ info->SetErrorCode(kFailHostAfterProxy);
1612 } else {
1613 ✗ if (failover_indefinitely_) {
1614 // Instead of giving up, reset the num_used_proxies counter,
1615 // switch proxy and try again
1616 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1617 "(manager '%s' - id %" PRId64 ") "
1618 "VerifyAndFinalize() would fail the download here. "
1619 "Instead switch proxy and retry download. "
1620 "typ=%s "
1621 "info->probe_hosts=%d host_chain=%p num_used_hosts=%d "
1622 "host_chain->size()=%lu same_url_retry=%d "
1623 "info->num_used_proxies=%d opt_num_proxies_=%d",
1624 name_.c_str(), info->id(), typ.c_str(),
1625 ✗ static_cast<int>(info->probe_hosts()), host_chain,
1626 ✗ num_used_hosts, host_chain ? host_chain->size() : -1,
1627 static_cast<int>(same_url_retry),
1628 ✗ info->num_used_proxies(), opt_num_proxies_);
1629 ✗ info->SetNumUsedProxies(1);
1630 ✗ RebalanceProxiesUnlocked(
1631 "download failed - failover indefinitely");
1632 ✗ try_again = !Interrupted(fqrn_, info);
1633 } else {
1634 ✗ try_again = false;
1635 }
1636 }
1637 } // Make a proxy failure a host failure
1638 } // Proxy failure assumed
1639 } // end !sharding
1640 890 }
1641
1642
2/2
✓ Branch 0 taken 488 times.
✓ Branch 1 taken 4864 times.
 5352 if (try_again) {
1643
1/2
✓ Branch 3 taken 488 times.
✗ Branch 4 not taken.
 488 LogCvmfs(kLogDownload, kLogDebug,
1644 "(manager '%s' - id %" PRId64 ") "
1645 "Trying again on same curl handle, same url: %d, "
1646 "error code %d no-cache %d",
1647 488 name_.c_str(), info->id(), same_url_retry, info->error_code(),
1648 488 info->nocache());
1649 // Reset internal state and destination
1650
4/8
✓ Branch 1 taken 488 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 488 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 488 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 488 times.
 488 if (info->sink() != NULL && info->sink()->Reset() != 0) {
1651 ✗ info->SetErrorCode(kFailLocalIO);
1652 ✗ goto verify_and_finalize_stop;
1653 }
1654
6/8
✓ Branch 1 taken 49 times.
✓ Branch 2 taken 439 times.
✓ Branch 5 taken 49 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 49 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 49 times.
✓ Branch 10 taken 439 times.
 488 if (info->interrupt_cue() && info->interrupt_cue()->IsCanceled()) {
1655 49 info->SetErrorCode(kFailCanceled);
1656 49 goto verify_and_finalize_stop;
1657 }
1658
1659
2/2
✓ Branch 1 taken 210 times.
✓ Branch 2 taken 229 times.
 439 if (info->expected_hash()) {
1660
1/2
✓ Branch 2 taken 210 times.
✗ Branch 3 not taken.
 210 shash::Init(info->hash_context());
1661 }
1662
2/2
✓ Branch 1 taken 210 times.
✓ Branch 2 taken 229 times.
 439 if (info->compressed()) {
1663
1/2
✓ Branch 2 taken 210 times.
✗ Branch 3 not taken.
 210 zlib::DecompressInit(info->GetZstreamPtr());
1664 }
1665
1666
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 439 times.
 439 if (sharding_policy_.UseCount() > 0) { // sharding policy
1667 ✗ ReleaseCredential(info);
1668 ✗ SetUrlOptions(info);
1669 } else { // no sharding policy
1670
1/2
✓ Branch 1 taken 439 times.
✗ Branch 2 not taken.
 439 SetRegularCache(info);
1671
1672 // Failure handling
1673 439 bool switch_proxy = false;
1674 439 bool switch_host = false;
1675
2/4
✓ Branch 1 taken 96 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 343 times.
 439 switch (info->error_code()) {
1676 96 case kFailBadData:
1677
1/2
✓ Branch 1 taken 96 times.
✗ Branch 2 not taken.
 96 SetNocache(info);
1678 96 break;
1679 ✗ case kFailProxyResolve:
1680 case kFailProxyHttp:
1681 ✗ switch_proxy = true;
1682 ✗ break;
1683 ✗ case kFailHostResolve:
1684 case kFailHostHttp:
1685 case kFailHostAfterProxy:
1686 ✗ switch_host = true;
1687 ✗ break;
1688 343 default:
1689
2/2
✓ Branch 2 taken 49 times.
✓ Branch 3 taken 294 times.
 343 if (IsProxyTransferError(info->error_code())) {
1690
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
 49 if (same_url_retry) {
1691 ✗ Backoff(info);
1692 } else {
1693 49 switch_proxy = true;
1694 }
1695
1/2
✓ Branch 2 taken 294 times.
✗ Branch 3 not taken.
 294 } else if (IsHostTransferError(info->error_code())) {
1696
2/2
✓ Branch 0 taken 196 times.
✓ Branch 1 taken 98 times.
 294 if (same_url_retry) {
1697
1/2
✓ Branch 1 taken 196 times.
✗ Branch 2 not taken.
 196 Backoff(info);
1698 } else {
1699 98 switch_host = true;
1700 }
1701 } else {
1702 // No other errors expected when retrying
1703 ✗ PANIC(NULL);
1704 }
1705 }
1706
2/2
✓ Branch 0 taken 49 times.
✓ Branch 1 taken 390 times.
 439 if (switch_proxy) {
1707
1/2
✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
 49 ReleaseCredential(info);
1708
1/2
✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
 49 SwitchProxy(info);
1709 49 info->SetNumUsedProxies(info->num_used_proxies() + 1);
1710
1/2
✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
 49 SetUrlOptions(info);
1711 }
1712
2/2
✓ Branch 0 taken 98 times.
✓ Branch 1 taken 341 times.
 439 if (switch_host) {
1713
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 ReleaseCredential(info);
1714
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 98 times.
 98 if (was_metalink) {
1715 ✗ SwitchMetalink(info);
1716 ✗ info->SetNumUsedMetalinks(num_used_hosts + 1);
1717 } else {
1718
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 SwitchHost(info);
1719 98 info->SetNumUsedHosts(num_used_hosts + 1);
1720 }
1721
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 SetUrlOptions(info);
1722 }
1723 } // end !sharding
1724
1725
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 439 times.
 439 if (failover_indefinitely_) {
1726 // try again, breaking if there's a cvmfs reload happening and we are in a
1727 // proxy failover. This will EIO the call application.
1728 ✗ return !Interrupted(fqrn_, info);
1729 }
1730 439 return true; // try again
1731 }
1732
1733 4864 verify_and_finalize_stop:
1734 // Finalize, flush destination file
1735
1/2
✓ Branch 1 taken 4913 times.
✗ Branch 2 not taken.
 4913 ReleaseCredential(info);
1736
4/8
✓ Branch 1 taken 4913 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 4913 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 4913 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 4913 times.
 4913 if (info->sink() != NULL && info->sink()->Flush() != 0) {
1737 ✗ info->SetErrorCode(kFailLocalIO);
1738 }
1739
1740
2/2
✓ Branch 1 taken 3173 times.
✓ Branch 2 taken 1740 times.
 4913 if (info->compressed())
1741
1/2
✓ Branch 2 taken 3173 times.
✗ Branch 3 not taken.
 3173 zlib::DecompressFini(info->GetZstreamPtr());
1742
1743
1/2
✓ Branch 1 taken 4913 times.
✗ Branch 2 not taken.
 4913 if (info->headers()) {
1744
1/2
✓ Branch 2 taken 4913 times.
✗ Branch 3 not taken.
 4913 header_lists_->PutList(info->headers());
1745 4913 info->SetHeaders(NULL);
1746 }
1747
1748 4913 return false; // stop transfer and return to Fetch()
1749 5352 }
1750
1751 3600 DownloadManager::~DownloadManager() {
1752 // cleaned up fini
1753
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 3600 times.
 3600 if (sharding_policy_.UseCount() > 0) {
1754 ✗ sharding_policy_.Reset();
1755 }
1756
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 3600 times.
 3600 if (health_check_.UseCount() > 0) {
1757 ✗ if (health_check_.Unique()) {
1758 ✗ LogCvmfs(kLogDownload, kLogDebug,
1759 "(manager '%s') Stopping healthcheck thread", name_.c_str());
1760 ✗ health_check_->StopHealthcheck();
1761 }
1762 ✗ health_check_.Reset();
1763 }
1764
1765
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 3600 times.
 3600 if (atomic_xadd32(&multi_threaded_, 0) == 1) {
1766 // Shutdown I/O thread
1767 ✗ pipe_terminate_->Write(kPipeTerminateSignal);
1768 ✗ pthread_join(thread_download_, NULL);
1769 // All handles are removed from the multi stack
1770 ✗ pipe_terminate_.Destroy();
1771 ✗ pipe_jobs_.Destroy();
1772 }
1773
1774 7200 for (set<CURL *>::iterator i = pool_handles_idle_->begin(),
1775 3600 iEnd = pool_handles_idle_->end();
1776
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 3600 times.
 3600 i != iEnd;
1777 ✗ ++i) {
1778 ✗ curl_easy_cleanup(*i);
1779 }
1780
1781
1/2
✓ Branch 0 taken 3600 times.
✗ Branch 1 not taken.
 3600 delete pool_handles_idle_;
1782
1/2
✓ Branch 0 taken 3600 times.
✗ Branch 1 not taken.
 3600 delete pool_handles_inuse_;
1783 3600 curl_multi_cleanup(curl_multi_);
1784
1785
1/2
✓ Branch 0 taken 3600 times.
✗ Branch 1 not taken.
 3600 delete header_lists_;
1786
1/2
✓ Branch 0 taken 3600 times.
✗ Branch 1 not taken.
 3600 if (user_agent_)
1787 3600 free(user_agent_);
1788
1789
1/2
✓ Branch 0 taken 3600 times.
✗ Branch 1 not taken.
 3600 delete counters_;
1790
2/2
✓ Branch 0 taken 1061 times.
✓ Branch 1 taken 2539 times.
 3600 delete opt_host_.chain;
1791
2/2
✓ Branch 0 taken 1061 times.
✓ Branch 1 taken 2539 times.
 3600 delete opt_host_chain_rtt_;
1792
2/2
✓ Branch 0 taken 961 times.
✓ Branch 1 taken 2639 times.
 3600 delete opt_proxy_groups_;
1793
1794 3600 curl_global_cleanup();
1795
1/2
✓ Branch 0 taken 3600 times.
✗ Branch 1 not taken.
 3600 delete resolver_;
1796
1797 // old destructor
1798 3600 pthread_mutex_destroy(lock_options_);
1799 3600 pthread_mutex_destroy(lock_synchronous_mode_);
1800 3600 free(lock_options_);
1801 3600 free(lock_synchronous_mode_);
1802 3600 }
1803
1804 3601 void DownloadManager::InitHeaders() {
1805 // User-Agent
1806
1/2
✓ Branch 2 taken 3601 times.
✗ Branch 3 not taken.
 3601 string cernvm_id = "User-Agent: cvmfs ";
1807 #ifdef CVMFS_LIBCVMFS
1808
1/2
✓ Branch 1 taken 3601 times.
✗ Branch 2 not taken.
 3601 cernvm_id += "libcvmfs ";
1809 #else
1810 ✗ cernvm_id += "Fuse ";
1811 #endif
1812
2/4
✓ Branch 2 taken 3601 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 3601 times.
✗ Branch 6 not taken.
 3601 cernvm_id += string(CVMFS_VERSION);
1813
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 3601 times.
 3601 if (getenv("CERNVM_UUID") != NULL) {
1814 cernvm_id += " "
1815 ✗ + sanitizer::InputSanitizer("az AZ 09 -")
1816 ✗ .Filter(getenv("CERNVM_UUID"));
1817 }
1818 3601 user_agent_ = strdup(cernvm_id.c_str());
1819
1820
1/2
✓ Branch 1 taken 3601 times.
✗ Branch 2 not taken.
 3601 header_lists_ = new HeaderLists();
1821
1822
1/2
✓ Branch 1 taken 3601 times.
✗ Branch 2 not taken.
 3601 default_headers_ = header_lists_->GetList("Connection: Keep-Alive");
1823
1/2
✓ Branch 1 taken 3601 times.
✗ Branch 2 not taken.
 3601 header_lists_->AppendHeader(default_headers_, "Pragma:");
1824
1/2
✓ Branch 1 taken 3601 times.
✗ Branch 2 not taken.
 3601 header_lists_->AppendHeader(default_headers_, user_agent_);
1825 3601 }
1826
1827 3601 DownloadManager::DownloadManager(const unsigned max_pool_handles,
1828 const perf::StatisticsTemplate &statistics,
1829 3601 const std::string &name)
1830 3601 : prng_(Prng())
1831 3601 , pool_handles_idle_(new set<CURL *>)
1832 3601 , pool_handles_inuse_(new set<CURL *>)
1833 3601 , pool_max_handles_(max_pool_handles)
1834 3601 , pipe_terminate_(NULL)
1835
1/2
✓ Branch 1 taken 3601 times.
✗ Branch 2 not taken.
 3601 , pipe_jobs_(NULL)
1836 3601 , watch_fds_(NULL)
1837 3601 , watch_fds_size_(0)
1838 3601 , watch_fds_inuse_(0)
1839 3601 , watch_fds_max_(4 * max_pool_handles)
1840 3601 , opt_timeout_proxy_(5)
1841 3601 , opt_timeout_direct_(10)
1842 3601 , opt_low_speed_limit_(1024)
1843 3601 , opt_max_retries_(0)
1844 3601 , opt_backoff_init_ms_(0)
1845 3601 , opt_backoff_max_ms_(0)
1846 3601 , enable_info_header_(false)
1847 3601 , opt_ipv4_only_(false)
1848 3601 , follow_redirects_(false)
1849 3601 , ignore_signature_failures_(false)
1850 3601 , enable_http_tracing_(false)
1851 3601 , opt_metalink_(NULL, 0, 0, 0)
1852 3601 , opt_metalink_timestamp_link_(0)
1853 3601 , opt_host_(NULL, 0, 0, 0)
1854 3601 , opt_host_chain_rtt_(NULL)
1855 3601 , opt_proxy_groups_(NULL)
1856 3601 , opt_proxy_groups_current_(0)
1857 3601 , opt_proxy_groups_current_burned_(0)
1858 3601 , opt_proxy_groups_fallback_(0)
1859 3601 , opt_num_proxies_(0)
1860 3601 , opt_proxy_shard_(false)
1861 3601 , failover_indefinitely_(false)
1862
1/2
✓ Branch 1 taken 3601 times.
✗ Branch 2 not taken.
 3601 , name_(name)
1863 3601 , opt_ip_preference_(dns::kIpPreferSystem)
1864 3601 , opt_timestamp_backup_proxies_(0)
1865 3601 , opt_timestamp_failover_proxies_(0)
1866 3601 , opt_proxy_groups_reset_after_(0)
1867 3601 , credentials_attachment_(NULL)
1868
4/8
✓ Branch 13 taken 3601 times.
✗ Branch 14 not taken.
✓ Branch 16 taken 3601 times.
✗ Branch 17 not taken.
✓ Branch 19 taken 3601 times.
✗ Branch 20 not taken.
✓ Branch 23 taken 3601 times.
✗ Branch 24 not taken.
 10803 , counters_(new Counters(statistics)) {
1869 3601 atomic_init32(&multi_threaded_);
1870
1871 3601 lock_options_ = reinterpret_cast<pthread_mutex_t *>(
1872 3601 smalloc(sizeof(pthread_mutex_t)));
1873 3601 int retval = pthread_mutex_init(lock_options_, NULL);
1874
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3601 times.
 3601 assert(retval == 0);
1875 3601 lock_synchronous_mode_ = reinterpret_cast<pthread_mutex_t *>(
1876 3601 smalloc(sizeof(pthread_mutex_t)));
1877 3601 retval = pthread_mutex_init(lock_synchronous_mode_, NULL);
1878
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3601 times.
 3601 assert(retval == 0);
1879
1880
1/2
✓ Branch 1 taken 3601 times.
✗ Branch 2 not taken.
 3601 retval = curl_global_init(CURL_GLOBAL_ALL);
1881
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3601 times.
 3601 assert(retval == CURLE_OK);
1882
1883
1/2
✓ Branch 1 taken 3601 times.
✗ Branch 2 not taken.
 3601 InitHeaders();
1884
1885
1/2
✓ Branch 1 taken 3601 times.
✗ Branch 2 not taken.
 3601 curl_multi_ = curl_multi_init();
1886
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3601 times.
 3601 assert(curl_multi_ != NULL);
1887
1/2
✓ Branch 1 taken 3601 times.
✗ Branch 2 not taken.
 3601 curl_multi_setopt(curl_multi_, CURLMOPT_SOCKETFUNCTION, CallbackCurlSocket);
1888
1/2
✓ Branch 1 taken 3601 times.
✗ Branch 2 not taken.
 3601 curl_multi_setopt(curl_multi_, CURLMOPT_SOCKETDATA,
1889 static_cast<void *>(this));
1890
1/2
✓ Branch 1 taken 3601 times.
✗ Branch 2 not taken.
 3601 curl_multi_setopt(curl_multi_, CURLMOPT_MAXCONNECTS, watch_fds_max_);
1891
1/2
✓ Branch 1 taken 3601 times.
✗ Branch 2 not taken.
 3601 curl_multi_setopt(curl_multi_, CURLMOPT_MAX_TOTAL_CONNECTIONS,
1892 pool_max_handles_);
1893
1894 3601 prng_.InitLocaltime();
1895
1896 // Name resolving
1897 3601 if ((getenv("CVMFS_IPV4_ONLY") != NULL)
1898
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 3601 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 3601 times.
 3601 && (strlen(getenv("CVMFS_IPV4_ONLY")) > 0)) {
1899 ✗ opt_ipv4_only_ = true;
1900 }
1901
1/2
✓ Branch 1 taken 3601 times.
✗ Branch 2 not taken.
 3601 resolver_ = dns::NormalResolver::Create(opt_ipv4_only_, kDnsDefaultRetries,
1902 kDnsDefaultTimeoutMs);
1903
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3601 times.
 3601 assert(resolver_);
1904 3601 }
1905
1906 /**
1907 * Spawns the I/O worker thread and switches the module in multi-threaded mode.
1908 * No way back except Fini(); Init();
1909 */
1910 ✗ void DownloadManager::Spawn() {
1911 ✗ pipe_terminate_ = new Pipe<kPipeThreadTerminator>();
1912 ✗ pipe_jobs_ = new Pipe<kPipeDownloadJobs>();
1913
1914 ✗ const int retval = pthread_create(&thread_download_, NULL, MainDownload,
1915 static_cast<void *>(this));
1916 ✗ assert(retval == 0);
1917
1918 ✗ atomic_inc32(&multi_threaded_);
1919
1920 ✗ if (health_check_.UseCount() > 0) {
1921 ✗ LogCvmfs(kLogDownload, kLogDebug,
1922 "(manager '%s') Starting healthcheck thread", name_.c_str());
1923 ✗ health_check_->StartHealthcheck();
1924 }
1925 }
1926
1927
1928 /**
1929 * Downloads data from an insecure outside channel (currently HTTP or file).
1930 */
1931 4913 Failures DownloadManager::Fetch(JobInfo *info) {
1932
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4913 times.
 4913 assert(info != NULL);
1933
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4913 times.
 4913 assert(info->url() != NULL);
1934
1935 Failures result;
1936
1/2
✓ Branch 1 taken 4913 times.
✗ Branch 2 not taken.
 4913 result = PrepareDownloadDestination(info);
1937
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4913 times.
 4913 if (result != kFailOk)
1938 ✗ return result;
1939
1940
2/2
✓ Branch 1 taken 3185 times.
✓ Branch 2 taken 1728 times.
 4913 if (info->expected_hash()) {
1941 3185 const shash::Algorithms algorithm = info->expected_hash()->algorithm;
1942 3185 info->GetHashContextPtr()->algorithm = algorithm;
1943
1/2
✓ Branch 1 taken 3185 times.
✗ Branch 2 not taken.
 3185 info->GetHashContextPtr()->size = shash::GetContextSize(algorithm);
1944 3185 info->GetHashContextPtr()->buffer = alloca(info->hash_context().size);
1945 }
1946
1947 // In case JobInfo object is being reused
1948
2/4
✓ Branch 2 taken 4913 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 4913 times.
✗ Branch 6 not taken.
 4913 info->SetLink("");
1949
1950 // Prepare cvmfs-info: header, allocate string on the stack
1951 4913 info->SetInfoHeader(NULL);
1952
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4913 times.
 4913 if (enable_info_header_) {
1953 ✗ const string header_info = info->GetInfoHeaderContents(info_header_template_);
1954 ✗ if (header_info != "") {
1955 ✗ const char * const header_name = "cvmfs-info: ";
1956 ✗ const size_t header_name_len = strlen(header_name);
1957 ✗ const size_t info_len = header_info.length();
1958 ✗ char *buf = static_cast<char *>(alloca(header_name_len + info_len + 1));
1959 ✗ memcpy(buf, header_name, header_name_len);
1960 ✗ memcpy(buf + header_name_len, header_info.c_str(), info_len);
1961 ✗ buf[header_name_len + info_len] = '\0';
1962 ✗ info->SetInfoHeader(buf);
1963 }
1964 }
1965
1966
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4913 times.
 4913 if (enable_http_tracing_) {
1967 ✗ const std::string str_pid = "X-CVMFS-PID: " + StringifyInt(info->pid());
1968 ✗ const std::string str_gid = "X-CVMFS-GID: " + StringifyUint(info->gid());
1969 ✗ const std::string str_uid = "X-CVMFS-UID: " + StringifyUint(info->uid());
1970
1971 // will be auto freed at the end of this function Fetch(JobInfo *info)
1972 ✗ info->SetTracingHeaderPid(static_cast<char *>(alloca(str_pid.size() + 1)));
1973 ✗ info->SetTracingHeaderGid(static_cast<char *>(alloca(str_gid.size() + 1)));
1974 ✗ info->SetTracingHeaderUid(static_cast<char *>(alloca(str_uid.size() + 1)));
1975
1976 ✗ memcpy(info->tracing_header_pid(), str_pid.c_str(), str_pid.size() + 1);
1977 ✗ memcpy(info->tracing_header_gid(), str_gid.c_str(), str_gid.size() + 1);
1978 ✗ memcpy(info->tracing_header_uid(), str_uid.c_str(), str_uid.size() + 1);
1979 }
1980
1981
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4913 times.
 4913 if (atomic_xadd32(&multi_threaded_, 0) == 1) {
1982 ✗ if (!info->IsValidPipeJobResults()) {
1983 ✗ info->CreatePipeJobResults();
1984 }
1985 ✗ if (!info->IsValidDataTube()) {
1986 ✗ info->CreateDataTube();
1987 }
1988
1989 // LogCvmfs(kLogDownload, kLogDebug, "send job to thread, pipe %d %d",
1990 // info->wait_at[0], info->wait_at[1]);
1991 ✗ pipe_jobs_->Write<JobInfo *>(info);
1992
1993 do {
1994 ✗ DataTubeElement *ele = info->GetDataTubePtr()->PopFront();
1995
1996 ✗ if (ele->action == kActionStop) {
1997 ✗ delete ele;
1998 ✗ break;
1999 }
2000 // TODO(heretherebedragons) add compression
2001 ✗ } while (true);
2002
2003 ✗ info->GetPipeJobResultPtr()->Read<download::Failures>(&result);
2004 // LogCvmfs(kLogDownload, kLogDebug, "got result %d", result);
2005 } else {
2006 4913 const MutexLockGuard l(lock_synchronous_mode_);
2007
1/2
✓ Branch 1 taken 4913 times.
✗ Branch 2 not taken.
 4913 CURL *handle = AcquireCurlHandle();
2008
1/2
✓ Branch 1 taken 4913 times.
✗ Branch 2 not taken.
 4913 InitializeRequest(info, handle);
2009
1/2
✓ Branch 1 taken 4913 times.
✗ Branch 2 not taken.
 4913 SetUrlOptions(info);
2010 // curl_easy_setopt(handle, CURLOPT_VERBOSE, 1);
2011 int retval;
2012 do {
2013
1/2
✓ Branch 1 taken 5352 times.
✗ Branch 2 not taken.
 5352 retval = curl_easy_perform(handle);
2014 5352 perf::Inc(counters_->n_requests);
2015 double elapsed;
2016
1/2
✓ Branch 1 taken 5352 times.
✗ Branch 2 not taken.
 5352 if (curl_easy_getinfo(handle, CURLINFO_TOTAL_TIME, &elapsed)
2017
1/2
✓ Branch 0 taken 5352 times.
✗ Branch 1 not taken.
 5352 == CURLE_OK) {
2018 5352 perf::Xadd(counters_->sz_transfer_time,
2019 5352 static_cast<int64_t>(elapsed * 1000));
2020 }
2021
3/4
✓ Branch 1 taken 5352 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 439 times.
✓ Branch 4 taken 4913 times.
 5352 } while (VerifyAndFinalize(retval, info));
2022 4913 result = info->error_code();
2023 // Prevent the handle from being added to the idle list, which
2024 // avoids that it is mistakenly picked up by the multi handle later
2025 // in multi-threaded context. This, in turn, would fail to wait for
2026 // for resolver threads that meanwhile vanished due to a fork()
2027 // in Daemonize()
2028
1/2
✓ Branch 2 taken 4913 times.
✗ Branch 3 not taken.
 4913 ReleaseCurlHandle(info->curl_handle(), false /* allow_reuse */);
2029 4913 }
2030
2031
2/2
✓ Branch 0 taken 451 times.
✓ Branch 1 taken 4462 times.
 4913 if (result != kFailOk) {
2032
1/2
✓ Branch 4 taken 451 times.
✗ Branch 5 not taken.
 451 LogCvmfs(kLogDownload, kLogDebug,
2033 "(manager '%s' - id %" PRId64 ") "
2034 "download failed (error %d - %s)",
2035 name_.c_str(), info->id(), result, Code2Ascii(result));
2036
2037
1/2
✓ Branch 1 taken 451 times.
✗ Branch 2 not taken.
 451 if (info->sink() != NULL) {
2038
1/2
✓ Branch 2 taken 451 times.
✗ Branch 3 not taken.
 451 info->sink()->Purge();
2039 }
2040 }
2041
2042 4913 return result;
2043 }
2044
2045
2046 /**
2047 * Used by the client to connect the authz session manager to the download
2048 * manager.
2049 */
2050 481 void DownloadManager::SetCredentialsAttachment(CredentialsAttachment *ca) {
2051 481 const MutexLockGuard m(lock_options_);
2052 481 credentials_attachment_ = ca;
2053 481 }
2054
2055 /**
2056 * Gets the DNS sever.
2057 */
2058 ✗ std::string DownloadManager::GetDnsServer() const { return opt_dns_server_; }
2059
2060 /**
2061 * Sets a DNS server. Only for testing as it cannot be reverted to the system
2062 * default.
2063 */
2064 ✗ void DownloadManager::SetDnsServer(const string &address) {
2065 ✗ if (!address.empty()) {
2066 ✗ const MutexLockGuard m(lock_options_);
2067 ✗ opt_dns_server_ = address;
2068 ✗ assert(!opt_dns_server_.empty());
2069
2070 ✗ vector<string> servers;
2071 ✗ servers.push_back(address);
2072 ✗ const bool retval = resolver_->SetResolvers(servers);
2073 ✗ assert(retval);
2074 }
2075 ✗ LogCvmfs(kLogDownload, kLogSyslog, "(manager '%s') set nameserver to %s",
2076 name_.c_str(), address.c_str());
2077 }
2078
2079
2080 /**
2081 * Sets the DNS query timeout parameters.
2082 */
2083 986 void DownloadManager::SetDnsParameters(const unsigned retries,
2084 const unsigned timeout_ms) {
2085 986 const MutexLockGuard m(lock_options_);
2086 986 if ((resolver_->retries() == retries)
2087
3/6
✓ Branch 0 taken 986 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 986 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 986 times.
✗ Branch 6 not taken.
 986 && (resolver_->timeout_ms() == timeout_ms)) {
2088 986 return;
2089 }
2090 ✗ delete resolver_;
2091 ✗ resolver_ = NULL;
2092 ✗ resolver_ = dns::NormalResolver::Create(opt_ipv4_only_, retries, timeout_ms);
2093 ✗ assert(resolver_);
2094
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 986 times.
 986 }
2095
2096
2097 986 void DownloadManager::SetDnsTtlLimits(const unsigned min_seconds,
2098 const unsigned max_seconds) {
2099 986 const MutexLockGuard m(lock_options_);
2100 986 resolver_->set_min_ttl(min_seconds);
2101 986 resolver_->set_max_ttl(max_seconds);
2102 986 }
2103
2104
2105 ✗ void DownloadManager::SetIpPreference(dns::IpPreference preference) {
2106 ✗ const MutexLockGuard m(lock_options_);
2107 ✗ opt_ip_preference_ = preference;
2108 }
2109
2110
2111 /**
2112 * Sets two timeout values for proxied and for direct connections, respectively.
2113 * The timeout counts for all sorts of connection phases,
2114 * DNS, HTTP connect, etc.
2115 */
2116 1456 void DownloadManager::SetTimeout(const unsigned seconds_proxy,
2117 const unsigned seconds_direct) {
2118 1456 const MutexLockGuard m(lock_options_);
2119 1456 opt_timeout_proxy_ = seconds_proxy;
2120 1456 opt_timeout_direct_ = seconds_direct;
2121 1456 }
2122
2123
2124 /**
2125 * Sets contains the average transfer speed in bytes per second that the
2126 * transfer should be below during CURLOPT_LOW_SPEED_TIME seconds for libcurl to
2127 * consider it to be too slow and abort. Only effective for new connections.
2128 */
2129 ✗ void DownloadManager::SetLowSpeedLimit(const unsigned low_speed_limit) {
2130 ✗ const MutexLockGuard m(lock_options_);
2131 ✗ opt_low_speed_limit_ = low_speed_limit;
2132 }
2133
2134
2135 /**
2136 * Receives the currently active timeout values.
2137 */
2138 407 void DownloadManager::GetTimeout(unsigned *seconds_proxy,
2139 unsigned *seconds_direct) {
2140 407 const MutexLockGuard m(lock_options_);
2141 407 *seconds_proxy = opt_timeout_proxy_;
2142 407 *seconds_direct = opt_timeout_direct_;
2143 407 }
2144
2145
2146 /**
2147 * Parses a list of ';'-separated hosts for the metalink chain. The empty
2148 * string removes the metalink list.
2149 */
2150 ✗ void DownloadManager::SetMetalinkChain(const string &metalink_list) {
2151 ✗ SetMetalinkChain(SplitString(metalink_list, ';'));
2152 }
2153
2154
2155 ✗ void DownloadManager::SetMetalinkChain(
2156 const std::vector<std::string> &metalink_list) {
2157 ✗ const MutexLockGuard m(lock_options_);
2158 ✗ opt_metalink_.timestamp_backup = 0;
2159 ✗ delete opt_metalink_.chain;
2160 ✗ opt_metalink_.current = 0;
2161
2162 ✗ if (metalink_list.empty()) {
2163 ✗ opt_metalink_.chain = NULL;
2164 ✗ return;
2165 }
2166
2167 ✗ opt_metalink_.chain = new vector<string>(metalink_list);
2168 }
2169
2170
2171 /**
2172 * Retrieves the currently set chain of metalink hosts and the currently
2173 * used metalink host.
2174 */
2175 ✗ void DownloadManager::GetMetalinkInfo(vector<string> *metalink_chain,
2176 unsigned *current_metalink) {
2177 ✗ const MutexLockGuard m(lock_options_);
2178 ✗ if (opt_metalink_.chain) {
2179 ✗ if (current_metalink) {
2180 ✗ *current_metalink = opt_metalink_.current;
2181 }
2182 ✗ if (metalink_chain) {
2183 ✗ *metalink_chain = *opt_metalink_.chain;
2184 }
2185 }
2186 }
2187
2188
2189 /**
2190 * Parses a list of ';'-separated hosts for the host chain. The empty string
2191 * removes the host list.
2192 */
2193 1073 void DownloadManager::SetHostChain(const string &host_list) {
2194
1/2
✓ Branch 2 taken 1073 times.
✗ Branch 3 not taken.
 1073 SetHostChain(SplitString(host_list, ';'));
2195 1073 }
2196
2197
2198 1077 void DownloadManager::SetHostChain(const std::vector<std::string> &host_list) {
2199 1077 const MutexLockGuard m(lock_options_);
2200 1077 opt_host_.timestamp_backup = 0;
2201
2/2
✓ Branch 0 taken 423 times.
✓ Branch 1 taken 654 times.
 1077 delete opt_host_.chain;
2202
2/2
✓ Branch 0 taken 423 times.
✓ Branch 1 taken 654 times.
 1077 delete opt_host_chain_rtt_;
2203 1077 opt_host_.current = 0;
2204
2205
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1077 times.
 1077 if (host_list.empty()) {
2206 ✗ opt_host_.chain = NULL;
2207 ✗ opt_host_chain_rtt_ = NULL;
2208 ✗ return;
2209 }
2210
2211
2/4
✓ Branch 1 taken 1077 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1077 times.
✗ Branch 5 not taken.
 1077 opt_host_.chain = new vector<string>(host_list);
2212 2154 opt_host_chain_rtt_ = new vector<int>(opt_host_.chain->size(),
2213
2/4
✓ Branch 1 taken 1077 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1077 times.
✗ Branch 5 not taken.
 1077 kProbeUnprobed);
2214 // LogCvmfs(kLogDownload, kLogSyslog, "using host %s",
2215 // (*opt_host_.chain)[0].c_str());
2216
1/2
✓ Branch 1 taken 1077 times.
✗ Branch 2 not taken.
 1077 }
2217
2218
2219 /**
2220 * Retrieves the currently set chain of hosts, their round trip times, and the
2221 * currently used host.
2222 */
2223 28 void DownloadManager::GetHostInfo(vector<string> *host_chain, vector<int> *rtt,
2224 unsigned *current_host) {
2225 28 const MutexLockGuard m(lock_options_);
2226
1/2
✓ Branch 0 taken 28 times.
✗ Branch 1 not taken.
 28 if (opt_host_.chain) {
2227
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 28 times.
 28 if (current_host) {
2228 ✗ *current_host = opt_host_.current;
2229 }
2230
1/2
✓ Branch 0 taken 28 times.
✗ Branch 1 not taken.
 28 if (host_chain) {
2231
1/2
✓ Branch 1 taken 28 times.
✗ Branch 2 not taken.
 28 *host_chain = *opt_host_.chain;
2232 }
2233
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 28 times.
 28 if (rtt) {
2234 ✗ *rtt = *opt_host_chain_rtt_;
2235 }
2236 }
2237 28 }
2238
2239
2240 /**
2241 * Jumps to the next proxy in the ring of forward proxy servers.
2242 * Selects one randomly from a load-balancing group.
2243 *
2244 * Allow for the fact that the proxy may have already been failed by
2245 * another transfer, or that the proxy may no longer be part of the
2246 * current load-balancing group.
2247 */
2248 49 void DownloadManager::SwitchProxy(JobInfo *info) {
2249 49 const MutexLockGuard m(lock_options_);
2250
2251
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
 49 if (!opt_proxy_groups_) {
2252 ✗ return;
2253 }
2254
2255 // Fail any matching proxies within the current load-balancing group
2256 49 vector<ProxyInfo> *group = current_proxy_group();
2257 49 const unsigned group_size = group->size();
2258 49 unsigned failed = 0;
2259
2/2
✓ Branch 0 taken 49 times.
✓ Branch 1 taken 49 times.
 98 for (unsigned i = 0; i < group_size - opt_proxy_groups_current_burned_; ++i) {
2260
5/14
✓ Branch 0 taken 49 times.
✗ Branch 1 not taken.
✓ Branch 4 taken 49 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 49 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 49 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✓ Branch 12 taken 49 times.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
 49 if (info && (info->proxy() == (*group)[i].url)) {
2261 // Move to list of failed proxies
2262 49 opt_proxy_groups_current_burned_++;
2263
1/2
✓ Branch 2 taken 49 times.
✗ Branch 3 not taken.
 49 swap((*group)[i],
2264 49 (*group)[group_size - opt_proxy_groups_current_burned_]);
2265 49 perf::Inc(counters_->n_proxy_failover);
2266 49 failed++;
2267 }
2268 }
2269
2270 // Do nothing more unless at least one proxy was marked as failed
2271
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
 49 if (!failed)
2272 ✗ return;
2273
2274 // If all proxies from the current load-balancing group are burned, switch to
2275 // another group
2276
1/2
✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
 49 if (opt_proxy_groups_current_burned_ == group->size()) {
2277 49 opt_proxy_groups_current_burned_ = 0;
2278
1/2
✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
 49 if (opt_proxy_groups_->size() > 1) {
2279 98 opt_proxy_groups_current_ = (opt_proxy_groups_current_ + 1)
2280 49 % opt_proxy_groups_->size();
2281 // Remember the timestamp of switching to backup proxies
2282
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
 49 if (opt_proxy_groups_reset_after_ > 0) {
2283 ✗ if (opt_proxy_groups_current_ > 0) {
2284 ✗ if (opt_timestamp_backup_proxies_ == 0)
2285 ✗ opt_timestamp_backup_proxies_ = time(NULL);
2286 // LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2287 // "switched to (another) backup proxy group");
2288 } else {
2289 ✗ opt_timestamp_backup_proxies_ = 0;
2290 // LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2291 // "switched back to primary proxy group");
2292 }
2293 ✗ opt_timestamp_failover_proxies_ = 0;
2294 }
2295 }
2296 } else {
2297 // Record failover time
2298 ✗ if (opt_proxy_groups_reset_after_ > 0) {
2299 ✗ if (opt_timestamp_failover_proxies_ == 0)
2300 ✗ opt_timestamp_failover_proxies_ = time(NULL);
2301 }
2302 }
2303
2304
2/4
✓ Branch 2 taken 49 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 49 times.
✗ Branch 6 not taken.
 49 UpdateProxiesUnlocked("failed proxy");
2305
1/2
✓ Branch 2 taken 49 times.
✗ Branch 3 not taken.
 49 LogCvmfs(kLogDownload, kLogDebug,
2306 "(manager '%s' - id %" PRId64 ") "
2307 "%lu proxies remain in group",
2308 name_.c_str(), info->id(),
2309 49 current_proxy_group()->size() - opt_proxy_groups_current_burned_);
2310
1/2
✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
 49 }
2311
2312
2313 /**
2314 * Switches to the next host in the chain. If jobinfo is set, switch only if
2315 * the current host is identical to the one used by jobinfo, otherwise another
2316 * transfer has already done the switch.
2317 */
2318 110 void DownloadManager::SwitchHostInfo(const std::string &typ,
2319 HostInfo &info,
2320 JobInfo *jobinfo) {
2321 110 const MutexLockGuard m(lock_options_);
2322
2323
5/6
✓ Branch 0 taken 110 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 12 times.
✓ Branch 4 taken 98 times.
✓ Branch 5 taken 12 times.
✓ Branch 6 taken 98 times.
 110 if (!info.chain || (info.chain->size() == 1)) {
2324 12 return;
2325 }
2326
2327
1/2
✓ Branch 0 taken 98 times.
✗ Branch 1 not taken.
 98 if (jobinfo) {
2328 int lastused;
2329
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 if (typ == "host") {
2330 98 lastused = jobinfo->current_host_chain_index();
2331 } else {
2332 ✗ lastused = jobinfo->current_metalink_chain_index();
2333 }
2334
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 98 times.
 98 if (lastused != info.current) {
2335 ✗ LogCvmfs(kLogDownload, kLogDebug,
2336 "(manager '%s' - id %" PRId64 ")"
2337 "don't switch %s, "
2338 "last used %s: %s, current %s: %s",
2339 name_.c_str(), jobinfo->id(), typ.c_str(), typ.c_str(),
2340 ✗ (*info.chain)[lastused].c_str(), typ.c_str(),
2341 ✗ (*info.chain)[info.current].c_str());
2342 ✗ return;
2343 }
2344 }
2345
2346
1/2
✓ Branch 2 taken 98 times.
✗ Branch 3 not taken.
 98 string reason = "manually triggered";
2347
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 string info_id = "(manager " + name_;
2348
1/2
✓ Branch 0 taken 98 times.
✗ Branch 1 not taken.
 98 if (jobinfo) {
2349
1/2
✓ Branch 3 taken 98 times.
✗ Branch 4 not taken.
 98 reason = download::Code2Ascii(jobinfo->error_code());
2350
2/4
✓ Branch 2 taken 98 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 98 times.
✗ Branch 6 not taken.
 98 info_id = " - id " + StringifyInt(jobinfo->id());
2351 }
2352
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 info_id += ")";
2353
2354
1/2
✓ Branch 2 taken 98 times.
✗ Branch 3 not taken.
 98 const std::string old_host = (*info.chain)[info.current];
2355 98 info.current = (info.current + 1) % static_cast<int>(info.chain->size());
2356
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 if (typ == "host") {
2357 98 perf::Inc(counters_->n_host_failover);
2358 } else {
2359 ✗ perf::Inc(counters_->n_metalink_failover);
2360 }
2361
1/3
✓ Branch 6 taken 98 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
 196 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2362 "%s switching %s from %s to %s (%s)", info_id.c_str(), typ.c_str(),
2363 98 old_host.c_str(), (*info.chain)[info.current].c_str(),
2364 reason.c_str());
2365
2366 // Remember the timestamp of switching to backup host
2367
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 98 times.
 98 if (info.reset_after > 0) {
2368 ✗ if (info.current != 0) {
2369 ✗ if (info.timestamp_backup == 0)
2370 ✗ info.timestamp_backup = time(NULL);
2371 } else {
2372 ✗ info.timestamp_backup = 0;
2373 }
2374 }
2375
2/2
✓ Branch 4 taken 98 times.
✓ Branch 5 taken 12 times.
 110 }
2376
2377 110 void DownloadManager::SwitchHost(JobInfo *info) {
2378
2/4
✓ Branch 2 taken 110 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 110 times.
✗ Branch 6 not taken.
 110 SwitchHostInfo("host", opt_host_, info);
2379 110 }
2380
2381 12 void DownloadManager::SwitchHost() { SwitchHost(NULL); }
2382
2383
2384 ✗ void DownloadManager::SwitchMetalink(JobInfo *info) {
2385 ✗ SwitchHostInfo("metalink", opt_metalink_, info);
2386 }
2387
2388
2389 ✗ void DownloadManager::SwitchMetalink() { SwitchMetalink(NULL); }
2390
2391 1209 bool DownloadManager::CheckMetalinkChain(time_t now) {
2392 1209 return (opt_metalink_.chain
2393
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1209 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 1209 && ((opt_metalink_timestamp_link_ == 0)
2394 ✗ || (static_cast<int64_t>((now == 0) ? time(NULL) : now)
2395 ✗ > static_cast<int64_t>(opt_metalink_timestamp_link_
2396
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 1209 + opt_metalink_.reset_after))));
2397 }
2398
2399
2400 /**
2401 * Orders the hostlist according to RTT of downloading .cvmfschecksum.
2402 * Sets the current host to the best-responsive host.
2403 * If you change the host list in between by SetHostChain(), it will be
2404 * overwritten by this function.
2405 */
2406 ✗ void DownloadManager::ProbeHosts() {
2407 ✗ vector<string> host_chain;
2408 ✗ vector<int> host_rtt;
2409 unsigned current_host;
2410
2411 ✗ GetHostInfo(&host_chain, &host_rtt, &current_host);
2412
2413 // Stopwatch, two times to fill caches first
2414 unsigned i, retries;
2415 ✗ string url;
2416
2417 ✗ cvmfs::MemSink memsink;
2418 ✗ JobInfo info(&url, false, false, NULL, &memsink);
2419 ✗ for (retries = 0; retries < 2; ++retries) {
2420 ✗ for (i = 0; i < host_chain.size(); ++i) {
2421 ✗ url = host_chain[i] + "/.cvmfspublished";
2422
2423 struct timeval tv_start, tv_end;
2424 ✗ gettimeofday(&tv_start, NULL);
2425 ✗ const Failures result = Fetch(&info);
2426 ✗ gettimeofday(&tv_end, NULL);
2427 ✗ memsink.Reset();
2428 ✗ if (result == kFailOk) {
2429 ✗ host_rtt[i] = static_cast<int>(DiffTimeSeconds(tv_start, tv_end)
2430 ✗ * 1000);
2431 ✗ LogCvmfs(kLogDownload, kLogDebug,
2432 "(manager '%s' - id %" PRId64 ") "
2433 "probing host %s had %dms rtt",
2434 ✗ name_.c_str(), info.id(), url.c_str(), host_rtt[i]);
2435 } else {
2436 ✗ LogCvmfs(kLogDownload, kLogDebug,
2437 "(manager '%s' - id %" PRId64 ") "
2438 "error while probing host %s: %d %s",
2439 name_.c_str(), info.id(), url.c_str(), result,
2440 Code2Ascii(result));
2441 ✗ host_rtt[i] = INT_MAX;
2442 }
2443 }
2444 }
2445
2446 ✗ SortTeam(&host_rtt, &host_chain);
2447 ✗ for (i = 0; i < host_chain.size(); ++i) {
2448 ✗ if (host_rtt[i] == INT_MAX)
2449 ✗ host_rtt[i] = kProbeDown;
2450 }
2451
2452 ✗ const MutexLockGuard m(lock_options_);
2453 ✗ delete opt_host_.chain;
2454 ✗ delete opt_host_chain_rtt_;
2455 ✗ opt_host_.chain = new vector<string>(host_chain);
2456 ✗ opt_host_chain_rtt_ = new vector<int>(host_rtt);
2457 ✗ opt_host_.current = 0;
2458 }
2459
2460 ✗ bool DownloadManager::GeoSortServers(std::vector<std::string> *servers,
2461 std::vector<uint64_t> *output_order) {
2462 ✗ if (!servers) {
2463 ✗ return false;
2464 }
2465 ✗ if (servers->size() == 1) {
2466 ✗ if (output_order) {
2467 ✗ output_order->clear();
2468 ✗ output_order->push_back(0);
2469 }
2470 ✗ return true;
2471 }
2472
2473 ✗ std::vector<std::string> host_chain;
2474 ✗ GetHostInfo(&host_chain, NULL, NULL);
2475
2476 ✗ std::vector<std::string> server_dns_names;
2477 ✗ server_dns_names.reserve(servers->size());
2478 ✗ for (unsigned i = 0; i < servers->size(); ++i) {
2479 ✗ const std::string host = dns::ExtractHost((*servers)[i]);
2480 ✗ server_dns_names.push_back(host.empty() ? (*servers)[i] : host);
2481 }
2482 ✗ const std::string host_list = JoinStrings(server_dns_names, ",");
2483
2484 ✗ vector<string> host_chain_shuffled;
2485 {
2486 // Protect against concurrent access to prng_
2487 ✗ const MutexLockGuard m(lock_options_);
2488 // Determine random hosts for the Geo-API query
2489 ✗ host_chain_shuffled = Shuffle(host_chain, &prng_);
2490 }
2491 // Request ordered list via Geo-API
2492 ✗ bool success = false;
2493 ✗ const unsigned max_attempts = std::min(host_chain_shuffled.size(), size_t(3));
2494 ✗ vector<uint64_t> geo_order(servers->size());
2495 ✗ for (unsigned i = 0; i < max_attempts; ++i) {
2496 ✗ const string url = host_chain_shuffled[i] + "/api/v1.0/geo/@proxy@/"
2497 ✗ + host_list;
2498 ✗ LogCvmfs(kLogDownload, kLogDebug,
2499 "(manager '%s') requesting ordered server list from %s",
2500 name_.c_str(), url.c_str());
2501 ✗ cvmfs::MemSink memsink;
2502 ✗ JobInfo info(&url, false, false, NULL, &memsink);
2503 ✗ const Failures result = Fetch(&info);
2504 ✗ if (result == kFailOk) {
2505 ✗ const string order(reinterpret_cast<char *>(memsink.data()),
2506 ✗ memsink.pos());
2507 ✗ memsink.Reset();
2508 ✗ const bool retval = ValidateGeoReply(order, servers->size(), &geo_order);
2509 ✗ if (!retval) {
2510 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2511 "(manager '%s') retrieved invalid GeoAPI reply from %s [%s]",
2512 name_.c_str(), url.c_str(), order.c_str());
2513 } else {
2514 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslog,
2515 "(manager '%s') "
2516 "geographic order of servers retrieved from %s",
2517 name_.c_str(),
2518 ✗ dns::ExtractHost(host_chain_shuffled[i]).c_str());
2519 // remove new line at end of "order"
2520 ✗ LogCvmfs(kLogDownload, kLogDebug, "order is %s",
2521 ✗ Trim(order, true /* trim_newline */).c_str());
2522 ✗ success = true;
2523 ✗ break;
2524 }
2525 ✗ } else {
2526 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2527 "(manager '%s') GeoAPI request for %s failed with error %d [%s]",
2528 name_.c_str(), url.c_str(), result, Code2Ascii(result));
2529 }
2530 }
2531 ✗ if (!success) {
2532 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2533 "(manager '%s') "
2534 "failed to retrieve geographic order from stratum 1 servers",
2535 name_.c_str());
2536 ✗ return false;
2537 }
2538
2539 ✗ if (output_order) {
2540 ✗ output_order->swap(geo_order);
2541 } else {
2542 ✗ std::vector<std::string> sorted_servers;
2543 ✗ sorted_servers.reserve(geo_order.size());
2544 ✗ for (unsigned i = 0; i < geo_order.size(); ++i) {
2545 ✗ const uint64_t orderval = geo_order[i];
2546 ✗ sorted_servers.push_back((*servers)[orderval]);
2547 }
2548 ✗ servers->swap(sorted_servers);
2549 }
2550 ✗ return true;
2551 }
2552
2553
2554 /**
2555 * Uses the Geo-API of Stratum 1s to let any of them order the list of servers
2556 * and fallback proxies (if any).
2557 * Tries at most three random Stratum 1s before giving up.
2558 * If you change the host list in between by SetHostChain() or the fallback
2559 * proxy list by SetProxyChain(), they will be overwritten by this function.
2560 */
2561 ✗ bool DownloadManager::ProbeGeo() {
2562 ✗ vector<string> host_chain;
2563 ✗ vector<int> host_rtt;
2564 unsigned current_host;
2565 ✗ vector<vector<ProxyInfo> > proxy_chain;
2566 unsigned fallback_group;
2567
2568 ✗ GetHostInfo(&host_chain, &host_rtt, &current_host);
2569 ✗ GetProxyInfo(&proxy_chain, NULL, &fallback_group);
2570 ✗ if ((host_chain.size() < 2) && ((proxy_chain.size() - fallback_group) < 2))
2571 ✗ return true;
2572
2573 ✗ vector<string> host_names;
2574 ✗ for (unsigned i = 0; i < host_chain.size(); ++i)
2575 ✗ host_names.push_back(dns::ExtractHost(host_chain[i]));
2576 ✗ SortTeam(&host_names, &host_chain);
2577 ✗ const unsigned last_geo_host = host_names.size();
2578
2579 ✗ if ((fallback_group == 0) && (last_geo_host > 1)) {
2580 // There are no non-fallback proxies, which means that the client
2581 // will always use the fallback proxies. Add a keyword separator
2582 // between the hosts and fallback proxies so the geosorting service
2583 // will know to sort the hosts based on the distance from the
2584 // closest fallback proxy rather than the distance from the client.
2585 ✗ host_names.push_back("+PXYSEP+");
2586 }
2587
2588 // Add fallback proxy names to the end of the host list
2589 ✗ const unsigned first_geo_fallback = host_names.size();
2590 ✗ for (unsigned i = fallback_group; i < proxy_chain.size(); ++i) {
2591 // We only take the first fallback proxy name from every group under the
2592 // assumption that load-balanced servers are at the same location
2593 ✗ host_names.push_back(proxy_chain[i][0].host.name());
2594 }
2595
2596 ✗ std::vector<uint64_t> geo_order;
2597 ✗ const bool success = GeoSortServers(&host_names, &geo_order);
2598 ✗ if (!success) {
2599 // GeoSortServers already logged a failure message.
2600 ✗ return false;
2601 }
2602
2603 // Re-install host chain and proxy chain
2604 ✗ const MutexLockGuard m(lock_options_);
2605 ✗ delete opt_host_.chain;
2606 ✗ opt_num_proxies_ = 0;
2607 ✗ opt_host_.chain = new vector<string>(host_chain.size());
2608
2609 // It's possible that opt_proxy_groups_fallback_ might have changed while
2610 // the lock wasn't held
2611 vector<vector<ProxyInfo> > *proxy_groups = new vector<vector<ProxyInfo> >(
2612 ✗ opt_proxy_groups_fallback_ + proxy_chain.size() - fallback_group);
2613 // First copy the non-fallback part of the current proxy chain
2614 ✗ for (unsigned i = 0; i < opt_proxy_groups_fallback_; ++i) {
2615 ✗ (*proxy_groups)[i] = (*opt_proxy_groups_)[i];
2616 ✗ opt_num_proxies_ += (*opt_proxy_groups_)[i].size();
2617 }
2618
2619 // Copy the host chain and fallback proxies by geo order. Array indices
2620 // in geo_order that are smaller than last_geo_host refer to a stratum 1,
2621 // and those indices greater than or equal to first_geo_fallback refer to
2622 // a fallback proxy.
2623 ✗ unsigned hosti = 0;
2624 ✗ unsigned proxyi = opt_proxy_groups_fallback_;
2625 ✗ for (unsigned i = 0; i < geo_order.size(); ++i) {
2626 ✗ const uint64_t orderval = geo_order[i];
2627 ✗ if (orderval < static_cast<uint64_t>(last_geo_host)) {
2628 // LogCvmfs(kLogCvmfs, kLogSyslog, "this is orderval %u at host index
2629 // %u", orderval, hosti);
2630 ✗ (*opt_host_.chain)[hosti++] = host_chain[orderval];
2631 ✗ } else if (orderval >= static_cast<uint64_t>(first_geo_fallback)) {
2632 // LogCvmfs(kLogCvmfs, kLogSyslog,
2633 // "this is orderval %u at proxy index %u, using proxy_chain index %u",
2634 // orderval, proxyi, fallback_group + orderval - first_geo_fallback);
2635 ✗ (*proxy_groups)[proxyi] = proxy_chain[fallback_group + orderval
2636 ✗ - first_geo_fallback];
2637 ✗ opt_num_proxies_ += (*proxy_groups)[proxyi].size();
2638 ✗ proxyi++;
2639 }
2640 }
2641
2642 ✗ opt_proxy_map_.clear();
2643 ✗ delete opt_proxy_groups_;
2644 ✗ opt_proxy_groups_ = proxy_groups;
2645 // In pathological cases, opt_proxy_groups_current_ can be larger now when
2646 // proxies changed in-between.
2647 ✗ if (opt_proxy_groups_current_ > opt_proxy_groups_->size()) {
2648 ✗ if (opt_proxy_groups_->size() == 0) {
2649 ✗ opt_proxy_groups_current_ = 0;
2650 } else {
2651 ✗ opt_proxy_groups_current_ = opt_proxy_groups_->size() - 1;
2652 }
2653 ✗ opt_proxy_groups_current_burned_ = 0;
2654 }
2655
2656 ✗ UpdateProxiesUnlocked("geosort");
2657
2658 ✗ delete opt_host_chain_rtt_;
2659 ✗ opt_host_chain_rtt_ = new vector<int>(host_chain.size(), kProbeGeo);
2660 ✗ opt_host_.current = 0;
2661
2662 ✗ return true;
2663 }
2664
2665
2666 /**
2667 * Validates a string of the form "1,4,2,3" representing in which order the
2668 * the expected_size number of hosts should be put for optimal geographic
2669 * proximity. Returns false if the reply_order string is invalid, otherwise
2670 * fills in the reply_vals array with zero-based order indexes (e.g.
2671 * [0,3,1,2]) and returns true.
2672 */
2673 686 bool DownloadManager::ValidateGeoReply(const string &reply_order,
2674 const unsigned expected_size,
2675 vector<uint64_t> *reply_vals) {
2676
2/2
✓ Branch 1 taken 49 times.
✓ Branch 2 taken 637 times.
 686 if (reply_order.empty())
2677 49 return false;
2678
2/4
✓ Branch 2 taken 637 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 637 times.
✗ Branch 6 not taken.
 1274 const sanitizer::InputSanitizer sanitizer("09 , \n");
2679
3/4
✓ Branch 1 taken 637 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 49 times.
✓ Branch 4 taken 588 times.
 637 if (!sanitizer.IsValid(reply_order))
2680 49 return false;
2681
2/4
✓ Branch 2 taken 588 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 588 times.
✗ Branch 6 not taken.
 1176 const sanitizer::InputSanitizer strip_newline("09 ,");
2682
1/2
✓ Branch 1 taken 588 times.
✗ Branch 2 not taken.
 588 vector<string> reply_strings = SplitString(strip_newline.Filter(reply_order),
2683
1/2
✓ Branch 1 taken 588 times.
✗ Branch 2 not taken.
 588 ',');
2684 588 vector<uint64_t> tmp_vals;
2685
2/2
✓ Branch 1 taken 1127 times.
✓ Branch 2 taken 490 times.
 1617 for (unsigned i = 0; i < reply_strings.size(); ++i) {
2686
2/2
✓ Branch 2 taken 98 times.
✓ Branch 3 taken 1029 times.
 1127 if (reply_strings[i].empty())
2687 98 return false;
2688
2/4
✓ Branch 2 taken 1029 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1029 times.
✗ Branch 6 not taken.
 1029 tmp_vals.push_back(String2Uint64(reply_strings[i]));
2689 }
2690
2/2
✓ Branch 1 taken 245 times.
✓ Branch 2 taken 245 times.
 490 if (tmp_vals.size() != expected_size)
2691 245 return false;
2692
2693 // Check if tmp_vals contains the number 1..n
2694
1/2
✓ Branch 3 taken 245 times.
✗ Branch 4 not taken.
 245 set<uint64_t> coverage(tmp_vals.begin(), tmp_vals.end());
2695
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 245 times.
 245 if (coverage.size() != tmp_vals.size())
2696 ✗ return false;
2697
5/6
✓ Branch 2 taken 196 times.
✓ Branch 3 taken 49 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 196 times.
✓ Branch 9 taken 49 times.
✓ Branch 10 taken 196 times.
 245 if ((*coverage.begin() != 1) || (*coverage.rbegin() != coverage.size()))
2698 49 return false;
2699
2700
2/2
✓ Branch 0 taken 441 times.
✓ Branch 1 taken 196 times.
 637 for (unsigned i = 0; i < expected_size; ++i) {
2701 441 (*reply_vals)[i] = tmp_vals[i] - 1;
2702 }
2703 196 return true;
2704 637 }
2705
2706
2707 /**
2708 * Removes DIRECT from a list of ';' and '|' separated proxies.
2709 * \return true if DIRECT was present, false otherwise
2710 */
2711 1451 bool DownloadManager::StripDirect(const string &proxy_list,
2712 string *cleaned_list) {
2713
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1451 times.
 1451 assert(cleaned_list);
2714
2/2
✓ Branch 1 taken 1010 times.
✓ Branch 2 taken 441 times.
 1451 if (proxy_list == "") {
2715
1/2
✓ Branch 1 taken 1010 times.
✗ Branch 2 not taken.
 1010 *cleaned_list = "";
2716 1010 return false;
2717 }
2718 441 bool result = false;
2719
2720
1/2
✓ Branch 1 taken 441 times.
✗ Branch 2 not taken.
 441 vector<string> proxy_groups = SplitString(proxy_list, ';');
2721 441 vector<string> cleaned_groups;
2722
2/2
✓ Branch 1 taken 1078 times.
✓ Branch 2 taken 441 times.
 1519 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2723
1/2
✓ Branch 2 taken 1078 times.
✗ Branch 3 not taken.
 1078 vector<string> group = SplitString(proxy_groups[i], '|');
2724 1078 vector<string> cleaned;
2725
2/2
✓ Branch 1 taken 1813 times.
✓ Branch 2 taken 1078 times.
 2891 for (unsigned j = 0; j < group.size(); ++j) {
2726
6/6
✓ Branch 2 taken 1323 times.
✓ Branch 3 taken 490 times.
✓ Branch 6 taken 637 times.
✓ Branch 7 taken 686 times.
✓ Branch 8 taken 1127 times.
✓ Branch 9 taken 686 times.
 1813 if ((group[j] == "DIRECT") || (group[j] == "")) {
2727 1127 result = true;
2728 } else {
2729
1/2
✓ Branch 2 taken 686 times.
✗ Branch 3 not taken.
 686 cleaned.push_back(group[j]);
2730 }
2731 }
2732
2/2
✓ Branch 1 taken 343 times.
✓ Branch 2 taken 735 times.
 1078 if (!cleaned.empty())
2733
3/6
✓ Branch 2 taken 343 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 343 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 343 times.
✗ Branch 9 not taken.
 343 cleaned_groups.push_back(JoinStrings(cleaned, "|"));
2734 1078 }
2735
2736
2/4
✓ Branch 2 taken 441 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 441 times.
✗ Branch 6 not taken.
 441 *cleaned_list = JoinStrings(cleaned_groups, ";");
2737 441 return result;
2738 441 }
2739
2740
2741 /**
2742 * Parses a list of ';'- and '|'-separated proxy servers and fallback proxy
2743 * servers for the proxy groups.
2744 * The empty string for both removes the proxy chain.
2745 * The set_mode parameter can be used to set either proxies (leaving fallback
2746 * proxies unchanged) or fallback proxies (leaving regular proxies unchanged)
2747 * or both.
2748 */
2749 961 void DownloadManager::SetProxyChain(const string &proxy_list,
2750 const string &fallback_proxy_list,
2751 const ProxySetModes set_mode) {
2752 961 const MutexLockGuard m(lock_options_);
2753
2754 961 opt_timestamp_backup_proxies_ = 0;
2755 961 opt_timestamp_failover_proxies_ = 0;
2756
1/2
✓ Branch 1 taken 961 times.
✗ Branch 2 not taken.
 961 string set_proxy_list = opt_proxy_list_;
2757
1/2
✓ Branch 1 taken 961 times.
✗ Branch 2 not taken.
 961 string set_proxy_fallback_list = opt_proxy_fallback_list_;
2758 bool contains_direct;
2759
3/4
✓ Branch 0 taken 961 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 814 times.
✓ Branch 3 taken 147 times.
 961 if ((set_mode == kSetProxyFallback) || (set_mode == kSetProxyBoth)) {
2760
1/2
✓ Branch 1 taken 814 times.
✗ Branch 2 not taken.
 814 opt_proxy_fallback_list_ = fallback_proxy_list;
2761 }
2762
3/4
✓ Branch 0 taken 814 times.
✓ Branch 1 taken 147 times.
✓ Branch 2 taken 814 times.
✗ Branch 3 not taken.
 961 if ((set_mode == kSetProxyRegular) || (set_mode == kSetProxyBoth)) {
2763
1/2
✓ Branch 1 taken 961 times.
✗ Branch 2 not taken.
 961 opt_proxy_list_ = proxy_list;
2764 }
2765
1/2
✓ Branch 1 taken 961 times.
✗ Branch 2 not taken.
 961 contains_direct = StripDirect(opt_proxy_fallback_list_,
2766 &set_proxy_fallback_list);
2767
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 961 times.
 961 if (contains_direct) {
2768 ✗ LogCvmfs(kLogDownload, kLogSyslogWarn | kLogDebug,
2769 "(manager '%s') fallback proxies do not support DIRECT, removing",
2770 name_.c_str());
2771 }
2772
1/2
✓ Branch 1 taken 961 times.
✗ Branch 2 not taken.
 961 if (set_proxy_fallback_list == "") {
2773
1/2
✓ Branch 1 taken 961 times.
✗ Branch 2 not taken.
 961 set_proxy_list = opt_proxy_list_;
2774 } else {
2775 ✗ const bool contains_direct = StripDirect(opt_proxy_list_, &set_proxy_list);
2776 ✗ if (contains_direct) {
2777 ✗ LogCvmfs(kLogDownload, kLogSyslog | kLogDebug,
2778 "(manager '%s') skipping DIRECT proxy to use fallback proxy",
2779 name_.c_str());
2780 }
2781 }
2782
2783 // From this point on, use set_proxy_list and set_fallback_proxy_list as
2784 // effective proxy lists!
2785
2786 961 opt_proxy_map_.clear();
2787
2/2
✓ Branch 0 taken 407 times.
✓ Branch 1 taken 554 times.
 961 delete opt_proxy_groups_;
2788
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 961 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 961 times.
 961 if ((set_proxy_list == "") && (set_proxy_fallback_list == "")) {
2789 ✗ opt_proxy_groups_ = NULL;
2790 ✗ opt_proxy_groups_current_ = 0;
2791 ✗ opt_proxy_groups_current_burned_ = 0;
2792 ✗ opt_proxy_groups_fallback_ = 0;
2793 ✗ opt_num_proxies_ = 0;
2794 ✗ return;
2795 }
2796
2797 // Determine number of regular proxy groups (== first fallback proxy group)
2798 961 opt_proxy_groups_fallback_ = 0;
2799
1/2
✓ Branch 1 taken 961 times.
✗ Branch 2 not taken.
 961 if (set_proxy_list != "") {
2800
1/2
✓ Branch 1 taken 961 times.
✗ Branch 2 not taken.
 961 opt_proxy_groups_fallback_ = SplitString(set_proxy_list, ';').size();
2801 }
2802
1/2
✓ Branch 2 taken 961 times.
✗ Branch 3 not taken.
 961 LogCvmfs(kLogDownload, kLogDebug,
2803 "(manager '%s') "
2804 "first fallback proxy group %u",
2805 name_.c_str(), opt_proxy_groups_fallback_);
2806
2807 // Concatenate regular proxies and fallback proxies, both of which can be
2808 // empty.
2809
1/2
✓ Branch 1 taken 961 times.
✗ Branch 2 not taken.
 961 string all_proxy_list = set_proxy_list;
2810
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 961 times.
 961 if (set_proxy_fallback_list != "") {
2811 ✗ if (all_proxy_list != "")
2812 ✗ all_proxy_list += ";";
2813 ✗ all_proxy_list += set_proxy_fallback_list;
2814 }
2815
1/2
✓ Branch 3 taken 961 times.
✗ Branch 4 not taken.
 961 LogCvmfs(kLogDownload, kLogDebug, "(manager '%s') full proxy list %s",
2816 name_.c_str(), all_proxy_list.c_str());
2817
2818 // Resolve server names in provided urls
2819 961 vector<string> hostnames; // All encountered hostnames
2820 961 vector<string> proxy_groups;
2821
1/2
✓ Branch 1 taken 961 times.
✗ Branch 2 not taken.
 961 if (all_proxy_list != "")
2822
1/2
✓ Branch 1 taken 961 times.
✗ Branch 2 not taken.
 961 proxy_groups = SplitString(all_proxy_list, ';');
2823
2/2
✓ Branch 1 taken 1010 times.
✓ Branch 2 taken 961 times.
 1971 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2824
1/2
✓ Branch 2 taken 1010 times.
✗ Branch 3 not taken.
 1010 vector<string> this_group = SplitString(proxy_groups[i], '|');
2825
2/2
✓ Branch 1 taken 1010 times.
✓ Branch 2 taken 1010 times.
 2020 for (unsigned j = 0; j < this_group.size(); ++j) {
2826
1/2
✓ Branch 2 taken 1010 times.
✗ Branch 3 not taken.
 1010 this_group[j] = dns::AddDefaultScheme(this_group[j]);
2827 // Note: DIRECT strings will be "extracted" to an empty string.
2828
1/2
✓ Branch 2 taken 1010 times.
✗ Branch 3 not taken.
 1010 const string hostname = dns::ExtractHost(this_group[j]);
2829 // Save the hostname. Leave empty (DIRECT) names so indexes will
2830 // match later.
2831
1/2
✓ Branch 1 taken 1010 times.
✗ Branch 2 not taken.
 1010 hostnames.push_back(hostname);
2832 1010 }
2833 1010 }
2834 961 vector<dns::Host> hosts;
2835
1/2
✓ Branch 3 taken 961 times.
✗ Branch 4 not taken.
 961 LogCvmfs(kLogDownload, kLogDebug,
2836 "(manager '%s') "
2837 "resolving %lu proxy addresses",
2838 name_.c_str(), hostnames.size());
2839
1/2
✓ Branch 1 taken 961 times.
✗ Branch 2 not taken.
 961 resolver_->ResolveMany(hostnames, &hosts);
2840
2841 // Construct opt_proxy_groups_: traverse proxy list in same order and expand
2842 // names to resolved IP addresses.
2843
1/2
✓ Branch 1 taken 961 times.
✗ Branch 2 not taken.
 961 opt_proxy_groups_ = new vector<vector<ProxyInfo> >();
2844 961 opt_num_proxies_ = 0;
2845 961 unsigned num_proxy = 0; // Combined i, j counter
2846
2/2
✓ Branch 1 taken 1010 times.
✓ Branch 2 taken 961 times.
 1971 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2847
1/2
✓ Branch 2 taken 1010 times.
✗ Branch 3 not taken.
 1010 vector<string> this_group = SplitString(proxy_groups[i], '|');
2848 // Construct ProxyInfo objects from proxy string and DNS resolver result for
2849 // every proxy in this_group. One URL can result in multiple ProxyInfo
2850 // objects, one for each IP address.
2851 1010 vector<ProxyInfo> infos;
2852
2/2
✓ Branch 1 taken 1010 times.
✓ Branch 2 taken 1010 times.
 2020 for (unsigned j = 0; j < this_group.size(); ++j, ++num_proxy) {
2853
1/2
✓ Branch 2 taken 1010 times.
✗ Branch 3 not taken.
 1010 this_group[j] = dns::AddDefaultScheme(this_group[j]);
2854
2/2
✓ Branch 2 taken 814 times.
✓ Branch 3 taken 196 times.
 1010 if (this_group[j] == "DIRECT") {
2855
3/6
✓ Branch 2 taken 814 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 814 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 814 times.
✗ Branch 9 not taken.
 814 infos.push_back(ProxyInfo("DIRECT"));
2856 814 continue;
2857 }
2858
2859
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 196 times.
 196 if (hosts[num_proxy].status() != dns::kFailOk) {
2860 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2861 "(manager '%s') "
2862 "failed to resolve IP addresses for %s (%d - %s)",
2863 ✗ name_.c_str(), hosts[num_proxy].name().c_str(),
2864 ✗ hosts[num_proxy].status(),
2865 ✗ dns::Code2Ascii(hosts[num_proxy].status()));
2866 const dns::Host failed_host = dns::Host::ExtendDeadline(
2867 ✗ hosts[num_proxy], resolver_->min_ttl());
2868 ✗ infos.push_back(ProxyInfo(failed_host, this_group[j]));
2869 ✗ continue;
2870 }
2871
2872 // IPv4 addresses have precedence
2873 196 set<string> best_addresses = hosts[num_proxy].ViewBestAddresses(
2874
2/4
✓ Branch 1 taken 196 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 196 times.
✗ Branch 5 not taken.
 196 opt_ip_preference_);
2875 196 set<string>::const_iterator iter_ips = best_addresses.begin();
2876
2/2
✓ Branch 3 taken 196 times.
✓ Branch 4 taken 196 times.
 392 for (; iter_ips != best_addresses.end(); ++iter_ips) {
2877
1/2
✓ Branch 3 taken 196 times.
✗ Branch 4 not taken.
 196 const string url_ip = dns::RewriteUrl(this_group[j], *iter_ips);
2878
2/4
✓ Branch 2 taken 196 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 196 times.
✗ Branch 6 not taken.
 196 infos.push_back(ProxyInfo(hosts[num_proxy], url_ip));
2879
2880
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 196 times.
 196 if (sharding_policy_.UseCount() > 0) {
2881 ✗ sharding_policy_->AddProxy(url_ip);
2882 }
2883 196 }
2884 196 }
2885
1/2
✓ Branch 1 taken 1010 times.
✗ Branch 2 not taken.
 1010 opt_proxy_groups_->push_back(infos);
2886 1010 opt_num_proxies_ += infos.size();
2887 1010 }
2888
1/2
✓ Branch 2 taken 961 times.
✗ Branch 3 not taken.
 961 LogCvmfs(kLogDownload, kLogDebug,
2889 "(manager '%s') installed %u proxies in %lu load-balance groups",
2890 961 name_.c_str(), opt_num_proxies_, opt_proxy_groups_->size());
2891 961 opt_proxy_groups_current_ = 0;
2892 961 opt_proxy_groups_current_burned_ = 0;
2893
2894 // Select random start proxy from the first group.
2895
1/2
✓ Branch 1 taken 961 times.
✗ Branch 2 not taken.
 961 if (opt_proxy_groups_->size() > 0) {
2896 // Select random start proxy from the first group.
2897
2/4
✓ Branch 2 taken 961 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 961 times.
✗ Branch 6 not taken.
 961 UpdateProxiesUnlocked("set random start proxy from the first proxy group");
2898 }
2899
3/6
✓ Branch 5 taken 961 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 961 times.
✗ Branch 9 not taken.
✓ Branch 11 taken 961 times.
✗ Branch 12 not taken.
 961 }
2900
2901
2902 /**
2903 * Retrieves the proxy chain, optionally the currently active load-balancing
2904 * group, and optionally the index of the first fallback proxy group.
2905 * If there are no fallback proxies, the index will equal the size of
2906 * the proxy chain.
2907 */
2908 ✗ void DownloadManager::GetProxyInfo(vector<vector<ProxyInfo> > *proxy_chain,
2909 unsigned *current_group,
2910 unsigned *fallback_group) {
2911 ✗ assert(proxy_chain != NULL);
2912 ✗ const MutexLockGuard m(lock_options_);
2913
2914 ✗ if (!opt_proxy_groups_) {
2915 ✗ const vector<vector<ProxyInfo> > empty_chain;
2916 ✗ *proxy_chain = empty_chain;
2917 ✗ if (current_group != NULL)
2918 ✗ *current_group = 0;
2919 ✗ if (fallback_group != NULL)
2920 ✗ *fallback_group = 0;
2921 ✗ return;
2922 }
2923
2924 ✗ *proxy_chain = *opt_proxy_groups_;
2925 ✗ if (current_group != NULL)
2926 ✗ *current_group = opt_proxy_groups_current_;
2927 ✗ if (fallback_group != NULL)
2928 ✗ *fallback_group = opt_proxy_groups_fallback_;
2929 }
2930
2931 ✗ string DownloadManager::GetProxyList() { return opt_proxy_list_; }
2932
2933 ✗ string DownloadManager::GetFallbackProxyList() {
2934 ✗ return opt_proxy_fallback_list_;
2935 }
2936
2937 /**
2938 * Choose proxy
2939 */
2940 5060 DownloadManager::ProxyInfo *DownloadManager::ChooseProxyUnlocked(
2941 const shash::Any *hash) {
2942
2/2
✓ Branch 0 taken 4071 times.
✓ Branch 1 taken 989 times.
 5060 if (!opt_proxy_groups_)
2943 4071 return NULL;
2944
2945
2/2
✓ Branch 0 taken 563 times.
✓ Branch 1 taken 426 times.
 989 const uint32_t key = (hash ? hash->Partial32() : 0);
2946
1/2
✓ Branch 1 taken 989 times.
✗ Branch 2 not taken.
 989 const map<uint32_t, ProxyInfo *>::iterator it = opt_proxy_map_.lower_bound(
2947 key);
2948 989 ProxyInfo *proxy = it->second;
2949
2950 989 return proxy;
2951 }
2952
2953 /**
2954 * Update currently selected proxy
2955 */
2956 1417 void DownloadManager::UpdateProxiesUnlocked(const string &reason) {
2957
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1417 times.
 1417 if (!opt_proxy_groups_)
2958 ✗ return;
2959
2960 // Identify number of non-burned proxies within the current group
2961 1417 vector<ProxyInfo> *group = current_proxy_group();
2962 1417 const unsigned num_alive = (group->size() - opt_proxy_groups_current_burned_);
2963
2/4
✓ Branch 2 taken 1417 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1417 times.
✗ Branch 6 not taken.
 2834 const string old_proxy = JoinStrings(opt_proxies_, "|");
2964
2965 // Rebuild proxy map and URL list
2966 1417 opt_proxy_map_.clear();
2967 1417 opt_proxies_.clear();
2968 1417 const uint32_t max_key = 0xffffffffUL;
2969
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1417 times.
 1417 if (opt_proxy_shard_) {
2970 // Build a consistent map with multiple entries for each proxy
2971 ✗ for (unsigned i = 0; i < num_alive; ++i) {
2972 ✗ ProxyInfo *proxy = &(*group)[i];
2973 ✗ shash::Any proxy_hash(shash::kSha1);
2974 ✗ HashString(proxy->url, &proxy_hash);
2975 ✗ Prng prng;
2976 ✗ prng.InitSeed(proxy_hash.Partial32());
2977 ✗ for (unsigned j = 0; j < kProxyMapScale; ++j) {
2978 ✗ const std::pair<uint32_t, ProxyInfo *> entry(prng.Next(max_key), proxy);
2979 ✗ opt_proxy_map_.insert(entry);
2980 }
2981 ✗ const std::string proxy_name = proxy->host.name().empty()
2982 ? ""
2983 ✗ : " (" + proxy->host.name() + ")";
2984 ✗ opt_proxies_.push_back(proxy->url + proxy_name);
2985 }
2986 // Ensure lower_bound() finds a value for all keys
2987 ✗ ProxyInfo *first_proxy = opt_proxy_map_.begin()->second;
2988 ✗ const std::pair<uint32_t, ProxyInfo *> last_entry(max_key, first_proxy);
2989 ✗ opt_proxy_map_.insert(last_entry);
2990 } else {
2991 // Build a map with a single entry for one randomly selected proxy
2992 1417 const unsigned select = prng_.Next(num_alive);
2993 1417 ProxyInfo *proxy = &(*group)[select];
2994 1417 const std::pair<uint32_t, ProxyInfo *> entry(max_key, proxy);
2995
1/2
✓ Branch 1 taken 1417 times.
✗ Branch 2 not taken.
 1417 opt_proxy_map_.insert(entry);
2996 1417 const std::string proxy_name = proxy->host.name().empty()
2997 ? ""
2998
9/18
✓ Branch 0 taken 1221 times.
✓ Branch 1 taken 196 times.
✓ Branch 4 taken 1221 times.
✗ Branch 5 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 196 times.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✓ Branch 11 taken 196 times.
✗ Branch 12 not taken.
✓ Branch 13 taken 196 times.
✓ Branch 14 taken 1221 times.
✓ Branch 15 taken 1221 times.
✓ Branch 16 taken 196 times.
✗ Branch 17 not taken.
✗ Branch 18 not taken.
✗ Branch 19 not taken.
✗ Branch 20 not taken.
 1613 : " (" + proxy->host.name() + ")";
2999
2/4
✓ Branch 1 taken 1417 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1417 times.
✗ Branch 5 not taken.
 1417 opt_proxies_.push_back(proxy->url + proxy_name);
3000 1417 }
3001
1/2
✓ Branch 3 taken 1417 times.
✗ Branch 4 not taken.
 1417 sort(opt_proxies_.begin(), opt_proxies_.end());
3002
3003 // Report any change in proxy usage
3004
2/4
✓ Branch 2 taken 1417 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1417 times.
✗ Branch 6 not taken.
 2834 const string new_proxy = JoinStrings(opt_proxies_, "|");
3005 const string curr_host = "Current host: "
3006 1417 + (opt_host_.chain
3007
6/12
✓ Branch 0 taken 1221 times.
✓ Branch 1 taken 196 times.
✓ Branch 4 taken 1221 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 196 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 196 times.
✓ Branch 11 taken 1221 times.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
 2834 ? (*opt_host_.chain)[opt_host_.current]
3008
1/2
✓ Branch 1 taken 1417 times.
✗ Branch 2 not taken.
 1417 : "");
3009
2/2
✓ Branch 1 taken 1010 times.
✓ Branch 2 taken 407 times.
 1417 if (new_proxy != old_proxy) {
3010
4/9
✗ Branch 2 not taken.
✓ Branch 3 taken 1010 times.
✓ Branch 4 taken 961 times.
✓ Branch 5 taken 49 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 1010 times.
✗ Branch 9 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
 3079 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
3011 "(manager '%s') switching proxy from %s to %s. Reason: %s [%s]",
3012 1059 name_.c_str(), (old_proxy.empty() ? "(none)" : old_proxy.c_str()),
3013 2020 (new_proxy.empty() ? "(none)" : new_proxy.c_str()), reason.c_str(),
3014 curr_host.c_str());
3015 }
3016 1417 }
3017
3018 /**
3019 * Enable proxy sharding
3020 */
3021 ✗ void DownloadManager::ShardProxies() {
3022 ✗ opt_proxy_shard_ = true;
3023 ✗ RebalanceProxiesUnlocked("enable sharding");
3024 }
3025
3026 /**
3027 * Selects a new random proxy in the current load-balancing group. Resets the
3028 * "burned" counter.
3029 */
3030 ✗ void DownloadManager::RebalanceProxiesUnlocked(const string &reason) {
3031 ✗ if (!opt_proxy_groups_)
3032 ✗ return;
3033
3034 ✗ opt_timestamp_failover_proxies_ = 0;
3035 ✗ opt_proxy_groups_current_burned_ = 0;
3036 ✗ UpdateProxiesUnlocked(reason);
3037 }
3038
3039
3040 ✗ void DownloadManager::RebalanceProxies() {
3041 ✗ const MutexLockGuard m(lock_options_);
3042 ✗ RebalanceProxiesUnlocked("rebalance invoked manually");
3043 }
3044
3045
3046 /**
3047 * Switches to the next load-balancing group of proxy servers.
3048 */
3049 ✗ void DownloadManager::SwitchProxyGroup() {
3050 ✗ const MutexLockGuard m(lock_options_);
3051
3052 ✗ if (!opt_proxy_groups_ || (opt_proxy_groups_->size() < 2)) {
3053 ✗ return;
3054 }
3055
3056 ✗ opt_proxy_groups_current_ = (opt_proxy_groups_current_ + 1)
3057 ✗ % opt_proxy_groups_->size();
3058 ✗ opt_timestamp_backup_proxies_ = time(NULL);
3059
3060 const std::string msg = "switch to proxy group "
3061 ✗ + StringifyUint(opt_proxy_groups_current_);
3062 ✗ RebalanceProxiesUnlocked(msg);
3063 }
3064
3065
3066 ✗ void DownloadManager::SetProxyGroupResetDelay(const unsigned seconds) {
3067 ✗ const MutexLockGuard m(lock_options_);
3068 ✗ opt_proxy_groups_reset_after_ = seconds;
3069 ✗ if (opt_proxy_groups_reset_after_ == 0) {
3070 ✗ opt_timestamp_backup_proxies_ = 0;
3071 ✗ opt_timestamp_failover_proxies_ = 0;
3072 }
3073 }
3074
3075
3076 ✗ void DownloadManager::SetMetalinkResetDelay(const unsigned seconds) {
3077 ✗ const MutexLockGuard m(lock_options_);
3078 ✗ opt_metalink_.reset_after = seconds;
3079 ✗ if (opt_metalink_.reset_after == 0)
3080 ✗ opt_metalink_.timestamp_backup = 0;
3081 }
3082
3083
3084 ✗ void DownloadManager::SetHostResetDelay(const unsigned seconds) {
3085 ✗ const MutexLockGuard m(lock_options_);
3086 ✗ opt_host_.reset_after = seconds;
3087 ✗ if (opt_host_.reset_after == 0)
3088 ✗ opt_host_.timestamp_backup = 0;
3089 }
3090
3091
3092 1049 void DownloadManager::SetRetryParameters(const unsigned max_retries,
3093 const unsigned backoff_init_ms,
3094 const unsigned backoff_max_ms) {
3095 1049 const MutexLockGuard m(lock_options_);
3096 1049 opt_max_retries_ = max_retries;
3097 1049 opt_backoff_init_ms_ = backoff_init_ms;
3098 1049 opt_backoff_max_ms_ = backoff_max_ms;
3099 1049 }
3100
3101
3102 505 void DownloadManager::SetMaxIpaddrPerProxy(unsigned limit) {
3103 505 const MutexLockGuard m(lock_options_);
3104 505 resolver_->set_throttle(limit);
3105 505 }
3106
3107
3108 481 void DownloadManager::SetProxyTemplates(const std::string &direct,
3109 const std::string &forced) {
3110 481 const MutexLockGuard m(lock_options_);
3111
1/2
✓ Branch 1 taken 481 times.
✗ Branch 2 not taken.
 481 proxy_template_direct_ = direct;
3112
1/2
✓ Branch 1 taken 481 times.
✗ Branch 2 not taken.
 481 proxy_template_forced_ = forced;
3113 481 }
3114
3115
3116 ✗ void DownloadManager::EnableInfoHeader() { enable_info_header_ = true; }
3117
3118
3119 715 void DownloadManager::EnableRedirects() { follow_redirects_ = true; }
3120
3121 ✗ void DownloadManager::EnableIgnoreSignatureFailures() {
3122 ✗ ignore_signature_failures_ = true;
3123 }
3124
3125 ✗ void DownloadManager::EnableHTTPTracing() { enable_http_tracing_ = true; }
3126
3127 ✗ void DownloadManager::AddHTTPTracingHeader(const std::string &header) {
3128 ✗ http_tracing_headers_.push_back(header);
3129 }
3130
3131 568 void DownloadManager::UseSystemCertificatePath() {
3132 568 ssl_certificate_store_.UseSystemCertificatePath();
3133 568 }
3134
3135 ✗ bool DownloadManager::SetShardingPolicy(const ShardingPolicySelector type) {
3136 ✗ const bool success = false;
3137 ✗ switch (type) {
3138 default:
3139 ✗ LogCvmfs(
3140 kLogDownload, kLogDebug | kLogSyslogErr,
3141 "(manager '%s') "
3142 "Proposed sharding policy does not exist. Falling back to default",
3143 name_.c_str());
3144 }
3145 ✗ return success;
3146 }
3147
3148 ✗ void DownloadManager::SetFailoverIndefinitely() {
3149 ✗ failover_indefinitely_ = true;
3150 }
3151
3152 /**
3153 * Creates a copy of the existing download manager. Must only be called in
3154 * single-threaded stage because it calls curl_global_init().
3155 */
3156 505 DownloadManager *DownloadManager::Clone(
3157 const perf::StatisticsTemplate &statistics,
3158 const std::string &cloned_name) {
3159 DownloadManager *clone = new DownloadManager(pool_max_handles_, statistics,
3160
1/2
✓ Branch 2 taken 505 times.
✗ Branch 3 not taken.
 505 cloned_name);
3161
3162 505 clone->SetDnsParameters(resolver_->retries(), resolver_->timeout_ms());
3163 505 clone->SetDnsTtlLimits(resolver_->min_ttl(), resolver_->max_ttl());
3164 505 clone->SetMaxIpaddrPerProxy(resolver_->throttle());
3165
3166
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 505 times.
 505 if (!opt_dns_server_.empty())
3167 ✗ clone->SetDnsServer(opt_dns_server_);
3168 505 clone->opt_timeout_proxy_ = opt_timeout_proxy_;
3169 505 clone->opt_timeout_direct_ = opt_timeout_direct_;
3170 505 clone->opt_low_speed_limit_ = opt_low_speed_limit_;
3171 505 clone->opt_max_retries_ = opt_max_retries_;
3172 505 clone->opt_backoff_init_ms_ = opt_backoff_init_ms_;
3173 505 clone->opt_backoff_max_ms_ = opt_backoff_max_ms_;
3174 505 clone->enable_info_header_ = enable_info_header_;
3175 505 clone->enable_http_tracing_ = enable_http_tracing_;
3176 505 clone->http_tracing_headers_ = http_tracing_headers_;
3177 505 clone->follow_redirects_ = follow_redirects_;
3178 505 clone->ignore_signature_failures_ = ignore_signature_failures_;
3179
2/2
✓ Branch 0 taken 407 times.
✓ Branch 1 taken 98 times.
 505 if (opt_host_.chain) {
3180
1/2
✓ Branch 2 taken 407 times.
✗ Branch 3 not taken.
 407 clone->opt_host_.chain = new vector<string>(*opt_host_.chain);
3181
1/2
✓ Branch 2 taken 407 times.
✗ Branch 3 not taken.
 407 clone->opt_host_chain_rtt_ = new vector<int>(*opt_host_chain_rtt_);
3182 }
3183
3184 505 CloneProxyConfig(clone);
3185 505 clone->opt_ip_preference_ = opt_ip_preference_;
3186 505 clone->proxy_template_direct_ = proxy_template_direct_;
3187 505 clone->proxy_template_forced_ = proxy_template_forced_;
3188 505 clone->opt_proxy_groups_reset_after_ = opt_proxy_groups_reset_after_;
3189 505 clone->opt_metalink_.reset_after = opt_metalink_.reset_after;
3190 505 clone->opt_host_.reset_after = opt_host_.reset_after;
3191 505 clone->credentials_attachment_ = credentials_attachment_;
3192 505 clone->ssl_certificate_store_ = ssl_certificate_store_;
3193
3194 505 clone->health_check_ = health_check_;
3195 505 clone->sharding_policy_ = sharding_policy_;
3196 505 clone->failover_indefinitely_ = failover_indefinitely_;
3197 505 clone->fqrn_ = fqrn_;
3198
3199 505 return clone;
3200 }
3201
3202
3203 505 void DownloadManager::CloneProxyConfig(DownloadManager *clone) {
3204 505 clone->opt_proxy_groups_current_ = opt_proxy_groups_current_;
3205 505 clone->opt_proxy_groups_current_burned_ = opt_proxy_groups_current_burned_;
3206 505 clone->opt_proxy_groups_fallback_ = opt_proxy_groups_fallback_;
3207 505 clone->opt_num_proxies_ = opt_num_proxies_;
3208 505 clone->opt_proxy_shard_ = opt_proxy_shard_;
3209 505 clone->opt_proxy_list_ = opt_proxy_list_;
3210 505 clone->opt_proxy_fallback_list_ = opt_proxy_fallback_list_;
3211
2/2
✓ Branch 0 taken 98 times.
✓ Branch 1 taken 407 times.
 505 if (opt_proxy_groups_ == NULL)
3212 98 return;
3213
3214
1/2
✓ Branch 2 taken 407 times.
✗ Branch 3 not taken.
 407 clone->opt_proxy_groups_ = new vector<vector<ProxyInfo> >(*opt_proxy_groups_);
3215
2/4
✓ Branch 2 taken 407 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 407 times.
✗ Branch 6 not taken.
 407 clone->UpdateProxiesUnlocked("cloned");
3216 }
3217
3218 } // namespace download
3219