GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/network/download.cc
Date: 2026-05-03 02:36:16
Exec Total Coverage
Lines: 877 1719 51.0%
Branches: 666 2194 30.4%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * The download module provides an interface for fetching files via HTTP
5 * and file. It is internally using libcurl and the asynchronous DNS resolver
6 * c-ares. The JobInfo struct describes a single file/url to download and
7 * keeps the state during the several phases of downloading.
8 *
9 * The module starts in single-threaded mode and can be switched to multi-
10 * threaded mode by Spawn(). In multi-threaded mode, the Fetch() function still
11 * blocks but there is a separate I/O thread using asynchronous I/O, which
12 * maintains all concurrent connections simultaneously. As there might be more
13 * than 1024 file descriptors for the CernVM-FS process, the I/O thread uses
14 * poll and the libcurl multi socket interface.
15 *
16 * While downloading, files can be decompressed and the secure hash can be
17 * calculated on the fly.
18 *
19 * The module also implements failure handling. If corrupted data has been
20 * downloaded, the transfer is restarted using HTTP "no-cache" pragma.
21 * A "host chain" can be configured. When a host fails, there is automatic
22 * fail-over to the next host in the chain until all hosts are probed.
23 * Similarly a chain of proxy sets can be configured. Inside a proxy set,
24 * proxies are selected randomly (load-balancing set).
25 */
26
27 // TODO(jblomer): MS for time summing
28
29 #include "download.h"
30
31 #include <alloca.h>
32 #include <errno.h>
33 #include <inttypes.h>
34 #include <poll.h>
35 #include <pthread.h>
36 #include <signal.h>
37 #include <stdint.h>
38 #include <sys/time.h>
39 #include <unistd.h>
40
41 #include <algorithm>
42 #include <cassert>
43 #include <cstdio>
44 #include <cstdlib>
45 #include <cstring>
46 #include <map>
47 #include <set>
48 #include <utility>
49
50 #include "compression/compression.h"
51 #include "crypto/hash.h"
52 #include "duplex_curl.h" // IWYU pragma: keep
53 #include "interrupt.h"
54 #include "network/sink_mem.h"
55 #include "network/sink_path.h"
56 #include "sanitizer.h"
57 #include "ssl.h"
58 #include "util/algorithm.h"
59 #include "util/atomic.h"
60 #include "util/exception.h"
61 #include "util/logging.h"
62 #include "util/posix.h"
63 #include "util/prng.h"
64 #include "util/smalloc.h"
65 #include "util/string.h"
66
67 using namespace std; // NOLINT
68
69 namespace download {
70
71 /**
72 * Returns the status if an interrupt happened for a given repository.
73 *
74 * Used only in case CVMFS_FAILOVER_INDEFINITELY (failover_indefinitely_) is set
75 * where failed downloads are retried indefinitely, unless an interrupt occurred
76 *
77 * @note If you use this functionality you need to change the source code of
78 * e.g. cvmfs_config reload to create a sentinel file. See comment below.
79 *
80 * @return true if an interrupt occurred
81 * false otherwise
82 */
83 ✗ bool Interrupted(const std::string &fqrn, JobInfo *info) {
84 ✗ if (info->allow_failure()) {
85 ✗ return true;
86 }
87
88 ✗ if (!fqrn.empty()) {
89 // it is up to the user the create this sentinel file ("pause_file") if
90 // CVMFS_FAILOVER_INDEFINITELY is used. It must be created during
91 // "cvmfs_config reload" and "cvmfs_config reload $fqrn"
92 ✗ const std::string pause_file = std::string("/var/run/cvmfs/interrupt.")
93 ✗ + fqrn;
94
95 ✗ LogCvmfs(kLogDownload, kLogDebug,
96 "(id %" PRId64 ") Interrupted(): checking for existence of %s",
97 info->id(), pause_file.c_str());
98 ✗ if (FileExists(pause_file)) {
99 ✗ LogCvmfs(kLogDownload, kLogDebug,
100 "(id %" PRId64 ") Interrupt marker found - "
101 "Interrupting current download, this will EIO outstanding IO.",
102 info->id());
103 ✗ if (0 != unlink(pause_file.c_str())) {
104 ✗ LogCvmfs(kLogDownload, kLogDebug,
105 "(id %" PRId64 ") Couldn't delete interrupt marker: errno=%d",
106 ✗ info->id(), errno);
107 }
108 ✗ return true;
109 }
110 }
111 ✗ return false;
112 }
113
114 7268 static Failures PrepareDownloadDestination(JobInfo *info) {
115
3/6
✓ Branch 1 taken 7268 times.
✗ Branch 2 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 7268 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 7268 times.
 7268 if (info->sink() != NULL && !info->sink()->IsValid()) {
116 ✗ cvmfs::PathSink *psink = dynamic_cast<cvmfs::PathSink *>(info->sink());
117 ✗ if (psink != NULL) {
118 ✗ LogCvmfs(kLogDownload, kLogDebug,
119 "(id %" PRId64 ") Failed to open path %s: %s (errno=%d).",
120 ✗ info->id(), psink->path().c_str(), strerror(errno), errno);
121 ✗ return kFailLocalIO;
122 } else {
123 ✗ LogCvmfs(kLogDownload, kLogDebug,
124 "(id %" PRId64 ") "
125 "Failed to create a valid sink: \n %s",
126 ✗ info->id(), info->sink()->Describe().c_str());
127 ✗ return kFailOther;
128 }
129 }
130
131 7268 return kFailOk;
132 }
133
134
135 /**
136 * Called by curl for every HTTP header. Not called for file:// transfers.
137 */
138 21383 static size_t CallbackCurlHeader(void *ptr, size_t size, size_t nmemb,
139 void *info_link) {
140 21383 const size_t num_bytes = size * nmemb;
141
1/2
✓ Branch 2 taken 21383 times.
✗ Branch 3 not taken.
 21383 const string header_line(static_cast<const char *>(ptr), num_bytes);
142 21383 JobInfo *info = static_cast<JobInfo *>(info_link);
143
144 // LogCvmfs(kLogDownload, kLogDebug, "REMOVE-ME: Header callback with %s",
145 // header_line.c_str());
146
147 // Check http status codes
148
4/6
✓ Branch 2 taken 21383 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 21383 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 408 times.
✓ Branch 10 taken 20975 times.
 21383 if (HasPrefix(header_line, "HTTP/1.", false)) {
149
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 408 times.
 408 if (header_line.length() < 10) {
150 ✗ return 0;
151 }
152
153 unsigned i;
154
5/6
✓ Branch 1 taken 816 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 408 times.
✓ Branch 5 taken 408 times.
✓ Branch 6 taken 408 times.
✓ Branch 7 taken 408 times.
 816 for (i = 8; (i < header_line.length()) && (header_line[i] == ' '); ++i) {
155 }
156
157 // Code is initialized to -1
158
1/2
✓ Branch 1 taken 408 times.
✗ Branch 2 not taken.
 408 if (header_line.length() > i + 2) {
159 408 info->SetHttpCode(DownloadManager::ParseHttpCode(&header_line[i]));
160 }
161
162
2/2
✓ Branch 1 taken 306 times.
✓ Branch 2 taken 102 times.
 408 if ((info->http_code() / 100) == 2) {
163 306 return num_bytes;
164
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 102 } else if ((info->http_code() == 301) || (info->http_code() == 302)
165
2/8
✗ Branch 0 not taken.
✓ Branch 1 taken 102 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 102 times.
✗ Branch 9 not taken.
 102 || (info->http_code() == 303) || (info->http_code() == 307)) {
166
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 102 times.
 102 if (!info->follow_redirects()) {
167 ✗ LogCvmfs(kLogDownload, kLogDebug,
168 "(id %" PRId64 ") redirect support not enabled: %s",
169 info->id(), header_line.c_str());
170 ✗ info->SetErrorCode(kFailHostHttp);
171 ✗ return 0;
172 }
173
1/2
✓ Branch 3 taken 102 times.
✗ Branch 4 not taken.
 102 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") http redirect: %s",
174 info->id(), header_line.c_str());
175 // libcurl will handle this because of CURLOPT_FOLLOWLOCATION
176 102 return num_bytes;
177 } else {
178 ✗ LogCvmfs(kLogDownload, kLogDebug,
179 "(id %" PRId64 ") http status error code: %s [%d]", info->id(),
180 header_line.c_str(), info->http_code());
181 ✗ if (((info->http_code() / 100) == 5) || (info->http_code() == 400)
182 ✗ || (info->http_code() == 404)) {
183 // 5XX returned by host
184 // 400: error from the GeoAPI module
185 // 404: the stratum 1 does not have the newest files
186 ✗ info->SetErrorCode(kFailHostHttp);
187 ✗ } else if (info->http_code() == 429) {
188 // 429: rate throttling (we ignore the backoff hint for the time being)
189 ✗ info->SetErrorCode(kFailHostConnection);
190 } else {
191 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostHttp
192 : kFailProxyHttp);
193 }
194 ✗ return 0;
195 }
196 }
197
198 // If needed: allocate space in sink
199
3/4
✓ Branch 2 taken 20975 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 7844 times.
✓ Branch 5 taken 13131 times.
 20975 if (info->sink() != NULL && info->sink()->RequiresReserve()
200
11/20
✓ Branch 1 taken 20975 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 7844 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 7844 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 2490 times.
✓ Branch 11 taken 5354 times.
✓ Branch 12 taken 7844 times.
✓ Branch 13 taken 13131 times.
✗ Branch 14 not taken.
✓ Branch 15 taken 7844 times.
✓ Branch 16 taken 13131 times.
✓ Branch 18 taken 2490 times.
✓ Branch 19 taken 18485 times.
✗ Branch 20 not taken.
✗ Branch 21 not taken.
✗ Branch 23 not taken.
✗ Branch 24 not taken.
 41950 && HasPrefix(header_line, "CONTENT-LENGTH:", true)) {
201 2490 char *tmp = reinterpret_cast<char *>(alloca(num_bytes + 1));
202 2490 uint64_t length = 0;
203 2490 sscanf(header_line.c_str(), "%s %" PRIu64, tmp, &length);
204
1/2
✓ Branch 0 taken 2490 times.
✗ Branch 1 not taken.
 2490 if (length > 0) {
205
2/4
✓ Branch 2 taken 2490 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 2490 times.
 2490 if (!info->sink()->Reserve(length)) {
206 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
207 "(id %" PRId64 ") "
208 "resource %s too large to store in memory (%" PRIu64 ")",
209 info->id(), info->url()->c_str(), length);
210 ✗ info->SetErrorCode(kFailTooBig);
211 ✗ return 0;
212 }
213 } else {
214 // Empty resource
215 ✗ info->sink()->Reserve(0);
216 }
217
4/6
✓ Branch 2 taken 18485 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 18485 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 102 times.
✓ Branch 10 taken 18383 times.
 18485 } else if (HasPrefix(header_line, "LOCATION:", true)) {
218 // This comes along with redirects
219
1/2
✓ Branch 3 taken 102 times.
✗ Branch 4 not taken.
 102 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") %s", info->id(),
220 header_line.c_str());
221
3/6
✓ Branch 2 taken 18383 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 18383 times.
✗ Branch 6 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 18383 times.
 18383 } else if (HasPrefix(header_line, "LINK:", true)) {
222 // This is metalink info
223 ✗ LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") %s", info->id(),
224 header_line.c_str());
225 ✗ std::string link = info->link();
226 ✗ if (link.size() != 0) {
227 // multiple LINK headers are allowed
228 ✗ link = link + ", " + header_line.substr(5);
229 } else {
230 ✗ link = header_line.substr(5);
231 }
232 ✗ info->SetLink(link);
233
3/6
✓ Branch 3 taken 18383 times.
✗ Branch 4 not taken.
✓ Branch 6 taken 18383 times.
✗ Branch 7 not taken.
✗ Branch 10 not taken.
✓ Branch 11 taken 18383 times.
 18383 } else if (HasPrefix(header_line, "X-SQUID-ERROR:", true)) {
234 // Reinterpret host error as proxy error
235 ✗ if (info->error_code() == kFailHostHttp) {
236 ✗ info->SetErrorCode(kFailProxyHttp);
237 }
238
3/6
✓ Branch 2 taken 18383 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 18383 times.
✗ Branch 6 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 18383 times.
 18383 } else if (HasPrefix(header_line, "PROXY-STATUS:", true)) {
239 // Reinterpret host error as proxy error if applicable
240 ✗ if ((info->error_code() == kFailHostHttp)
241 ✗ && (header_line.find("error=") != string::npos)) {
242 ✗ info->SetErrorCode(kFailProxyHttp);
243 }
244 }
245
246 20975 return num_bytes;
247 21383 }
248
249
250 /**
251 * Called by curl for every received data chunk.
252 */
253 8095 static size_t CallbackCurlData(void *ptr, size_t size, size_t nmemb,
254 void *info_link) {
255 8095 const size_t num_bytes = size * nmemb;
256 8095 JobInfo *info = static_cast<JobInfo *>(info_link);
257
258 // TODO(heretherebedragons) remove if no error comes up
259 // as this means only jobinfo data request (and not header only)
260 // come here
261
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 8095 times.
 8095 assert(info->sink() != NULL);
262
263 // LogCvmfs(kLogDownload, kLogDebug, "Data callback, %d bytes", num_bytes);
264
265
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 8095 times.
 8095 if (num_bytes == 0)
266 ✗ return 0;
267
268
2/2
✓ Branch 1 taken 4801 times.
✓ Branch 2 taken 3294 times.
 8095 if (info->expected_hash()) {
269 4801 shash::Update(reinterpret_cast<unsigned char *>(ptr), num_bytes,
270 info->hash_context());
271 }
272
273
2/2
✓ Branch 1 taken 4752 times.
✓ Branch 2 taken 3343 times.
 8095 if (info->compressed()) {
274 4752 const zlib::StreamStates retval = zlib::DecompressZStream2Sink(
275 ptr, static_cast<int64_t>(num_bytes), info->GetZstreamPtr(),
276 info->sink());
277
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4752 times.
 4752 if (retval == zlib::kStreamDataError) {
278 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
279 "(id %" PRId64 ") failed to decompress %s", info->id(),
280 info->url()->c_str());
281 ✗ info->SetErrorCode(kFailBadData);
282 ✗ return 0;
283
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4752 times.
 4752 } else if (retval == zlib::kStreamIOError) {
284 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
285 "(id %" PRId64 ") decompressing %s, local IO error", info->id(),
286 info->url()->c_str());
287 ✗ info->SetErrorCode(kFailLocalIO);
288 ✗ return 0;
289 }
290 } else {
291 3343 const int64_t written = info->sink()->Write(ptr, num_bytes);
292
2/4
✓ Branch 0 taken 3343 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 3343 times.
 3343 if (written < 0 || static_cast<uint64_t>(written) != num_bytes) {
293 ✗ LogCvmfs(kLogDownload, kLogDebug,
294 "(id %" PRId64 ") "
295 "Failed to perform write of %zu bytes to sink %s with errno %ld",
296 ✗ info->id(), num_bytes, info->sink()->Describe().c_str(),
297 written);
298 }
299 }
300
301 8095 return num_bytes;
302 }
303
304 #ifdef DEBUGMSG
305 3888 static int CallbackCurlDebug(CURL *handle,
306 curl_infotype type,
307 char *data,
308 size_t size,
309 void * /* clientp */) {
310 JobInfo *info;
311
1/2
✓ Branch 1 taken 3888 times.
✗ Branch 2 not taken.
 3888 curl_easy_getinfo(handle, CURLINFO_PRIVATE, &info);
312
313
3/6
✓ Branch 2 taken 3888 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 3888 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 3888 times.
✗ Branch 9 not taken.
 7776 std::string prefix = "(id " + StringifyInt(info->id()) + ") ";
314
4/8
✓ Branch 0 taken 658 times.
✓ Branch 1 taken 1598 times.
✓ Branch 2 taken 408 times.
✓ Branch 3 taken 1224 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
 3888 switch (type) {
315 658 case CURLINFO_TEXT:
316
1/2
✓ Branch 1 taken 658 times.
✗ Branch 2 not taken.
 658 prefix += "{info} ";
317 658 break;
318 1598 case CURLINFO_HEADER_IN:
319
1/2
✓ Branch 1 taken 1598 times.
✗ Branch 2 not taken.
 1598 prefix += "{header/recv} ";
320 1598 break;
321 408 case CURLINFO_HEADER_OUT:
322
1/2
✓ Branch 1 taken 408 times.
✗ Branch 2 not taken.
 408 prefix += "{header/sent} ";
323 408 break;
324 1224 case CURLINFO_DATA_IN:
325
2/2
✓ Branch 0 taken 136 times.
✓ Branch 1 taken 1088 times.
 1224 if (size < 50) {
326
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 prefix += "{data/recv} ";
327 136 break;
328 } else {
329
1/2
✓ Branch 2 taken 1088 times.
✗ Branch 3 not taken.
 1088 LogCvmfs(kLogCurl, kLogDebug, "%s{data/recv} <snip>", prefix.c_str());
330 1088 return 0;
331 }
332 ✗ case CURLINFO_DATA_OUT:
333 ✗ if (size < 50) {
334 ✗ prefix += "{data/sent} ";
335 ✗ break;
336 } else {
337 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{data/sent} <snip>", prefix.c_str());
338 ✗ return 0;
339 }
340 ✗ case CURLINFO_SSL_DATA_IN:
341 ✗ if (size < 50) {
342 ✗ prefix += "{ssldata/recv} ";
343 ✗ break;
344 } else {
345 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{ssldata/recv} <snip>",
346 prefix.c_str());
347 ✗ return 0;
348 }
349 ✗ case CURLINFO_SSL_DATA_OUT:
350 ✗ if (size < 50) {
351 ✗ prefix += "{ssldata/sent} ";
352 ✗ break;
353 } else {
354 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{ssldata/sent} <snip>",
355 prefix.c_str());
356 ✗ return 0;
357 }
358 ✗ default:
359 // just log the message
360 ✗ break;
361 }
362
363 2800 bool valid_char = true;
364
1/2
✓ Branch 2 taken 2800 times.
✗ Branch 3 not taken.
 2800 std::string msg(data, size);
365
2/2
✓ Branch 1 taken 145571 times.
✓ Branch 2 taken 2800 times.
 148371 for (size_t i = 0; i < msg.length(); ++i) {
366
2/4
✓ Branch 1 taken 145571 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 145571 times.
 145571 if (msg[i] == '\0') {
367 ✗ msg[i] = '~';
368 }
369
370 // verify that char is a valid printable char
371
3/6
✓ Branch 1 taken 145571 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 136549 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 136549 times.
 282120 if ((msg[i] < ' ' || msg[i] > '~')
372
6/8
✓ Branch 0 taken 136549 times.
✓ Branch 1 taken 9022 times.
✓ Branch 3 taken 9022 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 4182 times.
✓ Branch 6 taken 4840 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 145571 times.
 286302 && (msg[i] != 10 /*line feed*/
373
2/4
✓ Branch 1 taken 4182 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 4182 times.
 4182 && msg[i] != 13 /*carriage return*/)) {
374 ✗ valid_char = false;
375 }
376 }
377
378
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2800 times.
 2800 if (!valid_char) {
379 ✗ msg = "<Non-plaintext sequence>";
380 }
381
382
1/2
✓ Branch 3 taken 2800 times.
✗ Branch 4 not taken.
 2800 LogCvmfs(kLogCurl, kLogDebug, "%s%s", prefix.c_str(),
383
1/2
✓ Branch 1 taken 2800 times.
✗ Branch 2 not taken.
 5600 Trim(msg, true /* trim_newline */).c_str());
384 2800 return 0;
385 3888 }
386 #endif
387
388 //------------------------------------------------------------------------------
389
390
391 const int DownloadManager::kProbeUnprobed = -1;
392 const int DownloadManager::kProbeDown = -2;
393 const int DownloadManager::kProbeGeo = -3;
394
395 /**
396 * Escape special chars from the URL, except for ':' and '/',
397 * which should keep their meaning.
398 */
399 7404 string DownloadManager::EscapeUrl(const int64_t jobinfo_id, const string &url) {
400 7404 string escaped;
401
1/2
✓ Branch 2 taken 7404 times.
✗ Branch 3 not taken.
 7404 escaped.reserve(url.length());
402
403 char escaped_char[3];
404
2/2
✓ Branch 1 taken 619706 times.
✓ Branch 2 taken 7404 times.
 627110 for (unsigned i = 0, s = url.length(); i < s; ++i) {
405
3/4
✓ Branch 2 taken 619706 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 306 times.
✓ Branch 5 taken 619400 times.
 619706 if (JobInfo::EscapeUrlChar(url[i], escaped_char)) {
406
1/2
✓ Branch 1 taken 306 times.
✗ Branch 2 not taken.
 306 escaped.append(escaped_char, 3);
407 } else {
408
1/2
✓ Branch 1 taken 619400 times.
✗ Branch 2 not taken.
 619400 escaped.push_back(escaped_char[0]);
409 }
410 }
411
1/2
✓ Branch 3 taken 7404 times.
✗ Branch 4 not taken.
 7404 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") escaped %s to %s",
412 jobinfo_id, url.c_str(), escaped.c_str());
413
414 14808 return escaped;
415 }
416
417 /**
418 * -1 of digits is not a valid Http return code
419 */
420 578 int DownloadManager::ParseHttpCode(const char digits[3]) {
421 578 int result = 0;
422 578 int factor = 100;
423
2/2
✓ Branch 0 taken 1734 times.
✓ Branch 1 taken 544 times.
 2278 for (int i = 0; i < 3; ++i) {
424
3/4
✓ Branch 0 taken 1734 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 34 times.
✓ Branch 3 taken 1700 times.
 1734 if ((digits[i] < '0') || (digits[i] > '9'))
425 34 return -1;
426 1700 result += (digits[i] - '0') * factor;
427 1700 factor /= 10;
428 }
429 544 return result;
430 }
431
432
433 /**
434 * Called when new curl sockets arrive or existing curl sockets depart.
435 */
436 ✗ int DownloadManager::CallbackCurlSocket(CURL * /* easy */,
437 curl_socket_t s,
438 int action,
439 void *userp,
440 void * /* socketp */) {
441 // LogCvmfs(kLogDownload, kLogDebug, "CallbackCurlSocket called with easy "
442 // "handle %p, socket %d, action %d", easy, s, action);
443 ✗ DownloadManager *download_mgr = static_cast<DownloadManager *>(userp);
444 ✗ if (action == CURL_POLL_NONE)
445 ✗ return 0;
446
447 // Find s in watch_fds_
448 unsigned index;
449
450 // TODO(heretherebedragons) why start at index = 0 and not 2?
451 // fd[0] and fd[1] are fixed?
452 ✗ for (index = 0; index < download_mgr->watch_fds_inuse_; ++index) {
453 ✗ if (download_mgr->watch_fds_[index].fd == s)
454 ✗ break;
455 }
456 // Or create newly
457 ✗ if (index == download_mgr->watch_fds_inuse_) {
458 // Extend array if necessary
459 ✗ if (download_mgr->watch_fds_inuse_ == download_mgr->watch_fds_size_) {
460 ✗ assert(download_mgr->watch_fds_size_ > 0);
461 ✗ download_mgr->watch_fds_size_ *= 2;
462 ✗ download_mgr->watch_fds_ = static_cast<struct pollfd *>(
463 ✗ srealloc(download_mgr->watch_fds_,
464 ✗ download_mgr->watch_fds_size_ * sizeof(struct pollfd)));
465 }
466 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].fd = s;
467 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].events = 0;
468 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].revents = 0;
469 ✗ download_mgr->watch_fds_inuse_++;
470 }
471
472 ✗ switch (action) {
473 ✗ case CURL_POLL_IN:
474 ✗ download_mgr->watch_fds_[index].events = POLLIN | POLLPRI;
475 ✗ break;
476 ✗ case CURL_POLL_OUT:
477 ✗ download_mgr->watch_fds_[index].events = POLLOUT | POLLWRBAND;
478 ✗ break;
479 ✗ case CURL_POLL_INOUT:
480 ✗ download_mgr->watch_fds_[index].events = POLLIN | POLLPRI | POLLOUT
481 | POLLWRBAND;
482 ✗ break;
483 ✗ case CURL_POLL_REMOVE:
484 ✗ if (index < download_mgr->watch_fds_inuse_ - 1) {
485 download_mgr
486 ✗ ->watch_fds_[index] = download_mgr->watch_fds_
487 ✗ [download_mgr->watch_fds_inuse_ - 1];
488 }
489 ✗ download_mgr->watch_fds_inuse_--;
490 // Shrink array if necessary
491 ✗ if ((download_mgr->watch_fds_inuse_ > download_mgr->watch_fds_max_)
492 ✗ && (download_mgr->watch_fds_inuse_
493 ✗ < download_mgr->watch_fds_size_ / 2)) {
494 ✗ download_mgr->watch_fds_size_ /= 2;
495 // LogCvmfs(kLogDownload, kLogDebug, "shrinking watch_fds_ (%d)",
496 // watch_fds_size_);
497 ✗ download_mgr->watch_fds_ = static_cast<struct pollfd *>(
498 ✗ srealloc(download_mgr->watch_fds_,
499 ✗ download_mgr->watch_fds_size_ * sizeof(struct pollfd)));
500 // LogCvmfs(kLogDownload, kLogDebug, "shrinking watch_fds_ done",
501 // watch_fds_size_);
502 }
503 ✗ break;
504 ✗ default:
505 ✗ break;
506 }
507
508 ✗ return 0;
509 }
510
511
512 /**
513 * Worker thread event loop. Waits on new JobInfo structs on a pipe.
514 */
515 ✗ void *DownloadManager::MainDownload(void *data) {
516 ✗ DownloadManager *download_mgr = static_cast<DownloadManager *>(data);
517 ✗ LogCvmfs(kLogDownload, kLogDebug,
518 "download I/O thread of DownloadManager '%s' started",
519 download_mgr->name_.c_str());
520
521 ✗ const int kIdxPipeTerminate = 0;
522 ✗ const int kIdxPipeJobs = 1;
523
524 ✗ download_mgr->watch_fds_ = static_cast<struct pollfd *>(
525 ✗ smalloc(2 * sizeof(struct pollfd)));
526 ✗ download_mgr->watch_fds_size_ = 2;
527 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].fd = download_mgr->pipe_terminate_
528 ✗ ->GetReadFd();
529 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].events = POLLIN | POLLPRI;
530 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].revents = 0;
531 ✗ download_mgr->watch_fds_[kIdxPipeJobs].fd = download_mgr->pipe_jobs_
532 ✗ ->GetReadFd();
533 ✗ download_mgr->watch_fds_[kIdxPipeJobs].events = POLLIN | POLLPRI;
534 ✗ download_mgr->watch_fds_[kIdxPipeJobs].revents = 0;
535 ✗ download_mgr->watch_fds_inuse_ = 2;
536
537 ✗ int still_running = 0;
538 struct timeval timeval_start, timeval_stop;
539 ✗ gettimeofday(&timeval_start, NULL);
540 while (true) {
541 int timeout;
542 ✗ if (still_running) {
543 /* NOTE: The following might degrade the performance for many small files
544 * use case. TODO(jblomer): look into it.
545 // Specify a timeout for polling in ms; this allows us to return
546 // to libcurl once a second so it can look for internal operations
547 // which timed out. libcurl has a more elaborate mechanism
548 // (CURLMOPT_TIMERFUNCTION) that would inform us of the next potential
549 // timeout. TODO(bbockelm) we should switch to that in the future.
550 timeout = 100;
551 */
552 ✗ timeout = 1;
553 } else {
554 ✗ timeout = -1;
555 ✗ gettimeofday(&timeval_stop, NULL);
556 const int64_t delta = static_cast<int64_t>(
557 ✗ 1000 * DiffTimeSeconds(timeval_start, timeval_stop));
558 ✗ perf::Xadd(download_mgr->counters_->sz_transfer_time, delta);
559 }
560 ✗ const int retval = poll(download_mgr->watch_fds_,
561 ✗ download_mgr->watch_fds_inuse_, timeout);
562 ✗ if (retval < 0) {
563 ✗ continue;
564 }
565
566 // Handle timeout
567 ✗ if (retval == 0) {
568 ✗ curl_multi_socket_action(download_mgr->curl_multi_, CURL_SOCKET_TIMEOUT,
569 0, &still_running);
570 }
571
572 // Terminate I/O thread
573 ✗ if (download_mgr->watch_fds_[kIdxPipeTerminate].revents)
574 ✗ break;
575
576 // New job arrives
577 ✗ if (download_mgr->watch_fds_[kIdxPipeJobs].revents) {
578 ✗ download_mgr->watch_fds_[kIdxPipeJobs].revents = 0;
579 JobInfo *info;
580 ✗ download_mgr->pipe_jobs_->Read<JobInfo *>(&info);
581 ✗ if (!still_running) {
582 ✗ gettimeofday(&timeval_start, NULL);
583 }
584 ✗ CURL *handle = download_mgr->AcquireCurlHandle();
585 ✗ download_mgr->InitializeRequest(info, handle);
586 ✗ download_mgr->SetUrlOptions(info);
587 ✗ curl_multi_add_handle(download_mgr->curl_multi_, handle);
588 ✗ curl_multi_socket_action(download_mgr->curl_multi_, CURL_SOCKET_TIMEOUT,
589 0, &still_running);
590 }
591
592 // Activity on curl sockets
593 // Within this loop the curl_multi_socket_action() may cause socket(s)
594 // to be removed from watch_fds_. If a socket is removed it is replaced
595 // by the socket at the end of the array and the inuse count is decreased.
596 // Therefore loop over the array in reverse order.
597 ✗ for (int64_t i = download_mgr->watch_fds_inuse_ - 1; i >= 2; --i) {
598 ✗ if (i >= download_mgr->watch_fds_inuse_) {
599 ✗ continue;
600 }
601 ✗ if (download_mgr->watch_fds_[i].revents) {
602 ✗ int ev_bitmask = 0;
603 ✗ if (download_mgr->watch_fds_[i].revents & (POLLIN | POLLPRI))
604 ✗ ev_bitmask |= CURL_CSELECT_IN;
605 ✗ if (download_mgr->watch_fds_[i].revents & (POLLOUT | POLLWRBAND))
606 ✗ ev_bitmask |= CURL_CSELECT_OUT;
607 ✗ if (download_mgr->watch_fds_[i].revents
608 ✗ & (POLLERR | POLLHUP | POLLNVAL)) {
609 ✗ ev_bitmask |= CURL_CSELECT_ERR;
610 }
611 ✗ download_mgr->watch_fds_[i].revents = 0;
612
613 ✗ curl_multi_socket_action(download_mgr->curl_multi_,
614 ✗ download_mgr->watch_fds_[i].fd,
615 ev_bitmask,
616 &still_running);
617 }
618 }
619
620 // Check if transfers are completed
621 CURLMsg *curl_msg;
622 int msgs_in_queue;
623 ✗ while ((curl_msg = curl_multi_info_read(download_mgr->curl_multi_,
624 &msgs_in_queue))) {
625 ✗ if (curl_msg->msg == CURLMSG_DONE) {
626 ✗ perf::Inc(download_mgr->counters_->n_requests);
627 JobInfo *info;
628 ✗ CURL *easy_handle = curl_msg->easy_handle;
629 ✗ const int curl_error = curl_msg->data.result;
630 ✗ curl_easy_getinfo(easy_handle, CURLINFO_PRIVATE, &info);
631
632 int64_t redir_count;
633 ✗ curl_easy_getinfo(easy_handle, CURLINFO_REDIRECT_COUNT, &redir_count);
634 ✗ LogCvmfs(kLogDownload, kLogDebug,
635 "(manager '%s' - id %" PRId64 ") "
636 "Number of CURL redirects %" PRId64,
637 download_mgr->name_.c_str(), info->id(), redir_count);
638
639 ✗ curl_multi_remove_handle(download_mgr->curl_multi_, easy_handle);
640 ✗ if (download_mgr->VerifyAndFinalize(curl_error, info)) {
641 ✗ curl_multi_add_handle(download_mgr->curl_multi_, easy_handle);
642 ✗ curl_multi_socket_action(download_mgr->curl_multi_,
643 CURL_SOCKET_TIMEOUT,
644 0,
645 &still_running);
646 } else {
647 // Return easy handle into pool and write result back
648 ✗ download_mgr->ReleaseCurlHandle(easy_handle);
649
650 ✗ DataTubeElement *ele = new DataTubeElement(kActionStop);
651 ✗ info->GetDataTubePtr()->EnqueueBack(ele);
652 ✗ info->GetPipeJobResultPtr()->Write<download::Failures>(
653 ✗ info->error_code());
654 }
655 }
656 }
657 }
658
659 ✗ for (set<CURL *>::iterator i = download_mgr->pool_handles_inuse_->begin(),
660 ✗ iEnd = download_mgr->pool_handles_inuse_->end();
661 ✗ i != iEnd;
662 ✗ ++i) {
663 ✗ curl_multi_remove_handle(download_mgr->curl_multi_, *i);
664 ✗ curl_easy_cleanup(*i);
665 }
666 ✗ download_mgr->pool_handles_inuse_->clear();
667 ✗ free(download_mgr->watch_fds_);
668
669 ✗ LogCvmfs(kLogDownload, kLogDebug,
670 "download I/O thread of DownloadManager '%s' terminated",
671 download_mgr->name_.c_str());
672 ✗ return NULL;
673 }
674
675
676 //------------------------------------------------------------------------------
677
678
679 5217 HeaderLists::~HeaderLists() {
680
2/2
✓ Branch 1 taken 5220 times.
✓ Branch 2 taken 5217 times.
 10437 for (unsigned i = 0; i < blocks_.size(); ++i) {
681
1/2
✓ Branch 1 taken 5220 times.
✗ Branch 2 not taken.
 5220 delete[] blocks_[i];
682 }
683 5217 blocks_.clear();
684 5217 }
685
686
687 14037 curl_slist *HeaderLists::GetList(const char *header) { return Get(header); }
688
689
690 7268 curl_slist *HeaderLists::DuplicateList(curl_slist *slist) {
691
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7268 times.
 7268 assert(slist);
692 7268 curl_slist *copy = GetList(slist->data);
693 7268 copy->next = slist->next;
694 7268 curl_slist *prev = copy;
695 7268 slist = slist->next;
696
2/2
✓ Branch 0 taken 14536 times.
✓ Branch 1 taken 7268 times.
 21804 while (slist) {
697 14536 curl_slist *new_link = Get(slist->data);
698 14536 new_link->next = slist->next;
699 14536 prev->next = new_link;
700 14536 prev = new_link;
701 14536 slist = slist->next;
702 }
703 7268 return copy;
704 }
705
706
707 10870 void HeaderLists::AppendHeader(curl_slist *slist, const char *header) {
708
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 10870 times.
 10870 assert(slist);
709 10870 curl_slist *new_link = Get(header);
710 10870 new_link->next = NULL;
711
712
2/2
✓ Branch 0 taken 6300 times.
✓ Branch 1 taken 10870 times.
 17170 while (slist->next)
713 6300 slist = slist->next;
714 10870 slist->next = new_link;
715 10870 }
716
717
718 /**
719 * Ensures that a certain header string is _not_ part of slist on return.
720 * Note that if the first header element matches, the returned slist points
721 * to a different value.
722 */
723 15 void HeaderLists::CutHeader(const char *header, curl_slist **slist) {
724
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 15 times.
 15 assert(slist);
725 curl_slist head;
726 15 head.next = *slist;
727 15 curl_slist *prev = &head;
728 15 curl_slist *rover = *slist;
729
2/2
✓ Branch 0 taken 27 times.
✓ Branch 1 taken 15 times.
 42 while (rover) {
730
2/2
✓ Branch 0 taken 12 times.
✓ Branch 1 taken 15 times.
 27 if (strcmp(rover->data, header) == 0) {
731 12 prev->next = rover->next;
732 12 Put(rover);
733 12 rover = prev;
734 }
735 27 prev = rover;
736 27 rover = rover->next;
737 }
738 15 *slist = head.next;
739 15 }
740
741
742 8048 void HeaderLists::PutList(curl_slist *slist) {
743
2/2
✓ Branch 0 taken 23024 times.
✓ Branch 1 taken 8048 times.
 31072 while (slist) {
744 23024 curl_slist *next = slist->next;
745 23024 Put(slist);
746 23024 slist = next;
747 }
748 8048 }
749
750
751 12 string HeaderLists::Print(curl_slist *slist) {
752 12 string verbose;
753
2/2
✓ Branch 0 taken 24 times.
✓ Branch 1 taken 12 times.
 36 while (slist) {
754
3/6
✓ Branch 2 taken 24 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 24 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 24 times.
✗ Branch 9 not taken.
 24 verbose += string(slist->data) + "\n";
755 24 slist = slist->next;
756 }
757 12 return verbose;
758 }
759
760
761 39443 curl_slist *HeaderLists::Get(const char *header) {
762
2/2
✓ Branch 1 taken 34222 times.
✓ Branch 2 taken 5224 times.
 39446 for (unsigned i = 0; i < blocks_.size(); ++i) {
763
2/2
✓ Branch 0 taken 337262 times.
✓ Branch 1 taken 3 times.
 337265 for (unsigned j = 0; j < kBlockSize; ++j) {
764
2/2
✓ Branch 2 taken 34219 times.
✓ Branch 3 taken 303043 times.
 337262 if (!IsUsed(&(blocks_[i][j]))) {
765 34219 blocks_[i][j].data = const_cast<char *>(header);
766 34219 return &(blocks_[i][j]);
767 }
768 }
769 }
770
771 // All used, new block
772 5224 AddBlock();
773 5224 blocks_[blocks_.size() - 1][0].data = const_cast<char *>(header);
774 5224 return &(blocks_[blocks_.size() - 1][0]);
775 }
776
777
778 1360380 void HeaderLists::Put(curl_slist *slist) {
779 1360380 slist->data = NULL;
780 1360380 slist->next = NULL;
781 1360380 }
782
783
784 5224 void HeaderLists::AddBlock() {
785
1/2
✓ Branch 1 taken 5224 times.
✗ Branch 2 not taken.
 5224 curl_slist *new_block = new curl_slist[kBlockSize];
786
2/2
✓ Branch 0 taken 1337344 times.
✓ Branch 1 taken 5224 times.
 1342568 for (unsigned i = 0; i < kBlockSize; ++i) {
787 1337344 Put(&new_block[i]);
788 }
789
1/2
✓ Branch 1 taken 5224 times.
✗ Branch 2 not taken.
 5224 blocks_.push_back(new_block);
790 5224 }
791
792
793 //------------------------------------------------------------------------------
794
795
796 ✗ string DownloadManager::ProxyInfo::Print() {
797 ✗ if (url == "DIRECT")
798 ✗ return url;
799
800 ✗ string result = url;
801 ✗ const int remaining = static_cast<int>(host.deadline())
802 ✗ - static_cast<int>(time(NULL));
803 ✗ string expinfo = (remaining >= 0) ? "+" : "";
804 ✗ if (abs(remaining) >= 3600) {
805 ✗ expinfo += StringifyInt(remaining / 3600) + "h";
806 ✗ } else if (abs(remaining) >= 60) {
807 ✗ expinfo += StringifyInt(remaining / 60) + "m";
808 } else {
809 ✗ expinfo += StringifyInt(remaining) + "s";
810 }
811 ✗ if (host.status() == dns::kFailOk) {
812 ✗ result += " (" + host.name() + ", " + expinfo + ")";
813 } else {
814 ✗ result += " (:unresolved:, " + expinfo + ")";
815 }
816 ✗ return result;
817 }
818
819
820 /**
821 * Gets an idle CURL handle from the pool. Creates a new one and adds it to
822 * the pool if necessary.
823 */
824 7268 CURL *DownloadManager::AcquireCurlHandle() {
825 CURL *handle;
826
827
2/2
✓ Branch 1 taken 2839 times.
✓ Branch 2 taken 4429 times.
 7268 if (pool_handles_idle_->empty()) {
828 // Create a new handle
829
1/2
✓ Branch 1 taken 2839 times.
✗ Branch 2 not taken.
 2839 handle = curl_easy_init();
830
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2839 times.
 2839 assert(handle != NULL);
831
832
1/2
✓ Branch 1 taken 2839 times.
✗ Branch 2 not taken.
 2839 curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1);
833 // curl_easy_setopt(curl_default, CURLOPT_FAILONERROR, 1);
834
1/2
✓ Branch 1 taken 2839 times.
✗ Branch 2 not taken.
 2839 curl_easy_setopt(handle, CURLOPT_HEADERFUNCTION, CallbackCurlHeader);
835
1/2
✓ Branch 1 taken 2839 times.
✗ Branch 2 not taken.
 2839 curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, CallbackCurlData);
836 } else {
837 4429 handle = *(pool_handles_idle_->begin());
838
1/2
✓ Branch 2 taken 4429 times.
✗ Branch 3 not taken.
 4429 pool_handles_idle_->erase(pool_handles_idle_->begin());
839 }
840
841
1/2
✓ Branch 1 taken 7268 times.
✗ Branch 2 not taken.
 7268 pool_handles_inuse_->insert(handle);
842
843 7268 return handle;
844 }
845
846
847 7268 void DownloadManager::ReleaseCurlHandle(CURL *handle) {
848
1/2
✓ Branch 1 taken 7268 times.
✗ Branch 2 not taken.
 7268 const set<CURL *>::iterator elem = pool_handles_inuse_->find(handle);
849
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 7268 times.
 7268 assert(elem != pool_handles_inuse_->end());
850
851
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 7268 times.
 7268 if (pool_handles_idle_->size() > pool_max_handles_) {
852 ✗ curl_easy_cleanup(*elem);
853 } else {
854
1/2
✓ Branch 2 taken 7268 times.
✗ Branch 3 not taken.
 7268 pool_handles_idle_->insert(*elem);
855 }
856
857
1/2
✓ Branch 1 taken 7268 times.
✗ Branch 2 not taken.
 7268 pool_handles_inuse_->erase(elem);
858 7268 }
859
860
861 /**
862 * HTTP request options: set the URL and other options such as timeout and
863 * proxy.
864 */
865 7268 void DownloadManager::InitializeRequest(JobInfo *info, CURL *handle) {
866 // Initialize internal download state
867 7268 info->SetCurlHandle(handle);
868 7268 info->SetErrorCode(kFailOk);
869 7268 info->SetHttpCode(-1);
870 7268 info->SetFollowRedirects(follow_redirects_);
871 7268 info->SetNumUsedProxies(1);
872 7268 info->SetNumUsedMetalinks(1);
873 7268 info->SetNumUsedHosts(1);
874 7268 info->SetNumRetries(0);
875 7268 info->SetBackoffMs(0);
876 7268 info->SetHeaders(header_lists_->DuplicateList(default_headers_));
877
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 7268 times.
 7268 if (info->info_header()) {
878 ✗ header_lists_->AppendHeader(info->headers(), info->info_header());
879 }
880
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7268 times.
 7268 if (enable_http_tracing_) {
881 ✗ for (unsigned int i = 0; i < http_tracing_headers_.size(); i++) {
882 ✗ header_lists_->AppendHeader(info->headers(),
883 ✗ (http_tracing_headers_)[i].c_str());
884 }
885
886 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_pid());
887 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_gid());
888 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_uid());
889
890 ✗ LogCvmfs(kLogDownload, kLogDebug,
891 "(manager '%s' - id %" PRId64 ") "
892 "CURL Header for URL: %s is:\n %s",
893 name_.c_str(), info->id(), info->url()->c_str(),
894 ✗ header_lists_->Print(info->headers()).c_str());
895 }
896
897
2/2
✓ Branch 1 taken 99 times.
✓ Branch 2 taken 7169 times.
 7268 if (info->force_nocache()) {
898 99 SetNocache(info);
899 } else {
900 7169 info->SetNocache(false);
901 }
902
2/2
✓ Branch 1 taken 4962 times.
✓ Branch 2 taken 2306 times.
 7268 if (info->compressed()) {
903 4962 zlib::DecompressInit(info->GetZstreamPtr());
904 }
905
2/2
✓ Branch 1 taken 5011 times.
✓ Branch 2 taken 2257 times.
 7268 if (info->expected_hash()) {
906
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5011 times.
 5011 assert(info->hash_context().buffer != NULL);
907 5011 shash::Init(info->hash_context());
908 }
909
910
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 7268 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 7268 times.
 7268 if ((info->range_offset() != -1) && (info->range_size())) {
911 char byte_range_array[100];
912 ✗ const int64_t range_lower = static_cast<int64_t>(info->range_offset());
913 ✗ const int64_t range_upper = static_cast<int64_t>(info->range_offset()
914 ✗ + info->range_size() - 1);
915 ✗ if (snprintf(byte_range_array, sizeof(byte_range_array),
916 "%" PRId64 "-%" PRId64, range_lower, range_upper)
917 ✗ == 100) {
918 ✗ PANIC(NULL); // Should be impossible given limits on offset size.
919 }
920 ✗ curl_easy_setopt(handle, CURLOPT_RANGE, byte_range_array);
921 } else {
922 7268 curl_easy_setopt(handle, CURLOPT_RANGE, NULL);
923 }
924
925 // Set curl parameters
926 7268 curl_easy_setopt(handle, CURLOPT_PRIVATE, static_cast<void *>(info));
927 7268 curl_easy_setopt(handle, CURLOPT_WRITEHEADER, static_cast<void *>(info));
928 7268 curl_easy_setopt(handle, CURLOPT_WRITEDATA, static_cast<void *>(info));
929 7268 curl_easy_setopt(handle, CURLOPT_HTTPHEADER, info->headers());
930
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 7268 times.
 7268 if (info->head_request()) {
931 ✗ curl_easy_setopt(handle, CURLOPT_NOBODY, 1);
932 } else {
933 7268 curl_easy_setopt(handle, CURLOPT_HTTPGET, 1);
934 }
935
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7268 times.
 7268 if (opt_ipv4_only_) {
936 ✗ curl_easy_setopt(handle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
937 }
938
2/2
✓ Branch 0 taken 2320 times.
✓ Branch 1 taken 4948 times.
 7268 if (follow_redirects_) {
939 2320 curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1);
940 2320 curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 4);
941 }
942 #ifdef DEBUGMSG
943 7268 curl_easy_setopt(handle, CURLOPT_VERBOSE, 1);
944 7268 curl_easy_setopt(handle, CURLOPT_DEBUGFUNCTION, CallbackCurlDebug);
945 #endif
946 7268 }
947
948 14740 void DownloadManager::CheckHostInfoReset(const std::string &typ,
949 HostInfo &info,
950 JobInfo *jobinfo,
951 time_t &now) {
952
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 14740 times.
 14740 if (info.timestamp_backup > 0) {
953 ✗ if (now == 0)
954 ✗ now = time(NULL);
955 ✗ if (static_cast<int64_t>(now)
956 ✗ > static_cast<int64_t>(info.timestamp_backup + info.reset_after)) {
957 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
958 "(manager %s - id %" PRId64 ") "
959 "switching %s from %s to %s (reset %s)",
960 name_.c_str(), jobinfo->id(), typ.c_str(),
961 ✗ (*info.chain)[info.current].c_str(), (*info.chain)[0].c_str(),
962 typ.c_str());
963 ✗ info.current = 0;
964 ✗ info.timestamp_backup = 0;
965 }
966 }
967 14740 }
968
969
970 /**
971 * Sets the URL specific options such as host to use and timeout. It might also
972 * set an error code, in which case the further processing should react on.
973 */
974 7370 void DownloadManager::SetUrlOptions(JobInfo *info) {
975 7370 CURL *curl_handle = info->curl_handle();
976 7370 string url_prefix;
977 7370 time_t now = 0;
978
979 7370 const MutexLockGuard m(lock_options_);
980
981 // sharding policy
982
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 7370 times.
 7370 if (sharding_policy_.UseCount() > 0) {
983 ✗ if (info->proxy() != "") {
984 // proxy already set, so this is a failover event
985 ✗ perf::Inc(counters_->n_proxy_failover);
986 }
987 ✗ info->SetProxy(sharding_policy_->GetNextProxy(
988 ✗ info->url(), info->proxy(),
989 ✗ info->range_offset() == -1 ? 0 : info->range_offset()));
990
991 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, info->proxy().c_str());
992 } else { // no sharding policy
993 // Check if proxy group needs to be reset from backup to primary
994
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7370 times.
 7370 if (opt_timestamp_backup_proxies_ > 0) {
995 ✗ now = time(NULL);
996 ✗ if (static_cast<int64_t>(now) > static_cast<int64_t>(
997 ✗ opt_timestamp_backup_proxies_ + opt_proxy_groups_reset_after_)) {
998 ✗ opt_proxy_groups_current_ = 0;
999 ✗ opt_timestamp_backup_proxies_ = 0;
1000 ✗ RebalanceProxiesUnlocked("Reset proxy group from backup to primary");
1001 }
1002 }
1003 // Check if load-balanced proxies within the group need to be reset
1004
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7370 times.
 7370 if (opt_timestamp_failover_proxies_ > 0) {
1005 ✗ if (now == 0)
1006 ✗ now = time(NULL);
1007 ✗ if (static_cast<int64_t>(now)
1008 ✗ > static_cast<int64_t>(opt_timestamp_failover_proxies_
1009 ✗ + opt_proxy_groups_reset_after_)) {
1010 ✗ RebalanceProxiesUnlocked(
1011 "Reset load-balanced proxies within the active group");
1012 }
1013 }
1014
1015
1/2
✓ Branch 2 taken 7370 times.
✗ Branch 3 not taken.
 7370 ProxyInfo *proxy = ChooseProxyUnlocked(info->expected_hash());
1016
6/6
✓ Branch 0 taken 2671 times.
✓ Branch 1 taken 4699 times.
✓ Branch 3 taken 2535 times.
✓ Branch 4 taken 136 times.
✓ Branch 5 taken 7234 times.
✓ Branch 6 taken 136 times.
 7370 if (!proxy || (proxy->url == "DIRECT")) {
1017
2/4
✓ Branch 2 taken 7234 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 7234 times.
✗ Branch 6 not taken.
 7234 info->SetProxy("DIRECT");
1018
1/2
✓ Branch 2 taken 7234 times.
✗ Branch 3 not taken.
 7234 curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "");
1019 } else {
1020 // Note: inside ValidateProxyIpsUnlocked() we may change the proxy data
1021 // structure, so we must not pass proxy->... (== current_proxy())
1022 // parameters directly
1023
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 const std::string purl = proxy->url;
1024
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 const dns::Host phost = proxy->host;
1025
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 const bool changed = ValidateProxyIpsUnlocked(purl, phost);
1026 // Current proxy may have changed
1027
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 136 times.
 136 if (changed) {
1028 ✗ proxy = ChooseProxyUnlocked(info->expected_hash());
1029 }
1030
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 info->SetProxy(proxy->url);
1031
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 if (proxy->host.status() == dns::kFailOk) {
1032
2/4
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
✓ Branch 6 taken 136 times.
✗ Branch 7 not taken.
 136 curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY,
1033 info->proxy().c_str());
1034 } else {
1035 // We know it can't work, don't even try to download
1036 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "0.0.0.0");
1037 }
1038 136 }
1039 } // end !sharding
1040
1041 // Check if metalink and host chains need to be reset
1042
2/4
✓ Branch 2 taken 7370 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 7370 times.
✗ Branch 6 not taken.
 7370 CheckHostInfoReset("metalink", opt_metalink_, info, now);
1043
2/4
✓ Branch 2 taken 7370 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 7370 times.
✗ Branch 6 not taken.
 7370 CheckHostInfoReset("host", opt_host_, info, now);
1044
1045
1/2
✓ Branch 1 taken 7370 times.
✗ Branch 2 not taken.
 7370 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_LIMIT, opt_low_speed_limit_);
1046
3/6
✓ Branch 1 taken 7370 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 136 times.
✓ Branch 6 taken 7234 times.
 7370 if (info->proxy() != "DIRECT") {
1047
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT, opt_timeout_proxy_);
1048
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_TIME, opt_timeout_proxy_);
1049 } else {
1050
1/2
✓ Branch 1 taken 7234 times.
✗ Branch 2 not taken.
 7234 curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT, opt_timeout_direct_);
1051
1/2
✓ Branch 1 taken 7234 times.
✗ Branch 2 not taken.
 7234 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_TIME, opt_timeout_direct_);
1052 }
1053
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 7370 times.
 7370 if (!opt_dns_server_.empty())
1054 ✗ curl_easy_setopt(curl_handle, CURLOPT_DNS_SERVERS, opt_dns_server_.c_str());
1055
1056
2/2
✓ Branch 1 taken 3421 times.
✓ Branch 2 taken 3949 times.
 7370 if (info->probe_hosts()) {
1057
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 3421 times.
 3421 if (CheckMetalinkChain(now)) {
1058 ✗ url_prefix = (*opt_metalink_.chain)[opt_metalink_.current];
1059 ✗ info->SetCurrentMetalinkChainIndex(opt_metalink_.current);
1060 ✗ LogCvmfs(kLogDownload, kLogDebug,
1061 "(manager %s - id %" PRId64 ") "
1062 "reading from metalink %d",
1063 name_.c_str(), info->id(), opt_metalink_.current);
1064
1/2
✓ Branch 0 taken 3421 times.
✗ Branch 1 not taken.
 3421 } else if (opt_host_.chain) {
1065
1/2
✓ Branch 2 taken 3421 times.
✗ Branch 3 not taken.
 3421 url_prefix = (*opt_host_.chain)[opt_host_.current];
1066 3421 info->SetCurrentHostChainIndex(opt_host_.current);
1067
1/2
✓ Branch 3 taken 3421 times.
✗ Branch 4 not taken.
 3421 LogCvmfs(kLogDownload, kLogDebug,
1068 "(manager %s - id %" PRId64 ") "
1069 "reading from host %d",
1070 name_.c_str(), info->id(), opt_host_.current);
1071 }
1072 }
1073
1074
1/2
✓ Branch 2 taken 7370 times.
✗ Branch 3 not taken.
 7370 string url = url_prefix + *(info->url());
1075
1076
1/2
✓ Branch 1 taken 7370 times.
✗ Branch 2 not taken.
 7370 curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, 1L);
1077
2/6
✓ Branch 1 taken 7370 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 7370 times.
 7370 if (url.substr(0, 5) == "https") {
1078 ✗ const bool rvb = ssl_certificate_store_.ApplySslCertificatePath(
1079 curl_handle);
1080 ✗ if (!rvb) {
1081 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1082 "(manager %s - id %" PRId64 ") "
1083 "Failed to set SSL certificate path %s",
1084 name_.c_str(), info->id(),
1085 ✗ ssl_certificate_store_.GetCaPath().c_str());
1086 }
1087 ✗ if (info->pid() != -1) {
1088 ✗ if (credentials_attachment_ == NULL) {
1089 ✗ LogCvmfs(kLogDownload, kLogDebug,
1090 "(manager %s - id %" PRId64 ") "
1091 "uses secure downloads but no credentials attachment set",
1092 name_.c_str(), info->id());
1093 } else {
1094 ✗ const bool retval = credentials_attachment_->ConfigureCurlHandle(
1095 curl_handle, info->pid(), info->GetCredDataPtr());
1096 ✗ if (!retval) {
1097 ✗ LogCvmfs(kLogDownload, kLogDebug,
1098 "(manager %s - id %" PRId64 ") "
1099 "failed attaching credentials",
1100 name_.c_str(), info->id());
1101 }
1102 }
1103 }
1104 // The download manager disables signal handling in the curl library;
1105 // as OpenSSL's implementation of TLS will generate a sigpipe in some
1106 // error paths, we must explicitly disable SIGPIPE here.
1107 // TODO(jblomer): it should be enough to do this once
1108 ✗ signal(SIGPIPE, SIG_IGN);
1109 }
1110
1111
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 7370 times.
 7370 if (url.find("@proxy@") != string::npos) {
1112 // This is used in Geo-API requests (only), to replace a portion of the
1113 // URL with the current proxy name for the sake of caching the result.
1114 // Replace the @proxy@ either with a passed in "forced" template (which
1115 // is set from $CVMFS_PROXY_TEMPLATE) if there is one, or a "direct"
1116 // template (which is the uuid) if there's no proxy, or the name of the
1117 // proxy.
1118 ✗ string replacement;
1119 ✗ if (proxy_template_forced_ != "") {
1120 ✗ replacement = proxy_template_forced_;
1121 ✗ } else if (info->proxy() == "DIRECT") {
1122 ✗ replacement = proxy_template_direct_;
1123 } else {
1124 ✗ if (opt_proxy_groups_current_ >= opt_proxy_groups_fallback_) {
1125 // It doesn't make sense to use the fallback proxies in Geo-API requests
1126 // since the fallback proxies are supposed to get sorted, too.
1127 ✗ info->SetProxy("DIRECT");
1128 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "");
1129 ✗ replacement = proxy_template_direct_;
1130 } else {
1131 ✗ replacement = ChooseProxyUnlocked(info->expected_hash())->host.name();
1132 }
1133 }
1134 ✗ replacement = (replacement == "") ? proxy_template_direct_ : replacement;
1135 ✗ LogCvmfs(kLogDownload, kLogDebug,
1136 "(manager %s - id %" PRId64 ") "
1137 "replacing @proxy@ by %s",
1138 name_.c_str(), info->id(), replacement.c_str());
1139 ✗ url = ReplaceAll(url, "@proxy@", replacement);
1140 }
1141
1142 // TODO(heretherebedragons) before removing
1143 // static_cast<cvmfs::MemSink*>(info->sink)->size() == 0
1144 // and just always call info->sink->Reserve()
1145 // we should do a speed check
1146
3/4
✓ Branch 2 taken 7370 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2779 times.
✓ Branch 5 taken 4591 times.
 7370 if ((info->sink() != NULL) && info->sink()->RequiresReserve()
1147
1/2
✓ Branch 2 taken 2779 times.
✗ Branch 3 not taken.
 2779 && (static_cast<cvmfs::MemSink *>(info->sink())->size() == 0)
1148
11/20
✓ Branch 1 taken 7370 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 2779 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 2779 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 2336 times.
✓ Branch 11 taken 443 times.
✓ Branch 12 taken 2779 times.
✓ Branch 13 taken 4591 times.
✗ Branch 14 not taken.
✓ Branch 15 taken 2779 times.
✓ Branch 16 taken 4591 times.
✓ Branch 18 taken 2336 times.
✓ Branch 19 taken 5034 times.
✗ Branch 20 not taken.
✗ Branch 21 not taken.
✗ Branch 23 not taken.
✗ Branch 24 not taken.
 14740 && HasPrefix(url, "file://", false)) {
1149 platform_stat64 stat_buf;
1150 2336 const int retval = platform_stat(url.c_str(), &stat_buf);
1151
1/2
✓ Branch 0 taken 2336 times.
✗ Branch 1 not taken.
 2336 if (retval != 0) {
1152 // this is an error: file does not exist or out of memory
1153 // error is caught in other code section.
1154
1/2
✓ Branch 2 taken 2336 times.
✗ Branch 3 not taken.
 2336 info->sink()->Reserve(64ul * 1024ul);
1155 } else {
1156 ✗ info->sink()->Reserve(stat_buf.st_size);
1157 }
1158 }
1159
1160
2/4
✓ Branch 2 taken 7370 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 7370 times.
✗ Branch 7 not taken.
 7370 curl_easy_setopt(curl_handle, CURLOPT_URL,
1161 EscapeUrl(info->id(), url).c_str());
1162 7370 }
1163
1164
1165 /**
1166 * Checks if the name resolving information is still up to date. The host
1167 * object should be one from the current load-balance group. If the information
1168 * changed, gather new set of resolved IPs and, if different, exchange them in
1169 * the load-balance group on the fly. In the latter case, also rebalance the
1170 * proxies. The options mutex needs to be open.
1171 *
1172 * Returns true if proxies may have changed.
1173 */
1174 136 bool DownloadManager::ValidateProxyIpsUnlocked(const string &url,
1175 const dns::Host &host) {
1176
2/4
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 136 times.
✗ Branch 4 not taken.
 136 if (!host.IsExpired())
1177 136 return false;
1178 ✗ LogCvmfs(kLogDownload, kLogDebug, "(manager '%s') validate DNS entry for %s",
1179 ✗ name_.c_str(), host.name().c_str());
1180
1181 ✗ const unsigned group_idx = opt_proxy_groups_current_;
1182 ✗ dns::Host new_host = resolver_->Resolve(host.name());
1183
1184 ✗ bool update_only = true; // No changes to the list of IP addresses.
1185 ✗ if (new_host.status() != dns::kFailOk) {
1186 // Try again later in case resolving fails.
1187 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1188 "(manager '%s') failed to resolve IP addresses for %s (%d - %s)",
1189 ✗ name_.c_str(), host.name().c_str(), new_host.status(),
1190 dns::Code2Ascii(new_host.status()));
1191 ✗ new_host = dns::Host::ExtendDeadline(host, resolver_->min_ttl());
1192 ✗ } else if (!host.IsEquivalent(new_host)) {
1193 ✗ update_only = false;
1194 }
1195
1196 ✗ if (update_only) {
1197 ✗ for (unsigned i = 0; i < (*opt_proxy_groups_)[group_idx].size(); ++i) {
1198 ✗ if ((*opt_proxy_groups_)[group_idx][i].host.id() == host.id())
1199 ✗ (*opt_proxy_groups_)[group_idx][i].host = new_host;
1200 }
1201 ✗ return false;
1202 }
1203
1204 ✗ assert(new_host.status() == dns::kFailOk);
1205
1206 // Remove old host objects, insert new objects, and rebalance.
1207 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslog,
1208 "(manager '%s') DNS entries for proxy %s changed, adjusting",
1209 ✗ name_.c_str(), host.name().c_str());
1210 ✗ vector<ProxyInfo> *group = current_proxy_group();
1211 ✗ opt_num_proxies_ -= group->size();
1212 ✗ for (unsigned i = 0; i < group->size();) {
1213 ✗ if ((*group)[i].host.id() == host.id()) {
1214 ✗ group->erase(group->begin() + i);
1215 } else {
1216 ✗ i++;
1217 }
1218 }
1219 ✗ vector<ProxyInfo> new_infos;
1220 ✗ set<string> best_addresses = new_host.ViewBestAddresses(opt_ip_preference_);
1221 ✗ set<string>::const_iterator iter_ips = best_addresses.begin();
1222 ✗ for (; iter_ips != best_addresses.end(); ++iter_ips) {
1223 ✗ const string url_ip = dns::RewriteUrl(url, *iter_ips);
1224 ✗ new_infos.push_back(ProxyInfo(new_host, url_ip));
1225 }
1226 ✗ group->insert(group->end(), new_infos.begin(), new_infos.end());
1227 ✗ opt_num_proxies_ += new_infos.size();
1228
1229 ✗ const std::string msg = "DNS entries for proxy " + host.name() + " changed";
1230
1231 ✗ RebalanceProxiesUnlocked(msg);
1232 ✗ return true;
1233 }
1234
1235
1236 /**
1237 * Adds transfer time and downloaded bytes to the global counters.
1238 */
1239 7657 void DownloadManager::UpdateStatistics(CURL *handle) {
1240 double val;
1241 int retval;
1242 7657 int64_t sum = 0;
1243
1244
1/2
✓ Branch 1 taken 7657 times.
✗ Branch 2 not taken.
 7657 retval = curl_easy_getinfo(handle, CURLINFO_SIZE_DOWNLOAD, &val);
1245
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7657 times.
 7657 assert(retval == CURLE_OK);
1246 7657 sum += static_cast<int64_t>(val);
1247 /*retval = curl_easy_getinfo(handle, CURLINFO_HEADER_SIZE, &val);
1248 assert(retval == CURLE_OK);
1249 sum += static_cast<int64_t>(val);*/
1250 7657 perf::Xadd(counters_->sz_transferred_bytes, sum);
1251 7657 }
1252
1253
1254 /**
1255 * Retry if possible if not on no-cache and if not already done too often.
1256 */
1257 7657 bool DownloadManager::CanRetry(const JobInfo *info) {
1258 7657 const MutexLockGuard m(lock_options_);
1259 7657 const unsigned max_retries = opt_max_retries_;
1260
1261
2/2
✓ Branch 2 taken 2717 times.
✓ Branch 3 taken 4726 times.
 15100 return !(info->nocache()) && (info->num_retries() < max_retries)
1262
3/4
✓ Branch 0 taken 7443 times.
✓ Branch 1 taken 214 times.
✓ Branch 4 taken 2717 times.
✗ Branch 5 not taken.
 17817 && (IsProxyTransferError(info->error_code())
1263
2/2
✓ Branch 2 taken 172 times.
✓ Branch 3 taken 2545 times.
 2717 || IsHostTransferError(info->error_code()));
1264 7657 }
1265
1266 /**
1267 * Backoff for retry to introduce a jitter into a cluster of requesting
1268 * cvmfs nodes.
1269 * Retry only when HTTP caching is on.
1270 *
1271 * \return true if backoff has been performed, false otherwise
1272 */
1273 172 void DownloadManager::Backoff(JobInfo *info) {
1274 172 unsigned backoff_init_ms = 0;
1275 172 unsigned backoff_max_ms = 0;
1276 {
1277 172 const MutexLockGuard m(lock_options_);
1278 172 backoff_init_ms = opt_backoff_init_ms_;
1279 172 backoff_max_ms = opt_backoff_max_ms_;
1280 172 }
1281
1282 172 info->SetNumRetries(info->num_retries() + 1);
1283 172 perf::Inc(counters_->n_retries);
1284
2/2
✓ Branch 1 taken 69 times.
✓ Branch 2 taken 103 times.
 172 if (info->backoff_ms() == 0) {
1285 69 info->SetBackoffMs(prng_.Next(backoff_init_ms + 1)); // Must be != 0
1286 } else {
1287 103 info->SetBackoffMs(info->backoff_ms() * 2);
1288 }
1289
2/2
✓ Branch 1 taken 34 times.
✓ Branch 2 taken 138 times.
 172 if (info->backoff_ms() > backoff_max_ms) {
1290 34 info->SetBackoffMs(backoff_max_ms);
1291 }
1292
1293 172 LogCvmfs(kLogDownload, kLogDebug,
1294 "(manager '%s' - id %" PRId64 ") backing off for %d ms",
1295 name_.c_str(), info->id(), info->backoff_ms());
1296 172 SafeSleepMs(info->backoff_ms());
1297 172 }
1298
1299 214 void DownloadManager::SetNocache(JobInfo *info) {
1300
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 214 times.
 214 if (info->nocache())
1301 ✗ return;
1302 214 header_lists_->AppendHeader(info->headers(), "Pragma: no-cache");
1303 214 header_lists_->AppendHeader(info->headers(), "Cache-Control: no-cache");
1304 214 curl_easy_setopt(info->curl_handle(), CURLOPT_HTTPHEADER, info->headers());
1305 214 info->SetNocache(true);
1306 }
1307
1308
1309 /**
1310 * Reverse operation of SetNocache. Makes sure that "no-cache" header
1311 * disappears from the list of headers to let proxies work normally.
1312 */
1313 389 void DownloadManager::SetRegularCache(JobInfo *info) {
1314
1/2
✓ Branch 1 taken 389 times.
✗ Branch 2 not taken.
 389 if (info->nocache() == false)
1315 389 return;
1316 ✗ header_lists_->CutHeader("Pragma: no-cache", info->GetHeadersPtr());
1317 ✗ header_lists_->CutHeader("Cache-Control: no-cache", info->GetHeadersPtr());
1318 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_HTTPHEADER, info->headers());
1319 ✗ info->SetNocache(false);
1320 }
1321
1322
1323 /**
1324 * Frees the storage associated with the authz attachment from the job
1325 */
1326 7370 void DownloadManager::ReleaseCredential(JobInfo *info) {
1327
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 7370 times.
 7370 if (info->cred_data()) {
1328 ✗ assert(credentials_attachment_ != NULL); // Someone must have set it
1329 ✗ credentials_attachment_->ReleaseCurlHandle(info->curl_handle(),
1330 info->cred_data());
1331 ✗ info->SetCredData(NULL);
1332 }
1333 7370 }
1334
1335
1336 /* Sort links based on the "pri=" parameter */
1337 ✗ static bool sortlinks(const std::string &s1, const std::string &s2) {
1338 ✗ const size_t pos1 = s1.find("; pri=");
1339 ✗ const size_t pos2 = s2.find("; pri=");
1340 int pri1, pri2;
1341 ✗ if ((pos1 != std::string::npos) && (pos2 != std::string::npos)
1342 ✗ && (sscanf(s1.substr(pos1 + 6).c_str(), "%d", &pri1) == 1)
1343 ✗ && (sscanf(s2.substr(pos2 + 6).c_str(), "%d", &pri2) == 1)) {
1344 ✗ return pri1 < pri2;
1345 }
1346 ✗ return false;
1347 }
1348
1349 /**
1350 * Parses Link header and uses it to set a new host chain.
1351 * See rfc6249.
1352 */
1353 ✗ void DownloadManager::ProcessLink(JobInfo *info) {
1354 ✗ std::vector<std::string> links = SplitString(info->link(), ',');
1355 ✗ if (info->link().find("; pri=") != std::string::npos)
1356 ✗ std::sort(links.begin(), links.end(), sortlinks);
1357
1358 ✗ std::vector<std::string> host_list;
1359
1360 ✗ std::vector<std::string>::const_iterator il = links.begin();
1361 ✗ for (; il != links.end(); ++il) {
1362 ✗ const std::string &link = *il;
1363 ✗ if ((link.find("; rel=duplicate") == std::string::npos)
1364 ✗ && (link.find("; rel=\"duplicate\"") == std::string::npos)) {
1365 ✗ LogCvmfs(kLogDownload, kLogDebug,
1366 "skipping link '%s' because it does not contain rel=duplicate",
1367 link.c_str());
1368 ✗ continue;
1369 }
1370 // ignore depth= field since there's nothing useful we can do with it
1371
1372 ✗ size_t start = link.find('<');
1373 ✗ if (start == std::string::npos) {
1374 ✗ LogCvmfs(
1375 kLogDownload, kLogDebug,
1376 "skipping link '%s' because it does not have a left angle bracket",
1377 link.c_str());
1378 ✗ continue;
1379 }
1380
1381 ✗ start++;
1382 ✗ if ((link.substr(start, 7) != "http://")
1383 ✗ && (link.substr(start, 8) != "https://")) {
1384 ✗ LogCvmfs(kLogDownload, kLogDebug,
1385 "skipping link '%s' of unrecognized url protocol", link.c_str());
1386 ✗ continue;
1387 }
1388
1389 ✗ size_t end = link.find('/', start + 8);
1390 ✗ if (end == std::string::npos)
1391 ✗ end = link.find('>');
1392 ✗ if (end == std::string::npos) {
1393 ✗ LogCvmfs(kLogDownload, kLogDebug,
1394 "skipping link '%s' because no slash in url and no right angle "
1395 "bracket",
1396 link.c_str());
1397 ✗ continue;
1398 }
1399 ✗ const std::string host = link.substr(start, end - start);
1400 ✗ LogCvmfs(kLogDownload, kLogDebug, "adding linked host '%s'", host.c_str());
1401 ✗ host_list.push_back(host);
1402 }
1403
1404 ✗ if (host_list.size() > 0) {
1405 ✗ SetHostChain(host_list);
1406 ✗ opt_metalink_timestamp_link_ = time(NULL);
1407 }
1408 }
1409
1410
1411 /**
1412 * Checks the result of a curl download and implements the failure logic, such
1413 * as changing the proxy server. Takes care of cleanup.
1414 *
1415 * \return true if another download should be performed, false otherwise
1416 */
1417 7657 bool DownloadManager::VerifyAndFinalize(const int curl_error, JobInfo *info) {
1418
1/2
✓ Branch 6 taken 7657 times.
✗ Branch 7 not taken.
 7657 LogCvmfs(kLogDownload, kLogDebug,
1419 "(manager '%s' - id %" PRId64 ") "
1420 "Verify downloaded url %s, proxy %s (curl error %d)",
1421 name_.c_str(), info->id(), info->url()->c_str(),
1422
1/2
✓ Branch 1 taken 7657 times.
✗ Branch 2 not taken.
 15314 info->proxy().c_str(), curl_error);
1423
1/2
✓ Branch 2 taken 7657 times.
✗ Branch 3 not taken.
 7657 UpdateStatistics(info->curl_handle());
1424
1425 bool was_metalink;
1426 7657 std::string typ;
1427
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 7657 times.
 7657 if (info->current_metalink_chain_index() >= 0) {
1428 ✗ was_metalink = true;
1429 ✗ typ = "metalink";
1430 ✗ if (info->link() != "") {
1431 // process Link header whether or not the redirected URL got an error
1432 ✗ ProcessLink(info);
1433 }
1434 } else {
1435 7657 was_metalink = false;
1436
1/2
✓ Branch 1 taken 7657 times.
✗ Branch 2 not taken.
 7657 typ = "host";
1437 }
1438
1439
1440 // Verification and error classification
1441
3/14
✓ Branch 0 taken 6901 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 98 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 658 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
 7657 switch (curl_error) {
1442 6901 case CURLE_OK:
1443 // Verify content hash
1444
2/2
✓ Branch 1 taken 4797 times.
✓ Branch 2 taken 2104 times.
 6901 if (info->expected_hash()) {
1445
1/2
✓ Branch 1 taken 4797 times.
✗ Branch 2 not taken.
 4797 shash::Any match_hash;
1446
1/2
✓ Branch 2 taken 4797 times.
✗ Branch 3 not taken.
 4797 shash::Final(info->hash_context(), &match_hash);
1447
3/4
✓ Branch 2 taken 4797 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 230 times.
✓ Branch 5 taken 4567 times.
 4797 if (match_hash != *(info->expected_hash())) {
1448
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 230 times.
 230 if (ignore_signature_failures_) {
1449 ✗ LogCvmfs(
1450 kLogDownload, kLogDebug | kLogSyslogErr,
1451 "(manager '%s' - id %" PRId64 ") "
1452 "ignoring failed hash verification of %s (expected %s, got %s)",
1453 name_.c_str(), info->id(), info->url()->c_str(),
1454 ✗ info->expected_hash()->ToString().c_str(),
1455 ✗ match_hash.ToString().c_str());
1456 } else {
1457
1/2
✓ Branch 7 taken 230 times.
✗ Branch 8 not taken.
 460 LogCvmfs(kLogDownload, kLogDebug,
1458 "(manager '%s' - id %" PRId64 ") "
1459 "hash verification of %s failed (expected %s, got %s)",
1460 name_.c_str(), info->id(), info->url()->c_str(),
1461
1/2
✓ Branch 2 taken 230 times.
✗ Branch 3 not taken.
 460 info->expected_hash()->ToString().c_str(),
1462
1/2
✓ Branch 1 taken 230 times.
✗ Branch 2 not taken.
 460 match_hash.ToString().c_str());
1463 230 info->SetErrorCode(kFailBadData);
1464 230 break;
1465 }
1466 }
1467 }
1468
1469 6671 info->SetErrorCode(kFailOk);
1470 6671 break;
1471 ✗ case CURLE_UNSUPPORTED_PROTOCOL:
1472 ✗ info->SetErrorCode(kFailUnsupportedProtocol);
1473 ✗ break;
1474 98 case CURLE_URL_MALFORMAT:
1475 98 info->SetErrorCode(kFailBadUrl);
1476 98 break;
1477 ✗ case CURLE_COULDNT_RESOLVE_PROXY:
1478 ✗ info->SetErrorCode(kFailProxyResolve);
1479 ✗ break;
1480 ✗ case CURLE_COULDNT_RESOLVE_HOST:
1481 ✗ info->SetErrorCode(kFailHostResolve);
1482 ✗ break;
1483 ✗ case CURLE_OPERATION_TIMEDOUT:
1484 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostTooSlow
1485 : kFailProxyTooSlow);
1486 ✗ break;
1487 ✗ case CURLE_PARTIAL_FILE:
1488 case CURLE_GOT_NOTHING:
1489 case CURLE_RECV_ERROR:
1490 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostShortTransfer
1491 : kFailProxyShortTransfer);
1492 ✗ break;
1493 658 case CURLE_FILE_COULDNT_READ_FILE:
1494 case CURLE_COULDNT_CONNECT:
1495
3/6
✓ Branch 1 taken 658 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 34 times.
✓ Branch 6 taken 624 times.
 658 if (info->proxy() != "DIRECT") {
1496 // This is a guess. Fail-over can still change to switching host
1497 34 info->SetErrorCode(kFailProxyConnection);
1498 } else {
1499 624 info->SetErrorCode(kFailHostConnection);
1500 }
1501 658 break;
1502 ✗ case CURLE_TOO_MANY_REDIRECTS:
1503 ✗ info->SetErrorCode(kFailHostConnection);
1504 ✗ break;
1505 ✗ case CURLE_SSL_CACERT_BADFILE:
1506 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
1507 "(manager '%s' -id %" PRId64 ") "
1508 "Failed to load certificate bundle. "
1509 "X509_CERT_BUNDLE might point to the wrong location.",
1510 name_.c_str(), info->id());
1511 ✗ info->SetErrorCode(kFailHostConnection);
1512 ✗ break;
1513 // As of curl 7.62.0, CURLE_SSL_CACERT is the same as
1514 // CURLE_PEER_FAILED_VERIFICATION
1515 ✗ case CURLE_PEER_FAILED_VERIFICATION:
1516 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
1517 "(manager '%s' - id %" PRId64 ") "
1518 "invalid SSL certificate of remote host. "
1519 "X509_CERT_DIR and/or X509_CERT_BUNDLE might point to the wrong "
1520 "location.",
1521 name_.c_str(), info->id());
1522 ✗ info->SetErrorCode(kFailHostConnection);
1523 ✗ break;
1524 ✗ case CURLE_ABORTED_BY_CALLBACK:
1525 case CURLE_WRITE_ERROR:
1526 // Error set by callback
1527 ✗ break;
1528 ✗ case CURLE_SEND_ERROR:
1529 // The curl error CURLE_SEND_ERROR can be seen when a cache is misbehaving
1530 // and closing connections before the http request send is completed.
1531 // Handle this error, treating it as a short transfer error.
1532 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostShortTransfer
1533 : kFailProxyShortTransfer);
1534 ✗ break;
1535 ✗ default:
1536 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
1537 "(manager '%s' - id %" PRId64 ") "
1538 "unexpected curl error (%d) while trying to fetch %s",
1539 name_.c_str(), info->id(), curl_error, info->url()->c_str());
1540 ✗ info->SetErrorCode(kFailOther);
1541 ✗ break;
1542 }
1543
1544 std::vector<std::string> *host_chain;
1545 unsigned char num_used_hosts;
1546
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7657 times.
 7657 if (was_metalink) {
1547 ✗ host_chain = opt_metalink_.chain;
1548 ✗ num_used_hosts = info->num_used_metalinks();
1549 } else {
1550 7657 host_chain = opt_host_.chain;
1551 7657 num_used_hosts = info->num_used_hosts();
1552 }
1553
1554 // Determination if download should be repeated
1555 7657 bool try_again = false;
1556
1/2
✓ Branch 1 taken 7657 times.
✗ Branch 2 not taken.
 7657 bool same_url_retry = CanRetry(info);
1557
2/2
✓ Branch 1 taken 986 times.
✓ Branch 2 taken 6671 times.
 7657 if (info->error_code() != kFailOk) {
1558 986 const MutexLockGuard m(lock_options_);
1559
2/2
✓ Branch 1 taken 230 times.
✓ Branch 2 taken 756 times.
 986 if (info->error_code() == kFailBadData) {
1560
2/2
✓ Branch 1 taken 115 times.
✓ Branch 2 taken 115 times.
 230 if (!info->nocache()) {
1561 115 try_again = true;
1562 } else {
1563 // Make it a host failure
1564
1/2
✓ Branch 4 taken 115 times.
✗ Branch 5 not taken.
 115 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1565 "(manager '%s' - id %" PRId64 ") "
1566 "data corruption with no-cache header, try another %s",
1567 name_.c_str(), info->id(), typ.c_str());
1568
1569 115 info->SetErrorCode(kFailHostHttp);
1570 }
1571 }
1572 986 if (same_url_retry
1573
5/6
✓ Branch 0 taken 814 times.
✓ Branch 1 taken 172 times.
✓ Branch 3 taken 814 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 274 times.
✓ Branch 6 taken 712 times.
 1800 || (((info->error_code() == kFailHostResolve)
1574
2/2
✓ Branch 2 taken 362 times.
✓ Branch 3 taken 452 times.
 814 || IsHostTransferError(info->error_code())
1575
2/2
✓ Branch 1 taken 115 times.
✓ Branch 2 taken 247 times.
 362 || (info->error_code() == kFailHostHttp))
1576
3/4
✓ Branch 1 taken 368 times.
✓ Branch 2 taken 199 times.
✓ Branch 3 taken 368 times.
✗ Branch 4 not taken.
 567 && info->probe_hosts() && host_chain
1577
2/2
✓ Branch 1 taken 102 times.
✓ Branch 2 taken 266 times.
 368 && (num_used_hosts < host_chain->size()))) {
1578 274 try_again = true;
1579 }
1580 986 if (same_url_retry
1581
5/6
✓ Branch 0 taken 814 times.
✓ Branch 1 taken 172 times.
✓ Branch 3 taken 814 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 206 times.
✓ Branch 6 taken 780 times.
 1800 || (((info->error_code() == kFailProxyResolve)
1582
2/2
✓ Branch 2 taken 780 times.
✓ Branch 3 taken 34 times.
 814 || IsProxyTransferError(info->error_code())
1583
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 780 times.
 780 || (info->error_code() == kFailProxyHttp)))) {
1584
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 206 times.
 206 if (sharding_policy_.UseCount() > 0) { // sharding policy
1585 ✗ try_again = true;
1586 ✗ same_url_retry = false;
1587 } else { // no sharding
1588 206 try_again = true;
1589 // If all proxies failed, do a next round with the next host
1590
4/6
✓ Branch 0 taken 34 times.
✓ Branch 1 taken 172 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 34 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 206 times.
 206 if (!same_url_retry && (info->num_used_proxies() >= opt_num_proxies_)) {
1591 // Check if this can be made a host fail-over
1592 ✗ if (info->probe_hosts() && host_chain
1593 ✗ && (num_used_hosts < host_chain->size())) {
1594 // reset proxy group if not already performed by other handle
1595 ✗ if (opt_proxy_groups_) {
1596 ✗ if ((opt_proxy_groups_current_ > 0)
1597 ✗ || (opt_proxy_groups_current_burned_ > 0)) {
1598 ✗ opt_proxy_groups_current_ = 0;
1599 ✗ opt_timestamp_backup_proxies_ = 0;
1600 ✗ const std::string msg = "reset proxies for " + typ
1601 ✗ + " failover";
1602 ✗ RebalanceProxiesUnlocked(msg);
1603 }
1604 }
1605
1606 // Make it a host failure
1607 ✗ LogCvmfs(kLogDownload, kLogDebug,
1608 "(manager '%s' - id %" PRId64 ") make it a %s failure",
1609 name_.c_str(), info->id(), typ.c_str());
1610 ✗ info->SetNumUsedProxies(1);
1611 ✗ info->SetErrorCode(kFailHostAfterProxy);
1612 } else {
1613 ✗ if (failover_indefinitely_) {
1614 // Instead of giving up, reset the num_used_proxies counter,
1615 // switch proxy and try again
1616 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1617 "(manager '%s' - id %" PRId64 ") "
1618 "VerifyAndFinalize() would fail the download here. "
1619 "Instead switch proxy and retry download. "
1620 "typ=%s "
1621 "info->probe_hosts=%d host_chain=%p num_used_hosts=%d "
1622 "host_chain->size()=%lu same_url_retry=%d "
1623 "info->num_used_proxies=%d opt_num_proxies_=%d",
1624 name_.c_str(), info->id(), typ.c_str(),
1625 ✗ static_cast<int>(info->probe_hosts()), host_chain,
1626 ✗ num_used_hosts, host_chain ? host_chain->size() : -1,
1627 static_cast<int>(same_url_retry),
1628 ✗ info->num_used_proxies(), opt_num_proxies_);
1629 ✗ info->SetNumUsedProxies(1);
1630 ✗ RebalanceProxiesUnlocked(
1631 "download failed - failover indefinitely");
1632 ✗ try_again = !Interrupted(fqrn_, info);
1633 } else {
1634 ✗ try_again = false;
1635 }
1636 }
1637 } // Make a proxy failure a host failure
1638 } // Proxy failure assumed
1639 } // end !sharding
1640 986 }
1641
1642
2/2
✓ Branch 0 taken 423 times.
✓ Branch 1 taken 7234 times.
 7657 if (try_again) {
1643
1/2
✓ Branch 3 taken 423 times.
✗ Branch 4 not taken.
 423 LogCvmfs(kLogDownload, kLogDebug,
1644 "(manager '%s' - id %" PRId64 ") "
1645 "Trying again on same curl handle, same url: %d, "
1646 "error code %d no-cache %d",
1647 423 name_.c_str(), info->id(), same_url_retry, info->error_code(),
1648 423 info->nocache());
1649 // Reset internal state and destination
1650
4/8
✓ Branch 1 taken 423 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 423 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 423 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 423 times.
 423 if (info->sink() != NULL && info->sink()->Reset() != 0) {
1651 ✗ info->SetErrorCode(kFailLocalIO);
1652 ✗ goto verify_and_finalize_stop;
1653 }
1654
6/8
✓ Branch 1 taken 34 times.
✓ Branch 2 taken 389 times.
✓ Branch 5 taken 34 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 34 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 34 times.
✓ Branch 10 taken 389 times.
 423 if (info->interrupt_cue() && info->interrupt_cue()->IsCanceled()) {
1655 34 info->SetErrorCode(kFailCanceled);
1656 34 goto verify_and_finalize_stop;
1657 }
1658
1659
2/2
✓ Branch 1 taken 217 times.
✓ Branch 2 taken 172 times.
 389 if (info->expected_hash()) {
1660
1/2
✓ Branch 2 taken 217 times.
✗ Branch 3 not taken.
 217 shash::Init(info->hash_context());
1661 }
1662
2/2
✓ Branch 1 taken 217 times.
✓ Branch 2 taken 172 times.
 389 if (info->compressed()) {
1663
1/2
✓ Branch 2 taken 217 times.
✗ Branch 3 not taken.
 217 zlib::DecompressInit(info->GetZstreamPtr());
1664 }
1665
1666
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 389 times.
 389 if (sharding_policy_.UseCount() > 0) { // sharding policy
1667 ✗ ReleaseCredential(info);
1668 ✗ SetUrlOptions(info);
1669 } else { // no sharding policy
1670
1/2
✓ Branch 1 taken 389 times.
✗ Branch 2 not taken.
 389 SetRegularCache(info);
1671
1672 // Failure handling
1673 389 bool switch_proxy = false;
1674 389 bool switch_host = false;
1675
2/4
✓ Branch 1 taken 115 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 274 times.
 389 switch (info->error_code()) {
1676 115 case kFailBadData:
1677
1/2
✓ Branch 1 taken 115 times.
✗ Branch 2 not taken.
 115 SetNocache(info);
1678 115 break;
1679 ✗ case kFailProxyResolve:
1680 case kFailProxyHttp:
1681 ✗ switch_proxy = true;
1682 ✗ break;
1683 ✗ case kFailHostResolve:
1684 case kFailHostHttp:
1685 case kFailHostAfterProxy:
1686 ✗ switch_host = true;
1687 ✗ break;
1688 274 default:
1689
2/2
✓ Branch 2 taken 34 times.
✓ Branch 3 taken 240 times.
 274 if (IsProxyTransferError(info->error_code())) {
1690
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 34 times.
 34 if (same_url_retry) {
1691 ✗ Backoff(info);
1692 } else {
1693 34 switch_proxy = true;
1694 }
1695
1/2
✓ Branch 2 taken 240 times.
✗ Branch 3 not taken.
 240 } else if (IsHostTransferError(info->error_code())) {
1696
2/2
✓ Branch 0 taken 172 times.
✓ Branch 1 taken 68 times.
 240 if (same_url_retry) {
1697
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 Backoff(info);
1698 } else {
1699 68 switch_host = true;
1700 }
1701 } else {
1702 // No other errors expected when retrying
1703 ✗ PANIC(NULL);
1704 }
1705 }
1706
2/2
✓ Branch 0 taken 34 times.
✓ Branch 1 taken 355 times.
 389 if (switch_proxy) {
1707
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 ReleaseCredential(info);
1708
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 SwitchProxy(info);
1709 34 info->SetNumUsedProxies(info->num_used_proxies() + 1);
1710
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 SetUrlOptions(info);
1711 }
1712
2/2
✓ Branch 0 taken 68 times.
✓ Branch 1 taken 321 times.
 389 if (switch_host) {
1713
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 ReleaseCredential(info);
1714
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 68 times.
 68 if (was_metalink) {
1715 ✗ SwitchMetalink(info);
1716 ✗ info->SetNumUsedMetalinks(num_used_hosts + 1);
1717 } else {
1718
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 SwitchHost(info);
1719 68 info->SetNumUsedHosts(num_used_hosts + 1);
1720 }
1721
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 SetUrlOptions(info);
1722 }
1723 } // end !sharding
1724
1725
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 389 times.
 389 if (failover_indefinitely_) {
1726 // try again, breaking if there's a cvmfs reload happening and we are in a
1727 // proxy failover. This will EIO the call application.
1728 ✗ return !Interrupted(fqrn_, info);
1729 }
1730 389 return true; // try again
1731 }
1732
1733 7234 verify_and_finalize_stop:
1734 // Finalize, flush destination file
1735
1/2
✓ Branch 1 taken 7268 times.
✗ Branch 2 not taken.
 7268 ReleaseCredential(info);
1736
4/8
✓ Branch 1 taken 7268 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 7268 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 7268 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 7268 times.
 7268 if (info->sink() != NULL && info->sink()->Flush() != 0) {
1737 ✗ info->SetErrorCode(kFailLocalIO);
1738 }
1739
1740
2/2
✓ Branch 1 taken 4962 times.
✓ Branch 2 taken 2306 times.
 7268 if (info->compressed())
1741
1/2
✓ Branch 2 taken 4962 times.
✗ Branch 3 not taken.
 4962 zlib::DecompressFini(info->GetZstreamPtr());
1742
1743
1/2
✓ Branch 1 taken 7268 times.
✗ Branch 2 not taken.
 7268 if (info->headers()) {
1744
1/2
✓ Branch 2 taken 7268 times.
✗ Branch 3 not taken.
 7268 header_lists_->PutList(info->headers());
1745 7268 info->SetHeaders(NULL);
1746 }
1747
1748 7268 return false; // stop transfer and return to Fetch()
1749 7657 }
1750
1751 5211 DownloadManager::~DownloadManager() {
1752 // cleaned up fini
1753
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5211 times.
 5211 if (sharding_policy_.UseCount() > 0) {
1754 ✗ sharding_policy_.Reset();
1755 }
1756
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5211 times.
 5211 if (health_check_.UseCount() > 0) {
1757 ✗ if (health_check_.Unique()) {
1758 ✗ LogCvmfs(kLogDownload, kLogDebug,
1759 "(manager '%s') Stopping healthcheck thread", name_.c_str());
1760 ✗ health_check_->StopHealthcheck();
1761 }
1762 ✗ health_check_.Reset();
1763 }
1764
1765
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5211 times.
 5211 if (atomic_xadd32(&multi_threaded_, 0) == 1) {
1766 // Shutdown I/O thread
1767 ✗ pipe_terminate_->Write(kPipeTerminateSignal);
1768 ✗ pthread_join(thread_download_, NULL);
1769 // All handles are removed from the multi stack
1770 ✗ pipe_terminate_.Destroy();
1771 ✗ pipe_jobs_.Destroy();
1772 }
1773
1774 10422 for (set<CURL *>::iterator i = pool_handles_idle_->begin(),
1775 5211 iEnd = pool_handles_idle_->end();
1776
2/2
✓ Branch 1 taken 2838 times.
✓ Branch 2 taken 5211 times.
 8049 i != iEnd;
1777 2838 ++i) {
1778 2838 curl_easy_cleanup(*i);
1779 }
1780
1781
1/2
✓ Branch 0 taken 5211 times.
✗ Branch 1 not taken.
 5211 delete pool_handles_idle_;
1782
1/2
✓ Branch 0 taken 5211 times.
✗ Branch 1 not taken.
 5211 delete pool_handles_inuse_;
1783 5211 curl_multi_cleanup(curl_multi_);
1784
1785
1/2
✓ Branch 0 taken 5211 times.
✗ Branch 1 not taken.
 5211 delete header_lists_;
1786
1/2
✓ Branch 0 taken 5211 times.
✗ Branch 1 not taken.
 5211 if (user_agent_)
1787 5211 free(user_agent_);
1788
1789
1/2
✓ Branch 0 taken 5211 times.
✗ Branch 1 not taken.
 5211 delete counters_;
1790
2/2
✓ Branch 0 taken 2641 times.
✓ Branch 1 taken 2570 times.
 5211 delete opt_host_.chain;
1791
2/2
✓ Branch 0 taken 2641 times.
✓ Branch 1 taken 2570 times.
 5211 delete opt_host_chain_rtt_;
1792
2/2
✓ Branch 0 taken 2198 times.
✓ Branch 1 taken 3013 times.
 5211 delete opt_proxy_groups_;
1793
1794 5211 curl_global_cleanup();
1795
1/2
✓ Branch 0 taken 5211 times.
✗ Branch 1 not taken.
 5211 delete resolver_;
1796
1797 // old destructor
1798 5211 pthread_mutex_destroy(lock_options_);
1799 5211 pthread_mutex_destroy(lock_synchronous_mode_);
1800 5211 free(lock_options_);
1801 5211 free(lock_synchronous_mode_);
1802 5211 }
1803
1804 5212 void DownloadManager::InitHeaders() {
1805 // User-Agent
1806
1/2
✓ Branch 2 taken 5212 times.
✗ Branch 3 not taken.
 5212 string cernvm_id = "User-Agent: cvmfs ";
1807 #ifdef CVMFS_LIBCVMFS
1808
1/2
✓ Branch 1 taken 5212 times.
✗ Branch 2 not taken.
 5212 cernvm_id += "libcvmfs ";
1809 #else
1810 ✗ cernvm_id += "Fuse ";
1811 #endif
1812
2/4
✓ Branch 2 taken 5212 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 5212 times.
✗ Branch 6 not taken.
 5212 cernvm_id += string(CVMFS_VERSION);
1813
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5212 times.
 5212 if (getenv("CERNVM_UUID") != NULL) {
1814 cernvm_id += " "
1815 ✗ + sanitizer::InputSanitizer("az AZ 09 -")
1816 ✗ .Filter(getenv("CERNVM_UUID"));
1817 }
1818 5212 user_agent_ = strdup(cernvm_id.c_str());
1819
1820
1/2
✓ Branch 1 taken 5212 times.
✗ Branch 2 not taken.
 5212 header_lists_ = new HeaderLists();
1821
1822
1/2
✓ Branch 1 taken 5212 times.
✗ Branch 2 not taken.
 5212 default_headers_ = header_lists_->GetList("Connection: Keep-Alive");
1823
1/2
✓ Branch 1 taken 5212 times.
✗ Branch 2 not taken.
 5212 header_lists_->AppendHeader(default_headers_, "Pragma:");
1824
1/2
✓ Branch 1 taken 5212 times.
✗ Branch 2 not taken.
 5212 header_lists_->AppendHeader(default_headers_, user_agent_);
1825 5212 }
1826
1827 5212 DownloadManager::DownloadManager(const unsigned max_pool_handles,
1828 const perf::StatisticsTemplate &statistics,
1829 5212 const std::string &name)
1830 5212 : prng_(Prng())
1831 5212 , pool_handles_idle_(new set<CURL *>)
1832 5212 , pool_handles_inuse_(new set<CURL *>)
1833 5212 , pool_max_handles_(max_pool_handles)
1834 5212 , pipe_terminate_(NULL)
1835
1/2
✓ Branch 1 taken 5212 times.
✗ Branch 2 not taken.
 5212 , pipe_jobs_(NULL)
1836 5212 , watch_fds_(NULL)
1837 5212 , watch_fds_size_(0)
1838 5212 , watch_fds_inuse_(0)
1839 5212 , watch_fds_max_(4 * max_pool_handles)
1840 5212 , opt_timeout_proxy_(5)
1841 5212 , opt_timeout_direct_(10)
1842 5212 , opt_low_speed_limit_(1024)
1843 5212 , opt_max_retries_(0)
1844 5212 , opt_backoff_init_ms_(0)
1845 5212 , opt_backoff_max_ms_(0)
1846 5212 , enable_info_header_(false)
1847 5212 , opt_ipv4_only_(false)
1848 5212 , follow_redirects_(false)
1849 5212 , ignore_signature_failures_(false)
1850 5212 , enable_http_tracing_(false)
1851 5212 , opt_metalink_(NULL, 0, 0, 0)
1852 5212 , opt_metalink_timestamp_link_(0)
1853 5212 , opt_host_(NULL, 0, 0, 0)
1854 5212 , opt_host_chain_rtt_(NULL)
1855 5212 , opt_proxy_groups_(NULL)
1856 5212 , opt_proxy_groups_current_(0)
1857 5212 , opt_proxy_groups_current_burned_(0)
1858 5212 , opt_proxy_groups_fallback_(0)
1859 5212 , opt_num_proxies_(0)
1860 5212 , opt_proxy_shard_(false)
1861 5212 , failover_indefinitely_(false)
1862
1/2
✓ Branch 1 taken 5212 times.
✗ Branch 2 not taken.
 5212 , name_(name)
1863 5212 , opt_ip_preference_(dns::kIpPreferSystem)
1864 5212 , opt_timestamp_backup_proxies_(0)
1865 5212 , opt_timestamp_failover_proxies_(0)
1866 5212 , opt_proxy_groups_reset_after_(0)
1867 5212 , credentials_attachment_(NULL)
1868
4/8
✓ Branch 13 taken 5212 times.
✗ Branch 14 not taken.
✓ Branch 16 taken 5212 times.
✗ Branch 17 not taken.
✓ Branch 19 taken 5212 times.
✗ Branch 20 not taken.
✓ Branch 23 taken 5212 times.
✗ Branch 24 not taken.
 15636 , counters_(new Counters(statistics)) {
1869 5212 atomic_init32(&multi_threaded_);
1870
1871 5212 lock_options_ = reinterpret_cast<pthread_mutex_t *>(
1872 5212 smalloc(sizeof(pthread_mutex_t)));
1873 5212 int retval = pthread_mutex_init(lock_options_, NULL);
1874
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5212 times.
 5212 assert(retval == 0);
1875 5212 lock_synchronous_mode_ = reinterpret_cast<pthread_mutex_t *>(
1876 5212 smalloc(sizeof(pthread_mutex_t)));
1877 5212 retval = pthread_mutex_init(lock_synchronous_mode_, NULL);
1878
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5212 times.
 5212 assert(retval == 0);
1879
1880
1/2
✓ Branch 1 taken 5212 times.
✗ Branch 2 not taken.
 5212 retval = curl_global_init(CURL_GLOBAL_ALL);
1881
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5212 times.
 5212 assert(retval == CURLE_OK);
1882
1883
1/2
✓ Branch 1 taken 5212 times.
✗ Branch 2 not taken.
 5212 InitHeaders();
1884
1885
1/2
✓ Branch 1 taken 5212 times.
✗ Branch 2 not taken.
 5212 curl_multi_ = curl_multi_init();
1886
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5212 times.
 5212 assert(curl_multi_ != NULL);
1887
1/2
✓ Branch 1 taken 5212 times.
✗ Branch 2 not taken.
 5212 curl_multi_setopt(curl_multi_, CURLMOPT_SOCKETFUNCTION, CallbackCurlSocket);
1888
1/2
✓ Branch 1 taken 5212 times.
✗ Branch 2 not taken.
 5212 curl_multi_setopt(curl_multi_, CURLMOPT_SOCKETDATA,
1889 static_cast<void *>(this));
1890
1/2
✓ Branch 1 taken 5212 times.
✗ Branch 2 not taken.
 5212 curl_multi_setopt(curl_multi_, CURLMOPT_MAXCONNECTS, watch_fds_max_);
1891
1/2
✓ Branch 1 taken 5212 times.
✗ Branch 2 not taken.
 5212 curl_multi_setopt(curl_multi_, CURLMOPT_MAX_TOTAL_CONNECTIONS,
1892 pool_max_handles_);
1893
1894 5212 prng_.InitLocaltime();
1895
1896 // Name resolving
1897 5212 if ((getenv("CVMFS_IPV4_ONLY") != NULL)
1898
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 5212 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 5212 times.
 5212 && (strlen(getenv("CVMFS_IPV4_ONLY")) > 0)) {
1899 ✗ opt_ipv4_only_ = true;
1900 }
1901
1/2
✓ Branch 1 taken 5212 times.
✗ Branch 2 not taken.
 5212 resolver_ = dns::NormalResolver::Create(opt_ipv4_only_, kDnsDefaultRetries,
1902 kDnsDefaultTimeoutMs);
1903
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5212 times.
 5212 assert(resolver_);
1904 5212 }
1905
1906 /**
1907 * Spawns the I/O worker thread and switches the module in multi-threaded mode.
1908 * No way back except Fini(); Init();
1909 */
1910 ✗ void DownloadManager::Spawn() {
1911 ✗ pipe_terminate_ = new Pipe<kPipeThreadTerminator>();
1912 ✗ pipe_jobs_ = new Pipe<kPipeDownloadJobs>();
1913
1914 ✗ const int retval = pthread_create(&thread_download_, NULL, MainDownload,
1915 static_cast<void *>(this));
1916 ✗ assert(retval == 0);
1917
1918 ✗ atomic_inc32(&multi_threaded_);
1919
1920 ✗ if (health_check_.UseCount() > 0) {
1921 ✗ LogCvmfs(kLogDownload, kLogDebug,
1922 "(manager '%s') Starting healthcheck thread", name_.c_str());
1923 ✗ health_check_->StartHealthcheck();
1924 }
1925 }
1926
1927
1928 /**
1929 * Downloads data from an insecure outside channel (currently HTTP or file).
1930 */
1931 7268 Failures DownloadManager::Fetch(JobInfo *info) {
1932
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7268 times.
 7268 assert(info != NULL);
1933
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 7268 times.
 7268 assert(info->url() != NULL);
1934
1935 Failures result;
1936
1/2
✓ Branch 1 taken 7268 times.
✗ Branch 2 not taken.
 7268 result = PrepareDownloadDestination(info);
1937
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7268 times.
 7268 if (result != kFailOk)
1938 ✗ return result;
1939
1940
2/2
✓ Branch 1 taken 5011 times.
✓ Branch 2 taken 2257 times.
 7268 if (info->expected_hash()) {
1941 5011 const shash::Algorithms algorithm = info->expected_hash()->algorithm;
1942 5011 info->GetHashContextPtr()->algorithm = algorithm;
1943
1/2
✓ Branch 1 taken 5011 times.
✗ Branch 2 not taken.
 5011 info->GetHashContextPtr()->size = shash::GetContextSize(algorithm);
1944 5011 info->GetHashContextPtr()->buffer = alloca(info->hash_context().size);
1945 }
1946
1947 // In case JobInfo object is being reused
1948
2/4
✓ Branch 2 taken 7268 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 7268 times.
✗ Branch 6 not taken.
 7268 info->SetLink("");
1949
1950 // Prepare cvmfs-info: header, allocate string on the stack
1951 7268 info->SetInfoHeader(NULL);
1952
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7268 times.
 7268 if (enable_info_header_) {
1953 ✗ const string header_info = info->GetInfoHeaderContents(info_header_template_);
1954 ✗ if (header_info != "") {
1955 ✗ const char * const header_name = "cvmfs-info: ";
1956 ✗ const size_t header_name_len = strlen(header_name);
1957 ✗ const size_t info_len = header_info.length();
1958 ✗ char *buf = static_cast<char *>(alloca(header_name_len + info_len + 1));
1959 ✗ memcpy(buf, header_name, header_name_len);
1960 ✗ memcpy(buf + header_name_len, header_info.c_str(), info_len);
1961 ✗ buf[header_name_len + info_len] = '\0';
1962 ✗ info->SetInfoHeader(buf);
1963 }
1964 }
1965
1966
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7268 times.
 7268 if (enable_http_tracing_) {
1967 ✗ const std::string str_pid = "X-CVMFS-PID: " + StringifyInt(info->pid());
1968 ✗ const std::string str_gid = "X-CVMFS-GID: " + StringifyUint(info->gid());
1969 ✗ const std::string str_uid = "X-CVMFS-UID: " + StringifyUint(info->uid());
1970
1971 // will be auto freed at the end of this function Fetch(JobInfo *info)
1972 ✗ info->SetTracingHeaderPid(static_cast<char *>(alloca(str_pid.size() + 1)));
1973 ✗ info->SetTracingHeaderGid(static_cast<char *>(alloca(str_gid.size() + 1)));
1974 ✗ info->SetTracingHeaderUid(static_cast<char *>(alloca(str_uid.size() + 1)));
1975
1976 ✗ memcpy(info->tracing_header_pid(), str_pid.c_str(), str_pid.size() + 1);
1977 ✗ memcpy(info->tracing_header_gid(), str_gid.c_str(), str_gid.size() + 1);
1978 ✗ memcpy(info->tracing_header_uid(), str_uid.c_str(), str_uid.size() + 1);
1979 }
1980
1981
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 7268 times.
 7268 if (atomic_xadd32(&multi_threaded_, 0) == 1) {
1982 ✗ if (!info->IsValidPipeJobResults()) {
1983 ✗ info->CreatePipeJobResults();
1984 }
1985 ✗ if (!info->IsValidDataTube()) {
1986 ✗ info->CreateDataTube();
1987 }
1988
1989 // LogCvmfs(kLogDownload, kLogDebug, "send job to thread, pipe %d %d",
1990 // info->wait_at[0], info->wait_at[1]);
1991 ✗ pipe_jobs_->Write<JobInfo *>(info);
1992
1993 do {
1994 ✗ DataTubeElement *ele = info->GetDataTubePtr()->PopFront();
1995
1996 ✗ if (ele->action == kActionStop) {
1997 ✗ delete ele;
1998 ✗ break;
1999 }
2000 // TODO(heretherebedragons) add compression
2001 ✗ } while (true);
2002
2003 ✗ info->GetPipeJobResultPtr()->Read<download::Failures>(&result);
2004 // LogCvmfs(kLogDownload, kLogDebug, "got result %d", result);
2005 } else {
2006 7268 const MutexLockGuard l(lock_synchronous_mode_);
2007
1/2
✓ Branch 1 taken 7268 times.
✗ Branch 2 not taken.
 7268 CURL *handle = AcquireCurlHandle();
2008
1/2
✓ Branch 1 taken 7268 times.
✗ Branch 2 not taken.
 7268 InitializeRequest(info, handle);
2009
1/2
✓ Branch 1 taken 7268 times.
✗ Branch 2 not taken.
 7268 SetUrlOptions(info);
2010 // curl_easy_setopt(handle, CURLOPT_VERBOSE, 1);
2011 int retval;
2012 do {
2013
1/2
✓ Branch 1 taken 7657 times.
✗ Branch 2 not taken.
 7657 retval = curl_easy_perform(handle);
2014 7657 perf::Inc(counters_->n_requests);
2015 double elapsed;
2016
1/2
✓ Branch 1 taken 7657 times.
✗ Branch 2 not taken.
 7657 if (curl_easy_getinfo(handle, CURLINFO_TOTAL_TIME, &elapsed)
2017
1/2
✓ Branch 0 taken 7657 times.
✗ Branch 1 not taken.
 7657 == CURLE_OK) {
2018 7657 perf::Xadd(counters_->sz_transfer_time,
2019 7657 static_cast<int64_t>(elapsed * 1000));
2020 }
2021
3/4
✓ Branch 1 taken 7657 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 389 times.
✓ Branch 4 taken 7268 times.
 7657 } while (VerifyAndFinalize(retval, info));
2022 7268 result = info->error_code();
2023
1/2
✓ Branch 2 taken 7268 times.
✗ Branch 3 not taken.
 7268 ReleaseCurlHandle(info->curl_handle());
2024 7268 }
2025
2026
2/2
✓ Branch 0 taken 597 times.
✓ Branch 1 taken 6671 times.
 7268 if (result != kFailOk) {
2027
1/2
✓ Branch 4 taken 597 times.
✗ Branch 5 not taken.
 597 LogCvmfs(kLogDownload, kLogDebug,
2028 "(manager '%s' - id %" PRId64 ") "
2029 "download failed (error %d - %s)",
2030 name_.c_str(), info->id(), result, Code2Ascii(result));
2031
2032
1/2
✓ Branch 1 taken 597 times.
✗ Branch 2 not taken.
 597 if (info->sink() != NULL) {
2033
1/2
✓ Branch 2 taken 597 times.
✗ Branch 3 not taken.
 597 info->sink()->Purge();
2034 }
2035 }
2036
2037 7268 return result;
2038 }
2039
2040
2041 /**
2042 * Used by the client to connect the authz session manager to the download
2043 * manager.
2044 */
2045 1283 void DownloadManager::SetCredentialsAttachment(CredentialsAttachment *ca) {
2046 1283 const MutexLockGuard m(lock_options_);
2047 1283 credentials_attachment_ = ca;
2048 1283 }
2049
2050 /**
2051 * Gets the DNS sever.
2052 */
2053 ✗ std::string DownloadManager::GetDnsServer() const { return opt_dns_server_; }
2054
2055 /**
2056 * Sets a DNS server. Only for testing as it cannot be reverted to the system
2057 * default.
2058 */
2059 ✗ void DownloadManager::SetDnsServer(const string &address) {
2060 ✗ if (!address.empty()) {
2061 ✗ const MutexLockGuard m(lock_options_);
2062 ✗ opt_dns_server_ = address;
2063 ✗ assert(!opt_dns_server_.empty());
2064
2065 ✗ vector<string> servers;
2066 ✗ servers.push_back(address);
2067 ✗ const bool retval = resolver_->SetResolvers(servers);
2068 ✗ assert(retval);
2069 }
2070 ✗ LogCvmfs(kLogDownload, kLogSyslog, "(manager '%s') set nameserver to %s",
2071 name_.c_str(), address.c_str());
2072 }
2073
2074
2075 /**
2076 * Sets the DNS query timeout parameters.
2077 */
2078 2399 void DownloadManager::SetDnsParameters(const unsigned retries,
2079 const unsigned timeout_ms) {
2080 2399 const MutexLockGuard m(lock_options_);
2081 2399 if ((resolver_->retries() == retries)
2082
3/6
✓ Branch 0 taken 2399 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 2399 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 2399 times.
✗ Branch 6 not taken.
 2399 && (resolver_->timeout_ms() == timeout_ms)) {
2083 2399 return;
2084 }
2085 ✗ delete resolver_;
2086 ✗ resolver_ = NULL;
2087 ✗ resolver_ = dns::NormalResolver::Create(opt_ipv4_only_, retries, timeout_ms);
2088 ✗ assert(resolver_);
2089
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 2399 times.
 2399 }
2090
2091
2092 2399 void DownloadManager::SetDnsTtlLimits(const unsigned min_seconds,
2093 const unsigned max_seconds) {
2094 2399 const MutexLockGuard m(lock_options_);
2095 2399 resolver_->set_min_ttl(min_seconds);
2096 2399 resolver_->set_max_ttl(max_seconds);
2097 2399 }
2098
2099
2100 ✗ void DownloadManager::SetIpPreference(dns::IpPreference preference) {
2101 ✗ const MutexLockGuard m(lock_options_);
2102 ✗ opt_ip_preference_ = preference;
2103 }
2104
2105
2106 /**
2107 * Sets two timeout values for proxied and for direct connections, respectively.
2108 * The timeout counts for all sorts of connection phases,
2109 * DNS, HTTP connect, etc.
2110 */
2111 3308 void DownloadManager::SetTimeout(const unsigned seconds_proxy,
2112 const unsigned seconds_direct) {
2113 3308 const MutexLockGuard m(lock_options_);
2114 3308 opt_timeout_proxy_ = seconds_proxy;
2115 3308 opt_timeout_direct_ = seconds_direct;
2116 3308 }
2117
2118
2119 /**
2120 * Sets contains the average transfer speed in bytes per second that the
2121 * transfer should be below during CURLOPT_LOW_SPEED_TIME seconds for libcurl to
2122 * consider it to be too slow and abort. Only effective for new connections.
2123 */
2124 ✗ void DownloadManager::SetLowSpeedLimit(const unsigned low_speed_limit) {
2125 ✗ const MutexLockGuard m(lock_options_);
2126 ✗ opt_low_speed_limit_ = low_speed_limit;
2127 }
2128
2129
2130 /**
2131 * Receives the currently active timeout values.
2132 */
2133 1048 void DownloadManager::GetTimeout(unsigned *seconds_proxy,
2134 unsigned *seconds_direct) {
2135 1048 const MutexLockGuard m(lock_options_);
2136 1048 *seconds_proxy = opt_timeout_proxy_;
2137 1048 *seconds_direct = opt_timeout_direct_;
2138 1048 }
2139
2140
2141 /**
2142 * Parses a list of ';'-separated hosts for the metalink chain. The empty
2143 * string removes the metalink list.
2144 */
2145 ✗ void DownloadManager::SetMetalinkChain(const string &metalink_list) {
2146 ✗ SetMetalinkChain(SplitString(metalink_list, ';'));
2147 }
2148
2149
2150 ✗ void DownloadManager::SetMetalinkChain(
2151 const std::vector<std::string> &metalink_list) {
2152 ✗ const MutexLockGuard m(lock_options_);
2153 ✗ opt_metalink_.timestamp_backup = 0;
2154 ✗ delete opt_metalink_.chain;
2155 ✗ opt_metalink_.current = 0;
2156
2157 ✗ if (metalink_list.empty()) {
2158 ✗ opt_metalink_.chain = NULL;
2159 ✗ return;
2160 }
2161
2162 ✗ opt_metalink_.chain = new vector<string>(metalink_list);
2163 }
2164
2165
2166 /**
2167 * Retrieves the currently set chain of metalink hosts and the currently
2168 * used metalink host.
2169 */
2170 ✗ void DownloadManager::GetMetalinkInfo(vector<string> *metalink_chain,
2171 unsigned *current_metalink) {
2172 ✗ const MutexLockGuard m(lock_options_);
2173 ✗ if (opt_metalink_.chain) {
2174 ✗ if (current_metalink) {
2175 ✗ *current_metalink = opt_metalink_.current;
2176 }
2177 ✗ if (metalink_chain) {
2178 ✗ *metalink_chain = *opt_metalink_.chain;
2179 }
2180 }
2181 }
2182
2183
2184 /**
2185 * Parses a list of ';'-separated hosts for the host chain. The empty string
2186 * removes the host list.
2187 */
2188 2690 void DownloadManager::SetHostChain(const string &host_list) {
2189
1/2
✓ Branch 2 taken 2690 times.
✗ Branch 3 not taken.
 2690 SetHostChain(SplitString(host_list, ';'));
2190 2690 }
2191
2192
2193 2752 void DownloadManager::SetHostChain(const std::vector<std::string> &host_list) {
2194 2752 const MutexLockGuard m(lock_options_);
2195 2752 opt_host_.timestamp_backup = 0;
2196
2/2
✓ Branch 0 taken 1159 times.
✓ Branch 1 taken 1593 times.
 2752 delete opt_host_.chain;
2197
2/2
✓ Branch 0 taken 1159 times.
✓ Branch 1 taken 1593 times.
 2752 delete opt_host_chain_rtt_;
2198 2752 opt_host_.current = 0;
2199
2200
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 2752 times.
 2752 if (host_list.empty()) {
2201 ✗ opt_host_.chain = NULL;
2202 ✗ opt_host_chain_rtt_ = NULL;
2203 ✗ return;
2204 }
2205
2206
2/4
✓ Branch 1 taken 2752 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2752 times.
✗ Branch 5 not taken.
 2752 opt_host_.chain = new vector<string>(host_list);
2207 5504 opt_host_chain_rtt_ = new vector<int>(opt_host_.chain->size(),
2208
2/4
✓ Branch 1 taken 2752 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2752 times.
✗ Branch 5 not taken.
 2752 kProbeUnprobed);
2209 // LogCvmfs(kLogDownload, kLogSyslog, "using host %s",
2210 // (*opt_host_.chain)[0].c_str());
2211
1/2
✓ Branch 1 taken 2752 times.
✗ Branch 2 not taken.
 2752 }
2212
2213
2214 /**
2215 * Retrieves the currently set chain of hosts, their round trip times, and the
2216 * currently used host.
2217 */
2218 434 void DownloadManager::GetHostInfo(vector<string> *host_chain, vector<int> *rtt,
2219 unsigned *current_host) {
2220 434 const MutexLockGuard m(lock_options_);
2221
1/2
✓ Branch 0 taken 434 times.
✗ Branch 1 not taken.
 434 if (opt_host_.chain) {
2222
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 434 times.
 434 if (current_host) {
2223 ✗ *current_host = opt_host_.current;
2224 }
2225
1/2
✓ Branch 0 taken 434 times.
✗ Branch 1 not taken.
 434 if (host_chain) {
2226
1/2
✓ Branch 1 taken 434 times.
✗ Branch 2 not taken.
 434 *host_chain = *opt_host_.chain;
2227 }
2228
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 434 times.
 434 if (rtt) {
2229 ✗ *rtt = *opt_host_chain_rtt_;
2230 }
2231 }
2232 434 }
2233
2234
2235 /**
2236 * Jumps to the next proxy in the ring of forward proxy servers.
2237 * Selects one randomly from a load-balancing group.
2238 *
2239 * Allow for the fact that the proxy may have already been failed by
2240 * another transfer, or that the proxy may no longer be part of the
2241 * current load-balancing group.
2242 */
2243 34 void DownloadManager::SwitchProxy(JobInfo *info) {
2244 34 const MutexLockGuard m(lock_options_);
2245
2246
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 34 times.
 34 if (!opt_proxy_groups_) {
2247 ✗ return;
2248 }
2249
2250 // Fail any matching proxies within the current load-balancing group
2251 34 vector<ProxyInfo> *group = current_proxy_group();
2252 34 const unsigned group_size = group->size();
2253 34 unsigned failed = 0;
2254
2/2
✓ Branch 0 taken 34 times.
✓ Branch 1 taken 34 times.
 68 for (unsigned i = 0; i < group_size - opt_proxy_groups_current_burned_; ++i) {
2255
5/14
✓ Branch 0 taken 34 times.
✗ Branch 1 not taken.
✓ Branch 4 taken 34 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 34 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 34 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✓ Branch 12 taken 34 times.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
 34 if (info && (info->proxy() == (*group)[i].url)) {
2256 // Move to list of failed proxies
2257 34 opt_proxy_groups_current_burned_++;
2258
1/2
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
 34 swap((*group)[i],
2259 34 (*group)[group_size - opt_proxy_groups_current_burned_]);
2260 34 perf::Inc(counters_->n_proxy_failover);
2261 34 failed++;
2262 }
2263 }
2264
2265 // Do nothing more unless at least one proxy was marked as failed
2266
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 34 times.
 34 if (!failed)
2267 ✗ return;
2268
2269 // If all proxies from the current load-balancing group are burned, switch to
2270 // another group
2271
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 if (opt_proxy_groups_current_burned_ == group->size()) {
2272 34 opt_proxy_groups_current_burned_ = 0;
2273
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 if (opt_proxy_groups_->size() > 1) {
2274 68 opt_proxy_groups_current_ = (opt_proxy_groups_current_ + 1)
2275 34 % opt_proxy_groups_->size();
2276 // Remember the timestamp of switching to backup proxies
2277
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 34 times.
 34 if (opt_proxy_groups_reset_after_ > 0) {
2278 ✗ if (opt_proxy_groups_current_ > 0) {
2279 ✗ if (opt_timestamp_backup_proxies_ == 0)
2280 ✗ opt_timestamp_backup_proxies_ = time(NULL);
2281 // LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2282 // "switched to (another) backup proxy group");
2283 } else {
2284 ✗ opt_timestamp_backup_proxies_ = 0;
2285 // LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2286 // "switched back to primary proxy group");
2287 }
2288 ✗ opt_timestamp_failover_proxies_ = 0;
2289 }
2290 }
2291 } else {
2292 // Record failover time
2293 ✗ if (opt_proxy_groups_reset_after_ > 0) {
2294 ✗ if (opt_timestamp_failover_proxies_ == 0)
2295 ✗ opt_timestamp_failover_proxies_ = time(NULL);
2296 }
2297 }
2298
2299
2/4
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 34 times.
✗ Branch 6 not taken.
 34 UpdateProxiesUnlocked("failed proxy");
2300
1/2
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
 34 LogCvmfs(kLogDownload, kLogDebug,
2301 "(manager '%s' - id %" PRId64 ") "
2302 "%lu proxies remain in group",
2303 name_.c_str(), info->id(),
2304 34 current_proxy_group()->size() - opt_proxy_groups_current_burned_);
2305
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 }
2306
2307
2308 /**
2309 * Switches to the next host in the chain. If jobinfo is set, switch only if
2310 * the current host is identical to the one used by jobinfo, otherwise another
2311 * transfer has already done the switch.
2312 */
2313 117 void DownloadManager::SwitchHostInfo(const std::string &typ,
2314 HostInfo &info,
2315 JobInfo *jobinfo) {
2316 117 const MutexLockGuard m(lock_options_);
2317
2318
5/6
✓ Branch 0 taken 117 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 49 times.
✓ Branch 4 taken 68 times.
✓ Branch 5 taken 49 times.
✓ Branch 6 taken 68 times.
 117 if (!info.chain || (info.chain->size() == 1)) {
2319 49 return;
2320 }
2321
2322
1/2
✓ Branch 0 taken 68 times.
✗ Branch 1 not taken.
 68 if (jobinfo) {
2323 int lastused;
2324
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 if (typ == "host") {
2325 68 lastused = jobinfo->current_host_chain_index();
2326 } else {
2327 ✗ lastused = jobinfo->current_metalink_chain_index();
2328 }
2329
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 68 times.
 68 if (lastused != info.current) {
2330 ✗ LogCvmfs(kLogDownload, kLogDebug,
2331 "(manager '%s' - id %" PRId64 ")"
2332 "don't switch %s, "
2333 "last used %s: %s, current %s: %s",
2334 name_.c_str(), jobinfo->id(), typ.c_str(), typ.c_str(),
2335 ✗ (*info.chain)[lastused].c_str(), typ.c_str(),
2336 ✗ (*info.chain)[info.current].c_str());
2337 ✗ return;
2338 }
2339 }
2340
2341
1/2
✓ Branch 2 taken 68 times.
✗ Branch 3 not taken.
 68 string reason = "manually triggered";
2342
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 string info_id = "(manager " + name_;
2343
1/2
✓ Branch 0 taken 68 times.
✗ Branch 1 not taken.
 68 if (jobinfo) {
2344
1/2
✓ Branch 3 taken 68 times.
✗ Branch 4 not taken.
 68 reason = download::Code2Ascii(jobinfo->error_code());
2345
2/4
✓ Branch 2 taken 68 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 68 times.
✗ Branch 6 not taken.
 68 info_id = " - id " + StringifyInt(jobinfo->id());
2346 }
2347
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 info_id += ")";
2348
2349
1/2
✓ Branch 2 taken 68 times.
✗ Branch 3 not taken.
 68 const std::string old_host = (*info.chain)[info.current];
2350 68 info.current = (info.current + 1) % static_cast<int>(info.chain->size());
2351
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 if (typ == "host") {
2352 68 perf::Inc(counters_->n_host_failover);
2353 } else {
2354 ✗ perf::Inc(counters_->n_metalink_failover);
2355 }
2356
1/3
✓ Branch 6 taken 68 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
 136 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2357 "%s switching %s from %s to %s (%s)", info_id.c_str(), typ.c_str(),
2358 68 old_host.c_str(), (*info.chain)[info.current].c_str(),
2359 reason.c_str());
2360
2361 // Remember the timestamp of switching to backup host
2362
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 68 times.
 68 if (info.reset_after > 0) {
2363 ✗ if (info.current != 0) {
2364 ✗ if (info.timestamp_backup == 0)
2365 ✗ info.timestamp_backup = time(NULL);
2366 } else {
2367 ✗ info.timestamp_backup = 0;
2368 }
2369 }
2370
2/2
✓ Branch 4 taken 68 times.
✓ Branch 5 taken 49 times.
 117 }
2371
2372 117 void DownloadManager::SwitchHost(JobInfo *info) {
2373
2/4
✓ Branch 2 taken 117 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 117 times.
✗ Branch 6 not taken.
 117 SwitchHostInfo("host", opt_host_, info);
2374 117 }
2375
2376 49 void DownloadManager::SwitchHost() { SwitchHost(NULL); }
2377
2378
2379 ✗ void DownloadManager::SwitchMetalink(JobInfo *info) {
2380 ✗ SwitchHostInfo("metalink", opt_metalink_, info);
2381 }
2382
2383
2384 ✗ void DownloadManager::SwitchMetalink() { SwitchMetalink(NULL); }
2385
2386 3421 bool DownloadManager::CheckMetalinkChain(time_t now) {
2387 3421 return (opt_metalink_.chain
2388
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 3421 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 3421 && ((opt_metalink_timestamp_link_ == 0)
2389 ✗ || (static_cast<int64_t>((now == 0) ? time(NULL) : now)
2390 ✗ > static_cast<int64_t>(opt_metalink_timestamp_link_
2391
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 3421 + opt_metalink_.reset_after))));
2392 }
2393
2394
2395 /**
2396 * Orders the hostlist according to RTT of downloading .cvmfschecksum.
2397 * Sets the current host to the best-responsive host.
2398 * If you change the host list in between by SetHostChain(), it will be
2399 * overwritten by this function.
2400 */
2401 ✗ void DownloadManager::ProbeHosts() {
2402 ✗ vector<string> host_chain;
2403 ✗ vector<int> host_rtt;
2404 unsigned current_host;
2405
2406 ✗ GetHostInfo(&host_chain, &host_rtt, &current_host);
2407
2408 // Stopwatch, two times to fill caches first
2409 unsigned i, retries;
2410 ✗ string url;
2411
2412 ✗ cvmfs::MemSink memsink;
2413 ✗ JobInfo info(&url, false, false, NULL, &memsink);
2414 ✗ for (retries = 0; retries < 2; ++retries) {
2415 ✗ for (i = 0; i < host_chain.size(); ++i) {
2416 ✗ url = host_chain[i] + "/.cvmfspublished";
2417
2418 struct timeval tv_start, tv_end;
2419 ✗ gettimeofday(&tv_start, NULL);
2420 ✗ const Failures result = Fetch(&info);
2421 ✗ gettimeofday(&tv_end, NULL);
2422 ✗ memsink.Reset();
2423 ✗ if (result == kFailOk) {
2424 ✗ host_rtt[i] = static_cast<int>(DiffTimeSeconds(tv_start, tv_end)
2425 ✗ * 1000);
2426 ✗ LogCvmfs(kLogDownload, kLogDebug,
2427 "(manager '%s' - id %" PRId64 ") "
2428 "probing host %s had %dms rtt",
2429 ✗ name_.c_str(), info.id(), url.c_str(), host_rtt[i]);
2430 } else {
2431 ✗ LogCvmfs(kLogDownload, kLogDebug,
2432 "(manager '%s' - id %" PRId64 ") "
2433 "error while probing host %s: %d %s",
2434 name_.c_str(), info.id(), url.c_str(), result,
2435 Code2Ascii(result));
2436 ✗ host_rtt[i] = INT_MAX;
2437 }
2438 }
2439 }
2440
2441 ✗ SortTeam(&host_rtt, &host_chain);
2442 ✗ for (i = 0; i < host_chain.size(); ++i) {
2443 ✗ if (host_rtt[i] == INT_MAX)
2444 ✗ host_rtt[i] = kProbeDown;
2445 }
2446
2447 ✗ const MutexLockGuard m(lock_options_);
2448 ✗ delete opt_host_.chain;
2449 ✗ delete opt_host_chain_rtt_;
2450 ✗ opt_host_.chain = new vector<string>(host_chain);
2451 ✗ opt_host_chain_rtt_ = new vector<int>(host_rtt);
2452 ✗ opt_host_.current = 0;
2453 }
2454
2455 ✗ bool DownloadManager::GeoSortServers(std::vector<std::string> *servers,
2456 std::vector<uint64_t> *output_order) {
2457 ✗ if (!servers) {
2458 ✗ return false;
2459 }
2460 ✗ if (servers->size() == 1) {
2461 ✗ if (output_order) {
2462 ✗ output_order->clear();
2463 ✗ output_order->push_back(0);
2464 }
2465 ✗ return true;
2466 }
2467
2468 ✗ std::vector<std::string> host_chain;
2469 ✗ GetHostInfo(&host_chain, NULL, NULL);
2470
2471 ✗ std::vector<std::string> server_dns_names;
2472 ✗ server_dns_names.reserve(servers->size());
2473 ✗ for (unsigned i = 0; i < servers->size(); ++i) {
2474 ✗ const std::string host = dns::ExtractHost((*servers)[i]);
2475 ✗ server_dns_names.push_back(host.empty() ? (*servers)[i] : host);
2476 }
2477 ✗ const std::string host_list = JoinStrings(server_dns_names, ",");
2478
2479 ✗ vector<string> host_chain_shuffled;
2480 {
2481 // Protect against concurrent access to prng_
2482 ✗ const MutexLockGuard m(lock_options_);
2483 // Determine random hosts for the Geo-API query
2484 ✗ host_chain_shuffled = Shuffle(host_chain, &prng_);
2485 }
2486 // Request ordered list via Geo-API
2487 ✗ bool success = false;
2488 ✗ const unsigned max_attempts = std::min(host_chain_shuffled.size(), size_t(3));
2489 ✗ vector<uint64_t> geo_order(servers->size());
2490 ✗ for (unsigned i = 0; i < max_attempts; ++i) {
2491 ✗ const string url = host_chain_shuffled[i] + "/api/v1.0/geo/@proxy@/"
2492 ✗ + host_list;
2493 ✗ LogCvmfs(kLogDownload, kLogDebug,
2494 "(manager '%s') requesting ordered server list from %s",
2495 name_.c_str(), url.c_str());
2496 ✗ cvmfs::MemSink memsink;
2497 ✗ JobInfo info(&url, false, false, NULL, &memsink);
2498 ✗ const Failures result = Fetch(&info);
2499 ✗ if (result == kFailOk) {
2500 ✗ const string order(reinterpret_cast<char *>(memsink.data()),
2501 ✗ memsink.pos());
2502 ✗ memsink.Reset();
2503 ✗ const bool retval = ValidateGeoReply(order, servers->size(), &geo_order);
2504 ✗ if (!retval) {
2505 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2506 "(manager '%s') retrieved invalid GeoAPI reply from %s [%s]",
2507 name_.c_str(), url.c_str(), order.c_str());
2508 } else {
2509 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslog,
2510 "(manager '%s') "
2511 "geographic order of servers retrieved from %s",
2512 name_.c_str(),
2513 ✗ dns::ExtractHost(host_chain_shuffled[i]).c_str());
2514 // remove new line at end of "order"
2515 ✗ LogCvmfs(kLogDownload, kLogDebug, "order is %s",
2516 ✗ Trim(order, true /* trim_newline */).c_str());
2517 ✗ success = true;
2518 ✗ break;
2519 }
2520 ✗ } else {
2521 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2522 "(manager '%s') GeoAPI request for %s failed with error %d [%s]",
2523 name_.c_str(), url.c_str(), result, Code2Ascii(result));
2524 }
2525 }
2526 ✗ if (!success) {
2527 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2528 "(manager '%s') "
2529 "failed to retrieve geographic order from stratum 1 servers",
2530 name_.c_str());
2531 ✗ return false;
2532 }
2533
2534 ✗ if (output_order) {
2535 ✗ output_order->swap(geo_order);
2536 } else {
2537 ✗ std::vector<std::string> sorted_servers;
2538 ✗ sorted_servers.reserve(geo_order.size());
2539 ✗ for (unsigned i = 0; i < geo_order.size(); ++i) {
2540 ✗ const uint64_t orderval = geo_order[i];
2541 ✗ sorted_servers.push_back((*servers)[orderval]);
2542 }
2543 ✗ servers->swap(sorted_servers);
2544 }
2545 ✗ return true;
2546 }
2547
2548
2549 /**
2550 * Uses the Geo-API of Stratum 1s to let any of them order the list of servers
2551 * and fallback proxies (if any).
2552 * Tries at most three random Stratum 1s before giving up.
2553 * If you change the host list in between by SetHostChain() or the fallback
2554 * proxy list by SetProxyChain(), they will be overwritten by this function.
2555 */
2556 ✗ bool DownloadManager::ProbeGeo() {
2557 ✗ vector<string> host_chain;
2558 ✗ vector<int> host_rtt;
2559 unsigned current_host;
2560 ✗ vector<vector<ProxyInfo> > proxy_chain;
2561 unsigned fallback_group;
2562
2563 ✗ GetHostInfo(&host_chain, &host_rtt, &current_host);
2564 ✗ GetProxyInfo(&proxy_chain, NULL, &fallback_group);
2565 ✗ if ((host_chain.size() < 2) && ((proxy_chain.size() - fallback_group) < 2))
2566 ✗ return true;
2567
2568 ✗ vector<string> host_names;
2569 ✗ for (unsigned i = 0; i < host_chain.size(); ++i)
2570 ✗ host_names.push_back(dns::ExtractHost(host_chain[i]));
2571 ✗ SortTeam(&host_names, &host_chain);
2572 ✗ const unsigned last_geo_host = host_names.size();
2573
2574 ✗ if ((fallback_group == 0) && (last_geo_host > 1)) {
2575 // There are no non-fallback proxies, which means that the client
2576 // will always use the fallback proxies. Add a keyword separator
2577 // between the hosts and fallback proxies so the geosorting service
2578 // will know to sort the hosts based on the distance from the
2579 // closest fallback proxy rather than the distance from the client.
2580 ✗ host_names.push_back("+PXYSEP+");
2581 }
2582
2583 // Add fallback proxy names to the end of the host list
2584 ✗ const unsigned first_geo_fallback = host_names.size();
2585 ✗ for (unsigned i = fallback_group; i < proxy_chain.size(); ++i) {
2586 // We only take the first fallback proxy name from every group under the
2587 // assumption that load-balanced servers are at the same location
2588 ✗ host_names.push_back(proxy_chain[i][0].host.name());
2589 }
2590
2591 ✗ std::vector<uint64_t> geo_order;
2592 ✗ const bool success = GeoSortServers(&host_names, &geo_order);
2593 ✗ if (!success) {
2594 // GeoSortServers already logged a failure message.
2595 ✗ return false;
2596 }
2597
2598 // Re-install host chain and proxy chain
2599 ✗ const MutexLockGuard m(lock_options_);
2600 ✗ delete opt_host_.chain;
2601 ✗ opt_num_proxies_ = 0;
2602 ✗ opt_host_.chain = new vector<string>(host_chain.size());
2603
2604 // It's possible that opt_proxy_groups_fallback_ might have changed while
2605 // the lock wasn't held
2606 vector<vector<ProxyInfo> > *proxy_groups = new vector<vector<ProxyInfo> >(
2607 ✗ opt_proxy_groups_fallback_ + proxy_chain.size() - fallback_group);
2608 // First copy the non-fallback part of the current proxy chain
2609 ✗ for (unsigned i = 0; i < opt_proxy_groups_fallback_; ++i) {
2610 ✗ (*proxy_groups)[i] = (*opt_proxy_groups_)[i];
2611 ✗ opt_num_proxies_ += (*opt_proxy_groups_)[i].size();
2612 }
2613
2614 // Copy the host chain and fallback proxies by geo order. Array indices
2615 // in geo_order that are smaller than last_geo_host refer to a stratum 1,
2616 // and those indices greater than or equal to first_geo_fallback refer to
2617 // a fallback proxy.
2618 ✗ unsigned hosti = 0;
2619 ✗ unsigned proxyi = opt_proxy_groups_fallback_;
2620 ✗ for (unsigned i = 0; i < geo_order.size(); ++i) {
2621 ✗ const uint64_t orderval = geo_order[i];
2622 ✗ if (orderval < static_cast<uint64_t>(last_geo_host)) {
2623 // LogCvmfs(kLogCvmfs, kLogSyslog, "this is orderval %u at host index
2624 // %u", orderval, hosti);
2625 ✗ (*opt_host_.chain)[hosti++] = host_chain[orderval];
2626 ✗ } else if (orderval >= static_cast<uint64_t>(first_geo_fallback)) {
2627 // LogCvmfs(kLogCvmfs, kLogSyslog,
2628 // "this is orderval %u at proxy index %u, using proxy_chain index %u",
2629 // orderval, proxyi, fallback_group + orderval - first_geo_fallback);
2630 ✗ (*proxy_groups)[proxyi] = proxy_chain[fallback_group + orderval
2631 ✗ - first_geo_fallback];
2632 ✗ opt_num_proxies_ += (*proxy_groups)[proxyi].size();
2633 ✗ proxyi++;
2634 }
2635 }
2636
2637 ✗ opt_proxy_map_.clear();
2638 ✗ delete opt_proxy_groups_;
2639 ✗ opt_proxy_groups_ = proxy_groups;
2640 // In pathological cases, opt_proxy_groups_current_ can be larger now when
2641 // proxies changed in-between.
2642 ✗ if (opt_proxy_groups_current_ > opt_proxy_groups_->size()) {
2643 ✗ if (opt_proxy_groups_->size() == 0) {
2644 ✗ opt_proxy_groups_current_ = 0;
2645 } else {
2646 ✗ opt_proxy_groups_current_ = opt_proxy_groups_->size() - 1;
2647 }
2648 ✗ opt_proxy_groups_current_burned_ = 0;
2649 }
2650
2651 ✗ UpdateProxiesUnlocked("geosort");
2652
2653 ✗ delete opt_host_chain_rtt_;
2654 ✗ opt_host_chain_rtt_ = new vector<int>(host_chain.size(), kProbeGeo);
2655 ✗ opt_host_.current = 0;
2656
2657 ✗ return true;
2658 }
2659
2660
2661 /**
2662 * Validates a string of the form "1,4,2,3" representing in which order the
2663 * the expected_size number of hosts should be put for optimal geographic
2664 * proximity. Returns false if the reply_order string is invalid, otherwise
2665 * fills in the reply_vals array with zero-based order indexes (e.g.
2666 * [0,3,1,2]) and returns true.
2667 */
2668 476 bool DownloadManager::ValidateGeoReply(const string &reply_order,
2669 const unsigned expected_size,
2670 vector<uint64_t> *reply_vals) {
2671
2/2
✓ Branch 1 taken 34 times.
✓ Branch 2 taken 442 times.
 476 if (reply_order.empty())
2672 34 return false;
2673
2/4
✓ Branch 2 taken 442 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 442 times.
✗ Branch 6 not taken.
 884 const sanitizer::InputSanitizer sanitizer("09 , \n");
2674
3/4
✓ Branch 1 taken 442 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 34 times.
✓ Branch 4 taken 408 times.
 442 if (!sanitizer.IsValid(reply_order))
2675 34 return false;
2676
2/4
✓ Branch 2 taken 408 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 408 times.
✗ Branch 6 not taken.
 816 const sanitizer::InputSanitizer strip_newline("09 ,");
2677
1/2
✓ Branch 1 taken 408 times.
✗ Branch 2 not taken.
 408 vector<string> reply_strings = SplitString(strip_newline.Filter(reply_order),
2678
1/2
✓ Branch 1 taken 408 times.
✗ Branch 2 not taken.
 408 ',');
2679 408 vector<uint64_t> tmp_vals;
2680
2/2
✓ Branch 1 taken 782 times.
✓ Branch 2 taken 340 times.
 1122 for (unsigned i = 0; i < reply_strings.size(); ++i) {
2681
2/2
✓ Branch 2 taken 68 times.
✓ Branch 3 taken 714 times.
 782 if (reply_strings[i].empty())
2682 68 return false;
2683
2/4
✓ Branch 2 taken 714 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 714 times.
✗ Branch 6 not taken.
 714 tmp_vals.push_back(String2Uint64(reply_strings[i]));
2684 }
2685
2/2
✓ Branch 1 taken 170 times.
✓ Branch 2 taken 170 times.
 340 if (tmp_vals.size() != expected_size)
2686 170 return false;
2687
2688 // Check if tmp_vals contains the number 1..n
2689
1/2
✓ Branch 3 taken 170 times.
✗ Branch 4 not taken.
 170 set<uint64_t> coverage(tmp_vals.begin(), tmp_vals.end());
2690
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 170 times.
 170 if (coverage.size() != tmp_vals.size())
2691 ✗ return false;
2692
5/6
✓ Branch 2 taken 136 times.
✓ Branch 3 taken 34 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 136 times.
✓ Branch 9 taken 34 times.
✓ Branch 10 taken 136 times.
 170 if ((*coverage.begin() != 1) || (*coverage.rbegin() != coverage.size()))
2693 34 return false;
2694
2695
2/2
✓ Branch 0 taken 306 times.
✓ Branch 1 taken 136 times.
 442 for (unsigned i = 0; i < expected_size; ++i) {
2696 306 (*reply_vals)[i] = tmp_vals[i] - 1;
2697 }
2698 136 return true;
2699 442 }
2700
2701
2702 /**
2703 * Removes DIRECT from a list of ';' and '|' separated proxies.
2704 * \return true if DIRECT was present, false otherwise
2705 */
2706 2538 bool DownloadManager::StripDirect(const string &proxy_list,
2707 string *cleaned_list) {
2708
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2538 times.
 2538 assert(cleaned_list);
2709
2/2
✓ Branch 1 taken 2232 times.
✓ Branch 2 taken 306 times.
 2538 if (proxy_list == "") {
2710
1/2
✓ Branch 1 taken 2232 times.
✗ Branch 2 not taken.
 2232 *cleaned_list = "";
2711 2232 return false;
2712 }
2713 306 bool result = false;
2714
2715
1/2
✓ Branch 1 taken 306 times.
✗ Branch 2 not taken.
 306 vector<string> proxy_groups = SplitString(proxy_list, ';');
2716 306 vector<string> cleaned_groups;
2717
2/2
✓ Branch 1 taken 748 times.
✓ Branch 2 taken 306 times.
 1054 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2718
1/2
✓ Branch 2 taken 748 times.
✗ Branch 3 not taken.
 748 vector<string> group = SplitString(proxy_groups[i], '|');
2719 748 vector<string> cleaned;
2720
2/2
✓ Branch 1 taken 1258 times.
✓ Branch 2 taken 748 times.
 2006 for (unsigned j = 0; j < group.size(); ++j) {
2721
6/6
✓ Branch 2 taken 918 times.
✓ Branch 3 taken 340 times.
✓ Branch 6 taken 442 times.
✓ Branch 7 taken 476 times.
✓ Branch 8 taken 782 times.
✓ Branch 9 taken 476 times.
 1258 if ((group[j] == "DIRECT") || (group[j] == "")) {
2722 782 result = true;
2723 } else {
2724
1/2
✓ Branch 2 taken 476 times.
✗ Branch 3 not taken.
 476 cleaned.push_back(group[j]);
2725 }
2726 }
2727
2/2
✓ Branch 1 taken 238 times.
✓ Branch 2 taken 510 times.
 748 if (!cleaned.empty())
2728
3/6
✓ Branch 2 taken 238 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 238 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 238 times.
✗ Branch 9 not taken.
 238 cleaned_groups.push_back(JoinStrings(cleaned, "|"));
2729 748 }
2730
2731
2/4
✓ Branch 2 taken 306 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 306 times.
✗ Branch 6 not taken.
 306 *cleaned_list = JoinStrings(cleaned_groups, ";");
2732 306 return result;
2733 306 }
2734
2735
2736 /**
2737 * Parses a list of ';'- and '|'-separated proxy servers and fallback proxy
2738 * servers for the proxy groups.
2739 * The empty string for both removes the proxy chain.
2740 * The set_mode parameter can be used to set either proxies (leaving fallback
2741 * proxies unchanged) or fallback proxies (leaving regular proxies unchanged)
2742 * or both.
2743 */
2744 2198 void DownloadManager::SetProxyChain(const string &proxy_list,
2745 const string &fallback_proxy_list,
2746 const ProxySetModes set_mode) {
2747 2198 const MutexLockGuard m(lock_options_);
2748
2749 2198 opt_timestamp_backup_proxies_ = 0;
2750 2198 opt_timestamp_failover_proxies_ = 0;
2751
1/2
✓ Branch 1 taken 2198 times.
✗ Branch 2 not taken.
 2198 string set_proxy_list = opt_proxy_list_;
2752
1/2
✓ Branch 1 taken 2198 times.
✗ Branch 2 not taken.
 2198 string set_proxy_fallback_list = opt_proxy_fallback_list_;
2753 bool contains_direct;
2754
3/4
✓ Branch 0 taken 2198 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2096 times.
✓ Branch 3 taken 102 times.
 2198 if ((set_mode == kSetProxyFallback) || (set_mode == kSetProxyBoth)) {
2755
1/2
✓ Branch 1 taken 2096 times.
✗ Branch 2 not taken.
 2096 opt_proxy_fallback_list_ = fallback_proxy_list;
2756 }
2757
3/4
✓ Branch 0 taken 2096 times.
✓ Branch 1 taken 102 times.
✓ Branch 2 taken 2096 times.
✗ Branch 3 not taken.
 2198 if ((set_mode == kSetProxyRegular) || (set_mode == kSetProxyBoth)) {
2758
1/2
✓ Branch 1 taken 2198 times.
✗ Branch 2 not taken.
 2198 opt_proxy_list_ = proxy_list;
2759 }
2760
1/2
✓ Branch 1 taken 2198 times.
✗ Branch 2 not taken.
 2198 contains_direct = StripDirect(opt_proxy_fallback_list_,
2761 &set_proxy_fallback_list);
2762
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2198 times.
 2198 if (contains_direct) {
2763 ✗ LogCvmfs(kLogDownload, kLogSyslogWarn | kLogDebug,
2764 "(manager '%s') fallback proxies do not support DIRECT, removing",
2765 name_.c_str());
2766 }
2767
1/2
✓ Branch 1 taken 2198 times.
✗ Branch 2 not taken.
 2198 if (set_proxy_fallback_list == "") {
2768
1/2
✓ Branch 1 taken 2198 times.
✗ Branch 2 not taken.
 2198 set_proxy_list = opt_proxy_list_;
2769 } else {
2770 ✗ const bool contains_direct = StripDirect(opt_proxy_list_, &set_proxy_list);
2771 ✗ if (contains_direct) {
2772 ✗ LogCvmfs(kLogDownload, kLogSyslog | kLogDebug,
2773 "(manager '%s') skipping DIRECT proxy to use fallback proxy",
2774 name_.c_str());
2775 }
2776 }
2777
2778 // From this point on, use set_proxy_list and set_fallback_proxy_list as
2779 // effective proxy lists!
2780
2781 2198 opt_proxy_map_.clear();
2782
2/2
✓ Branch 0 taken 1048 times.
✓ Branch 1 taken 1150 times.
 2198 delete opt_proxy_groups_;
2783
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 2198 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 2198 times.
 2198 if ((set_proxy_list == "") && (set_proxy_fallback_list == "")) {
2784 ✗ opt_proxy_groups_ = NULL;
2785 ✗ opt_proxy_groups_current_ = 0;
2786 ✗ opt_proxy_groups_current_burned_ = 0;
2787 ✗ opt_proxy_groups_fallback_ = 0;
2788 ✗ opt_num_proxies_ = 0;
2789 ✗ return;
2790 }
2791
2792 // Determine number of regular proxy groups (== first fallback proxy group)
2793 2198 opt_proxy_groups_fallback_ = 0;
2794
1/2
✓ Branch 1 taken 2198 times.
✗ Branch 2 not taken.
 2198 if (set_proxy_list != "") {
2795
1/2
✓ Branch 1 taken 2198 times.
✗ Branch 2 not taken.
 2198 opt_proxy_groups_fallback_ = SplitString(set_proxy_list, ';').size();
2796 }
2797
1/2
✓ Branch 2 taken 2198 times.
✗ Branch 3 not taken.
 2198 LogCvmfs(kLogDownload, kLogDebug,
2798 "(manager '%s') "
2799 "first fallback proxy group %u",
2800 name_.c_str(), opt_proxy_groups_fallback_);
2801
2802 // Concatenate regular proxies and fallback proxies, both of which can be
2803 // empty.
2804
1/2
✓ Branch 1 taken 2198 times.
✗ Branch 2 not taken.
 2198 string all_proxy_list = set_proxy_list;
2805
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 2198 times.
 2198 if (set_proxy_fallback_list != "") {
2806 ✗ if (all_proxy_list != "")
2807 ✗ all_proxy_list += ";";
2808 ✗ all_proxy_list += set_proxy_fallback_list;
2809 }
2810
1/2
✓ Branch 3 taken 2198 times.
✗ Branch 4 not taken.
 2198 LogCvmfs(kLogDownload, kLogDebug, "(manager '%s') full proxy list %s",
2811 name_.c_str(), all_proxy_list.c_str());
2812
2813 // Resolve server names in provided urls
2814 2198 vector<string> hostnames; // All encountered hostnames
2815 2198 vector<string> proxy_groups;
2816
1/2
✓ Branch 1 taken 2198 times.
✗ Branch 2 not taken.
 2198 if (all_proxy_list != "")
2817
1/2
✓ Branch 1 taken 2198 times.
✗ Branch 2 not taken.
 2198 proxy_groups = SplitString(all_proxy_list, ';');
2818
2/2
✓ Branch 1 taken 2232 times.
✓ Branch 2 taken 2198 times.
 4430 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2819
1/2
✓ Branch 2 taken 2232 times.
✗ Branch 3 not taken.
 2232 vector<string> this_group = SplitString(proxy_groups[i], '|');
2820
2/2
✓ Branch 1 taken 2232 times.
✓ Branch 2 taken 2232 times.
 4464 for (unsigned j = 0; j < this_group.size(); ++j) {
2821
1/2
✓ Branch 2 taken 2232 times.
✗ Branch 3 not taken.
 2232 this_group[j] = dns::AddDefaultScheme(this_group[j]);
2822 // Note: DIRECT strings will be "extracted" to an empty string.
2823
1/2
✓ Branch 2 taken 2232 times.
✗ Branch 3 not taken.
 2232 const string hostname = dns::ExtractHost(this_group[j]);
2824 // Save the hostname. Leave empty (DIRECT) names so indexes will
2825 // match later.
2826
1/2
✓ Branch 1 taken 2232 times.
✗ Branch 2 not taken.
 2232 hostnames.push_back(hostname);
2827 2232 }
2828 2232 }
2829 2198 vector<dns::Host> hosts;
2830
1/2
✓ Branch 3 taken 2198 times.
✗ Branch 4 not taken.
 2198 LogCvmfs(kLogDownload, kLogDebug,
2831 "(manager '%s') "
2832 "resolving %lu proxy addresses",
2833 name_.c_str(), hostnames.size());
2834
1/2
✓ Branch 1 taken 2198 times.
✗ Branch 2 not taken.
 2198 resolver_->ResolveMany(hostnames, &hosts);
2835
2836 // Construct opt_proxy_groups_: traverse proxy list in same order and expand
2837 // names to resolved IP addresses.
2838
1/2
✓ Branch 1 taken 2198 times.
✗ Branch 2 not taken.
 2198 opt_proxy_groups_ = new vector<vector<ProxyInfo> >();
2839 2198 opt_num_proxies_ = 0;
2840 2198 unsigned num_proxy = 0; // Combined i, j counter
2841
2/2
✓ Branch 1 taken 2232 times.
✓ Branch 2 taken 2198 times.
 4430 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2842
1/2
✓ Branch 2 taken 2232 times.
✗ Branch 3 not taken.
 2232 vector<string> this_group = SplitString(proxy_groups[i], '|');
2843 // Construct ProxyInfo objects from proxy string and DNS resolver result for
2844 // every proxy in this_group. One URL can result in multiple ProxyInfo
2845 // objects, one for each IP address.
2846 2232 vector<ProxyInfo> infos;
2847
2/2
✓ Branch 1 taken 2232 times.
✓ Branch 2 taken 2232 times.
 4464 for (unsigned j = 0; j < this_group.size(); ++j, ++num_proxy) {
2848
1/2
✓ Branch 2 taken 2232 times.
✗ Branch 3 not taken.
 2232 this_group[j] = dns::AddDefaultScheme(this_group[j]);
2849
2/2
✓ Branch 2 taken 2096 times.
✓ Branch 3 taken 136 times.
 2232 if (this_group[j] == "DIRECT") {
2850
3/6
✓ Branch 2 taken 2096 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2096 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 2096 times.
✗ Branch 9 not taken.
 2096 infos.push_back(ProxyInfo("DIRECT"));
2851 2096 continue;
2852 }
2853
2854
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 136 times.
 136 if (hosts[num_proxy].status() != dns::kFailOk) {
2855 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2856 "(manager '%s') "
2857 "failed to resolve IP addresses for %s (%d - %s)",
2858 ✗ name_.c_str(), hosts[num_proxy].name().c_str(),
2859 ✗ hosts[num_proxy].status(),
2860 ✗ dns::Code2Ascii(hosts[num_proxy].status()));
2861 const dns::Host failed_host = dns::Host::ExtendDeadline(
2862 ✗ hosts[num_proxy], resolver_->min_ttl());
2863 ✗ infos.push_back(ProxyInfo(failed_host, this_group[j]));
2864 ✗ continue;
2865 }
2866
2867 // IPv4 addresses have precedence
2868 136 set<string> best_addresses = hosts[num_proxy].ViewBestAddresses(
2869
2/4
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 136 times.
✗ Branch 5 not taken.
 136 opt_ip_preference_);
2870 136 set<string>::const_iterator iter_ips = best_addresses.begin();
2871
2/2
✓ Branch 3 taken 136 times.
✓ Branch 4 taken 136 times.
 272 for (; iter_ips != best_addresses.end(); ++iter_ips) {
2872
1/2
✓ Branch 3 taken 136 times.
✗ Branch 4 not taken.
 136 const string url_ip = dns::RewriteUrl(this_group[j], *iter_ips);
2873
2/4
✓ Branch 2 taken 136 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 136 times.
✗ Branch 6 not taken.
 136 infos.push_back(ProxyInfo(hosts[num_proxy], url_ip));
2874
2875
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 136 times.
 136 if (sharding_policy_.UseCount() > 0) {
2876 ✗ sharding_policy_->AddProxy(url_ip);
2877 }
2878 136 }
2879 136 }
2880
1/2
✓ Branch 1 taken 2232 times.
✗ Branch 2 not taken.
 2232 opt_proxy_groups_->push_back(infos);
2881 2232 opt_num_proxies_ += infos.size();
2882 2232 }
2883
1/2
✓ Branch 2 taken 2198 times.
✗ Branch 3 not taken.
 2198 LogCvmfs(kLogDownload, kLogDebug,
2884 "(manager '%s') installed %u proxies in %lu load-balance groups",
2885 2198 name_.c_str(), opt_num_proxies_, opt_proxy_groups_->size());
2886 2198 opt_proxy_groups_current_ = 0;
2887 2198 opt_proxy_groups_current_burned_ = 0;
2888
2889 // Select random start proxy from the first group.
2890
1/2
✓ Branch 1 taken 2198 times.
✗ Branch 2 not taken.
 2198 if (opt_proxy_groups_->size() > 0) {
2891 // Select random start proxy from the first group.
2892
2/4
✓ Branch 2 taken 2198 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2198 times.
✗ Branch 6 not taken.
 2198 UpdateProxiesUnlocked("set random start proxy from the first proxy group");
2893 }
2894
3/6
✓ Branch 5 taken 2198 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 2198 times.
✗ Branch 9 not taken.
✓ Branch 11 taken 2198 times.
✗ Branch 12 not taken.
 2198 }
2895
2896
2897 /**
2898 * Retrieves the proxy chain, optionally the currently active load-balancing
2899 * group, and optionally the index of the first fallback proxy group.
2900 * If there are no fallback proxies, the index will equal the size of
2901 * the proxy chain.
2902 */
2903 ✗ void DownloadManager::GetProxyInfo(vector<vector<ProxyInfo> > *proxy_chain,
2904 unsigned *current_group,
2905 unsigned *fallback_group) {
2906 ✗ assert(proxy_chain != NULL);
2907 ✗ const MutexLockGuard m(lock_options_);
2908
2909 ✗ if (!opt_proxy_groups_) {
2910 ✗ const vector<vector<ProxyInfo> > empty_chain;
2911 ✗ *proxy_chain = empty_chain;
2912 ✗ if (current_group != NULL)
2913 ✗ *current_group = 0;
2914 ✗ if (fallback_group != NULL)
2915 ✗ *fallback_group = 0;
2916 ✗ return;
2917 }
2918
2919 ✗ *proxy_chain = *opt_proxy_groups_;
2920 ✗ if (current_group != NULL)
2921 ✗ *current_group = opt_proxy_groups_current_;
2922 ✗ if (fallback_group != NULL)
2923 ✗ *fallback_group = opt_proxy_groups_fallback_;
2924 }
2925
2926 ✗ string DownloadManager::GetProxyList() { return opt_proxy_list_; }
2927
2928 ✗ string DownloadManager::GetFallbackProxyList() {
2929 ✗ return opt_proxy_fallback_list_;
2930 }
2931
2932 /**
2933 * Choose proxy
2934 */
2935 7370 DownloadManager::ProxyInfo *DownloadManager::ChooseProxyUnlocked(
2936 const shash::Any *hash) {
2937
2/2
✓ Branch 0 taken 4699 times.
✓ Branch 1 taken 2671 times.
 7370 if (!opt_proxy_groups_)
2938 4699 return NULL;
2939
2940
2/2
✓ Branch 0 taken 1383 times.
✓ Branch 1 taken 1288 times.
 2671 const uint32_t key = (hash ? hash->Partial32() : 0);
2941
1/2
✓ Branch 1 taken 2671 times.
✗ Branch 2 not taken.
 2671 const map<uint32_t, ProxyInfo *>::iterator it = opt_proxy_map_.lower_bound(
2942 key);
2943 2671 ProxyInfo *proxy = it->second;
2944
2945 2671 return proxy;
2946 }
2947
2948 /**
2949 * Update currently selected proxy
2950 */
2951 3280 void DownloadManager::UpdateProxiesUnlocked(const string &reason) {
2952
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3280 times.
 3280 if (!opt_proxy_groups_)
2953 ✗ return;
2954
2955 // Identify number of non-burned proxies within the current group
2956 3280 vector<ProxyInfo> *group = current_proxy_group();
2957 3280 const unsigned num_alive = (group->size() - opt_proxy_groups_current_burned_);
2958
2/4
✓ Branch 2 taken 3280 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 3280 times.
✗ Branch 6 not taken.
 6560 const string old_proxy = JoinStrings(opt_proxies_, "|");
2959
2960 // Rebuild proxy map and URL list
2961 3280 opt_proxy_map_.clear();
2962 3280 opt_proxies_.clear();
2963 3280 const uint32_t max_key = 0xffffffffUL;
2964
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3280 times.
 3280 if (opt_proxy_shard_) {
2965 // Build a consistent map with multiple entries for each proxy
2966 ✗ for (unsigned i = 0; i < num_alive; ++i) {
2967 ✗ ProxyInfo *proxy = &(*group)[i];
2968 ✗ shash::Any proxy_hash(shash::kSha1);
2969 ✗ HashString(proxy->url, &proxy_hash);
2970 ✗ Prng prng;
2971 ✗ prng.InitSeed(proxy_hash.Partial32());
2972 ✗ for (unsigned j = 0; j < kProxyMapScale; ++j) {
2973 ✗ const std::pair<uint32_t, ProxyInfo *> entry(prng.Next(max_key), proxy);
2974 ✗ opt_proxy_map_.insert(entry);
2975 }
2976 ✗ const std::string proxy_name = proxy->host.name().empty()
2977 ? ""
2978 ✗ : " (" + proxy->host.name() + ")";
2979 ✗ opt_proxies_.push_back(proxy->url + proxy_name);
2980 }
2981 // Ensure lower_bound() finds a value for all keys
2982 ✗ ProxyInfo *first_proxy = opt_proxy_map_.begin()->second;
2983 ✗ const std::pair<uint32_t, ProxyInfo *> last_entry(max_key, first_proxy);
2984 ✗ opt_proxy_map_.insert(last_entry);
2985 } else {
2986 // Build a map with a single entry for one randomly selected proxy
2987 3280 const unsigned select = prng_.Next(num_alive);
2988 3280 ProxyInfo *proxy = &(*group)[select];
2989 3280 const std::pair<uint32_t, ProxyInfo *> entry(max_key, proxy);
2990
1/2
✓ Branch 1 taken 3280 times.
✗ Branch 2 not taken.
 3280 opt_proxy_map_.insert(entry);
2991 3280 const std::string proxy_name = proxy->host.name().empty()
2992 ? ""
2993
9/18
✓ Branch 0 taken 3144 times.
✓ Branch 1 taken 136 times.
✓ Branch 4 taken 3144 times.
✗ Branch 5 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 136 times.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✓ Branch 11 taken 136 times.
✗ Branch 12 not taken.
✓ Branch 13 taken 136 times.
✓ Branch 14 taken 3144 times.
✓ Branch 15 taken 3144 times.
✓ Branch 16 taken 136 times.
✗ Branch 17 not taken.
✗ Branch 18 not taken.
✗ Branch 19 not taken.
✗ Branch 20 not taken.
 3416 : " (" + proxy->host.name() + ")";
2994
2/4
✓ Branch 1 taken 3280 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3280 times.
✗ Branch 5 not taken.
 3280 opt_proxies_.push_back(proxy->url + proxy_name);
2995 3280 }
2996
1/2
✓ Branch 3 taken 3280 times.
✗ Branch 4 not taken.
 3280 sort(opt_proxies_.begin(), opt_proxies_.end());
2997
2998 // Report any change in proxy usage
2999
2/4
✓ Branch 2 taken 3280 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 3280 times.
✗ Branch 6 not taken.
 6560 const string new_proxy = JoinStrings(opt_proxies_, "|");
3000 const string curr_host = "Current host: "
3001 3280 + (opt_host_.chain
3002
6/12
✓ Branch 0 taken 3144 times.
✓ Branch 1 taken 136 times.
✓ Branch 4 taken 3144 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 136 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 136 times.
✓ Branch 11 taken 3144 times.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
 6560 ? (*opt_host_.chain)[opt_host_.current]
3003
1/2
✓ Branch 1 taken 3280 times.
✗ Branch 2 not taken.
 3280 : "");
3004
2/2
✓ Branch 1 taken 2232 times.
✓ Branch 2 taken 1048 times.
 3280 if (new_proxy != old_proxy) {
3005
4/9
✗ Branch 2 not taken.
✓ Branch 3 taken 2232 times.
✓ Branch 4 taken 2198 times.
✓ Branch 5 taken 34 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 2232 times.
✗ Branch 9 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
 6730 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
3006 "(manager '%s') switching proxy from %s to %s. Reason: %s [%s]",
3007 2266 name_.c_str(), (old_proxy.empty() ? "(none)" : old_proxy.c_str()),
3008 4464 (new_proxy.empty() ? "(none)" : new_proxy.c_str()), reason.c_str(),
3009 curr_host.c_str());
3010 }
3011 3280 }
3012
3013 /**
3014 * Enable proxy sharding
3015 */
3016 ✗ void DownloadManager::ShardProxies() {
3017 ✗ opt_proxy_shard_ = true;
3018 ✗ RebalanceProxiesUnlocked("enable sharding");
3019 }
3020
3021 /**
3022 * Selects a new random proxy in the current load-balancing group. Resets the
3023 * "burned" counter.
3024 */
3025 ✗ void DownloadManager::RebalanceProxiesUnlocked(const string &reason) {
3026 ✗ if (!opt_proxy_groups_)
3027 ✗ return;
3028
3029 ✗ opt_timestamp_failover_proxies_ = 0;
3030 ✗ opt_proxy_groups_current_burned_ = 0;
3031 ✗ UpdateProxiesUnlocked(reason);
3032 }
3033
3034
3035 ✗ void DownloadManager::RebalanceProxies() {
3036 ✗ const MutexLockGuard m(lock_options_);
3037 ✗ RebalanceProxiesUnlocked("rebalance invoked manually");
3038 }
3039
3040
3041 /**
3042 * Switches to the next load-balancing group of proxy servers.
3043 */
3044 ✗ void DownloadManager::SwitchProxyGroup() {
3045 ✗ const MutexLockGuard m(lock_options_);
3046
3047 ✗ if (!opt_proxy_groups_ || (opt_proxy_groups_->size() < 2)) {
3048 ✗ return;
3049 }
3050
3051 ✗ opt_proxy_groups_current_ = (opt_proxy_groups_current_ + 1)
3052 ✗ % opt_proxy_groups_->size();
3053 ✗ opt_timestamp_backup_proxies_ = time(NULL);
3054
3055 const std::string msg = "switch to proxy group "
3056 ✗ + StringifyUint(opt_proxy_groups_current_);
3057 ✗ RebalanceProxiesUnlocked(msg);
3058 }
3059
3060
3061 ✗ void DownloadManager::SetProxyGroupResetDelay(const unsigned seconds) {
3062 ✗ const MutexLockGuard m(lock_options_);
3063 ✗ opt_proxy_groups_reset_after_ = seconds;
3064 ✗ if (opt_proxy_groups_reset_after_ == 0) {
3065 ✗ opt_timestamp_backup_proxies_ = 0;
3066 ✗ opt_timestamp_failover_proxies_ = 0;
3067 }
3068 }
3069
3070
3071 ✗ void DownloadManager::SetMetalinkResetDelay(const unsigned seconds) {
3072 ✗ const MutexLockGuard m(lock_options_);
3073 ✗ opt_metalink_.reset_after = seconds;
3074 ✗ if (opt_metalink_.reset_after == 0)
3075 ✗ opt_metalink_.timestamp_backup = 0;
3076 }
3077
3078
3079 ✗ void DownloadManager::SetHostResetDelay(const unsigned seconds) {
3080 ✗ const MutexLockGuard m(lock_options_);
3081 ✗ opt_host_.reset_after = seconds;
3082 ✗ if (opt_host_.reset_after == 0)
3083 ✗ opt_host_.timestamp_backup = 0;
3084 }
3085
3086
3087 2260 void DownloadManager::SetRetryParameters(const unsigned max_retries,
3088 const unsigned backoff_init_ms,
3089 const unsigned backoff_max_ms) {
3090 2260 const MutexLockGuard m(lock_options_);
3091 2260 opt_max_retries_ = max_retries;
3092 2260 opt_backoff_init_ms_ = backoff_init_ms;
3093 2260 opt_backoff_max_ms_ = backoff_max_ms;
3094 2260 }
3095
3096
3097 1116 void DownloadManager::SetMaxIpaddrPerProxy(unsigned limit) {
3098 1116 const MutexLockGuard m(lock_options_);
3099 1116 resolver_->set_throttle(limit);
3100 1116 }
3101
3102
3103 1283 void DownloadManager::SetProxyTemplates(const std::string &direct,
3104 const std::string &forced) {
3105 1283 const MutexLockGuard m(lock_options_);
3106
1/2
✓ Branch 1 taken 1283 times.
✗ Branch 2 not taken.
 1283 proxy_template_direct_ = direct;
3107
1/2
✓ Branch 1 taken 1283 times.
✗ Branch 2 not taken.
 1283 proxy_template_forced_ = forced;
3108 1283 }
3109
3110
3111 ✗ void DownloadManager::EnableInfoHeader() { enable_info_header_ = true; }
3112
3113
3114 1079 void DownloadManager::EnableRedirects() { follow_redirects_ = true; }
3115
3116 ✗ void DownloadManager::EnableIgnoreSignatureFailures() {
3117 ✗ ignore_signature_failures_ = true;
3118 }
3119
3120 ✗ void DownloadManager::EnableHTTPTracing() { enable_http_tracing_ = true; }
3121
3122 ✗ void DownloadManager::AddHTTPTracingHeader(const std::string &header) {
3123 ✗ http_tracing_headers_.push_back(header);
3124 }
3125
3126 977 void DownloadManager::UseSystemCertificatePath() {
3127 977 ssl_certificate_store_.UseSystemCertificatePath();
3128 977 }
3129
3130 ✗ bool DownloadManager::SetShardingPolicy(const ShardingPolicySelector type) {
3131 ✗ const bool success = false;
3132 ✗ switch (type) {
3133 default:
3134 ✗ LogCvmfs(
3135 kLogDownload, kLogDebug | kLogSyslogErr,
3136 "(manager '%s') "
3137 "Proposed sharding policy does not exist. Falling back to default",
3138 name_.c_str());
3139 }
3140 ✗ return success;
3141 }
3142
3143 ✗ void DownloadManager::SetFailoverIndefinitely() {
3144 ✗ failover_indefinitely_ = true;
3145 }
3146
3147 /**
3148 * Creates a copy of the existing download manager. Must only be called in
3149 * single-threaded stage because it calls curl_global_init().
3150 */
3151 1116 DownloadManager *DownloadManager::Clone(
3152 const perf::StatisticsTemplate &statistics,
3153 const std::string &cloned_name) {
3154 DownloadManager *clone = new DownloadManager(pool_max_handles_, statistics,
3155
1/2
✓ Branch 2 taken 1116 times.
✗ Branch 3 not taken.
 1116 cloned_name);
3156
3157 1116 clone->SetDnsParameters(resolver_->retries(), resolver_->timeout_ms());
3158 1116 clone->SetDnsTtlLimits(resolver_->min_ttl(), resolver_->max_ttl());
3159 1116 clone->SetMaxIpaddrPerProxy(resolver_->throttle());
3160
3161
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1116 times.
 1116 if (!opt_dns_server_.empty())
3162 ✗ clone->SetDnsServer(opt_dns_server_);
3163 1116 clone->opt_timeout_proxy_ = opt_timeout_proxy_;
3164 1116 clone->opt_timeout_direct_ = opt_timeout_direct_;
3165 1116 clone->opt_low_speed_limit_ = opt_low_speed_limit_;
3166 1116 clone->opt_max_retries_ = opt_max_retries_;
3167 1116 clone->opt_backoff_init_ms_ = opt_backoff_init_ms_;
3168 1116 clone->opt_backoff_max_ms_ = opt_backoff_max_ms_;
3169 1116 clone->enable_info_header_ = enable_info_header_;
3170 1116 clone->enable_http_tracing_ = enable_http_tracing_;
3171 1116 clone->http_tracing_headers_ = http_tracing_headers_;
3172 1116 clone->follow_redirects_ = follow_redirects_;
3173 1116 clone->ignore_signature_failures_ = ignore_signature_failures_;
3174
2/2
✓ Branch 0 taken 1048 times.
✓ Branch 1 taken 68 times.
 1116 if (opt_host_.chain) {
3175
1/2
✓ Branch 2 taken 1048 times.
✗ Branch 3 not taken.
 1048 clone->opt_host_.chain = new vector<string>(*opt_host_.chain);
3176
1/2
✓ Branch 2 taken 1048 times.
✗ Branch 3 not taken.
 1048 clone->opt_host_chain_rtt_ = new vector<int>(*opt_host_chain_rtt_);
3177 }
3178
3179 1116 CloneProxyConfig(clone);
3180 1116 clone->opt_ip_preference_ = opt_ip_preference_;
3181 1116 clone->proxy_template_direct_ = proxy_template_direct_;
3182 1116 clone->proxy_template_forced_ = proxy_template_forced_;
3183 1116 clone->opt_proxy_groups_reset_after_ = opt_proxy_groups_reset_after_;
3184 1116 clone->opt_metalink_.reset_after = opt_metalink_.reset_after;
3185 1116 clone->opt_host_.reset_after = opt_host_.reset_after;
3186 1116 clone->credentials_attachment_ = credentials_attachment_;
3187 1116 clone->ssl_certificate_store_ = ssl_certificate_store_;
3188
3189 1116 clone->health_check_ = health_check_;
3190 1116 clone->sharding_policy_ = sharding_policy_;
3191 1116 clone->failover_indefinitely_ = failover_indefinitely_;
3192 1116 clone->fqrn_ = fqrn_;
3193
3194 1116 return clone;
3195 }
3196
3197
3198 1116 void DownloadManager::CloneProxyConfig(DownloadManager *clone) {
3199 1116 clone->opt_proxy_groups_current_ = opt_proxy_groups_current_;
3200 1116 clone->opt_proxy_groups_current_burned_ = opt_proxy_groups_current_burned_;
3201 1116 clone->opt_proxy_groups_fallback_ = opt_proxy_groups_fallback_;
3202 1116 clone->opt_num_proxies_ = opt_num_proxies_;
3203 1116 clone->opt_proxy_shard_ = opt_proxy_shard_;
3204 1116 clone->opt_proxy_list_ = opt_proxy_list_;
3205 1116 clone->opt_proxy_fallback_list_ = opt_proxy_fallback_list_;
3206
2/2
✓ Branch 0 taken 68 times.
✓ Branch 1 taken 1048 times.
 1116 if (opt_proxy_groups_ == NULL)
3207 68 return;
3208
3209
1/2
✓ Branch 2 taken 1048 times.
✗ Branch 3 not taken.
 1048 clone->opt_proxy_groups_ = new vector<vector<ProxyInfo> >(*opt_proxy_groups_);
3210
2/4
✓ Branch 2 taken 1048 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1048 times.
✗ Branch 6 not taken.
 1048 clone->UpdateProxiesUnlocked("cloned");
3211 }
3212
3213 } // namespace download
3214