GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/network/download.cc
Date: 2026-04-19 02:41:37
Exec Total Coverage
Lines: 877 1719 51.0%
Branches: 666 2194 30.4%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * The download module provides an interface for fetching files via HTTP
5 * and file. It is internally using libcurl and the asynchronous DNS resolver
6 * c-ares. The JobInfo struct describes a single file/url to download and
7 * keeps the state during the several phases of downloading.
8 *
9 * The module starts in single-threaded mode and can be switched to multi-
10 * threaded mode by Spawn(). In multi-threaded mode, the Fetch() function still
11 * blocks but there is a separate I/O thread using asynchronous I/O, which
12 * maintains all concurrent connections simultaneously. As there might be more
13 * than 1024 file descriptors for the CernVM-FS process, the I/O thread uses
14 * poll and the libcurl multi socket interface.
15 *
16 * While downloading, files can be decompressed and the secure hash can be
17 * calculated on the fly.
18 *
19 * The module also implements failure handling. If corrupted data has been
20 * downloaded, the transfer is restarted using HTTP "no-cache" pragma.
21 * A "host chain" can be configured. When a host fails, there is automatic
22 * fail-over to the next host in the chain until all hosts are probed.
23 * Similarly a chain of proxy sets can be configured. Inside a proxy set,
24 * proxies are selected randomly (load-balancing set).
25 */
26
27 // TODO(jblomer): MS for time summing
28 // NOLINTNEXTLINE
29 #define __STDC_FORMAT_MACROS
30
31
32 #include "download.h"
33
34 #include <alloca.h>
35 #include <errno.h>
36 #include <inttypes.h>
37 #include <poll.h>
38 #include <pthread.h>
39 #include <signal.h>
40 #include <stdint.h>
41 #include <sys/time.h>
42 #include <unistd.h>
43
44 #include <algorithm>
45 #include <cassert>
46 #include <cstdio>
47 #include <cstdlib>
48 #include <cstring>
49 #include <map>
50 #include <set>
51 #include <utility>
52
53 #include "compression/compression.h"
54 #include "crypto/hash.h"
55 #include "duplex_curl.h" // IWYU pragma: keep
56 #include "interrupt.h"
57 #include "network/sink_mem.h"
58 #include "network/sink_path.h"
59 #include "sanitizer.h"
60 #include "ssl.h"
61 #include "util/algorithm.h"
62 #include "util/atomic.h"
63 #include "util/exception.h"
64 #include "util/logging.h"
65 #include "util/posix.h"
66 #include "util/prng.h"
67 #include "util/smalloc.h"
68 #include "util/string.h"
69
70 using namespace std; // NOLINT
71
72 namespace download {
73
74 /**
75 * Returns the status if an interrupt happened for a given repository.
76 *
77 * Used only in case CVMFS_FAILOVER_INDEFINITELY (failover_indefinitely_) is set
78 * where failed downloads are retried indefinitely, unless an interrupt occurred
79 *
80 * @note If you use this functionality you need to change the source code of
81 * e.g. cvmfs_config reload to create a sentinel file. See comment below.
82 *
83 * @return true if an interrupt occurred
84 * false otherwise
85 */
86 ✗ bool Interrupted(const std::string &fqrn, JobInfo *info) {
87 ✗ if (info->allow_failure()) {
88 ✗ return true;
89 }
90
91 ✗ if (!fqrn.empty()) {
92 // it is up to the user the create this sentinel file ("pause_file") if
93 // CVMFS_FAILOVER_INDEFINITELY is used. It must be created during
94 // "cvmfs_config reload" and "cvmfs_config reload $fqrn"
95 ✗ const std::string pause_file = std::string("/var/run/cvmfs/interrupt.")
96 ✗ + fqrn;
97
98 ✗ LogCvmfs(kLogDownload, kLogDebug,
99 "(id %" PRId64 ") Interrupted(): checking for existence of %s",
100 info->id(), pause_file.c_str());
101 ✗ if (FileExists(pause_file)) {
102 ✗ LogCvmfs(kLogDownload, kLogDebug,
103 "(id %" PRId64 ") Interrupt marker found - "
104 "Interrupting current download, this will EIO outstanding IO.",
105 info->id());
106 ✗ if (0 != unlink(pause_file.c_str())) {
107 ✗ LogCvmfs(kLogDownload, kLogDebug,
108 "(id %" PRId64 ") Couldn't delete interrupt marker: errno=%d",
109 ✗ info->id(), errno);
110 }
111 ✗ return true;
112 }
113 }
114 ✗ return false;
115 }
116
117 5999 static Failures PrepareDownloadDestination(JobInfo *info) {
118
3/6
✓ Branch 1 taken 5999 times.
✗ Branch 2 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 5999 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 5999 times.
 5999 if (info->sink() != NULL && !info->sink()->IsValid()) {
119 ✗ cvmfs::PathSink *psink = dynamic_cast<cvmfs::PathSink *>(info->sink());
120 ✗ if (psink != NULL) {
121 ✗ LogCvmfs(kLogDownload, kLogDebug,
122 "(id %" PRId64 ") Failed to open path %s: %s (errno=%d).",
123 ✗ info->id(), psink->path().c_str(), strerror(errno), errno);
124 ✗ return kFailLocalIO;
125 } else {
126 ✗ LogCvmfs(kLogDownload, kLogDebug,
127 "(id %" PRId64 ") "
128 "Failed to create a valid sink: \n %s",
129 ✗ info->id(), info->sink()->Describe().c_str());
130 ✗ return kFailOther;
131 }
132 }
133
134 5999 return kFailOk;
135 }
136
137
138 /**
139 * Called by curl for every HTTP header. Not called for file:// transfers.
140 */
141 18029 static size_t CallbackCurlHeader(void *ptr, size_t size, size_t nmemb,
142 void *info_link) {
143 18029 const size_t num_bytes = size * nmemb;
144
1/2
✓ Branch 2 taken 18029 times.
✗ Branch 3 not taken.
 18029 const string header_line(static_cast<const char *>(ptr), num_bytes);
145 18029 JobInfo *info = static_cast<JobInfo *>(info_link);
146
147 // LogCvmfs(kLogDownload, kLogDebug, "REMOVE-ME: Header callback with %s",
148 // header_line.c_str());
149
150 // Check http status codes
151
4/6
✓ Branch 2 taken 18029 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 18029 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 516 times.
✓ Branch 10 taken 17513 times.
 18029 if (HasPrefix(header_line, "HTTP/1.", false)) {
152
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 516 times.
 516 if (header_line.length() < 10) {
153 ✗ return 0;
154 }
155
156 unsigned i;
157
5/6
✓ Branch 1 taken 1032 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 516 times.
✓ Branch 5 taken 516 times.
✓ Branch 6 taken 516 times.
✓ Branch 7 taken 516 times.
 1032 for (i = 8; (i < header_line.length()) && (header_line[i] == ' '); ++i) {
158 }
159
160 // Code is initialized to -1
161
1/2
✓ Branch 1 taken 516 times.
✗ Branch 2 not taken.
 516 if (header_line.length() > i + 2) {
162 516 info->SetHttpCode(DownloadManager::ParseHttpCode(&header_line[i]));
163 }
164
165
2/2
✓ Branch 1 taken 387 times.
✓ Branch 2 taken 129 times.
 516 if ((info->http_code() / 100) == 2) {
166 387 return num_bytes;
167
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 129 } else if ((info->http_code() == 301) || (info->http_code() == 302)
168
2/8
✗ Branch 0 not taken.
✓ Branch 1 taken 129 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 129 times.
✗ Branch 9 not taken.
 129 || (info->http_code() == 303) || (info->http_code() == 307)) {
169
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 129 times.
 129 if (!info->follow_redirects()) {
170 ✗ LogCvmfs(kLogDownload, kLogDebug,
171 "(id %" PRId64 ") redirect support not enabled: %s",
172 info->id(), header_line.c_str());
173 ✗ info->SetErrorCode(kFailHostHttp);
174 ✗ return 0;
175 }
176
1/2
✓ Branch 3 taken 129 times.
✗ Branch 4 not taken.
 129 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") http redirect: %s",
177 info->id(), header_line.c_str());
178 // libcurl will handle this because of CURLOPT_FOLLOWLOCATION
179 129 return num_bytes;
180 } else {
181 ✗ LogCvmfs(kLogDownload, kLogDebug,
182 "(id %" PRId64 ") http status error code: %s [%d]", info->id(),
183 header_line.c_str(), info->http_code());
184 ✗ if (((info->http_code() / 100) == 5) || (info->http_code() == 400)
185 ✗ || (info->http_code() == 404)) {
186 // 5XX returned by host
187 // 400: error from the GeoAPI module
188 // 404: the stratum 1 does not have the newest files
189 ✗ info->SetErrorCode(kFailHostHttp);
190 ✗ } else if (info->http_code() == 429) {
191 // 429: rate throttling (we ignore the backoff hint for the time being)
192 ✗ info->SetErrorCode(kFailHostConnection);
193 } else {
194 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostHttp
195 : kFailProxyHttp);
196 }
197 ✗ return 0;
198 }
199 }
200
201 // If needed: allocate space in sink
202
3/4
✓ Branch 2 taken 17513 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 5519 times.
✓ Branch 5 taken 11994 times.
 17513 if (info->sink() != NULL && info->sink()->RequiresReserve()
203
11/20
✓ Branch 1 taken 17513 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 5519 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 5519 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 1682 times.
✓ Branch 11 taken 3837 times.
✓ Branch 12 taken 5519 times.
✓ Branch 13 taken 11994 times.
✗ Branch 14 not taken.
✓ Branch 15 taken 5519 times.
✓ Branch 16 taken 11994 times.
✓ Branch 18 taken 1682 times.
✓ Branch 19 taken 15831 times.
✗ Branch 20 not taken.
✗ Branch 21 not taken.
✗ Branch 23 not taken.
✗ Branch 24 not taken.
 35026 && HasPrefix(header_line, "CONTENT-LENGTH:", true)) {
204 1682 char *tmp = reinterpret_cast<char *>(alloca(num_bytes + 1));
205 1682 uint64_t length = 0;
206 1682 sscanf(header_line.c_str(), "%s %" PRIu64, tmp, &length);
207
1/2
✓ Branch 0 taken 1682 times.
✗ Branch 1 not taken.
 1682 if (length > 0) {
208
2/4
✓ Branch 2 taken 1682 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 1682 times.
 1682 if (!info->sink()->Reserve(length)) {
209 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
210 "(id %" PRId64 ") "
211 "resource %s too large to store in memory (%" PRIu64 ")",
212 info->id(), info->url()->c_str(), length);
213 ✗ info->SetErrorCode(kFailTooBig);
214 ✗ return 0;
215 }
216 } else {
217 // Empty resource
218 ✗ info->sink()->Reserve(0);
219 }
220
4/6
✓ Branch 2 taken 15831 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 15831 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 129 times.
✓ Branch 10 taken 15702 times.
 15831 } else if (HasPrefix(header_line, "LOCATION:", true)) {
221 // This comes along with redirects
222
1/2
✓ Branch 3 taken 129 times.
✗ Branch 4 not taken.
 129 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") %s", info->id(),
223 header_line.c_str());
224
3/6
✓ Branch 2 taken 15702 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 15702 times.
✗ Branch 6 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 15702 times.
 15702 } else if (HasPrefix(header_line, "LINK:", true)) {
225 // This is metalink info
226 ✗ LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") %s", info->id(),
227 header_line.c_str());
228 ✗ std::string link = info->link();
229 ✗ if (link.size() != 0) {
230 // multiple LINK headers are allowed
231 ✗ link = link + ", " + header_line.substr(5);
232 } else {
233 ✗ link = header_line.substr(5);
234 }
235 ✗ info->SetLink(link);
236
3/6
✓ Branch 3 taken 15702 times.
✗ Branch 4 not taken.
✓ Branch 6 taken 15702 times.
✗ Branch 7 not taken.
✗ Branch 10 not taken.
✓ Branch 11 taken 15702 times.
 15702 } else if (HasPrefix(header_line, "X-SQUID-ERROR:", true)) {
237 // Reinterpret host error as proxy error
238 ✗ if (info->error_code() == kFailHostHttp) {
239 ✗ info->SetErrorCode(kFailProxyHttp);
240 }
241
3/6
✓ Branch 2 taken 15702 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 15702 times.
✗ Branch 6 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 15702 times.
 15702 } else if (HasPrefix(header_line, "PROXY-STATUS:", true)) {
242 // Reinterpret host error as proxy error if applicable
243 ✗ if ((info->error_code() == kFailHostHttp)
244 ✗ && (header_line.find("error=") != string::npos)) {
245 ✗ info->SetErrorCode(kFailProxyHttp);
246 }
247 }
248
249 17513 return num_bytes;
250 18029 }
251
252
253 /**
254 * Called by curl for every received data chunk.
255 */
256 7240 static size_t CallbackCurlData(void *ptr, size_t size, size_t nmemb,
257 void *info_link) {
258 7240 const size_t num_bytes = size * nmemb;
259 7240 JobInfo *info = static_cast<JobInfo *>(info_link);
260
261 // TODO(heretherebedragons) remove if no error comes up
262 // as this means only jobinfo data request (and not header only)
263 // come here
264
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 7240 times.
 7240 assert(info->sink() != NULL);
265
266 // LogCvmfs(kLogDownload, kLogDebug, "Data callback, %d bytes", num_bytes);
267
268
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7240 times.
 7240 if (num_bytes == 0)
269 ✗ return 0;
270
271
2/2
✓ Branch 1 taken 4095 times.
✓ Branch 2 taken 3145 times.
 7240 if (info->expected_hash()) {
272 4095 shash::Update(reinterpret_cast<unsigned char *>(ptr), num_bytes,
273 info->hash_context());
274 }
275
276
2/2
✓ Branch 1 taken 4052 times.
✓ Branch 2 taken 3188 times.
 7240 if (info->compressed()) {
277 4052 const zlib::StreamStates retval = zlib::DecompressZStream2Sink(
278 ptr, static_cast<int64_t>(num_bytes), info->GetZstreamPtr(),
279 info->sink());
280
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4052 times.
 4052 if (retval == zlib::kStreamDataError) {
281 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
282 "(id %" PRId64 ") failed to decompress %s", info->id(),
283 info->url()->c_str());
284 ✗ info->SetErrorCode(kFailBadData);
285 ✗ return 0;
286
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4052 times.
 4052 } else if (retval == zlib::kStreamIOError) {
287 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
288 "(id %" PRId64 ") decompressing %s, local IO error", info->id(),
289 info->url()->c_str());
290 ✗ info->SetErrorCode(kFailLocalIO);
291 ✗ return 0;
292 }
293 } else {
294 3188 const int64_t written = info->sink()->Write(ptr, num_bytes);
295
2/4
✓ Branch 0 taken 3188 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 3188 times.
 3188 if (written < 0 || static_cast<uint64_t>(written) != num_bytes) {
296 ✗ LogCvmfs(kLogDownload, kLogDebug,
297 "(id %" PRId64 ") "
298 "Failed to perform write of %zu bytes to sink %s with errno %ld",
299 ✗ info->id(), num_bytes, info->sink()->Describe().c_str(),
300 written);
301 }
302 }
303
304 7240 return num_bytes;
305 }
306
307 #ifdef DEBUGMSG
308 4774 static int CallbackCurlDebug(CURL *handle,
309 curl_infotype type,
310 char *data,
311 size_t size,
312 void * /* clientp */) {
313 JobInfo *info;
314
1/2
✓ Branch 1 taken 4774 times.
✗ Branch 2 not taken.
 4774 curl_easy_getinfo(handle, CURLINFO_PRIVATE, &info);
315
316
3/6
✓ Branch 2 taken 4774 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 4774 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 4774 times.
✗ Branch 9 not taken.
 9548 std::string prefix = "(id " + StringifyInt(info->id()) + ") ";
317
4/8
✓ Branch 0 taken 689 times.
✓ Branch 1 taken 2021 times.
✓ Branch 2 taken 516 times.
✓ Branch 3 taken 1548 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
 4774 switch (type) {
318 689 case CURLINFO_TEXT:
319
1/2
✓ Branch 1 taken 689 times.
✗ Branch 2 not taken.
 689 prefix += "{info} ";
320 689 break;
321 2021 case CURLINFO_HEADER_IN:
322
1/2
✓ Branch 1 taken 2021 times.
✗ Branch 2 not taken.
 2021 prefix += "{header/recv} ";
323 2021 break;
324 516 case CURLINFO_HEADER_OUT:
325
1/2
✓ Branch 1 taken 516 times.
✗ Branch 2 not taken.
 516 prefix += "{header/sent} ";
326 516 break;
327 1548 case CURLINFO_DATA_IN:
328
2/2
✓ Branch 0 taken 172 times.
✓ Branch 1 taken 1376 times.
 1548 if (size < 50) {
329
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 prefix += "{data/recv} ";
330 172 break;
331 } else {
332
1/2
✓ Branch 2 taken 1376 times.
✗ Branch 3 not taken.
 1376 LogCvmfs(kLogCurl, kLogDebug, "%s{data/recv} <snip>", prefix.c_str());
333 1376 return 0;
334 }
335 ✗ case CURLINFO_DATA_OUT:
336 ✗ if (size < 50) {
337 ✗ prefix += "{data/sent} ";
338 ✗ break;
339 } else {
340 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{data/sent} <snip>", prefix.c_str());
341 ✗ return 0;
342 }
343 ✗ case CURLINFO_SSL_DATA_IN:
344 ✗ if (size < 50) {
345 ✗ prefix += "{ssldata/recv} ";
346 ✗ break;
347 } else {
348 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{ssldata/recv} <snip>",
349 prefix.c_str());
350 ✗ return 0;
351 }
352 ✗ case CURLINFO_SSL_DATA_OUT:
353 ✗ if (size < 50) {
354 ✗ prefix += "{ssldata/sent} ";
355 ✗ break;
356 } else {
357 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{ssldata/sent} <snip>",
358 prefix.c_str());
359 ✗ return 0;
360 }
361 ✗ default:
362 // just log the message
363 ✗ break;
364 }
365
366 3398 bool valid_char = true;
367
1/2
✓ Branch 2 taken 3398 times.
✗ Branch 3 not taken.
 3398 std::string msg(data, size);
368
2/2
✓ Branch 1 taken 170440 times.
✓ Branch 2 taken 3398 times.
 173838 for (size_t i = 0; i < msg.length(); ++i) {
369
2/4
✓ Branch 1 taken 170440 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 170440 times.
 170440 if (msg[i] == '\0') {
370 ✗ msg[i] = '~';
371 }
372
373 // verify that char is a valid printable char
374
3/6
✓ Branch 1 taken 170440 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 159173 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 159173 times.
 329613 if ((msg[i] < ' ' || msg[i] > '~')
375
6/8
✓ Branch 0 taken 159173 times.
✓ Branch 1 taken 11267 times.
✓ Branch 3 taken 11267 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 5289 times.
✓ Branch 6 taken 5978 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 170440 times.
 334902 && (msg[i] != 10 /*line feed*/
376
2/4
✓ Branch 1 taken 5289 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 5289 times.
 5289 && msg[i] != 13 /*carriage return*/)) {
377 ✗ valid_char = false;
378 }
379 }
380
381
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3398 times.
 3398 if (!valid_char) {
382 ✗ msg = "<Non-plaintext sequence>";
383 }
384
385
1/2
✓ Branch 3 taken 3398 times.
✗ Branch 4 not taken.
 3398 LogCvmfs(kLogCurl, kLogDebug, "%s%s", prefix.c_str(),
386
1/2
✓ Branch 1 taken 3398 times.
✗ Branch 2 not taken.
 6796 Trim(msg, true /* trim_newline */).c_str());
387 3398 return 0;
388 4774 }
389 #endif
390
391 //------------------------------------------------------------------------------
392
393
394 const int DownloadManager::kProbeUnprobed = -1;
395 const int DownloadManager::kProbeDown = -2;
396 const int DownloadManager::kProbeGeo = -3;
397
398 /**
399 * Escape special chars from the URL, except for ':' and '/',
400 * which should keep their meaning.
401 */
402 6171 string DownloadManager::EscapeUrl(const int64_t jobinfo_id, const string &url) {
403 6171 string escaped;
404
1/2
✓ Branch 2 taken 6171 times.
✗ Branch 3 not taken.
 6171 escaped.reserve(url.length());
405
406 char escaped_char[3];
407
2/2
✓ Branch 1 taken 532550 times.
✓ Branch 2 taken 6171 times.
 538721 for (unsigned i = 0, s = url.length(); i < s; ++i) {
408
3/4
✓ Branch 2 taken 532550 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 387 times.
✓ Branch 5 taken 532163 times.
 532550 if (JobInfo::EscapeUrlChar(url[i], escaped_char)) {
409
1/2
✓ Branch 1 taken 387 times.
✗ Branch 2 not taken.
 387 escaped.append(escaped_char, 3);
410 } else {
411
1/2
✓ Branch 1 taken 532163 times.
✗ Branch 2 not taken.
 532163 escaped.push_back(escaped_char[0]);
412 }
413 }
414
1/2
✓ Branch 3 taken 6171 times.
✗ Branch 4 not taken.
 6171 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") escaped %s to %s",
415 jobinfo_id, url.c_str(), escaped.c_str());
416
417 12342 return escaped;
418 }
419
420 /**
421 * -1 of digits is not a valid Http return code
422 */
423 731 int DownloadManager::ParseHttpCode(const char digits[3]) {
424 731 int result = 0;
425 731 int factor = 100;
426
2/2
✓ Branch 0 taken 2193 times.
✓ Branch 1 taken 688 times.
 2881 for (int i = 0; i < 3; ++i) {
427
3/4
✓ Branch 0 taken 2193 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 43 times.
✓ Branch 3 taken 2150 times.
 2193 if ((digits[i] < '0') || (digits[i] > '9'))
428 43 return -1;
429 2150 result += (digits[i] - '0') * factor;
430 2150 factor /= 10;
431 }
432 688 return result;
433 }
434
435
436 /**
437 * Called when new curl sockets arrive or existing curl sockets depart.
438 */
439 ✗ int DownloadManager::CallbackCurlSocket(CURL * /* easy */,
440 curl_socket_t s,
441 int action,
442 void *userp,
443 void * /* socketp */) {
444 // LogCvmfs(kLogDownload, kLogDebug, "CallbackCurlSocket called with easy "
445 // "handle %p, socket %d, action %d", easy, s, action);
446 ✗ DownloadManager *download_mgr = static_cast<DownloadManager *>(userp);
447 ✗ if (action == CURL_POLL_NONE)
448 ✗ return 0;
449
450 // Find s in watch_fds_
451 unsigned index;
452
453 // TODO(heretherebedragons) why start at index = 0 and not 2?
454 // fd[0] and fd[1] are fixed?
455 ✗ for (index = 0; index < download_mgr->watch_fds_inuse_; ++index) {
456 ✗ if (download_mgr->watch_fds_[index].fd == s)
457 ✗ break;
458 }
459 // Or create newly
460 ✗ if (index == download_mgr->watch_fds_inuse_) {
461 // Extend array if necessary
462 ✗ if (download_mgr->watch_fds_inuse_ == download_mgr->watch_fds_size_) {
463 ✗ assert(download_mgr->watch_fds_size_ > 0);
464 ✗ download_mgr->watch_fds_size_ *= 2;
465 ✗ download_mgr->watch_fds_ = static_cast<struct pollfd *>(
466 ✗ srealloc(download_mgr->watch_fds_,
467 ✗ download_mgr->watch_fds_size_ * sizeof(struct pollfd)));
468 }
469 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].fd = s;
470 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].events = 0;
471 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].revents = 0;
472 ✗ download_mgr->watch_fds_inuse_++;
473 }
474
475 ✗ switch (action) {
476 ✗ case CURL_POLL_IN:
477 ✗ download_mgr->watch_fds_[index].events = POLLIN | POLLPRI;
478 ✗ break;
479 ✗ case CURL_POLL_OUT:
480 ✗ download_mgr->watch_fds_[index].events = POLLOUT | POLLWRBAND;
481 ✗ break;
482 ✗ case CURL_POLL_INOUT:
483 ✗ download_mgr->watch_fds_[index].events = POLLIN | POLLPRI | POLLOUT
484 | POLLWRBAND;
485 ✗ break;
486 ✗ case CURL_POLL_REMOVE:
487 ✗ if (index < download_mgr->watch_fds_inuse_ - 1) {
488 download_mgr
489 ✗ ->watch_fds_[index] = download_mgr->watch_fds_
490 ✗ [download_mgr->watch_fds_inuse_ - 1];
491 }
492 ✗ download_mgr->watch_fds_inuse_--;
493 // Shrink array if necessary
494 ✗ if ((download_mgr->watch_fds_inuse_ > download_mgr->watch_fds_max_)
495 ✗ && (download_mgr->watch_fds_inuse_
496 ✗ < download_mgr->watch_fds_size_ / 2)) {
497 ✗ download_mgr->watch_fds_size_ /= 2;
498 // LogCvmfs(kLogDownload, kLogDebug, "shrinking watch_fds_ (%d)",
499 // watch_fds_size_);
500 ✗ download_mgr->watch_fds_ = static_cast<struct pollfd *>(
501 ✗ srealloc(download_mgr->watch_fds_,
502 ✗ download_mgr->watch_fds_size_ * sizeof(struct pollfd)));
503 // LogCvmfs(kLogDownload, kLogDebug, "shrinking watch_fds_ done",
504 // watch_fds_size_);
505 }
506 ✗ break;
507 ✗ default:
508 ✗ break;
509 }
510
511 ✗ return 0;
512 }
513
514
515 /**
516 * Worker thread event loop. Waits on new JobInfo structs on a pipe.
517 */
518 ✗ void *DownloadManager::MainDownload(void *data) {
519 ✗ DownloadManager *download_mgr = static_cast<DownloadManager *>(data);
520 ✗ LogCvmfs(kLogDownload, kLogDebug,
521 "download I/O thread of DownloadManager '%s' started",
522 download_mgr->name_.c_str());
523
524 ✗ const int kIdxPipeTerminate = 0;
525 ✗ const int kIdxPipeJobs = 1;
526
527 ✗ download_mgr->watch_fds_ = static_cast<struct pollfd *>(
528 ✗ smalloc(2 * sizeof(struct pollfd)));
529 ✗ download_mgr->watch_fds_size_ = 2;
530 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].fd = download_mgr->pipe_terminate_
531 ✗ ->GetReadFd();
532 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].events = POLLIN | POLLPRI;
533 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].revents = 0;
534 ✗ download_mgr->watch_fds_[kIdxPipeJobs].fd = download_mgr->pipe_jobs_
535 ✗ ->GetReadFd();
536 ✗ download_mgr->watch_fds_[kIdxPipeJobs].events = POLLIN | POLLPRI;
537 ✗ download_mgr->watch_fds_[kIdxPipeJobs].revents = 0;
538 ✗ download_mgr->watch_fds_inuse_ = 2;
539
540 ✗ int still_running = 0;
541 struct timeval timeval_start, timeval_stop;
542 ✗ gettimeofday(&timeval_start, NULL);
543 while (true) {
544 int timeout;
545 ✗ if (still_running) {
546 /* NOTE: The following might degrade the performance for many small files
547 * use case. TODO(jblomer): look into it.
548 // Specify a timeout for polling in ms; this allows us to return
549 // to libcurl once a second so it can look for internal operations
550 // which timed out. libcurl has a more elaborate mechanism
551 // (CURLMOPT_TIMERFUNCTION) that would inform us of the next potential
552 // timeout. TODO(bbockelm) we should switch to that in the future.
553 timeout = 100;
554 */
555 ✗ timeout = 1;
556 } else {
557 ✗ timeout = -1;
558 ✗ gettimeofday(&timeval_stop, NULL);
559 const int64_t delta = static_cast<int64_t>(
560 ✗ 1000 * DiffTimeSeconds(timeval_start, timeval_stop));
561 ✗ perf::Xadd(download_mgr->counters_->sz_transfer_time, delta);
562 }
563 ✗ const int retval = poll(download_mgr->watch_fds_,
564 ✗ download_mgr->watch_fds_inuse_, timeout);
565 ✗ if (retval < 0) {
566 ✗ continue;
567 }
568
569 // Handle timeout
570 ✗ if (retval == 0) {
571 ✗ curl_multi_socket_action(download_mgr->curl_multi_, CURL_SOCKET_TIMEOUT,
572 0, &still_running);
573 }
574
575 // Terminate I/O thread
576 ✗ if (download_mgr->watch_fds_[kIdxPipeTerminate].revents)
577 ✗ break;
578
579 // New job arrives
580 ✗ if (download_mgr->watch_fds_[kIdxPipeJobs].revents) {
581 ✗ download_mgr->watch_fds_[kIdxPipeJobs].revents = 0;
582 JobInfo *info;
583 ✗ download_mgr->pipe_jobs_->Read<JobInfo *>(&info);
584 ✗ if (!still_running) {
585 ✗ gettimeofday(&timeval_start, NULL);
586 }
587 ✗ CURL *handle = download_mgr->AcquireCurlHandle();
588 ✗ download_mgr->InitializeRequest(info, handle);
589 ✗ download_mgr->SetUrlOptions(info);
590 ✗ curl_multi_add_handle(download_mgr->curl_multi_, handle);
591 ✗ curl_multi_socket_action(download_mgr->curl_multi_, CURL_SOCKET_TIMEOUT,
592 0, &still_running);
593 }
594
595 // Activity on curl sockets
596 // Within this loop the curl_multi_socket_action() may cause socket(s)
597 // to be removed from watch_fds_. If a socket is removed it is replaced
598 // by the socket at the end of the array and the inuse count is decreased.
599 // Therefore loop over the array in reverse order.
600 ✗ for (int64_t i = download_mgr->watch_fds_inuse_ - 1; i >= 2; --i) {
601 ✗ if (i >= download_mgr->watch_fds_inuse_) {
602 ✗ continue;
603 }
604 ✗ if (download_mgr->watch_fds_[i].revents) {
605 ✗ int ev_bitmask = 0;
606 ✗ if (download_mgr->watch_fds_[i].revents & (POLLIN | POLLPRI))
607 ✗ ev_bitmask |= CURL_CSELECT_IN;
608 ✗ if (download_mgr->watch_fds_[i].revents & (POLLOUT | POLLWRBAND))
609 ✗ ev_bitmask |= CURL_CSELECT_OUT;
610 ✗ if (download_mgr->watch_fds_[i].revents
611 ✗ & (POLLERR | POLLHUP | POLLNVAL)) {
612 ✗ ev_bitmask |= CURL_CSELECT_ERR;
613 }
614 ✗ download_mgr->watch_fds_[i].revents = 0;
615
616 ✗ curl_multi_socket_action(download_mgr->curl_multi_,
617 ✗ download_mgr->watch_fds_[i].fd,
618 ev_bitmask,
619 &still_running);
620 }
621 }
622
623 // Check if transfers are completed
624 CURLMsg *curl_msg;
625 int msgs_in_queue;
626 ✗ while ((curl_msg = curl_multi_info_read(download_mgr->curl_multi_,
627 &msgs_in_queue))) {
628 ✗ if (curl_msg->msg == CURLMSG_DONE) {
629 ✗ perf::Inc(download_mgr->counters_->n_requests);
630 JobInfo *info;
631 ✗ CURL *easy_handle = curl_msg->easy_handle;
632 ✗ const int curl_error = curl_msg->data.result;
633 ✗ curl_easy_getinfo(easy_handle, CURLINFO_PRIVATE, &info);
634
635 int64_t redir_count;
636 ✗ curl_easy_getinfo(easy_handle, CURLINFO_REDIRECT_COUNT, &redir_count);
637 ✗ LogCvmfs(kLogDownload, kLogDebug,
638 "(manager '%s' - id %" PRId64 ") "
639 "Number of CURL redirects %" PRId64,
640 download_mgr->name_.c_str(), info->id(), redir_count);
641
642 ✗ curl_multi_remove_handle(download_mgr->curl_multi_, easy_handle);
643 ✗ if (download_mgr->VerifyAndFinalize(curl_error, info)) {
644 ✗ curl_multi_add_handle(download_mgr->curl_multi_, easy_handle);
645 ✗ curl_multi_socket_action(download_mgr->curl_multi_,
646 CURL_SOCKET_TIMEOUT,
647 0,
648 &still_running);
649 } else {
650 // Return easy handle into pool and write result back
651 ✗ download_mgr->ReleaseCurlHandle(easy_handle);
652
653 ✗ DataTubeElement *ele = new DataTubeElement(kActionStop);
654 ✗ info->GetDataTubePtr()->EnqueueBack(ele);
655 ✗ info->GetPipeJobResultPtr()->Write<download::Failures>(
656 ✗ info->error_code());
657 }
658 }
659 }
660 }
661
662 ✗ for (set<CURL *>::iterator i = download_mgr->pool_handles_inuse_->begin(),
663 ✗ iEnd = download_mgr->pool_handles_inuse_->end();
664 ✗ i != iEnd;
665 ✗ ++i) {
666 ✗ curl_multi_remove_handle(download_mgr->curl_multi_, *i);
667 ✗ curl_easy_cleanup(*i);
668 }
669 ✗ download_mgr->pool_handles_inuse_->clear();
670 ✗ free(download_mgr->watch_fds_);
671
672 ✗ LogCvmfs(kLogDownload, kLogDebug,
673 "download I/O thread of DownloadManager '%s' terminated",
674 download_mgr->name_.c_str());
675 ✗ return NULL;
676 }
677
678
679 //------------------------------------------------------------------------------
680
681
682 4678 HeaderLists::~HeaderLists() {
683
2/2
✓ Branch 1 taken 4721 times.
✓ Branch 2 taken 4678 times.
 9399 for (unsigned i = 0; i < blocks_.size(); ++i) {
684
1/2
✓ Branch 1 taken 4721 times.
✗ Branch 2 not taken.
 4721 delete[] blocks_[i];
685 }
686 4678 blocks_.clear();
687 4678 }
688
689
690 32909 curl_slist *HeaderLists::GetList(const char *header) { return Get(header); }
691
692
693 5999 curl_slist *HeaderLists::DuplicateList(curl_slist *slist) {
694
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5999 times.
 5999 assert(slist);
695 5999 curl_slist *copy = GetList(slist->data);
696 5999 copy->next = slist->next;
697 5999 curl_slist *prev = copy;
698 5999 slist = slist->next;
699
2/2
✓ Branch 0 taken 11998 times.
✓ Branch 1 taken 5999 times.
 17997 while (slist) {
700 11998 curl_slist *new_link = Get(slist->data);
701 11998 new_link->next = slist->next;
702 11998 prev->next = new_link;
703 11998 prev = new_link;
704 11998 slist = slist->next;
705 }
706 5999 return copy;
707 }
708
709
710 9930 void HeaderLists::AppendHeader(curl_slist *slist, const char *header) {
711
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9930 times.
 9930 assert(slist);
712 9930 curl_slist *new_link = Get(header);
713 9930 new_link->next = NULL;
714
715
2/2
✓ Branch 0 taken 6066 times.
✓ Branch 1 taken 9930 times.
 15996 while (slist->next)
716 6066 slist = slist->next;
717 9930 slist->next = new_link;
718 9930 }
719
720
721 /**
722 * Ensures that a certain header string is _not_ part of slist on return.
723 * Note that if the first header element matches, the returned slist points
724 * to a different value.
725 */
726 215 void HeaderLists::CutHeader(const char *header, curl_slist **slist) {
727
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 215 times.
 215 assert(slist);
728 curl_slist head;
729 215 head.next = *slist;
730 215 curl_slist *prev = &head;
731 215 curl_slist *rover = *slist;
732
2/2
✓ Branch 0 taken 387 times.
✓ Branch 1 taken 215 times.
 602 while (rover) {
733
2/2
✓ Branch 0 taken 172 times.
✓ Branch 1 taken 215 times.
 387 if (strcmp(rover->data, header) == 0) {
734 172 prev->next = rover->next;
735 172 Put(rover);
736 172 rover = prev;
737 }
738 387 prev = rover;
739 387 rover = rover->next;
740 }
741 215 *slist = head.next;
742 215 }
743
744
745 17179 void HeaderLists::PutList(curl_slist *slist) {
746
2/2
✓ Branch 0 taken 29835 times.
✓ Branch 1 taken 17179 times.
 47014 while (slist) {
747 29835 curl_slist *next = slist->next;
748 29835 Put(slist);
749 29835 slist = next;
750 }
751 17179 }
752
753
754 172 string HeaderLists::Print(curl_slist *slist) {
755 172 string verbose;
756
2/2
✓ Branch 0 taken 344 times.
✓ Branch 1 taken 172 times.
 516 while (slist) {
757
3/6
✓ Branch 2 taken 344 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 344 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 344 times.
✗ Branch 9 not taken.
 344 verbose += string(slist->data) + "\n";
758 344 slist = slist->next;
759 }
760 172 return verbose;
761 }
762
763
764 54837 curl_slist *HeaderLists::Get(const char *header) {
765
2/2
✓ Branch 1 taken 50115 times.
✓ Branch 2 taken 4765 times.
 54880 for (unsigned i = 0; i < blocks_.size(); ++i) {
766
2/2
✓ Branch 0 taken 2968527 times.
✓ Branch 1 taken 43 times.
 2968570 for (unsigned j = 0; j < kBlockSize; ++j) {
767
2/2
✓ Branch 2 taken 50072 times.
✓ Branch 3 taken 2918455 times.
 2968527 if (!IsUsed(&(blocks_[i][j]))) {
768 50072 blocks_[i][j].data = const_cast<char *>(header);
769 50072 return &(blocks_[i][j]);
770 }
771 }
772 }
773
774 // All used, new block
775 4765 AddBlock();
776 4765 blocks_[blocks_.size() - 1][0].data = const_cast<char *>(header);
777 4765 return &(blocks_[blocks_.size() - 1][0]);
778 }
779
780
781 1249847 void HeaderLists::Put(curl_slist *slist) {
782 1249847 slist->data = NULL;
783 1249847 slist->next = NULL;
784 1249847 }
785
786
787 4765 void HeaderLists::AddBlock() {
788
1/2
✓ Branch 1 taken 4765 times.
✗ Branch 2 not taken.
 4765 curl_slist *new_block = new curl_slist[kBlockSize];
789
2/2
✓ Branch 0 taken 1219840 times.
✓ Branch 1 taken 4765 times.
 1224605 for (unsigned i = 0; i < kBlockSize; ++i) {
790 1219840 Put(&new_block[i]);
791 }
792
1/2
✓ Branch 1 taken 4765 times.
✗ Branch 2 not taken.
 4765 blocks_.push_back(new_block);
793 4765 }
794
795
796 //------------------------------------------------------------------------------
797
798
799 ✗ string DownloadManager::ProxyInfo::Print() {
800 ✗ if (url == "DIRECT")
801 ✗ return url;
802
803 ✗ string result = url;
804 ✗ const int remaining = static_cast<int>(host.deadline())
805 ✗ - static_cast<int>(time(NULL));
806 ✗ string expinfo = (remaining >= 0) ? "+" : "";
807 ✗ if (abs(remaining) >= 3600) {
808 ✗ expinfo += StringifyInt(remaining / 3600) + "h";
809 ✗ } else if (abs(remaining) >= 60) {
810 ✗ expinfo += StringifyInt(remaining / 60) + "m";
811 } else {
812 ✗ expinfo += StringifyInt(remaining) + "s";
813 }
814 ✗ if (host.status() == dns::kFailOk) {
815 ✗ result += " (" + host.name() + ", " + expinfo + ")";
816 } else {
817 ✗ result += " (:unresolved:, " + expinfo + ")";
818 }
819 ✗ return result;
820 }
821
822
823 /**
824 * Gets an idle CURL handle from the pool. Creates a new one and adds it to
825 * the pool if necessary.
826 */
827 5999 CURL *DownloadManager::AcquireCurlHandle() {
828 CURL *handle;
829
830
2/2
✓ Branch 1 taken 2513 times.
✓ Branch 2 taken 3486 times.
 5999 if (pool_handles_idle_->empty()) {
831 // Create a new handle
832
1/2
✓ Branch 1 taken 2513 times.
✗ Branch 2 not taken.
 2513 handle = curl_easy_init();
833
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2513 times.
 2513 assert(handle != NULL);
834
835
1/2
✓ Branch 1 taken 2513 times.
✗ Branch 2 not taken.
 2513 curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1);
836 // curl_easy_setopt(curl_default, CURLOPT_FAILONERROR, 1);
837
1/2
✓ Branch 1 taken 2513 times.
✗ Branch 2 not taken.
 2513 curl_easy_setopt(handle, CURLOPT_HEADERFUNCTION, CallbackCurlHeader);
838
1/2
✓ Branch 1 taken 2513 times.
✗ Branch 2 not taken.
 2513 curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, CallbackCurlData);
839 } else {
840 3486 handle = *(pool_handles_idle_->begin());
841
1/2
✓ Branch 2 taken 3486 times.
✗ Branch 3 not taken.
 3486 pool_handles_idle_->erase(pool_handles_idle_->begin());
842 }
843
844
1/2
✓ Branch 1 taken 5999 times.
✗ Branch 2 not taken.
 5999 pool_handles_inuse_->insert(handle);
845
846 5999 return handle;
847 }
848
849
850 5999 void DownloadManager::ReleaseCurlHandle(CURL *handle) {
851
1/2
✓ Branch 1 taken 5999 times.
✗ Branch 2 not taken.
 5999 const set<CURL *>::iterator elem = pool_handles_inuse_->find(handle);
852
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 5999 times.
 5999 assert(elem != pool_handles_inuse_->end());
853
854
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5999 times.
 5999 if (pool_handles_idle_->size() > pool_max_handles_) {
855 ✗ curl_easy_cleanup(*elem);
856 } else {
857
1/2
✓ Branch 2 taken 5999 times.
✗ Branch 3 not taken.
 5999 pool_handles_idle_->insert(*elem);
858 }
859
860
1/2
✓ Branch 1 taken 5999 times.
✗ Branch 2 not taken.
 5999 pool_handles_inuse_->erase(elem);
861 5999 }
862
863
864 /**
865 * HTTP request options: set the URL and other options such as timeout and
866 * proxy.
867 */
868 5999 void DownloadManager::InitializeRequest(JobInfo *info, CURL *handle) {
869 // Initialize internal download state
870 5999 info->SetCurlHandle(handle);
871 5999 info->SetErrorCode(kFailOk);
872 5999 info->SetHttpCode(-1);
873 5999 info->SetFollowRedirects(follow_redirects_);
874 5999 info->SetNumUsedProxies(1);
875 5999 info->SetNumUsedMetalinks(1);
876 5999 info->SetNumUsedHosts(1);
877 5999 info->SetNumRetries(0);
878 5999 info->SetBackoffMs(0);
879 5999 info->SetHeaders(header_lists_->DuplicateList(default_headers_));
880
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5999 times.
 5999 if (info->info_header()) {
881 ✗ header_lists_->AppendHeader(info->headers(), info->info_header());
882 }
883
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5999 times.
 5999 if (enable_http_tracing_) {
884 ✗ for (unsigned int i = 0; i < http_tracing_headers_.size(); i++) {
885 ✗ header_lists_->AppendHeader(info->headers(),
886 ✗ (http_tracing_headers_)[i].c_str());
887 }
888
889 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_pid());
890 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_gid());
891 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_uid());
892
893 ✗ LogCvmfs(kLogDownload, kLogDebug,
894 "(manager '%s' - id %" PRId64 ") "
895 "CURL Header for URL: %s is:\n %s",
896 name_.c_str(), info->id(), info->url()->c_str(),
897 ✗ header_lists_->Print(info->headers()).c_str());
898 }
899
900
2/2
✓ Branch 1 taken 120 times.
✓ Branch 2 taken 5879 times.
 5999 if (info->force_nocache()) {
901 120 SetNocache(info);
902 } else {
903 5879 info->SetNocache(false);
904 }
905
2/2
✓ Branch 1 taken 4202 times.
✓ Branch 2 taken 1797 times.
 5999 if (info->compressed()) {
906 4202 zlib::DecompressInit(info->GetZstreamPtr());
907 }
908
2/2
✓ Branch 1 taken 4245 times.
✓ Branch 2 taken 1754 times.
 5999 if (info->expected_hash()) {
909
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4245 times.
 4245 assert(info->hash_context().buffer != NULL);
910 4245 shash::Init(info->hash_context());
911 }
912
913
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 5999 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 5999 times.
 5999 if ((info->range_offset() != -1) && (info->range_size())) {
914 char byte_range_array[100];
915 ✗ const int64_t range_lower = static_cast<int64_t>(info->range_offset());
916 ✗ const int64_t range_upper = static_cast<int64_t>(info->range_offset()
917 ✗ + info->range_size() - 1);
918 ✗ if (snprintf(byte_range_array, sizeof(byte_range_array),
919 "%" PRId64 "-%" PRId64, range_lower, range_upper)
920 ✗ == 100) {
921 ✗ PANIC(NULL); // Should be impossible given limits on offset size.
922 }
923 ✗ curl_easy_setopt(handle, CURLOPT_RANGE, byte_range_array);
924 } else {
925 5999 curl_easy_setopt(handle, CURLOPT_RANGE, NULL);
926 }
927
928 // Set curl parameters
929 5999 curl_easy_setopt(handle, CURLOPT_PRIVATE, static_cast<void *>(info));
930 5999 curl_easy_setopt(handle, CURLOPT_WRITEHEADER, static_cast<void *>(info));
931 5999 curl_easy_setopt(handle, CURLOPT_WRITEDATA, static_cast<void *>(info));
932 5999 curl_easy_setopt(handle, CURLOPT_HTTPHEADER, info->headers());
933
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5999 times.
 5999 if (info->head_request()) {
934 ✗ curl_easy_setopt(handle, CURLOPT_NOBODY, 1);
935 } else {
936 5999 curl_easy_setopt(handle, CURLOPT_HTTPGET, 1);
937 }
938
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5999 times.
 5999 if (opt_ipv4_only_) {
939 ✗ curl_easy_setopt(handle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
940 }
941
2/2
✓ Branch 0 taken 2180 times.
✓ Branch 1 taken 3819 times.
 5999 if (follow_redirects_) {
942 2180 curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1);
943 2180 curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 4);
944 }
945 #ifdef DEBUGMSG
946 5999 curl_easy_setopt(handle, CURLOPT_VERBOSE, 1);
947 5999 curl_easy_setopt(handle, CURLOPT_DEBUGFUNCTION, CallbackCurlDebug);
948 #endif
949 5999 }
950
951 12256 void DownloadManager::CheckHostInfoReset(const std::string &typ,
952 HostInfo &info,
953 JobInfo *jobinfo,
954 time_t &now) {
955
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 12256 times.
 12256 if (info.timestamp_backup > 0) {
956 ✗ if (now == 0)
957 ✗ now = time(NULL);
958 ✗ if (static_cast<int64_t>(now)
959 ✗ > static_cast<int64_t>(info.timestamp_backup + info.reset_after)) {
960 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
961 "(manager %s - id %" PRId64 ") "
962 "switching %s from %s to %s (reset %s)",
963 name_.c_str(), jobinfo->id(), typ.c_str(),
964 ✗ (*info.chain)[info.current].c_str(), (*info.chain)[0].c_str(),
965 typ.c_str());
966 ✗ info.current = 0;
967 ✗ info.timestamp_backup = 0;
968 }
969 }
970 12256 }
971
972
973 /**
974 * Sets the URL specific options such as host to use and timeout. It might also
975 * set an error code, in which case the further processing should react on.
976 */
977 6128 void DownloadManager::SetUrlOptions(JobInfo *info) {
978 6128 CURL *curl_handle = info->curl_handle();
979 6128 string url_prefix;
980 6128 time_t now = 0;
981
982 6128 const MutexLockGuard m(lock_options_);
983
984 // sharding policy
985
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6128 times.
 6128 if (sharding_policy_.UseCount() > 0) {
986 ✗ if (info->proxy() != "") {
987 // proxy already set, so this is a failover event
988 ✗ perf::Inc(counters_->n_proxy_failover);
989 }
990 ✗ info->SetProxy(sharding_policy_->GetNextProxy(
991 ✗ info->url(), info->proxy(),
992 ✗ info->range_offset() == -1 ? 0 : info->range_offset()));
993
994 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, info->proxy().c_str());
995 } else { // no sharding policy
996 // Check if proxy group needs to be reset from backup to primary
997
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6128 times.
 6128 if (opt_timestamp_backup_proxies_ > 0) {
998 ✗ now = time(NULL);
999 ✗ if (static_cast<int64_t>(now) > static_cast<int64_t>(
1000 ✗ opt_timestamp_backup_proxies_ + opt_proxy_groups_reset_after_)) {
1001 ✗ opt_proxy_groups_current_ = 0;
1002 ✗ opt_timestamp_backup_proxies_ = 0;
1003 ✗ RebalanceProxiesUnlocked("Reset proxy group from backup to primary");
1004 }
1005 }
1006 // Check if load-balanced proxies within the group need to be reset
1007
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6128 times.
 6128 if (opt_timestamp_failover_proxies_ > 0) {
1008 ✗ if (now == 0)
1009 ✗ now = time(NULL);
1010 ✗ if (static_cast<int64_t>(now)
1011 ✗ > static_cast<int64_t>(opt_timestamp_failover_proxies_
1012 ✗ + opt_proxy_groups_reset_after_)) {
1013 ✗ RebalanceProxiesUnlocked(
1014 "Reset load-balanced proxies within the active group");
1015 }
1016 }
1017
1018
1/2
✓ Branch 2 taken 6128 times.
✗ Branch 3 not taken.
 6128 ProxyInfo *proxy = ChooseProxyUnlocked(info->expected_hash());
1019
6/6
✓ Branch 0 taken 1178 times.
✓ Branch 1 taken 4950 times.
✓ Branch 3 taken 1006 times.
✓ Branch 4 taken 172 times.
✓ Branch 5 taken 5956 times.
✓ Branch 6 taken 172 times.
 6128 if (!proxy || (proxy->url == "DIRECT")) {
1020
2/4
✓ Branch 2 taken 5956 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 5956 times.
✗ Branch 6 not taken.
 5956 info->SetProxy("DIRECT");
1021
1/2
✓ Branch 2 taken 5956 times.
✗ Branch 3 not taken.
 5956 curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "");
1022 } else {
1023 // Note: inside ValidateProxyIpsUnlocked() we may change the proxy data
1024 // structure, so we must not pass proxy->... (== current_proxy())
1025 // parameters directly
1026
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 const std::string purl = proxy->url;
1027
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 const dns::Host phost = proxy->host;
1028
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 const bool changed = ValidateProxyIpsUnlocked(purl, phost);
1029 // Current proxy may have changed
1030
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 172 times.
 172 if (changed) {
1031 ✗ proxy = ChooseProxyUnlocked(info->expected_hash());
1032 }
1033
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 info->SetProxy(proxy->url);
1034
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 if (proxy->host.status() == dns::kFailOk) {
1035
2/4
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
✓ Branch 6 taken 172 times.
✗ Branch 7 not taken.
 172 curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY,
1036 info->proxy().c_str());
1037 } else {
1038 // We know it can't work, don't even try to download
1039 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "0.0.0.0");
1040 }
1041 172 }
1042 } // end !sharding
1043
1044 // Check if metalink and host chains need to be reset
1045
2/4
✓ Branch 2 taken 6128 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 6128 times.
✗ Branch 6 not taken.
 6128 CheckHostInfoReset("metalink", opt_metalink_, info, now);
1046
2/4
✓ Branch 2 taken 6128 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 6128 times.
✗ Branch 6 not taken.
 6128 CheckHostInfoReset("host", opt_host_, info, now);
1047
1048
1/2
✓ Branch 1 taken 6128 times.
✗ Branch 2 not taken.
 6128 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_LIMIT, opt_low_speed_limit_);
1049
3/6
✓ Branch 1 taken 6128 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 172 times.
✓ Branch 6 taken 5956 times.
 6128 if (info->proxy() != "DIRECT") {
1050
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT, opt_timeout_proxy_);
1051
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_TIME, opt_timeout_proxy_);
1052 } else {
1053
1/2
✓ Branch 1 taken 5956 times.
✗ Branch 2 not taken.
 5956 curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT, opt_timeout_direct_);
1054
1/2
✓ Branch 1 taken 5956 times.
✗ Branch 2 not taken.
 5956 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_TIME, opt_timeout_direct_);
1055 }
1056
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6128 times.
 6128 if (!opt_dns_server_.empty())
1057 ✗ curl_easy_setopt(curl_handle, CURLOPT_DNS_SERVERS, opt_dns_server_.c_str());
1058
1059
2/2
✓ Branch 1 taken 1952 times.
✓ Branch 2 taken 4176 times.
 6128 if (info->probe_hosts()) {
1060
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1952 times.
 1952 if (CheckMetalinkChain(now)) {
1061 ✗ url_prefix = (*opt_metalink_.chain)[opt_metalink_.current];
1062 ✗ info->SetCurrentMetalinkChainIndex(opt_metalink_.current);
1063 ✗ LogCvmfs(kLogDownload, kLogDebug,
1064 "(manager %s - id %" PRId64 ") "
1065 "reading from metalink %d",
1066 name_.c_str(), info->id(), opt_metalink_.current);
1067
1/2
✓ Branch 0 taken 1952 times.
✗ Branch 1 not taken.
 1952 } else if (opt_host_.chain) {
1068
1/2
✓ Branch 2 taken 1952 times.
✗ Branch 3 not taken.
 1952 url_prefix = (*opt_host_.chain)[opt_host_.current];
1069 1952 info->SetCurrentHostChainIndex(opt_host_.current);
1070
1/2
✓ Branch 3 taken 1952 times.
✗ Branch 4 not taken.
 1952 LogCvmfs(kLogDownload, kLogDebug,
1071 "(manager %s - id %" PRId64 ") "
1072 "reading from host %d",
1073 name_.c_str(), info->id(), opt_host_.current);
1074 }
1075 }
1076
1077
1/2
✓ Branch 2 taken 6128 times.
✗ Branch 3 not taken.
 6128 string url = url_prefix + *(info->url());
1078
1079
1/2
✓ Branch 1 taken 6128 times.
✗ Branch 2 not taken.
 6128 curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, 1L);
1080
2/6
✓ Branch 1 taken 6128 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 6128 times.
 6128 if (url.substr(0, 5) == "https") {
1081 ✗ const bool rvb = ssl_certificate_store_.ApplySslCertificatePath(
1082 curl_handle);
1083 ✗ if (!rvb) {
1084 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1085 "(manager %s - id %" PRId64 ") "
1086 "Failed to set SSL certificate path %s",
1087 name_.c_str(), info->id(),
1088 ✗ ssl_certificate_store_.GetCaPath().c_str());
1089 }
1090 ✗ if (info->pid() != -1) {
1091 ✗ if (credentials_attachment_ == NULL) {
1092 ✗ LogCvmfs(kLogDownload, kLogDebug,
1093 "(manager %s - id %" PRId64 ") "
1094 "uses secure downloads but no credentials attachment set",
1095 name_.c_str(), info->id());
1096 } else {
1097 ✗ const bool retval = credentials_attachment_->ConfigureCurlHandle(
1098 curl_handle, info->pid(), info->GetCredDataPtr());
1099 ✗ if (!retval) {
1100 ✗ LogCvmfs(kLogDownload, kLogDebug,
1101 "(manager %s - id %" PRId64 ") "
1102 "failed attaching credentials",
1103 name_.c_str(), info->id());
1104 }
1105 }
1106 }
1107 // The download manager disables signal handling in the curl library;
1108 // as OpenSSL's implementation of TLS will generate a sigpipe in some
1109 // error paths, we must explicitly disable SIGPIPE here.
1110 // TODO(jblomer): it should be enough to do this once
1111 ✗ signal(SIGPIPE, SIG_IGN);
1112 }
1113
1114
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6128 times.
 6128 if (url.find("@proxy@") != string::npos) {
1115 // This is used in Geo-API requests (only), to replace a portion of the
1116 // URL with the current proxy name for the sake of caching the result.
1117 // Replace the @proxy@ either with a passed in "forced" template (which
1118 // is set from $CVMFS_PROXY_TEMPLATE) if there is one, or a "direct"
1119 // template (which is the uuid) if there's no proxy, or the name of the
1120 // proxy.
1121 ✗ string replacement;
1122 ✗ if (proxy_template_forced_ != "") {
1123 ✗ replacement = proxy_template_forced_;
1124 ✗ } else if (info->proxy() == "DIRECT") {
1125 ✗ replacement = proxy_template_direct_;
1126 } else {
1127 ✗ if (opt_proxy_groups_current_ >= opt_proxy_groups_fallback_) {
1128 // It doesn't make sense to use the fallback proxies in Geo-API requests
1129 // since the fallback proxies are supposed to get sorted, too.
1130 ✗ info->SetProxy("DIRECT");
1131 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "");
1132 ✗ replacement = proxy_template_direct_;
1133 } else {
1134 ✗ replacement = ChooseProxyUnlocked(info->expected_hash())->host.name();
1135 }
1136 }
1137 ✗ replacement = (replacement == "") ? proxy_template_direct_ : replacement;
1138 ✗ LogCvmfs(kLogDownload, kLogDebug,
1139 "(manager %s - id %" PRId64 ") "
1140 "replacing @proxy@ by %s",
1141 name_.c_str(), info->id(), replacement.c_str());
1142 ✗ url = ReplaceAll(url, "@proxy@", replacement);
1143 }
1144
1145 // TODO(heretherebedragons) before removing
1146 // static_cast<cvmfs::MemSink*>(info->sink)->size() == 0
1147 // and just always call info->sink->Reserve()
1148 // we should do a speed check
1149
3/4
✓ Branch 2 taken 6128 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1968 times.
✓ Branch 5 taken 4160 times.
 6128 if ((info->sink() != NULL) && info->sink()->RequiresReserve()
1150
1/2
✓ Branch 2 taken 1968 times.
✗ Branch 3 not taken.
 1968 && (static_cast<cvmfs::MemSink *>(info->sink())->size() == 0)
1151
11/20
✓ Branch 1 taken 6128 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 1968 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 1968 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 1418 times.
✓ Branch 11 taken 550 times.
✓ Branch 12 taken 1968 times.
✓ Branch 13 taken 4160 times.
✗ Branch 14 not taken.
✓ Branch 15 taken 1968 times.
✓ Branch 16 taken 4160 times.
✓ Branch 18 taken 1418 times.
✓ Branch 19 taken 4710 times.
✗ Branch 20 not taken.
✗ Branch 21 not taken.
✗ Branch 23 not taken.
✗ Branch 24 not taken.
 12256 && HasPrefix(url, "file://", false)) {
1152 platform_stat64 stat_buf;
1153 1418 const int retval = platform_stat(url.c_str(), &stat_buf);
1154
1/2
✓ Branch 0 taken 1418 times.
✗ Branch 1 not taken.
 1418 if (retval != 0) {
1155 // this is an error: file does not exist or out of memory
1156 // error is caught in other code section.
1157
1/2
✓ Branch 2 taken 1418 times.
✗ Branch 3 not taken.
 1418 info->sink()->Reserve(64ul * 1024ul);
1158 } else {
1159 ✗ info->sink()->Reserve(stat_buf.st_size);
1160 }
1161 }
1162
1163
2/4
✓ Branch 2 taken 6128 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 6128 times.
✗ Branch 7 not taken.
 6128 curl_easy_setopt(curl_handle, CURLOPT_URL,
1164 EscapeUrl(info->id(), url).c_str());
1165 6128 }
1166
1167
1168 /**
1169 * Checks if the name resolving information is still up to date. The host
1170 * object should be one from the current load-balance group. If the information
1171 * changed, gather new set of resolved IPs and, if different, exchange them in
1172 * the load-balance group on the fly. In the latter case, also rebalance the
1173 * proxies. The options mutex needs to be open.
1174 *
1175 * Returns true if proxies may have changed.
1176 */
1177 172 bool DownloadManager::ValidateProxyIpsUnlocked(const string &url,
1178 const dns::Host &host) {
1179
2/4
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 172 times.
✗ Branch 4 not taken.
 172 if (!host.IsExpired())
1180 172 return false;
1181 ✗ LogCvmfs(kLogDownload, kLogDebug, "(manager '%s') validate DNS entry for %s",
1182 ✗ name_.c_str(), host.name().c_str());
1183
1184 ✗ const unsigned group_idx = opt_proxy_groups_current_;
1185 ✗ dns::Host new_host = resolver_->Resolve(host.name());
1186
1187 ✗ bool update_only = true; // No changes to the list of IP addresses.
1188 ✗ if (new_host.status() != dns::kFailOk) {
1189 // Try again later in case resolving fails.
1190 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1191 "(manager '%s') failed to resolve IP addresses for %s (%d - %s)",
1192 ✗ name_.c_str(), host.name().c_str(), new_host.status(),
1193 dns::Code2Ascii(new_host.status()));
1194 ✗ new_host = dns::Host::ExtendDeadline(host, resolver_->min_ttl());
1195 ✗ } else if (!host.IsEquivalent(new_host)) {
1196 ✗ update_only = false;
1197 }
1198
1199 ✗ if (update_only) {
1200 ✗ for (unsigned i = 0; i < (*opt_proxy_groups_)[group_idx].size(); ++i) {
1201 ✗ if ((*opt_proxy_groups_)[group_idx][i].host.id() == host.id())
1202 ✗ (*opt_proxy_groups_)[group_idx][i].host = new_host;
1203 }
1204 ✗ return false;
1205 }
1206
1207 ✗ assert(new_host.status() == dns::kFailOk);
1208
1209 // Remove old host objects, insert new objects, and rebalance.
1210 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslog,
1211 "(manager '%s') DNS entries for proxy %s changed, adjusting",
1212 ✗ name_.c_str(), host.name().c_str());
1213 ✗ vector<ProxyInfo> *group = current_proxy_group();
1214 ✗ opt_num_proxies_ -= group->size();
1215 ✗ for (unsigned i = 0; i < group->size();) {
1216 ✗ if ((*group)[i].host.id() == host.id()) {
1217 ✗ group->erase(group->begin() + i);
1218 } else {
1219 ✗ i++;
1220 }
1221 }
1222 ✗ vector<ProxyInfo> new_infos;
1223 ✗ set<string> best_addresses = new_host.ViewBestAddresses(opt_ip_preference_);
1224 ✗ set<string>::const_iterator iter_ips = best_addresses.begin();
1225 ✗ for (; iter_ips != best_addresses.end(); ++iter_ips) {
1226 ✗ const string url_ip = dns::RewriteUrl(url, *iter_ips);
1227 ✗ new_infos.push_back(ProxyInfo(new_host, url_ip));
1228 }
1229 ✗ group->insert(group->end(), new_infos.begin(), new_infos.end());
1230 ✗ opt_num_proxies_ += new_infos.size();
1231
1232 ✗ const std::string msg = "DNS entries for proxy " + host.name() + " changed";
1233
1234 ✗ RebalanceProxiesUnlocked(msg);
1235 ✗ return true;
1236 }
1237
1238
1239 /**
1240 * Adds transfer time and downloaded bytes to the global counters.
1241 */
1242 6418 void DownloadManager::UpdateStatistics(CURL *handle) {
1243 double val;
1244 int retval;
1245 6418 int64_t sum = 0;
1246
1247
1/2
✓ Branch 1 taken 6418 times.
✗ Branch 2 not taken.
 6418 retval = curl_easy_getinfo(handle, CURLINFO_SIZE_DOWNLOAD, &val);
1248
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6418 times.
 6418 assert(retval == CURLE_OK);
1249 6418 sum += static_cast<int64_t>(val);
1250 /*retval = curl_easy_getinfo(handle, CURLINFO_HEADER_SIZE, &val);
1251 assert(retval == CURLE_OK);
1252 sum += static_cast<int64_t>(val);*/
1253 6418 perf::Xadd(counters_->sz_transferred_bytes, sum);
1254 6418 }
1255
1256
1257 /**
1258 * Retry if possible if not on no-cache and if not already done too often.
1259 */
1260 6418 bool DownloadManager::CanRetry(const JobInfo *info) {
1261 6418 const MutexLockGuard m(lock_options_);
1262 6418 const unsigned max_retries = opt_max_retries_;
1263
1264
2/2
✓ Branch 2 taken 2539 times.
✓ Branch 3 taken 3636 times.
 12593 return !(info->nocache()) && (info->num_retries() < max_retries)
1265
3/4
✓ Branch 0 taken 6175 times.
✓ Branch 1 taken 243 times.
✓ Branch 4 taken 2539 times.
✗ Branch 5 not taken.
 15132 && (IsProxyTransferError(info->error_code())
1266
2/2
✓ Branch 2 taken 167 times.
✓ Branch 3 taken 2372 times.
 2539 || IsHostTransferError(info->error_code()));
1267 6418 }
1268
1269 /**
1270 * Backoff for retry to introduce a jitter into a cluster of requesting
1271 * cvmfs nodes.
1272 * Retry only when HTTP caching is on.
1273 *
1274 * \return true if backoff has been performed, false otherwise
1275 */
1276 167 void DownloadManager::Backoff(JobInfo *info) {
1277 167 unsigned backoff_init_ms = 0;
1278 167 unsigned backoff_max_ms = 0;
1279 {
1280 167 const MutexLockGuard m(lock_options_);
1281 167 backoff_init_ms = opt_backoff_init_ms_;
1282 167 backoff_max_ms = opt_backoff_max_ms_;
1283 167 }
1284
1285 167 info->SetNumRetries(info->num_retries() + 1);
1286 167 perf::Inc(counters_->n_retries);
1287
2/2
✓ Branch 1 taken 67 times.
✓ Branch 2 taken 100 times.
 167 if (info->backoff_ms() == 0) {
1288 67 info->SetBackoffMs(prng_.Next(backoff_init_ms + 1)); // Must be != 0
1289 } else {
1290 100 info->SetBackoffMs(info->backoff_ms() * 2);
1291 }
1292
2/2
✓ Branch 1 taken 33 times.
✓ Branch 2 taken 134 times.
 167 if (info->backoff_ms() > backoff_max_ms) {
1293 33 info->SetBackoffMs(backoff_max_ms);
1294 }
1295
1296 167 LogCvmfs(kLogDownload, kLogDebug,
1297 "(manager '%s' - id %" PRId64 ") backing off for %d ms",
1298 name_.c_str(), info->id(), info->backoff_ms());
1299 167 SafeSleepMs(info->backoff_ms());
1300 167 }
1301
1302 243 void DownloadManager::SetNocache(JobInfo *info) {
1303
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 243 times.
 243 if (info->nocache())
1304 ✗ return;
1305 243 header_lists_->AppendHeader(info->headers(), "Pragma: no-cache");
1306 243 header_lists_->AppendHeader(info->headers(), "Cache-Control: no-cache");
1307 243 curl_easy_setopt(info->curl_handle(), CURLOPT_HTTPHEADER, info->headers());
1308 243 info->SetNocache(true);
1309 }
1310
1311
1312 /**
1313 * Reverse operation of SetNocache. Makes sure that "no-cache" header
1314 * disappears from the list of headers to let proxies work normally.
1315 */
1316 419 void DownloadManager::SetRegularCache(JobInfo *info) {
1317
1/2
✓ Branch 1 taken 419 times.
✗ Branch 2 not taken.
 419 if (info->nocache() == false)
1318 419 return;
1319 ✗ header_lists_->CutHeader("Pragma: no-cache", info->GetHeadersPtr());
1320 ✗ header_lists_->CutHeader("Cache-Control: no-cache", info->GetHeadersPtr());
1321 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_HTTPHEADER, info->headers());
1322 ✗ info->SetNocache(false);
1323 }
1324
1325
1326 /**
1327 * Frees the storage associated with the authz attachment from the job
1328 */
1329 6128 void DownloadManager::ReleaseCredential(JobInfo *info) {
1330
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6128 times.
 6128 if (info->cred_data()) {
1331 ✗ assert(credentials_attachment_ != NULL); // Someone must have set it
1332 ✗ credentials_attachment_->ReleaseCurlHandle(info->curl_handle(),
1333 info->cred_data());
1334 ✗ info->SetCredData(NULL);
1335 }
1336 6128 }
1337
1338
1339 /* Sort links based on the "pri=" parameter */
1340 ✗ static bool sortlinks(const std::string &s1, const std::string &s2) {
1341 ✗ const size_t pos1 = s1.find("; pri=");
1342 ✗ const size_t pos2 = s2.find("; pri=");
1343 int pri1, pri2;
1344 ✗ if ((pos1 != std::string::npos) && (pos2 != std::string::npos)
1345 ✗ && (sscanf(s1.substr(pos1 + 6).c_str(), "%d", &pri1) == 1)
1346 ✗ && (sscanf(s2.substr(pos2 + 6).c_str(), "%d", &pri2) == 1)) {
1347 ✗ return pri1 < pri2;
1348 }
1349 ✗ return false;
1350 }
1351
1352 /**
1353 * Parses Link header and uses it to set a new host chain.
1354 * See rfc6249.
1355 */
1356 ✗ void DownloadManager::ProcessLink(JobInfo *info) {
1357 ✗ std::vector<std::string> links = SplitString(info->link(), ',');
1358 ✗ if (info->link().find("; pri=") != std::string::npos)
1359 ✗ std::sort(links.begin(), links.end(), sortlinks);
1360
1361 ✗ std::vector<std::string> host_list;
1362
1363 ✗ std::vector<std::string>::const_iterator il = links.begin();
1364 ✗ for (; il != links.end(); ++il) {
1365 ✗ const std::string &link = *il;
1366 ✗ if ((link.find("; rel=duplicate") == std::string::npos)
1367 ✗ && (link.find("; rel=\"duplicate\"") == std::string::npos)) {
1368 ✗ LogCvmfs(kLogDownload, kLogDebug,
1369 "skipping link '%s' because it does not contain rel=duplicate",
1370 link.c_str());
1371 ✗ continue;
1372 }
1373 // ignore depth= field since there's nothing useful we can do with it
1374
1375 ✗ size_t start = link.find('<');
1376 ✗ if (start == std::string::npos) {
1377 ✗ LogCvmfs(
1378 kLogDownload, kLogDebug,
1379 "skipping link '%s' because it does not have a left angle bracket",
1380 link.c_str());
1381 ✗ continue;
1382 }
1383
1384 ✗ start++;
1385 ✗ if ((link.substr(start, 7) != "http://")
1386 ✗ && (link.substr(start, 8) != "https://")) {
1387 ✗ LogCvmfs(kLogDownload, kLogDebug,
1388 "skipping link '%s' of unrecognized url protocol", link.c_str());
1389 ✗ continue;
1390 }
1391
1392 ✗ size_t end = link.find('/', start + 8);
1393 ✗ if (end == std::string::npos)
1394 ✗ end = link.find('>');
1395 ✗ if (end == std::string::npos) {
1396 ✗ LogCvmfs(kLogDownload, kLogDebug,
1397 "skipping link '%s' because no slash in url and no right angle "
1398 "bracket",
1399 link.c_str());
1400 ✗ continue;
1401 }
1402 ✗ const std::string host = link.substr(start, end - start);
1403 ✗ LogCvmfs(kLogDownload, kLogDebug, "adding linked host '%s'", host.c_str());
1404 ✗ host_list.push_back(host);
1405 }
1406
1407 ✗ if (host_list.size() > 0) {
1408 ✗ SetHostChain(host_list);
1409 ✗ opt_metalink_timestamp_link_ = time(NULL);
1410 }
1411 }
1412
1413
1414 /**
1415 * Checks the result of a curl download and implements the failure logic, such
1416 * as changing the proxy server. Takes care of cleanup.
1417 *
1418 * \return true if another download should be performed, false otherwise
1419 */
1420 6418 bool DownloadManager::VerifyAndFinalize(const int curl_error, JobInfo *info) {
1421
1/2
✓ Branch 6 taken 6418 times.
✗ Branch 7 not taken.
 6418 LogCvmfs(kLogDownload, kLogDebug,
1422 "(manager '%s' - id %" PRId64 ") "
1423 "Verify downloaded url %s, proxy %s (curl error %d)",
1424 name_.c_str(), info->id(), info->url()->c_str(),
1425
1/2
✓ Branch 1 taken 6418 times.
✗ Branch 2 not taken.
 12836 info->proxy().c_str(), curl_error);
1426
1/2
✓ Branch 2 taken 6418 times.
✗ Branch 3 not taken.
 6418 UpdateStatistics(info->curl_handle());
1427
1428 bool was_metalink;
1429 6418 std::string typ;
1430
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6418 times.
 6418 if (info->current_metalink_chain_index() >= 0) {
1431 ✗ was_metalink = true;
1432 ✗ typ = "metalink";
1433 ✗ if (info->link() != "") {
1434 // process Link header whether or not the redirected URL got an error
1435 ✗ ProcessLink(info);
1436 }
1437 } else {
1438 6418 was_metalink = false;
1439
1/2
✓ Branch 1 taken 6418 times.
✗ Branch 2 not taken.
 6418 typ = "host";
1440 }
1441
1442
1443 // Verification and error classification
1444
3/14
✓ Branch 0 taken 5723 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 6 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 689 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
 6418 switch (curl_error) {
1445 5723 case CURLE_OK:
1446 // Verify content hash
1447
2/2
✓ Branch 1 taken 4083 times.
✓ Branch 2 taken 1640 times.
 5723 if (info->expected_hash()) {
1448
1/2
✓ Branch 1 taken 4083 times.
✗ Branch 2 not taken.
 4083 shash::Any match_hash;
1449
1/2
✓ Branch 2 taken 4083 times.
✗ Branch 3 not taken.
 4083 shash::Final(info->hash_context(), &match_hash);
1450
3/4
✓ Branch 2 taken 4083 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 246 times.
✓ Branch 5 taken 3837 times.
 4083 if (match_hash != *(info->expected_hash())) {
1451
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 246 times.
 246 if (ignore_signature_failures_) {
1452 ✗ LogCvmfs(
1453 kLogDownload, kLogDebug | kLogSyslogErr,
1454 "(manager '%s' - id %" PRId64 ") "
1455 "ignoring failed hash verification of %s (expected %s, got %s)",
1456 name_.c_str(), info->id(), info->url()->c_str(),
1457 ✗ info->expected_hash()->ToString().c_str(),
1458 ✗ match_hash.ToString().c_str());
1459 } else {
1460
1/2
✓ Branch 7 taken 246 times.
✗ Branch 8 not taken.
 492 LogCvmfs(kLogDownload, kLogDebug,
1461 "(manager '%s' - id %" PRId64 ") "
1462 "hash verification of %s failed (expected %s, got %s)",
1463 name_.c_str(), info->id(), info->url()->c_str(),
1464
1/2
✓ Branch 2 taken 246 times.
✗ Branch 3 not taken.
 492 info->expected_hash()->ToString().c_str(),
1465
1/2
✓ Branch 1 taken 246 times.
✗ Branch 2 not taken.
 492 match_hash.ToString().c_str());
1466 246 info->SetErrorCode(kFailBadData);
1467 246 break;
1468 }
1469 }
1470 }
1471
1472 5477 info->SetErrorCode(kFailOk);
1473 5477 break;
1474 ✗ case CURLE_UNSUPPORTED_PROTOCOL:
1475 ✗ info->SetErrorCode(kFailUnsupportedProtocol);
1476 ✗ break;
1477 6 case CURLE_URL_MALFORMAT:
1478 6 info->SetErrorCode(kFailBadUrl);
1479 6 break;
1480 ✗ case CURLE_COULDNT_RESOLVE_PROXY:
1481 ✗ info->SetErrorCode(kFailProxyResolve);
1482 ✗ break;
1483 ✗ case CURLE_COULDNT_RESOLVE_HOST:
1484 ✗ info->SetErrorCode(kFailHostResolve);
1485 ✗ break;
1486 ✗ case CURLE_OPERATION_TIMEDOUT:
1487 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostTooSlow
1488 : kFailProxyTooSlow);
1489 ✗ break;
1490 ✗ case CURLE_PARTIAL_FILE:
1491 case CURLE_GOT_NOTHING:
1492 case CURLE_RECV_ERROR:
1493 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostShortTransfer
1494 : kFailProxyShortTransfer);
1495 ✗ break;
1496 689 case CURLE_FILE_COULDNT_READ_FILE:
1497 case CURLE_COULDNT_CONNECT:
1498
3/6
✓ Branch 1 taken 689 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 43 times.
✓ Branch 6 taken 646 times.
 689 if (info->proxy() != "DIRECT") {
1499 // This is a guess. Fail-over can still change to switching host
1500 43 info->SetErrorCode(kFailProxyConnection);
1501 } else {
1502 646 info->SetErrorCode(kFailHostConnection);
1503 }
1504 689 break;
1505 ✗ case CURLE_TOO_MANY_REDIRECTS:
1506 ✗ info->SetErrorCode(kFailHostConnection);
1507 ✗ break;
1508 ✗ case CURLE_SSL_CACERT_BADFILE:
1509 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
1510 "(manager '%s' -id %" PRId64 ") "
1511 "Failed to load certificate bundle. "
1512 "X509_CERT_BUNDLE might point to the wrong location.",
1513 name_.c_str(), info->id());
1514 ✗ info->SetErrorCode(kFailHostConnection);
1515 ✗ break;
1516 // As of curl 7.62.0, CURLE_SSL_CACERT is the same as
1517 // CURLE_PEER_FAILED_VERIFICATION
1518 ✗ case CURLE_PEER_FAILED_VERIFICATION:
1519 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
1520 "(manager '%s' - id %" PRId64 ") "
1521 "invalid SSL certificate of remote host. "
1522 "X509_CERT_DIR and/or X509_CERT_BUNDLE might point to the wrong "
1523 "location.",
1524 name_.c_str(), info->id());
1525 ✗ info->SetErrorCode(kFailHostConnection);
1526 ✗ break;
1527 ✗ case CURLE_ABORTED_BY_CALLBACK:
1528 case CURLE_WRITE_ERROR:
1529 // Error set by callback
1530 ✗ break;
1531 ✗ case CURLE_SEND_ERROR:
1532 // The curl error CURLE_SEND_ERROR can be seen when a cache is misbehaving
1533 // and closing connections before the http request send is completed.
1534 // Handle this error, treating it as a short transfer error.
1535 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostShortTransfer
1536 : kFailProxyShortTransfer);
1537 ✗ break;
1538 ✗ default:
1539 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
1540 "(manager '%s' - id %" PRId64 ") "
1541 "unexpected curl error (%d) while trying to fetch %s",
1542 name_.c_str(), info->id(), curl_error, info->url()->c_str());
1543 ✗ info->SetErrorCode(kFailOther);
1544 ✗ break;
1545 }
1546
1547 std::vector<std::string> *host_chain;
1548 unsigned char num_used_hosts;
1549
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6418 times.
 6418 if (was_metalink) {
1550 ✗ host_chain = opt_metalink_.chain;
1551 ✗ num_used_hosts = info->num_used_metalinks();
1552 } else {
1553 6418 host_chain = opt_host_.chain;
1554 6418 num_used_hosts = info->num_used_hosts();
1555 }
1556
1557 // Determination if download should be repeated
1558 6418 bool try_again = false;
1559
1/2
✓ Branch 1 taken 6418 times.
✗ Branch 2 not taken.
 6418 bool same_url_retry = CanRetry(info);
1560
2/2
✓ Branch 1 taken 941 times.
✓ Branch 2 taken 5477 times.
 6418 if (info->error_code() != kFailOk) {
1561 941 const MutexLockGuard m(lock_options_);
1562
2/2
✓ Branch 1 taken 246 times.
✓ Branch 2 taken 695 times.
 941 if (info->error_code() == kFailBadData) {
1563
2/2
✓ Branch 1 taken 123 times.
✓ Branch 2 taken 123 times.
 246 if (!info->nocache()) {
1564 123 try_again = true;
1565 } else {
1566 // Make it a host failure
1567
1/2
✓ Branch 4 taken 123 times.
✗ Branch 5 not taken.
 123 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1568 "(manager '%s' - id %" PRId64 ") "
1569 "data corruption with no-cache header, try another %s",
1570 name_.c_str(), info->id(), typ.c_str());
1571
1572 123 info->SetErrorCode(kFailHostHttp);
1573 }
1574 }
1575 941 if (same_url_retry
1576
5/6
✓ Branch 0 taken 774 times.
✓ Branch 1 taken 167 times.
✓ Branch 3 taken 774 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 296 times.
✓ Branch 6 taken 645 times.
 1715 || (((info->error_code() == kFailHostResolve)
1577
2/2
✓ Branch 2 taken 295 times.
✓ Branch 3 taken 479 times.
 774 || IsHostTransferError(info->error_code())
1578
2/2
✓ Branch 1 taken 123 times.
✓ Branch 2 taken 172 times.
 295 || (info->error_code() == kFailHostHttp))
1579
3/4
✓ Branch 1 taken 369 times.
✓ Branch 2 taken 233 times.
✓ Branch 3 taken 369 times.
✗ Branch 4 not taken.
 602 && info->probe_hosts() && host_chain
1580
2/2
✓ Branch 1 taken 129 times.
✓ Branch 2 taken 240 times.
 369 && (num_used_hosts < host_chain->size()))) {
1581 296 try_again = true;
1582 }
1583 941 if (same_url_retry
1584
5/6
✓ Branch 0 taken 774 times.
✓ Branch 1 taken 167 times.
✓ Branch 3 taken 774 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 210 times.
✓ Branch 6 taken 731 times.
 1715 || (((info->error_code() == kFailProxyResolve)
1585
2/2
✓ Branch 2 taken 731 times.
✓ Branch 3 taken 43 times.
 774 || IsProxyTransferError(info->error_code())
1586
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 731 times.
 731 || (info->error_code() == kFailProxyHttp)))) {
1587
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 210 times.
 210 if (sharding_policy_.UseCount() > 0) { // sharding policy
1588 ✗ try_again = true;
1589 ✗ same_url_retry = false;
1590 } else { // no sharding
1591 210 try_again = true;
1592 // If all proxies failed, do a next round with the next host
1593
4/6
✓ Branch 0 taken 43 times.
✓ Branch 1 taken 167 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 43 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 210 times.
 210 if (!same_url_retry && (info->num_used_proxies() >= opt_num_proxies_)) {
1594 // Check if this can be made a host fail-over
1595 ✗ if (info->probe_hosts() && host_chain
1596 ✗ && (num_used_hosts < host_chain->size())) {
1597 // reset proxy group if not already performed by other handle
1598 ✗ if (opt_proxy_groups_) {
1599 ✗ if ((opt_proxy_groups_current_ > 0)
1600 ✗ || (opt_proxy_groups_current_burned_ > 0)) {
1601 ✗ opt_proxy_groups_current_ = 0;
1602 ✗ opt_timestamp_backup_proxies_ = 0;
1603 ✗ const std::string msg = "reset proxies for " + typ
1604 ✗ + " failover";
1605 ✗ RebalanceProxiesUnlocked(msg);
1606 }
1607 }
1608
1609 // Make it a host failure
1610 ✗ LogCvmfs(kLogDownload, kLogDebug,
1611 "(manager '%s' - id %" PRId64 ") make it a %s failure",
1612 name_.c_str(), info->id(), typ.c_str());
1613 ✗ info->SetNumUsedProxies(1);
1614 ✗ info->SetErrorCode(kFailHostAfterProxy);
1615 } else {
1616 ✗ if (failover_indefinitely_) {
1617 // Instead of giving up, reset the num_used_proxies counter,
1618 // switch proxy and try again
1619 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1620 "(manager '%s' - id %" PRId64 ") "
1621 "VerifyAndFinalize() would fail the download here. "
1622 "Instead switch proxy and retry download. "
1623 "typ=%s "
1624 "info->probe_hosts=%d host_chain=%p num_used_hosts=%d "
1625 "host_chain->size()=%lu same_url_retry=%d "
1626 "info->num_used_proxies=%d opt_num_proxies_=%d",
1627 name_.c_str(), info->id(), typ.c_str(),
1628 ✗ static_cast<int>(info->probe_hosts()), host_chain,
1629 ✗ num_used_hosts, host_chain ? host_chain->size() : -1,
1630 static_cast<int>(same_url_retry),
1631 ✗ info->num_used_proxies(), opt_num_proxies_);
1632 ✗ info->SetNumUsedProxies(1);
1633 ✗ RebalanceProxiesUnlocked(
1634 "download failed - failover indefinitely");
1635 ✗ try_again = !Interrupted(fqrn_, info);
1636 } else {
1637 ✗ try_again = false;
1638 }
1639 }
1640 } // Make a proxy failure a host failure
1641 } // Proxy failure assumed
1642 } // end !sharding
1643 941 }
1644
1645
2/2
✓ Branch 0 taken 462 times.
✓ Branch 1 taken 5956 times.
 6418 if (try_again) {
1646
1/2
✓ Branch 3 taken 462 times.
✗ Branch 4 not taken.
 462 LogCvmfs(kLogDownload, kLogDebug,
1647 "(manager '%s' - id %" PRId64 ") "
1648 "Trying again on same curl handle, same url: %d, "
1649 "error code %d no-cache %d",
1650 462 name_.c_str(), info->id(), same_url_retry, info->error_code(),
1651 462 info->nocache());
1652 // Reset internal state and destination
1653
4/8
✓ Branch 1 taken 462 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 462 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 462 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 462 times.
 462 if (info->sink() != NULL && info->sink()->Reset() != 0) {
1654 ✗ info->SetErrorCode(kFailLocalIO);
1655 ✗ goto verify_and_finalize_stop;
1656 }
1657
6/8
✓ Branch 1 taken 43 times.
✓ Branch 2 taken 419 times.
✓ Branch 5 taken 43 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 43 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 43 times.
✓ Branch 10 taken 419 times.
 462 if (info->interrupt_cue() && info->interrupt_cue()->IsCanceled()) {
1658 43 info->SetErrorCode(kFailCanceled);
1659 43 goto verify_and_finalize_stop;
1660 }
1661
1662
2/2
✓ Branch 1 taken 222 times.
✓ Branch 2 taken 197 times.
 419 if (info->expected_hash()) {
1663
1/2
✓ Branch 2 taken 222 times.
✗ Branch 3 not taken.
 222 shash::Init(info->hash_context());
1664 }
1665
2/2
✓ Branch 1 taken 222 times.
✓ Branch 2 taken 197 times.
 419 if (info->compressed()) {
1666
1/2
✓ Branch 2 taken 222 times.
✗ Branch 3 not taken.
 222 zlib::DecompressInit(info->GetZstreamPtr());
1667 }
1668
1669
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 419 times.
 419 if (sharding_policy_.UseCount() > 0) { // sharding policy
1670 ✗ ReleaseCredential(info);
1671 ✗ SetUrlOptions(info);
1672 } else { // no sharding policy
1673
1/2
✓ Branch 1 taken 419 times.
✗ Branch 2 not taken.
 419 SetRegularCache(info);
1674
1675 // Failure handling
1676 419 bool switch_proxy = false;
1677 419 bool switch_host = false;
1678
2/4
✓ Branch 1 taken 123 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 296 times.
 419 switch (info->error_code()) {
1679 123 case kFailBadData:
1680
1/2
✓ Branch 1 taken 123 times.
✗ Branch 2 not taken.
 123 SetNocache(info);
1681 123 break;
1682 ✗ case kFailProxyResolve:
1683 case kFailProxyHttp:
1684 ✗ switch_proxy = true;
1685 ✗ break;
1686 ✗ case kFailHostResolve:
1687 case kFailHostHttp:
1688 case kFailHostAfterProxy:
1689 ✗ switch_host = true;
1690 ✗ break;
1691 296 default:
1692
2/2
✓ Branch 2 taken 43 times.
✓ Branch 3 taken 253 times.
 296 if (IsProxyTransferError(info->error_code())) {
1693
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 43 times.
 43 if (same_url_retry) {
1694 ✗ Backoff(info);
1695 } else {
1696 43 switch_proxy = true;
1697 }
1698
1/2
✓ Branch 2 taken 253 times.
✗ Branch 3 not taken.
 253 } else if (IsHostTransferError(info->error_code())) {
1699
2/2
✓ Branch 0 taken 167 times.
✓ Branch 1 taken 86 times.
 253 if (same_url_retry) {
1700
1/2
✓ Branch 1 taken 167 times.
✗ Branch 2 not taken.
 167 Backoff(info);
1701 } else {
1702 86 switch_host = true;
1703 }
1704 } else {
1705 // No other errors expected when retrying
1706 ✗ PANIC(NULL);
1707 }
1708 }
1709
2/2
✓ Branch 0 taken 43 times.
✓ Branch 1 taken 376 times.
 419 if (switch_proxy) {
1710
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 ReleaseCredential(info);
1711
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 SwitchProxy(info);
1712 43 info->SetNumUsedProxies(info->num_used_proxies() + 1);
1713
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 SetUrlOptions(info);
1714 }
1715
2/2
✓ Branch 0 taken 86 times.
✓ Branch 1 taken 333 times.
 419 if (switch_host) {
1716
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 86 ReleaseCredential(info);
1717
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 86 times.
 86 if (was_metalink) {
1718 ✗ SwitchMetalink(info);
1719 ✗ info->SetNumUsedMetalinks(num_used_hosts + 1);
1720 } else {
1721
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 86 SwitchHost(info);
1722 86 info->SetNumUsedHosts(num_used_hosts + 1);
1723 }
1724
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 86 SetUrlOptions(info);
1725 }
1726 } // end !sharding
1727
1728
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 419 times.
 419 if (failover_indefinitely_) {
1729 // try again, breaking if there's a cvmfs reload happening and we are in a
1730 // proxy failover. This will EIO the call application.
1731 ✗ return !Interrupted(fqrn_, info);
1732 }
1733 419 return true; // try again
1734 }
1735
1736 5956 verify_and_finalize_stop:
1737 // Finalize, flush destination file
1738
1/2
✓ Branch 1 taken 5999 times.
✗ Branch 2 not taken.
 5999 ReleaseCredential(info);
1739
4/8
✓ Branch 1 taken 5999 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 5999 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 5999 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 5999 times.
 5999 if (info->sink() != NULL && info->sink()->Flush() != 0) {
1740 ✗ info->SetErrorCode(kFailLocalIO);
1741 }
1742
1743
2/2
✓ Branch 1 taken 4202 times.
✓ Branch 2 taken 1797 times.
 5999 if (info->compressed())
1744
1/2
✓ Branch 2 taken 4202 times.
✗ Branch 3 not taken.
 4202 zlib::DecompressFini(info->GetZstreamPtr());
1745
1746
1/2
✓ Branch 1 taken 5999 times.
✗ Branch 2 not taken.
 5999 if (info->headers()) {
1747
1/2
✓ Branch 2 taken 5999 times.
✗ Branch 3 not taken.
 5999 header_lists_->PutList(info->headers());
1748 5999 info->SetHeaders(NULL);
1749 }
1750
1751 5999 return false; // stop transfer and return to Fetch()
1752 6418 }
1753
1754 4592 DownloadManager::~DownloadManager() {
1755 // cleaned up fini
1756
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4592 times.
 4592 if (sharding_policy_.UseCount() > 0) {
1757 ✗ sharding_policy_.Reset();
1758 }
1759
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4592 times.
 4592 if (health_check_.UseCount() > 0) {
1760 ✗ if (health_check_.Unique()) {
1761 ✗ LogCvmfs(kLogDownload, kLogDebug,
1762 "(manager '%s') Stopping healthcheck thread", name_.c_str());
1763 ✗ health_check_->StopHealthcheck();
1764 }
1765 ✗ health_check_.Reset();
1766 }
1767
1768
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4592 times.
 4592 if (atomic_xadd32(&multi_threaded_, 0) == 1) {
1769 // Shutdown I/O thread
1770 ✗ pipe_terminate_->Write(kPipeTerminateSignal);
1771 ✗ pthread_join(thread_download_, NULL);
1772 // All handles are removed from the multi stack
1773 ✗ pipe_terminate_.Destroy();
1774 ✗ pipe_jobs_.Destroy();
1775 }
1776
1777 9184 for (set<CURL *>::iterator i = pool_handles_idle_->begin(),
1778 4592 iEnd = pool_handles_idle_->end();
1779
2/2
✓ Branch 1 taken 2512 times.
✓ Branch 2 taken 4592 times.
 7104 i != iEnd;
1780 2512 ++i) {
1781 2512 curl_easy_cleanup(*i);
1782 }
1783
1784
1/2
✓ Branch 0 taken 4592 times.
✗ Branch 1 not taken.
 4592 delete pool_handles_idle_;
1785
1/2
✓ Branch 0 taken 4592 times.
✗ Branch 1 not taken.
 4592 delete pool_handles_inuse_;
1786 4592 curl_multi_cleanup(curl_multi_);
1787
1788
1/2
✓ Branch 0 taken 4592 times.
✗ Branch 1 not taken.
 4592 delete header_lists_;
1789
1/2
✓ Branch 0 taken 4592 times.
✗ Branch 1 not taken.
 4592 if (user_agent_)
1790 4592 free(user_agent_);
1791
1792
1/2
✓ Branch 0 taken 4592 times.
✗ Branch 1 not taken.
 4592 delete counters_;
1793
2/2
✓ Branch 0 taken 1658 times.
✓ Branch 1 taken 2934 times.
 4592 delete opt_host_.chain;
1794
2/2
✓ Branch 0 taken 1658 times.
✓ Branch 1 taken 2934 times.
 4592 delete opt_host_chain_rtt_;
1795
2/2
✓ Branch 0 taken 1215 times.
✓ Branch 1 taken 3377 times.
 4592 delete opt_proxy_groups_;
1796
1797 4592 curl_global_cleanup();
1798
1/2
✓ Branch 0 taken 4592 times.
✗ Branch 1 not taken.
 4592 delete resolver_;
1799
1800 // old destructor
1801 4592 pthread_mutex_destroy(lock_options_);
1802 4592 pthread_mutex_destroy(lock_synchronous_mode_);
1803 4592 free(lock_options_);
1804 4592 free(lock_synchronous_mode_);
1805 4592 }
1806
1807 4593 void DownloadManager::InitHeaders() {
1808 // User-Agent
1809
1/2
✓ Branch 2 taken 4593 times.
✗ Branch 3 not taken.
 4593 string cernvm_id = "User-Agent: cvmfs ";
1810 #ifdef CVMFS_LIBCVMFS
1811
1/2
✓ Branch 1 taken 4593 times.
✗ Branch 2 not taken.
 4593 cernvm_id += "libcvmfs ";
1812 #else
1813 ✗ cernvm_id += "Fuse ";
1814 #endif
1815
2/4
✓ Branch 2 taken 4593 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 4593 times.
✗ Branch 6 not taken.
 4593 cernvm_id += string(CVMFS_VERSION);
1816
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4593 times.
 4593 if (getenv("CERNVM_UUID") != NULL) {
1817 cernvm_id += " "
1818 ✗ + sanitizer::InputSanitizer("az AZ 09 -")
1819 ✗ .Filter(getenv("CERNVM_UUID"));
1820 }
1821 4593 user_agent_ = strdup(cernvm_id.c_str());
1822
1823
1/2
✓ Branch 1 taken 4593 times.
✗ Branch 2 not taken.
 4593 header_lists_ = new HeaderLists();
1824
1825
1/2
✓ Branch 1 taken 4593 times.
✗ Branch 2 not taken.
 4593 default_headers_ = header_lists_->GetList("Connection: Keep-Alive");
1826
1/2
✓ Branch 1 taken 4593 times.
✗ Branch 2 not taken.
 4593 header_lists_->AppendHeader(default_headers_, "Pragma:");
1827
1/2
✓ Branch 1 taken 4593 times.
✗ Branch 2 not taken.
 4593 header_lists_->AppendHeader(default_headers_, user_agent_);
1828 4593 }
1829
1830 4593 DownloadManager::DownloadManager(const unsigned max_pool_handles,
1831 const perf::StatisticsTemplate &statistics,
1832 4593 const std::string &name)
1833 4593 : prng_(Prng())
1834 4593 , pool_handles_idle_(new set<CURL *>)
1835 4593 , pool_handles_inuse_(new set<CURL *>)
1836 4593 , pool_max_handles_(max_pool_handles)
1837 4593 , pipe_terminate_(NULL)
1838
1/2
✓ Branch 1 taken 4593 times.
✗ Branch 2 not taken.
 4593 , pipe_jobs_(NULL)
1839 4593 , watch_fds_(NULL)
1840 4593 , watch_fds_size_(0)
1841 4593 , watch_fds_inuse_(0)
1842 4593 , watch_fds_max_(4 * max_pool_handles)
1843 4593 , opt_timeout_proxy_(5)
1844 4593 , opt_timeout_direct_(10)
1845 4593 , opt_low_speed_limit_(1024)
1846 4593 , opt_max_retries_(0)
1847 4593 , opt_backoff_init_ms_(0)
1848 4593 , opt_backoff_max_ms_(0)
1849 4593 , enable_info_header_(false)
1850 4593 , opt_ipv4_only_(false)
1851 4593 , follow_redirects_(false)
1852 4593 , ignore_signature_failures_(false)
1853 4593 , enable_http_tracing_(false)
1854 4593 , opt_metalink_(NULL, 0, 0, 0)
1855 4593 , opt_metalink_timestamp_link_(0)
1856 4593 , opt_host_(NULL, 0, 0, 0)
1857 4593 , opt_host_chain_rtt_(NULL)
1858 4593 , opt_proxy_groups_(NULL)
1859 4593 , opt_proxy_groups_current_(0)
1860 4593 , opt_proxy_groups_current_burned_(0)
1861 4593 , opt_proxy_groups_fallback_(0)
1862 4593 , opt_num_proxies_(0)
1863 4593 , opt_proxy_shard_(false)
1864 4593 , failover_indefinitely_(false)
1865
1/2
✓ Branch 1 taken 4593 times.
✗ Branch 2 not taken.
 4593 , name_(name)
1866 4593 , opt_ip_preference_(dns::kIpPreferSystem)
1867 4593 , opt_timestamp_backup_proxies_(0)
1868 4593 , opt_timestamp_failover_proxies_(0)
1869 4593 , opt_proxy_groups_reset_after_(0)
1870 4593 , credentials_attachment_(NULL)
1871
4/8
✓ Branch 13 taken 4593 times.
✗ Branch 14 not taken.
✓ Branch 16 taken 4593 times.
✗ Branch 17 not taken.
✓ Branch 19 taken 4593 times.
✗ Branch 20 not taken.
✓ Branch 23 taken 4593 times.
✗ Branch 24 not taken.
 13779 , counters_(new Counters(statistics)) {
1872 4593 atomic_init32(&multi_threaded_);
1873
1874 4593 lock_options_ = reinterpret_cast<pthread_mutex_t *>(
1875 4593 smalloc(sizeof(pthread_mutex_t)));
1876 4593 int retval = pthread_mutex_init(lock_options_, NULL);
1877
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4593 times.
 4593 assert(retval == 0);
1878 4593 lock_synchronous_mode_ = reinterpret_cast<pthread_mutex_t *>(
1879 4593 smalloc(sizeof(pthread_mutex_t)));
1880 4593 retval = pthread_mutex_init(lock_synchronous_mode_, NULL);
1881
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4593 times.
 4593 assert(retval == 0);
1882
1883
1/2
✓ Branch 1 taken 4593 times.
✗ Branch 2 not taken.
 4593 retval = curl_global_init(CURL_GLOBAL_ALL);
1884
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4593 times.
 4593 assert(retval == CURLE_OK);
1885
1886
1/2
✓ Branch 1 taken 4593 times.
✗ Branch 2 not taken.
 4593 InitHeaders();
1887
1888
1/2
✓ Branch 1 taken 4593 times.
✗ Branch 2 not taken.
 4593 curl_multi_ = curl_multi_init();
1889
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4593 times.
 4593 assert(curl_multi_ != NULL);
1890
1/2
✓ Branch 1 taken 4593 times.
✗ Branch 2 not taken.
 4593 curl_multi_setopt(curl_multi_, CURLMOPT_SOCKETFUNCTION, CallbackCurlSocket);
1891
1/2
✓ Branch 1 taken 4593 times.
✗ Branch 2 not taken.
 4593 curl_multi_setopt(curl_multi_, CURLMOPT_SOCKETDATA,
1892 static_cast<void *>(this));
1893
1/2
✓ Branch 1 taken 4593 times.
✗ Branch 2 not taken.
 4593 curl_multi_setopt(curl_multi_, CURLMOPT_MAXCONNECTS, watch_fds_max_);
1894
1/2
✓ Branch 1 taken 4593 times.
✗ Branch 2 not taken.
 4593 curl_multi_setopt(curl_multi_, CURLMOPT_MAX_TOTAL_CONNECTIONS,
1895 pool_max_handles_);
1896
1897 4593 prng_.InitLocaltime();
1898
1899 // Name resolving
1900 4593 if ((getenv("CVMFS_IPV4_ONLY") != NULL)
1901
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 4593 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 4593 times.
 4593 && (strlen(getenv("CVMFS_IPV4_ONLY")) > 0)) {
1902 ✗ opt_ipv4_only_ = true;
1903 }
1904
1/2
✓ Branch 1 taken 4593 times.
✗ Branch 2 not taken.
 4593 resolver_ = dns::NormalResolver::Create(opt_ipv4_only_, kDnsDefaultRetries,
1905 kDnsDefaultTimeoutMs);
1906
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4593 times.
 4593 assert(resolver_);
1907 4593 }
1908
1909 /**
1910 * Spawns the I/O worker thread and switches the module in multi-threaded mode.
1911 * No way back except Fini(); Init();
1912 */
1913 ✗ void DownloadManager::Spawn() {
1914 ✗ pipe_terminate_ = new Pipe<kPipeThreadTerminator>();
1915 ✗ pipe_jobs_ = new Pipe<kPipeDownloadJobs>();
1916
1917 ✗ const int retval = pthread_create(&thread_download_, NULL, MainDownload,
1918 static_cast<void *>(this));
1919 ✗ assert(retval == 0);
1920
1921 ✗ atomic_inc32(&multi_threaded_);
1922
1923 ✗ if (health_check_.UseCount() > 0) {
1924 ✗ LogCvmfs(kLogDownload, kLogDebug,
1925 "(manager '%s') Starting healthcheck thread", name_.c_str());
1926 ✗ health_check_->StartHealthcheck();
1927 }
1928 }
1929
1930
1931 /**
1932 * Downloads data from an insecure outside channel (currently HTTP or file).
1933 */
1934 5999 Failures DownloadManager::Fetch(JobInfo *info) {
1935
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5999 times.
 5999 assert(info != NULL);
1936
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5999 times.
 5999 assert(info->url() != NULL);
1937
1938 Failures result;
1939
1/2
✓ Branch 1 taken 5999 times.
✗ Branch 2 not taken.
 5999 result = PrepareDownloadDestination(info);
1940
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5999 times.
 5999 if (result != kFailOk)
1941 ✗ return result;
1942
1943
2/2
✓ Branch 1 taken 4245 times.
✓ Branch 2 taken 1754 times.
 5999 if (info->expected_hash()) {
1944 4245 const shash::Algorithms algorithm = info->expected_hash()->algorithm;
1945 4245 info->GetHashContextPtr()->algorithm = algorithm;
1946
1/2
✓ Branch 1 taken 4245 times.
✗ Branch 2 not taken.
 4245 info->GetHashContextPtr()->size = shash::GetContextSize(algorithm);
1947 4245 info->GetHashContextPtr()->buffer = alloca(info->hash_context().size);
1948 }
1949
1950 // In case JobInfo object is being reused
1951
2/4
✓ Branch 2 taken 5999 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 5999 times.
✗ Branch 6 not taken.
 5999 info->SetLink("");
1952
1953 // Prepare cvmfs-info: header, allocate string on the stack
1954 5999 info->SetInfoHeader(NULL);
1955
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5999 times.
 5999 if (enable_info_header_) {
1956 ✗ const string header_info = info->GetInfoHeaderContents(info_header_template_);
1957 ✗ if (header_info != "") {
1958 ✗ const char * const header_name = "cvmfs-info: ";
1959 ✗ const size_t header_name_len = strlen(header_name);
1960 ✗ const size_t info_len = header_info.length();
1961 ✗ char *buf = static_cast<char *>(alloca(header_name_len + info_len + 1));
1962 ✗ memcpy(buf, header_name, header_name_len);
1963 ✗ memcpy(buf + header_name_len, header_info.c_str(), info_len);
1964 ✗ buf[header_name_len + info_len] = '\0';
1965 ✗ info->SetInfoHeader(buf);
1966 }
1967 }
1968
1969
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5999 times.
 5999 if (enable_http_tracing_) {
1970 ✗ const std::string str_pid = "X-CVMFS-PID: " + StringifyInt(info->pid());
1971 ✗ const std::string str_gid = "X-CVMFS-GID: " + StringifyUint(info->gid());
1972 ✗ const std::string str_uid = "X-CVMFS-UID: " + StringifyUint(info->uid());
1973
1974 // will be auto freed at the end of this function Fetch(JobInfo *info)
1975 ✗ info->SetTracingHeaderPid(static_cast<char *>(alloca(str_pid.size() + 1)));
1976 ✗ info->SetTracingHeaderGid(static_cast<char *>(alloca(str_gid.size() + 1)));
1977 ✗ info->SetTracingHeaderUid(static_cast<char *>(alloca(str_uid.size() + 1)));
1978
1979 ✗ memcpy(info->tracing_header_pid(), str_pid.c_str(), str_pid.size() + 1);
1980 ✗ memcpy(info->tracing_header_gid(), str_gid.c_str(), str_gid.size() + 1);
1981 ✗ memcpy(info->tracing_header_uid(), str_uid.c_str(), str_uid.size() + 1);
1982 }
1983
1984
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5999 times.
 5999 if (atomic_xadd32(&multi_threaded_, 0) == 1) {
1985 ✗ if (!info->IsValidPipeJobResults()) {
1986 ✗ info->CreatePipeJobResults();
1987 }
1988 ✗ if (!info->IsValidDataTube()) {
1989 ✗ info->CreateDataTube();
1990 }
1991
1992 // LogCvmfs(kLogDownload, kLogDebug, "send job to thread, pipe %d %d",
1993 // info->wait_at[0], info->wait_at[1]);
1994 ✗ pipe_jobs_->Write<JobInfo *>(info);
1995
1996 do {
1997 ✗ DataTubeElement *ele = info->GetDataTubePtr()->PopFront();
1998
1999 ✗ if (ele->action == kActionStop) {
2000 ✗ delete ele;
2001 ✗ break;
2002 }
2003 // TODO(heretherebedragons) add compression
2004 ✗ } while (true);
2005
2006 ✗ info->GetPipeJobResultPtr()->Read<download::Failures>(&result);
2007 // LogCvmfs(kLogDownload, kLogDebug, "got result %d", result);
2008 } else {
2009 5999 const MutexLockGuard l(lock_synchronous_mode_);
2010
1/2
✓ Branch 1 taken 5999 times.
✗ Branch 2 not taken.
 5999 CURL *handle = AcquireCurlHandle();
2011
1/2
✓ Branch 1 taken 5999 times.
✗ Branch 2 not taken.
 5999 InitializeRequest(info, handle);
2012
1/2
✓ Branch 1 taken 5999 times.
✗ Branch 2 not taken.
 5999 SetUrlOptions(info);
2013 // curl_easy_setopt(handle, CURLOPT_VERBOSE, 1);
2014 int retval;
2015 do {
2016
1/2
✓ Branch 1 taken 6418 times.
✗ Branch 2 not taken.
 6418 retval = curl_easy_perform(handle);
2017 6418 perf::Inc(counters_->n_requests);
2018 double elapsed;
2019
1/2
✓ Branch 1 taken 6418 times.
✗ Branch 2 not taken.
 6418 if (curl_easy_getinfo(handle, CURLINFO_TOTAL_TIME, &elapsed)
2020
1/2
✓ Branch 0 taken 6418 times.
✗ Branch 1 not taken.
 6418 == CURLE_OK) {
2021 6418 perf::Xadd(counters_->sz_transfer_time,
2022 6418 static_cast<int64_t>(elapsed * 1000));
2023 }
2024
3/4
✓ Branch 1 taken 6418 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 419 times.
✓ Branch 4 taken 5999 times.
 6418 } while (VerifyAndFinalize(retval, info));
2025 5999 result = info->error_code();
2026
1/2
✓ Branch 2 taken 5999 times.
✗ Branch 3 not taken.
 5999 ReleaseCurlHandle(info->curl_handle());
2027 5999 }
2028
2029
2/2
✓ Branch 0 taken 522 times.
✓ Branch 1 taken 5477 times.
 5999 if (result != kFailOk) {
2030
1/2
✓ Branch 4 taken 522 times.
✗ Branch 5 not taken.
 522 LogCvmfs(kLogDownload, kLogDebug,
2031 "(manager '%s' - id %" PRId64 ") "
2032 "download failed (error %d - %s)",
2033 name_.c_str(), info->id(), result, Code2Ascii(result));
2034
2035
1/2
✓ Branch 1 taken 522 times.
✗ Branch 2 not taken.
 522 if (info->sink() != NULL) {
2036
1/2
✓ Branch 2 taken 522 times.
✗ Branch 3 not taken.
 522 info->sink()->Purge();
2037 }
2038 }
2039
2040 5999 return result;
2041 }
2042
2043
2044 /**
2045 * Used by the client to connect the authz session manager to the download
2046 * manager.
2047 */
2048 760 void DownloadManager::SetCredentialsAttachment(CredentialsAttachment *ca) {
2049 760 const MutexLockGuard m(lock_options_);
2050 760 credentials_attachment_ = ca;
2051 760 }
2052
2053 /**
2054 * Gets the DNS sever.
2055 */
2056 ✗ std::string DownloadManager::GetDnsServer() const { return opt_dns_server_; }
2057
2058 /**
2059 * Sets a DNS server. Only for testing as it cannot be reverted to the system
2060 * default.
2061 */
2062 ✗ void DownloadManager::SetDnsServer(const string &address) {
2063 ✗ if (!address.empty()) {
2064 ✗ const MutexLockGuard m(lock_options_);
2065 ✗ opt_dns_server_ = address;
2066 ✗ assert(!opt_dns_server_.empty());
2067
2068 ✗ vector<string> servers;
2069 ✗ servers.push_back(address);
2070 ✗ const bool retval = resolver_->SetResolvers(servers);
2071 ✗ assert(retval);
2072 }
2073 ✗ LogCvmfs(kLogDownload, kLogSyslog, "(manager '%s') set nameserver to %s",
2074 name_.c_str(), address.c_str());
2075 }
2076
2077
2078 /**
2079 * Sets the DNS query timeout parameters.
2080 */
2081 1389 void DownloadManager::SetDnsParameters(const unsigned retries,
2082 const unsigned timeout_ms) {
2083 1389 const MutexLockGuard m(lock_options_);
2084 1389 if ((resolver_->retries() == retries)
2085
3/6
✓ Branch 0 taken 1389 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 1389 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 1389 times.
✗ Branch 6 not taken.
 1389 && (resolver_->timeout_ms() == timeout_ms)) {
2086 1389 return;
2087 }
2088 ✗ delete resolver_;
2089 ✗ resolver_ = NULL;
2090 ✗ resolver_ = dns::NormalResolver::Create(opt_ipv4_only_, retries, timeout_ms);
2091 ✗ assert(resolver_);
2092
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1389 times.
 1389 }
2093
2094
2095 1389 void DownloadManager::SetDnsTtlLimits(const unsigned min_seconds,
2096 const unsigned max_seconds) {
2097 1389 const MutexLockGuard m(lock_options_);
2098 1389 resolver_->set_min_ttl(min_seconds);
2099 1389 resolver_->set_max_ttl(max_seconds);
2100 1389 }
2101
2102
2103 ✗ void DownloadManager::SetIpPreference(dns::IpPreference preference) {
2104 ✗ const MutexLockGuard m(lock_options_);
2105 ✗ opt_ip_preference_ = preference;
2106 }
2107
2108
2109 /**
2110 * Sets two timeout values for proxied and for direct connections, respectively.
2111 * The timeout counts for all sorts of connection phases,
2112 * DNS, HTTP connect, etc.
2113 */
2114 2024 void DownloadManager::SetTimeout(const unsigned seconds_proxy,
2115 const unsigned seconds_direct) {
2116 2024 const MutexLockGuard m(lock_options_);
2117 2024 opt_timeout_proxy_ = seconds_proxy;
2118 2024 opt_timeout_direct_ = seconds_direct;
2119 2024 }
2120
2121
2122 /**
2123 * Sets contains the average transfer speed in bytes per second that the
2124 * transfer should be below during CURLOPT_LOW_SPEED_TIME seconds for libcurl to
2125 * consider it to be too slow and abort. Only effective for new connections.
2126 */
2127 ✗ void DownloadManager::SetLowSpeedLimit(const unsigned low_speed_limit) {
2128 ✗ const MutexLockGuard m(lock_options_);
2129 ✗ opt_low_speed_limit_ = low_speed_limit;
2130 }
2131
2132
2133 /**
2134 * Receives the currently active timeout values.
2135 */
2136 543 void DownloadManager::GetTimeout(unsigned *seconds_proxy,
2137 unsigned *seconds_direct) {
2138 543 const MutexLockGuard m(lock_options_);
2139 543 *seconds_proxy = opt_timeout_proxy_;
2140 543 *seconds_direct = opt_timeout_direct_;
2141 543 }
2142
2143
2144 /**
2145 * Parses a list of ';'-separated hosts for the metalink chain. The empty
2146 * string removes the metalink list.
2147 */
2148 ✗ void DownloadManager::SetMetalinkChain(const string &metalink_list) {
2149 ✗ SetMetalinkChain(SplitString(metalink_list, ';'));
2150 }
2151
2152
2153 ✗ void DownloadManager::SetMetalinkChain(
2154 const std::vector<std::string> &metalink_list) {
2155 ✗ const MutexLockGuard m(lock_options_);
2156 ✗ opt_metalink_.timestamp_backup = 0;
2157 ✗ delete opt_metalink_.chain;
2158 ✗ opt_metalink_.current = 0;
2159
2160 ✗ if (metalink_list.empty()) {
2161 ✗ opt_metalink_.chain = NULL;
2162 ✗ return;
2163 }
2164
2165 ✗ opt_metalink_.chain = new vector<string>(metalink_list);
2166 }
2167
2168
2169 /**
2170 * Retrieves the currently set chain of metalink hosts and the currently
2171 * used metalink host.
2172 */
2173 ✗ void DownloadManager::GetMetalinkInfo(vector<string> *metalink_chain,
2174 unsigned *current_metalink) {
2175 ✗ const MutexLockGuard m(lock_options_);
2176 ✗ if (opt_metalink_.chain) {
2177 ✗ if (current_metalink) {
2178 ✗ *current_metalink = opt_metalink_.current;
2179 }
2180 ✗ if (metalink_chain) {
2181 ✗ *metalink_chain = *opt_metalink_.chain;
2182 }
2183 }
2184 }
2185
2186
2187 /**
2188 * Parses a list of ';'-separated hosts for the host chain. The empty string
2189 * removes the host list.
2190 */
2191 1661 void DownloadManager::SetHostChain(const string &host_list) {
2192
1/2
✓ Branch 2 taken 1661 times.
✗ Branch 3 not taken.
 1661 SetHostChain(SplitString(host_list, ';'));
2193 1661 }
2194
2195
2196 1687 void DownloadManager::SetHostChain(const std::vector<std::string> &host_list) {
2197 1687 const MutexLockGuard m(lock_options_);
2198 1687 opt_host_.timestamp_backup = 0;
2199
2/2
✓ Branch 0 taken 572 times.
✓ Branch 1 taken 1115 times.
 1687 delete opt_host_.chain;
2200
2/2
✓ Branch 0 taken 572 times.
✓ Branch 1 taken 1115 times.
 1687 delete opt_host_chain_rtt_;
2201 1687 opt_host_.current = 0;
2202
2203
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1687 times.
 1687 if (host_list.empty()) {
2204 ✗ opt_host_.chain = NULL;
2205 ✗ opt_host_chain_rtt_ = NULL;
2206 ✗ return;
2207 }
2208
2209
2/4
✓ Branch 1 taken 1687 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1687 times.
✗ Branch 5 not taken.
 1687 opt_host_.chain = new vector<string>(host_list);
2210 3374 opt_host_chain_rtt_ = new vector<int>(opt_host_.chain->size(),
2211
2/4
✓ Branch 1 taken 1687 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1687 times.
✗ Branch 5 not taken.
 1687 kProbeUnprobed);
2212 // LogCvmfs(kLogDownload, kLogSyslog, "using host %s",
2213 // (*opt_host_.chain)[0].c_str());
2214
1/2
✓ Branch 1 taken 1687 times.
✗ Branch 2 not taken.
 1687 }
2215
2216
2217 /**
2218 * Retrieves the currently set chain of hosts, their round trip times, and the
2219 * currently used host.
2220 */
2221 182 void DownloadManager::GetHostInfo(vector<string> *host_chain, vector<int> *rtt,
2222 unsigned *current_host) {
2223 182 const MutexLockGuard m(lock_options_);
2224
1/2
✓ Branch 0 taken 182 times.
✗ Branch 1 not taken.
 182 if (opt_host_.chain) {
2225
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 182 times.
 182 if (current_host) {
2226 ✗ *current_host = opt_host_.current;
2227 }
2228
1/2
✓ Branch 0 taken 182 times.
✗ Branch 1 not taken.
 182 if (host_chain) {
2229
1/2
✓ Branch 1 taken 182 times.
✗ Branch 2 not taken.
 182 *host_chain = *opt_host_.chain;
2230 }
2231
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 182 times.
 182 if (rtt) {
2232 ✗ *rtt = *opt_host_chain_rtt_;
2233 }
2234 }
2235 182 }
2236
2237
2238 /**
2239 * Jumps to the next proxy in the ring of forward proxy servers.
2240 * Selects one randomly from a load-balancing group.
2241 *
2242 * Allow for the fact that the proxy may have already been failed by
2243 * another transfer, or that the proxy may no longer be part of the
2244 * current load-balancing group.
2245 */
2246 43 void DownloadManager::SwitchProxy(JobInfo *info) {
2247 43 const MutexLockGuard m(lock_options_);
2248
2249
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 43 times.
 43 if (!opt_proxy_groups_) {
2250 ✗ return;
2251 }
2252
2253 // Fail any matching proxies within the current load-balancing group
2254 43 vector<ProxyInfo> *group = current_proxy_group();
2255 43 const unsigned group_size = group->size();
2256 43 unsigned failed = 0;
2257
2/2
✓ Branch 0 taken 43 times.
✓ Branch 1 taken 43 times.
 86 for (unsigned i = 0; i < group_size - opt_proxy_groups_current_burned_; ++i) {
2258
5/14
✓ Branch 0 taken 43 times.
✗ Branch 1 not taken.
✓ Branch 4 taken 43 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 43 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 43 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✓ Branch 12 taken 43 times.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
 43 if (info && (info->proxy() == (*group)[i].url)) {
2259 // Move to list of failed proxies
2260 43 opt_proxy_groups_current_burned_++;
2261
1/2
✓ Branch 2 taken 43 times.
✗ Branch 3 not taken.
 43 swap((*group)[i],
2262 43 (*group)[group_size - opt_proxy_groups_current_burned_]);
2263 43 perf::Inc(counters_->n_proxy_failover);
2264 43 failed++;
2265 }
2266 }
2267
2268 // Do nothing more unless at least one proxy was marked as failed
2269
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 43 times.
 43 if (!failed)
2270 ✗ return;
2271
2272 // If all proxies from the current load-balancing group are burned, switch to
2273 // another group
2274
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 if (opt_proxy_groups_current_burned_ == group->size()) {
2275 43 opt_proxy_groups_current_burned_ = 0;
2276
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 if (opt_proxy_groups_->size() > 1) {
2277 86 opt_proxy_groups_current_ = (opt_proxy_groups_current_ + 1)
2278 43 % opt_proxy_groups_->size();
2279 // Remember the timestamp of switching to backup proxies
2280
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 43 times.
 43 if (opt_proxy_groups_reset_after_ > 0) {
2281 ✗ if (opt_proxy_groups_current_ > 0) {
2282 ✗ if (opt_timestamp_backup_proxies_ == 0)
2283 ✗ opt_timestamp_backup_proxies_ = time(NULL);
2284 // LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2285 // "switched to (another) backup proxy group");
2286 } else {
2287 ✗ opt_timestamp_backup_proxies_ = 0;
2288 // LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2289 // "switched back to primary proxy group");
2290 }
2291 ✗ opt_timestamp_failover_proxies_ = 0;
2292 }
2293 }
2294 } else {
2295 // Record failover time
2296 ✗ if (opt_proxy_groups_reset_after_ > 0) {
2297 ✗ if (opt_timestamp_failover_proxies_ == 0)
2298 ✗ opt_timestamp_failover_proxies_ = time(NULL);
2299 }
2300 }
2301
2302
2/4
✓ Branch 2 taken 43 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 43 times.
✗ Branch 6 not taken.
 43 UpdateProxiesUnlocked("failed proxy");
2303
1/2
✓ Branch 2 taken 43 times.
✗ Branch 3 not taken.
 43 LogCvmfs(kLogDownload, kLogDebug,
2304 "(manager '%s' - id %" PRId64 ") "
2305 "%lu proxies remain in group",
2306 name_.c_str(), info->id(),
2307 43 current_proxy_group()->size() - opt_proxy_groups_current_burned_);
2308
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 }
2309
2310
2311 /**
2312 * Switches to the next host in the chain. If jobinfo is set, switch only if
2313 * the current host is identical to the one used by jobinfo, otherwise another
2314 * transfer has already done the switch.
2315 */
2316 89 void DownloadManager::SwitchHostInfo(const std::string &typ,
2317 HostInfo &info,
2318 JobInfo *jobinfo) {
2319 89 const MutexLockGuard m(lock_options_);
2320
2321
5/6
✓ Branch 0 taken 89 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 3 times.
✓ Branch 4 taken 86 times.
✓ Branch 5 taken 3 times.
✓ Branch 6 taken 86 times.
 89 if (!info.chain || (info.chain->size() == 1)) {
2322 3 return;
2323 }
2324
2325
1/2
✓ Branch 0 taken 86 times.
✗ Branch 1 not taken.
 86 if (jobinfo) {
2326 int lastused;
2327
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 86 if (typ == "host") {
2328 86 lastused = jobinfo->current_host_chain_index();
2329 } else {
2330 ✗ lastused = jobinfo->current_metalink_chain_index();
2331 }
2332
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 86 times.
 86 if (lastused != info.current) {
2333 ✗ LogCvmfs(kLogDownload, kLogDebug,
2334 "(manager '%s' - id %" PRId64 ")"
2335 "don't switch %s, "
2336 "last used %s: %s, current %s: %s",
2337 name_.c_str(), jobinfo->id(), typ.c_str(), typ.c_str(),
2338 ✗ (*info.chain)[lastused].c_str(), typ.c_str(),
2339 ✗ (*info.chain)[info.current].c_str());
2340 ✗ return;
2341 }
2342 }
2343
2344
1/2
✓ Branch 2 taken 86 times.
✗ Branch 3 not taken.
 86 string reason = "manually triggered";
2345
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 86 string info_id = "(manager " + name_;
2346
1/2
✓ Branch 0 taken 86 times.
✗ Branch 1 not taken.
 86 if (jobinfo) {
2347
1/2
✓ Branch 3 taken 86 times.
✗ Branch 4 not taken.
 86 reason = download::Code2Ascii(jobinfo->error_code());
2348
2/4
✓ Branch 2 taken 86 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 86 times.
✗ Branch 6 not taken.
 86 info_id = " - id " + StringifyInt(jobinfo->id());
2349 }
2350
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 86 info_id += ")";
2351
2352
1/2
✓ Branch 2 taken 86 times.
✗ Branch 3 not taken.
 86 const std::string old_host = (*info.chain)[info.current];
2353 86 info.current = (info.current + 1) % static_cast<int>(info.chain->size());
2354
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 86 if (typ == "host") {
2355 86 perf::Inc(counters_->n_host_failover);
2356 } else {
2357 ✗ perf::Inc(counters_->n_metalink_failover);
2358 }
2359
1/3
✓ Branch 6 taken 86 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
 172 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2360 "%s switching %s from %s to %s (%s)", info_id.c_str(), typ.c_str(),
2361 86 old_host.c_str(), (*info.chain)[info.current].c_str(),
2362 reason.c_str());
2363
2364 // Remember the timestamp of switching to backup host
2365
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 86 times.
 86 if (info.reset_after > 0) {
2366 ✗ if (info.current != 0) {
2367 ✗ if (info.timestamp_backup == 0)
2368 ✗ info.timestamp_backup = time(NULL);
2369 } else {
2370 ✗ info.timestamp_backup = 0;
2371 }
2372 }
2373
2/2
✓ Branch 4 taken 86 times.
✓ Branch 5 taken 3 times.
 89 }
2374
2375 89 void DownloadManager::SwitchHost(JobInfo *info) {
2376
2/4
✓ Branch 2 taken 89 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 89 times.
✗ Branch 6 not taken.
 89 SwitchHostInfo("host", opt_host_, info);
2377 89 }
2378
2379 3 void DownloadManager::SwitchHost() { SwitchHost(NULL); }
2380
2381
2382 ✗ void DownloadManager::SwitchMetalink(JobInfo *info) {
2383 ✗ SwitchHostInfo("metalink", opt_metalink_, info);
2384 }
2385
2386
2387 ✗ void DownloadManager::SwitchMetalink() { SwitchMetalink(NULL); }
2388
2389 1952 bool DownloadManager::CheckMetalinkChain(time_t now) {
2390 1952 return (opt_metalink_.chain
2391
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1952 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 1952 && ((opt_metalink_timestamp_link_ == 0)
2392 ✗ || (static_cast<int64_t>((now == 0) ? time(NULL) : now)
2393 ✗ > static_cast<int64_t>(opt_metalink_timestamp_link_
2394
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 1952 + opt_metalink_.reset_after))));
2395 }
2396
2397
2398 /**
2399 * Orders the hostlist according to RTT of downloading .cvmfschecksum.
2400 * Sets the current host to the best-responsive host.
2401 * If you change the host list in between by SetHostChain(), it will be
2402 * overwritten by this function.
2403 */
2404 ✗ void DownloadManager::ProbeHosts() {
2405 ✗ vector<string> host_chain;
2406 ✗ vector<int> host_rtt;
2407 unsigned current_host;
2408
2409 ✗ GetHostInfo(&host_chain, &host_rtt, &current_host);
2410
2411 // Stopwatch, two times to fill caches first
2412 unsigned i, retries;
2413 ✗ string url;
2414
2415 ✗ cvmfs::MemSink memsink;
2416 ✗ JobInfo info(&url, false, false, NULL, &memsink);
2417 ✗ for (retries = 0; retries < 2; ++retries) {
2418 ✗ for (i = 0; i < host_chain.size(); ++i) {
2419 ✗ url = host_chain[i] + "/.cvmfspublished";
2420
2421 struct timeval tv_start, tv_end;
2422 ✗ gettimeofday(&tv_start, NULL);
2423 ✗ const Failures result = Fetch(&info);
2424 ✗ gettimeofday(&tv_end, NULL);
2425 ✗ memsink.Reset();
2426 ✗ if (result == kFailOk) {
2427 ✗ host_rtt[i] = static_cast<int>(DiffTimeSeconds(tv_start, tv_end)
2428 ✗ * 1000);
2429 ✗ LogCvmfs(kLogDownload, kLogDebug,
2430 "(manager '%s' - id %" PRId64 ") "
2431 "probing host %s had %dms rtt",
2432 ✗ name_.c_str(), info.id(), url.c_str(), host_rtt[i]);
2433 } else {
2434 ✗ LogCvmfs(kLogDownload, kLogDebug,
2435 "(manager '%s' - id %" PRId64 ") "
2436 "error while probing host %s: %d %s",
2437 name_.c_str(), info.id(), url.c_str(), result,
2438 Code2Ascii(result));
2439 ✗ host_rtt[i] = INT_MAX;
2440 }
2441 }
2442 }
2443
2444 ✗ SortTeam(&host_rtt, &host_chain);
2445 ✗ for (i = 0; i < host_chain.size(); ++i) {
2446 ✗ if (host_rtt[i] == INT_MAX)
2447 ✗ host_rtt[i] = kProbeDown;
2448 }
2449
2450 ✗ const MutexLockGuard m(lock_options_);
2451 ✗ delete opt_host_.chain;
2452 ✗ delete opt_host_chain_rtt_;
2453 ✗ opt_host_.chain = new vector<string>(host_chain);
2454 ✗ opt_host_chain_rtt_ = new vector<int>(host_rtt);
2455 ✗ opt_host_.current = 0;
2456 }
2457
2458 ✗ bool DownloadManager::GeoSortServers(std::vector<std::string> *servers,
2459 std::vector<uint64_t> *output_order) {
2460 ✗ if (!servers) {
2461 ✗ return false;
2462 }
2463 ✗ if (servers->size() == 1) {
2464 ✗ if (output_order) {
2465 ✗ output_order->clear();
2466 ✗ output_order->push_back(0);
2467 }
2468 ✗ return true;
2469 }
2470
2471 ✗ std::vector<std::string> host_chain;
2472 ✗ GetHostInfo(&host_chain, NULL, NULL);
2473
2474 ✗ std::vector<std::string> server_dns_names;
2475 ✗ server_dns_names.reserve(servers->size());
2476 ✗ for (unsigned i = 0; i < servers->size(); ++i) {
2477 ✗ const std::string host = dns::ExtractHost((*servers)[i]);
2478 ✗ server_dns_names.push_back(host.empty() ? (*servers)[i] : host);
2479 }
2480 ✗ const std::string host_list = JoinStrings(server_dns_names, ",");
2481
2482 ✗ vector<string> host_chain_shuffled;
2483 {
2484 // Protect against concurrent access to prng_
2485 ✗ const MutexLockGuard m(lock_options_);
2486 // Determine random hosts for the Geo-API query
2487 ✗ host_chain_shuffled = Shuffle(host_chain, &prng_);
2488 }
2489 // Request ordered list via Geo-API
2490 ✗ bool success = false;
2491 ✗ const unsigned max_attempts = std::min(host_chain_shuffled.size(), size_t(3));
2492 ✗ vector<uint64_t> geo_order(servers->size());
2493 ✗ for (unsigned i = 0; i < max_attempts; ++i) {
2494 ✗ const string url = host_chain_shuffled[i] + "/api/v1.0/geo/@proxy@/"
2495 ✗ + host_list;
2496 ✗ LogCvmfs(kLogDownload, kLogDebug,
2497 "(manager '%s') requesting ordered server list from %s",
2498 name_.c_str(), url.c_str());
2499 ✗ cvmfs::MemSink memsink;
2500 ✗ JobInfo info(&url, false, false, NULL, &memsink);
2501 ✗ const Failures result = Fetch(&info);
2502 ✗ if (result == kFailOk) {
2503 ✗ const string order(reinterpret_cast<char *>(memsink.data()),
2504 ✗ memsink.pos());
2505 ✗ memsink.Reset();
2506 ✗ const bool retval = ValidateGeoReply(order, servers->size(), &geo_order);
2507 ✗ if (!retval) {
2508 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2509 "(manager '%s') retrieved invalid GeoAPI reply from %s [%s]",
2510 name_.c_str(), url.c_str(), order.c_str());
2511 } else {
2512 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslog,
2513 "(manager '%s') "
2514 "geographic order of servers retrieved from %s",
2515 name_.c_str(),
2516 ✗ dns::ExtractHost(host_chain_shuffled[i]).c_str());
2517 // remove new line at end of "order"
2518 ✗ LogCvmfs(kLogDownload, kLogDebug, "order is %s",
2519 ✗ Trim(order, true /* trim_newline */).c_str());
2520 ✗ success = true;
2521 ✗ break;
2522 }
2523 ✗ } else {
2524 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2525 "(manager '%s') GeoAPI request for %s failed with error %d [%s]",
2526 name_.c_str(), url.c_str(), result, Code2Ascii(result));
2527 }
2528 }
2529 ✗ if (!success) {
2530 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2531 "(manager '%s') "
2532 "failed to retrieve geographic order from stratum 1 servers",
2533 name_.c_str());
2534 ✗ return false;
2535 }
2536
2537 ✗ if (output_order) {
2538 ✗ output_order->swap(geo_order);
2539 } else {
2540 ✗ std::vector<std::string> sorted_servers;
2541 ✗ sorted_servers.reserve(geo_order.size());
2542 ✗ for (unsigned i = 0; i < geo_order.size(); ++i) {
2543 ✗ const uint64_t orderval = geo_order[i];
2544 ✗ sorted_servers.push_back((*servers)[orderval]);
2545 }
2546 ✗ servers->swap(sorted_servers);
2547 }
2548 ✗ return true;
2549 }
2550
2551
2552 /**
2553 * Uses the Geo-API of Stratum 1s to let any of them order the list of servers
2554 * and fallback proxies (if any).
2555 * Tries at most three random Stratum 1s before giving up.
2556 * If you change the host list in between by SetHostChain() or the fallback
2557 * proxy list by SetProxyChain(), they will be overwritten by this function.
2558 */
2559 ✗ bool DownloadManager::ProbeGeo() {
2560 ✗ vector<string> host_chain;
2561 ✗ vector<int> host_rtt;
2562 unsigned current_host;
2563 ✗ vector<vector<ProxyInfo> > proxy_chain;
2564 unsigned fallback_group;
2565
2566 ✗ GetHostInfo(&host_chain, &host_rtt, &current_host);
2567 ✗ GetProxyInfo(&proxy_chain, NULL, &fallback_group);
2568 ✗ if ((host_chain.size() < 2) && ((proxy_chain.size() - fallback_group) < 2))
2569 ✗ return true;
2570
2571 ✗ vector<string> host_names;
2572 ✗ for (unsigned i = 0; i < host_chain.size(); ++i)
2573 ✗ host_names.push_back(dns::ExtractHost(host_chain[i]));
2574 ✗ SortTeam(&host_names, &host_chain);
2575 ✗ const unsigned last_geo_host = host_names.size();
2576
2577 ✗ if ((fallback_group == 0) && (last_geo_host > 1)) {
2578 // There are no non-fallback proxies, which means that the client
2579 // will always use the fallback proxies. Add a keyword separator
2580 // between the hosts and fallback proxies so the geosorting service
2581 // will know to sort the hosts based on the distance from the
2582 // closest fallback proxy rather than the distance from the client.
2583 ✗ host_names.push_back("+PXYSEP+");
2584 }
2585
2586 // Add fallback proxy names to the end of the host list
2587 ✗ const unsigned first_geo_fallback = host_names.size();
2588 ✗ for (unsigned i = fallback_group; i < proxy_chain.size(); ++i) {
2589 // We only take the first fallback proxy name from every group under the
2590 // assumption that load-balanced servers are at the same location
2591 ✗ host_names.push_back(proxy_chain[i][0].host.name());
2592 }
2593
2594 ✗ std::vector<uint64_t> geo_order;
2595 ✗ const bool success = GeoSortServers(&host_names, &geo_order);
2596 ✗ if (!success) {
2597 // GeoSortServers already logged a failure message.
2598 ✗ return false;
2599 }
2600
2601 // Re-install host chain and proxy chain
2602 ✗ const MutexLockGuard m(lock_options_);
2603 ✗ delete opt_host_.chain;
2604 ✗ opt_num_proxies_ = 0;
2605 ✗ opt_host_.chain = new vector<string>(host_chain.size());
2606
2607 // It's possible that opt_proxy_groups_fallback_ might have changed while
2608 // the lock wasn't held
2609 vector<vector<ProxyInfo> > *proxy_groups = new vector<vector<ProxyInfo> >(
2610 ✗ opt_proxy_groups_fallback_ + proxy_chain.size() - fallback_group);
2611 // First copy the non-fallback part of the current proxy chain
2612 ✗ for (unsigned i = 0; i < opt_proxy_groups_fallback_; ++i) {
2613 ✗ (*proxy_groups)[i] = (*opt_proxy_groups_)[i];
2614 ✗ opt_num_proxies_ += (*opt_proxy_groups_)[i].size();
2615 }
2616
2617 // Copy the host chain and fallback proxies by geo order. Array indices
2618 // in geo_order that are smaller than last_geo_host refer to a stratum 1,
2619 // and those indices greater than or equal to first_geo_fallback refer to
2620 // a fallback proxy.
2621 ✗ unsigned hosti = 0;
2622 ✗ unsigned proxyi = opt_proxy_groups_fallback_;
2623 ✗ for (unsigned i = 0; i < geo_order.size(); ++i) {
2624 ✗ const uint64_t orderval = geo_order[i];
2625 ✗ if (orderval < static_cast<uint64_t>(last_geo_host)) {
2626 // LogCvmfs(kLogCvmfs, kLogSyslog, "this is orderval %u at host index
2627 // %u", orderval, hosti);
2628 ✗ (*opt_host_.chain)[hosti++] = host_chain[orderval];
2629 ✗ } else if (orderval >= static_cast<uint64_t>(first_geo_fallback)) {
2630 // LogCvmfs(kLogCvmfs, kLogSyslog,
2631 // "this is orderval %u at proxy index %u, using proxy_chain index %u",
2632 // orderval, proxyi, fallback_group + orderval - first_geo_fallback);
2633 ✗ (*proxy_groups)[proxyi] = proxy_chain[fallback_group + orderval
2634 ✗ - first_geo_fallback];
2635 ✗ opt_num_proxies_ += (*proxy_groups)[proxyi].size();
2636 ✗ proxyi++;
2637 }
2638 }
2639
2640 ✗ opt_proxy_map_.clear();
2641 ✗ delete opt_proxy_groups_;
2642 ✗ opt_proxy_groups_ = proxy_groups;
2643 // In pathological cases, opt_proxy_groups_current_ can be larger now when
2644 // proxies changed in-between.
2645 ✗ if (opt_proxy_groups_current_ > opt_proxy_groups_->size()) {
2646 ✗ if (opt_proxy_groups_->size() == 0) {
2647 ✗ opt_proxy_groups_current_ = 0;
2648 } else {
2649 ✗ opt_proxy_groups_current_ = opt_proxy_groups_->size() - 1;
2650 }
2651 ✗ opt_proxy_groups_current_burned_ = 0;
2652 }
2653
2654 ✗ UpdateProxiesUnlocked("geosort");
2655
2656 ✗ delete opt_host_chain_rtt_;
2657 ✗ opt_host_chain_rtt_ = new vector<int>(host_chain.size(), kProbeGeo);
2658 ✗ opt_host_.current = 0;
2659
2660 ✗ return true;
2661 }
2662
2663
2664 /**
2665 * Validates a string of the form "1,4,2,3" representing in which order the
2666 * the expected_size number of hosts should be put for optimal geographic
2667 * proximity. Returns false if the reply_order string is invalid, otherwise
2668 * fills in the reply_vals array with zero-based order indexes (e.g.
2669 * [0,3,1,2]) and returns true.
2670 */
2671 602 bool DownloadManager::ValidateGeoReply(const string &reply_order,
2672 const unsigned expected_size,
2673 vector<uint64_t> *reply_vals) {
2674
2/2
✓ Branch 1 taken 43 times.
✓ Branch 2 taken 559 times.
 602 if (reply_order.empty())
2675 43 return false;
2676
2/4
✓ Branch 2 taken 559 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 559 times.
✗ Branch 6 not taken.
 1118 const sanitizer::InputSanitizer sanitizer("09 , \n");
2677
3/4
✓ Branch 1 taken 559 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 43 times.
✓ Branch 4 taken 516 times.
 559 if (!sanitizer.IsValid(reply_order))
2678 43 return false;
2679
2/4
✓ Branch 2 taken 516 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 516 times.
✗ Branch 6 not taken.
 1032 const sanitizer::InputSanitizer strip_newline("09 ,");
2680
1/2
✓ Branch 1 taken 516 times.
✗ Branch 2 not taken.
 516 vector<string> reply_strings = SplitString(strip_newline.Filter(reply_order),
2681
1/2
✓ Branch 1 taken 516 times.
✗ Branch 2 not taken.
 516 ',');
2682 516 vector<uint64_t> tmp_vals;
2683
2/2
✓ Branch 1 taken 989 times.
✓ Branch 2 taken 430 times.
 1419 for (unsigned i = 0; i < reply_strings.size(); ++i) {
2684
2/2
✓ Branch 2 taken 86 times.
✓ Branch 3 taken 903 times.
 989 if (reply_strings[i].empty())
2685 86 return false;
2686
2/4
✓ Branch 2 taken 903 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 903 times.
✗ Branch 6 not taken.
 903 tmp_vals.push_back(String2Uint64(reply_strings[i]));
2687 }
2688
2/2
✓ Branch 1 taken 215 times.
✓ Branch 2 taken 215 times.
 430 if (tmp_vals.size() != expected_size)
2689 215 return false;
2690
2691 // Check if tmp_vals contains the number 1..n
2692
1/2
✓ Branch 3 taken 215 times.
✗ Branch 4 not taken.
 215 set<uint64_t> coverage(tmp_vals.begin(), tmp_vals.end());
2693
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 215 times.
 215 if (coverage.size() != tmp_vals.size())
2694 ✗ return false;
2695
5/6
✓ Branch 2 taken 172 times.
✓ Branch 3 taken 43 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 172 times.
✓ Branch 9 taken 43 times.
✓ Branch 10 taken 172 times.
 215 if ((*coverage.begin() != 1) || (*coverage.rbegin() != coverage.size()))
2696 43 return false;
2697
2698
2/2
✓ Branch 0 taken 387 times.
✓ Branch 1 taken 172 times.
 559 for (unsigned i = 0; i < expected_size; ++i) {
2699 387 (*reply_vals)[i] = tmp_vals[i] - 1;
2700 }
2701 172 return true;
2702 559 }
2703
2704
2705 /**
2706 * Removes DIRECT from a list of ';' and '|' separated proxies.
2707 * \return true if DIRECT was present, false otherwise
2708 */
2709 1645 bool DownloadManager::StripDirect(const string &proxy_list,
2710 string *cleaned_list) {
2711
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1645 times.
 1645 assert(cleaned_list);
2712
2/2
✓ Branch 1 taken 1258 times.
✓ Branch 2 taken 387 times.
 1645 if (proxy_list == "") {
2713
1/2
✓ Branch 1 taken 1258 times.
✗ Branch 2 not taken.
 1258 *cleaned_list = "";
2714 1258 return false;
2715 }
2716 387 bool result = false;
2717
2718
1/2
✓ Branch 1 taken 387 times.
✗ Branch 2 not taken.
 387 vector<string> proxy_groups = SplitString(proxy_list, ';');
2719 387 vector<string> cleaned_groups;
2720
2/2
✓ Branch 1 taken 946 times.
✓ Branch 2 taken 387 times.
 1333 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2721
1/2
✓ Branch 2 taken 946 times.
✗ Branch 3 not taken.
 946 vector<string> group = SplitString(proxy_groups[i], '|');
2722 946 vector<string> cleaned;
2723
2/2
✓ Branch 1 taken 1591 times.
✓ Branch 2 taken 946 times.
 2537 for (unsigned j = 0; j < group.size(); ++j) {
2724
6/6
✓ Branch 2 taken 1161 times.
✓ Branch 3 taken 430 times.
✓ Branch 6 taken 559 times.
✓ Branch 7 taken 602 times.
✓ Branch 8 taken 989 times.
✓ Branch 9 taken 602 times.
 1591 if ((group[j] == "DIRECT") || (group[j] == "")) {
2725 989 result = true;
2726 } else {
2727
1/2
✓ Branch 2 taken 602 times.
✗ Branch 3 not taken.
 602 cleaned.push_back(group[j]);
2728 }
2729 }
2730
2/2
✓ Branch 1 taken 301 times.
✓ Branch 2 taken 645 times.
 946 if (!cleaned.empty())
2731
3/6
✓ Branch 2 taken 301 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 301 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 301 times.
✗ Branch 9 not taken.
 301 cleaned_groups.push_back(JoinStrings(cleaned, "|"));
2732 946 }
2733
2734
2/4
✓ Branch 2 taken 387 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 387 times.
✗ Branch 6 not taken.
 387 *cleaned_list = JoinStrings(cleaned_groups, ";");
2735 387 return result;
2736 387 }
2737
2738
2739 /**
2740 * Parses a list of ';'- and '|'-separated proxy servers and fallback proxy
2741 * servers for the proxy groups.
2742 * The empty string for both removes the proxy chain.
2743 * The set_mode parameter can be used to set either proxies (leaving fallback
2744 * proxies unchanged) or fallback proxies (leaving regular proxies unchanged)
2745 * or both.
2746 */
2747 1215 void DownloadManager::SetProxyChain(const string &proxy_list,
2748 const string &fallback_proxy_list,
2749 const ProxySetModes set_mode) {
2750 1215 const MutexLockGuard m(lock_options_);
2751
2752 1215 opt_timestamp_backup_proxies_ = 0;
2753 1215 opt_timestamp_failover_proxies_ = 0;
2754
1/2
✓ Branch 1 taken 1215 times.
✗ Branch 2 not taken.
 1215 string set_proxy_list = opt_proxy_list_;
2755
1/2
✓ Branch 1 taken 1215 times.
✗ Branch 2 not taken.
 1215 string set_proxy_fallback_list = opt_proxy_fallback_list_;
2756 bool contains_direct;
2757
3/4
✓ Branch 0 taken 1215 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1086 times.
✓ Branch 3 taken 129 times.
 1215 if ((set_mode == kSetProxyFallback) || (set_mode == kSetProxyBoth)) {
2758
1/2
✓ Branch 1 taken 1086 times.
✗ Branch 2 not taken.
 1086 opt_proxy_fallback_list_ = fallback_proxy_list;
2759 }
2760
3/4
✓ Branch 0 taken 1086 times.
✓ Branch 1 taken 129 times.
✓ Branch 2 taken 1086 times.
✗ Branch 3 not taken.
 1215 if ((set_mode == kSetProxyRegular) || (set_mode == kSetProxyBoth)) {
2761
1/2
✓ Branch 1 taken 1215 times.
✗ Branch 2 not taken.
 1215 opt_proxy_list_ = proxy_list;
2762 }
2763
1/2
✓ Branch 1 taken 1215 times.
✗ Branch 2 not taken.
 1215 contains_direct = StripDirect(opt_proxy_fallback_list_,
2764 &set_proxy_fallback_list);
2765
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1215 times.
 1215 if (contains_direct) {
2766 ✗ LogCvmfs(kLogDownload, kLogSyslogWarn | kLogDebug,
2767 "(manager '%s') fallback proxies do not support DIRECT, removing",
2768 name_.c_str());
2769 }
2770
1/2
✓ Branch 1 taken 1215 times.
✗ Branch 2 not taken.
 1215 if (set_proxy_fallback_list == "") {
2771
1/2
✓ Branch 1 taken 1215 times.
✗ Branch 2 not taken.
 1215 set_proxy_list = opt_proxy_list_;
2772 } else {
2773 ✗ const bool contains_direct = StripDirect(opt_proxy_list_, &set_proxy_list);
2774 ✗ if (contains_direct) {
2775 ✗ LogCvmfs(kLogDownload, kLogSyslog | kLogDebug,
2776 "(manager '%s') skipping DIRECT proxy to use fallback proxy",
2777 name_.c_str());
2778 }
2779 }
2780
2781 // From this point on, use set_proxy_list and set_fallback_proxy_list as
2782 // effective proxy lists!
2783
2784 1215 opt_proxy_map_.clear();
2785
2/2
✓ Branch 0 taken 543 times.
✓ Branch 1 taken 672 times.
 1215 delete opt_proxy_groups_;
2786
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 1215 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 1215 times.
 1215 if ((set_proxy_list == "") && (set_proxy_fallback_list == "")) {
2787 ✗ opt_proxy_groups_ = NULL;
2788 ✗ opt_proxy_groups_current_ = 0;
2789 ✗ opt_proxy_groups_current_burned_ = 0;
2790 ✗ opt_proxy_groups_fallback_ = 0;
2791 ✗ opt_num_proxies_ = 0;
2792 ✗ return;
2793 }
2794
2795 // Determine number of regular proxy groups (== first fallback proxy group)
2796 1215 opt_proxy_groups_fallback_ = 0;
2797
1/2
✓ Branch 1 taken 1215 times.
✗ Branch 2 not taken.
 1215 if (set_proxy_list != "") {
2798
1/2
✓ Branch 1 taken 1215 times.
✗ Branch 2 not taken.
 1215 opt_proxy_groups_fallback_ = SplitString(set_proxy_list, ';').size();
2799 }
2800
1/2
✓ Branch 2 taken 1215 times.
✗ Branch 3 not taken.
 1215 LogCvmfs(kLogDownload, kLogDebug,
2801 "(manager '%s') "
2802 "first fallback proxy group %u",
2803 name_.c_str(), opt_proxy_groups_fallback_);
2804
2805 // Concatenate regular proxies and fallback proxies, both of which can be
2806 // empty.
2807
1/2
✓ Branch 1 taken 1215 times.
✗ Branch 2 not taken.
 1215 string all_proxy_list = set_proxy_list;
2808
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1215 times.
 1215 if (set_proxy_fallback_list != "") {
2809 ✗ if (all_proxy_list != "")
2810 ✗ all_proxy_list += ";";
2811 ✗ all_proxy_list += set_proxy_fallback_list;
2812 }
2813
1/2
✓ Branch 3 taken 1215 times.
✗ Branch 4 not taken.
 1215 LogCvmfs(kLogDownload, kLogDebug, "(manager '%s') full proxy list %s",
2814 name_.c_str(), all_proxy_list.c_str());
2815
2816 // Resolve server names in provided urls
2817 1215 vector<string> hostnames; // All encountered hostnames
2818 1215 vector<string> proxy_groups;
2819
1/2
✓ Branch 1 taken 1215 times.
✗ Branch 2 not taken.
 1215 if (all_proxy_list != "")
2820
1/2
✓ Branch 1 taken 1215 times.
✗ Branch 2 not taken.
 1215 proxy_groups = SplitString(all_proxy_list, ';');
2821
2/2
✓ Branch 1 taken 1258 times.
✓ Branch 2 taken 1215 times.
 2473 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2822
1/2
✓ Branch 2 taken 1258 times.
✗ Branch 3 not taken.
 1258 vector<string> this_group = SplitString(proxy_groups[i], '|');
2823
2/2
✓ Branch 1 taken 1258 times.
✓ Branch 2 taken 1258 times.
 2516 for (unsigned j = 0; j < this_group.size(); ++j) {
2824
1/2
✓ Branch 2 taken 1258 times.
✗ Branch 3 not taken.
 1258 this_group[j] = dns::AddDefaultScheme(this_group[j]);
2825 // Note: DIRECT strings will be "extracted" to an empty string.
2826
1/2
✓ Branch 2 taken 1258 times.
✗ Branch 3 not taken.
 1258 const string hostname = dns::ExtractHost(this_group[j]);
2827 // Save the hostname. Leave empty (DIRECT) names so indexes will
2828 // match later.
2829
1/2
✓ Branch 1 taken 1258 times.
✗ Branch 2 not taken.
 1258 hostnames.push_back(hostname);
2830 1258 }
2831 1258 }
2832 1215 vector<dns::Host> hosts;
2833
1/2
✓ Branch 3 taken 1215 times.
✗ Branch 4 not taken.
 1215 LogCvmfs(kLogDownload, kLogDebug,
2834 "(manager '%s') "
2835 "resolving %lu proxy addresses",
2836 name_.c_str(), hostnames.size());
2837
1/2
✓ Branch 1 taken 1215 times.
✗ Branch 2 not taken.
 1215 resolver_->ResolveMany(hostnames, &hosts);
2838
2839 // Construct opt_proxy_groups_: traverse proxy list in same order and expand
2840 // names to resolved IP addresses.
2841
1/2
✓ Branch 1 taken 1215 times.
✗ Branch 2 not taken.
 1215 opt_proxy_groups_ = new vector<vector<ProxyInfo> >();
2842 1215 opt_num_proxies_ = 0;
2843 1215 unsigned num_proxy = 0; // Combined i, j counter
2844
2/2
✓ Branch 1 taken 1258 times.
✓ Branch 2 taken 1215 times.
 2473 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2845
1/2
✓ Branch 2 taken 1258 times.
✗ Branch 3 not taken.
 1258 vector<string> this_group = SplitString(proxy_groups[i], '|');
2846 // Construct ProxyInfo objects from proxy string and DNS resolver result for
2847 // every proxy in this_group. One URL can result in multiple ProxyInfo
2848 // objects, one for each IP address.
2849 1258 vector<ProxyInfo> infos;
2850
2/2
✓ Branch 1 taken 1258 times.
✓ Branch 2 taken 1258 times.
 2516 for (unsigned j = 0; j < this_group.size(); ++j, ++num_proxy) {
2851
1/2
✓ Branch 2 taken 1258 times.
✗ Branch 3 not taken.
 1258 this_group[j] = dns::AddDefaultScheme(this_group[j]);
2852
2/2
✓ Branch 2 taken 1086 times.
✓ Branch 3 taken 172 times.
 1258 if (this_group[j] == "DIRECT") {
2853
3/6
✓ Branch 2 taken 1086 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1086 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1086 times.
✗ Branch 9 not taken.
 1086 infos.push_back(ProxyInfo("DIRECT"));
2854 1086 continue;
2855 }
2856
2857
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 172 times.
 172 if (hosts[num_proxy].status() != dns::kFailOk) {
2858 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2859 "(manager '%s') "
2860 "failed to resolve IP addresses for %s (%d - %s)",
2861 ✗ name_.c_str(), hosts[num_proxy].name().c_str(),
2862 ✗ hosts[num_proxy].status(),
2863 ✗ dns::Code2Ascii(hosts[num_proxy].status()));
2864 const dns::Host failed_host = dns::Host::ExtendDeadline(
2865 ✗ hosts[num_proxy], resolver_->min_ttl());
2866 ✗ infos.push_back(ProxyInfo(failed_host, this_group[j]));
2867 ✗ continue;
2868 }
2869
2870 // IPv4 addresses have precedence
2871 172 set<string> best_addresses = hosts[num_proxy].ViewBestAddresses(
2872
2/4
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 172 times.
✗ Branch 5 not taken.
 172 opt_ip_preference_);
2873 172 set<string>::const_iterator iter_ips = best_addresses.begin();
2874
2/2
✓ Branch 3 taken 172 times.
✓ Branch 4 taken 172 times.
 344 for (; iter_ips != best_addresses.end(); ++iter_ips) {
2875
1/2
✓ Branch 3 taken 172 times.
✗ Branch 4 not taken.
 172 const string url_ip = dns::RewriteUrl(this_group[j], *iter_ips);
2876
2/4
✓ Branch 2 taken 172 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 172 times.
✗ Branch 6 not taken.
 172 infos.push_back(ProxyInfo(hosts[num_proxy], url_ip));
2877
2878
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 172 times.
 172 if (sharding_policy_.UseCount() > 0) {
2879 ✗ sharding_policy_->AddProxy(url_ip);
2880 }
2881 172 }
2882 172 }
2883
1/2
✓ Branch 1 taken 1258 times.
✗ Branch 2 not taken.
 1258 opt_proxy_groups_->push_back(infos);
2884 1258 opt_num_proxies_ += infos.size();
2885 1258 }
2886
1/2
✓ Branch 2 taken 1215 times.
✗ Branch 3 not taken.
 1215 LogCvmfs(kLogDownload, kLogDebug,
2887 "(manager '%s') installed %u proxies in %lu load-balance groups",
2888 1215 name_.c_str(), opt_num_proxies_, opt_proxy_groups_->size());
2889 1215 opt_proxy_groups_current_ = 0;
2890 1215 opt_proxy_groups_current_burned_ = 0;
2891
2892 // Select random start proxy from the first group.
2893
1/2
✓ Branch 1 taken 1215 times.
✗ Branch 2 not taken.
 1215 if (opt_proxy_groups_->size() > 0) {
2894 // Select random start proxy from the first group.
2895
2/4
✓ Branch 2 taken 1215 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1215 times.
✗ Branch 6 not taken.
 1215 UpdateProxiesUnlocked("set random start proxy from the first proxy group");
2896 }
2897
3/6
✓ Branch 5 taken 1215 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1215 times.
✗ Branch 9 not taken.
✓ Branch 11 taken 1215 times.
✗ Branch 12 not taken.
 1215 }
2898
2899
2900 /**
2901 * Retrieves the proxy chain, optionally the currently active load-balancing
2902 * group, and optionally the index of the first fallback proxy group.
2903 * If there are no fallback proxies, the index will equal the size of
2904 * the proxy chain.
2905 */
2906 ✗ void DownloadManager::GetProxyInfo(vector<vector<ProxyInfo> > *proxy_chain,
2907 unsigned *current_group,
2908 unsigned *fallback_group) {
2909 ✗ assert(proxy_chain != NULL);
2910 ✗ const MutexLockGuard m(lock_options_);
2911
2912 ✗ if (!opt_proxy_groups_) {
2913 ✗ const vector<vector<ProxyInfo> > empty_chain;
2914 ✗ *proxy_chain = empty_chain;
2915 ✗ if (current_group != NULL)
2916 ✗ *current_group = 0;
2917 ✗ if (fallback_group != NULL)
2918 ✗ *fallback_group = 0;
2919 ✗ return;
2920 }
2921
2922 ✗ *proxy_chain = *opt_proxy_groups_;
2923 ✗ if (current_group != NULL)
2924 ✗ *current_group = opt_proxy_groups_current_;
2925 ✗ if (fallback_group != NULL)
2926 ✗ *fallback_group = opt_proxy_groups_fallback_;
2927 }
2928
2929 ✗ string DownloadManager::GetProxyList() { return opt_proxy_list_; }
2930
2931 ✗ string DownloadManager::GetFallbackProxyList() {
2932 ✗ return opt_proxy_fallback_list_;
2933 }
2934
2935 /**
2936 * Choose proxy
2937 */
2938 6128 DownloadManager::ProxyInfo *DownloadManager::ChooseProxyUnlocked(
2939 const shash::Any *hash) {
2940
2/2
✓ Branch 0 taken 4950 times.
✓ Branch 1 taken 1178 times.
 6128 if (!opt_proxy_groups_)
2941 4950 return NULL;
2942
2943
2/2
✓ Branch 0 taken 620 times.
✓ Branch 1 taken 558 times.
 1178 const uint32_t key = (hash ? hash->Partial32() : 0);
2944
1/2
✓ Branch 1 taken 1178 times.
✗ Branch 2 not taken.
 1178 const map<uint32_t, ProxyInfo *>::iterator it = opt_proxy_map_.lower_bound(
2945 key);
2946 1178 ProxyInfo *proxy = it->second;
2947
2948 1178 return proxy;
2949 }
2950
2951 /**
2952 * Update currently selected proxy
2953 */
2954 1801 void DownloadManager::UpdateProxiesUnlocked(const string &reason) {
2955
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1801 times.
 1801 if (!opt_proxy_groups_)
2956 ✗ return;
2957
2958 // Identify number of non-burned proxies within the current group
2959 1801 vector<ProxyInfo> *group = current_proxy_group();
2960 1801 const unsigned num_alive = (group->size() - opt_proxy_groups_current_burned_);
2961
2/4
✓ Branch 2 taken 1801 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1801 times.
✗ Branch 6 not taken.
 3602 const string old_proxy = JoinStrings(opt_proxies_, "|");
2962
2963 // Rebuild proxy map and URL list
2964 1801 opt_proxy_map_.clear();
2965 1801 opt_proxies_.clear();
2966 1801 const uint32_t max_key = 0xffffffffUL;
2967
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1801 times.
 1801 if (opt_proxy_shard_) {
2968 // Build a consistent map with multiple entries for each proxy
2969 ✗ for (unsigned i = 0; i < num_alive; ++i) {
2970 ✗ ProxyInfo *proxy = &(*group)[i];
2971 ✗ shash::Any proxy_hash(shash::kSha1);
2972 ✗ HashString(proxy->url, &proxy_hash);
2973 ✗ Prng prng;
2974 ✗ prng.InitSeed(proxy_hash.Partial32());
2975 ✗ for (unsigned j = 0; j < kProxyMapScale; ++j) {
2976 ✗ const std::pair<uint32_t, ProxyInfo *> entry(prng.Next(max_key), proxy);
2977 ✗ opt_proxy_map_.insert(entry);
2978 }
2979 ✗ const std::string proxy_name = proxy->host.name().empty()
2980 ? ""
2981 ✗ : " (" + proxy->host.name() + ")";
2982 ✗ opt_proxies_.push_back(proxy->url + proxy_name);
2983 }
2984 // Ensure lower_bound() finds a value for all keys
2985 ✗ ProxyInfo *first_proxy = opt_proxy_map_.begin()->second;
2986 ✗ const std::pair<uint32_t, ProxyInfo *> last_entry(max_key, first_proxy);
2987 ✗ opt_proxy_map_.insert(last_entry);
2988 } else {
2989 // Build a map with a single entry for one randomly selected proxy
2990 1801 const unsigned select = prng_.Next(num_alive);
2991 1801 ProxyInfo *proxy = &(*group)[select];
2992 1801 const std::pair<uint32_t, ProxyInfo *> entry(max_key, proxy);
2993
1/2
✓ Branch 1 taken 1801 times.
✗ Branch 2 not taken.
 1801 opt_proxy_map_.insert(entry);
2994 1801 const std::string proxy_name = proxy->host.name().empty()
2995 ? ""
2996
9/18
✓ Branch 0 taken 1629 times.
✓ Branch 1 taken 172 times.
✓ Branch 4 taken 1629 times.
✗ Branch 5 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 172 times.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✓ Branch 11 taken 172 times.
✗ Branch 12 not taken.
✓ Branch 13 taken 172 times.
✓ Branch 14 taken 1629 times.
✓ Branch 15 taken 1629 times.
✓ Branch 16 taken 172 times.
✗ Branch 17 not taken.
✗ Branch 18 not taken.
✗ Branch 19 not taken.
✗ Branch 20 not taken.
 1973 : " (" + proxy->host.name() + ")";
2997
2/4
✓ Branch 1 taken 1801 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1801 times.
✗ Branch 5 not taken.
 1801 opt_proxies_.push_back(proxy->url + proxy_name);
2998 1801 }
2999
1/2
✓ Branch 3 taken 1801 times.
✗ Branch 4 not taken.
 1801 sort(opt_proxies_.begin(), opt_proxies_.end());
3000
3001 // Report any change in proxy usage
3002
2/4
✓ Branch 2 taken 1801 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1801 times.
✗ Branch 6 not taken.
 3602 const string new_proxy = JoinStrings(opt_proxies_, "|");
3003 const string curr_host = "Current host: "
3004 1801 + (opt_host_.chain
3005
6/12
✓ Branch 0 taken 1629 times.
✓ Branch 1 taken 172 times.
✓ Branch 4 taken 1629 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 172 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 172 times.
✓ Branch 11 taken 1629 times.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
 3602 ? (*opt_host_.chain)[opt_host_.current]
3006
1/2
✓ Branch 1 taken 1801 times.
✗ Branch 2 not taken.
 1801 : "");
3007
2/2
✓ Branch 1 taken 1258 times.
✓ Branch 2 taken 543 times.
 1801 if (new_proxy != old_proxy) {
3008
4/9
✗ Branch 2 not taken.
✓ Branch 3 taken 1258 times.
✓ Branch 4 taken 1215 times.
✓ Branch 5 taken 43 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 1258 times.
✗ Branch 9 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
 3817 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
3009 "(manager '%s') switching proxy from %s to %s. Reason: %s [%s]",
3010 1301 name_.c_str(), (old_proxy.empty() ? "(none)" : old_proxy.c_str()),
3011 2516 (new_proxy.empty() ? "(none)" : new_proxy.c_str()), reason.c_str(),
3012 curr_host.c_str());
3013 }
3014 1801 }
3015
3016 /**
3017 * Enable proxy sharding
3018 */
3019 ✗ void DownloadManager::ShardProxies() {
3020 ✗ opt_proxy_shard_ = true;
3021 ✗ RebalanceProxiesUnlocked("enable sharding");
3022 }
3023
3024 /**
3025 * Selects a new random proxy in the current load-balancing group. Resets the
3026 * "burned" counter.
3027 */
3028 ✗ void DownloadManager::RebalanceProxiesUnlocked(const string &reason) {
3029 ✗ if (!opt_proxy_groups_)
3030 ✗ return;
3031
3032 ✗ opt_timestamp_failover_proxies_ = 0;
3033 ✗ opt_proxy_groups_current_burned_ = 0;
3034 ✗ UpdateProxiesUnlocked(reason);
3035 }
3036
3037
3038 ✗ void DownloadManager::RebalanceProxies() {
3039 ✗ const MutexLockGuard m(lock_options_);
3040 ✗ RebalanceProxiesUnlocked("rebalance invoked manually");
3041 }
3042
3043
3044 /**
3045 * Switches to the next load-balancing group of proxy servers.
3046 */
3047 ✗ void DownloadManager::SwitchProxyGroup() {
3048 ✗ const MutexLockGuard m(lock_options_);
3049
3050 ✗ if (!opt_proxy_groups_ || (opt_proxy_groups_->size() < 2)) {
3051 ✗ return;
3052 }
3053
3054 ✗ opt_proxy_groups_current_ = (opt_proxy_groups_current_ + 1)
3055 ✗ % opt_proxy_groups_->size();
3056 ✗ opt_timestamp_backup_proxies_ = time(NULL);
3057
3058 const std::string msg = "switch to proxy group "
3059 ✗ + StringifyUint(opt_proxy_groups_current_);
3060 ✗ RebalanceProxiesUnlocked(msg);
3061 }
3062
3063
3064 ✗ void DownloadManager::SetProxyGroupResetDelay(const unsigned seconds) {
3065 ✗ const MutexLockGuard m(lock_options_);
3066 ✗ opt_proxy_groups_reset_after_ = seconds;
3067 ✗ if (opt_proxy_groups_reset_after_ == 0) {
3068 ✗ opt_timestamp_backup_proxies_ = 0;
3069 ✗ opt_timestamp_failover_proxies_ = 0;
3070 }
3071 }
3072
3073
3074 ✗ void DownloadManager::SetMetalinkResetDelay(const unsigned seconds) {
3075 ✗ const MutexLockGuard m(lock_options_);
3076 ✗ opt_metalink_.reset_after = seconds;
3077 ✗ if (opt_metalink_.reset_after == 0)
3078 ✗ opt_metalink_.timestamp_backup = 0;
3079 }
3080
3081
3082 ✗ void DownloadManager::SetHostResetDelay(const unsigned seconds) {
3083 ✗ const MutexLockGuard m(lock_options_);
3084 ✗ opt_host_.reset_after = seconds;
3085 ✗ if (opt_host_.reset_after == 0)
3086 ✗ opt_host_.timestamp_backup = 0;
3087 }
3088
3089
3090 1481 void DownloadManager::SetRetryParameters(const unsigned max_retries,
3091 const unsigned backoff_init_ms,
3092 const unsigned backoff_max_ms) {
3093 1481 const MutexLockGuard m(lock_options_);
3094 1481 opt_max_retries_ = max_retries;
3095 1481 opt_backoff_init_ms_ = backoff_init_ms;
3096 1481 opt_backoff_max_ms_ = backoff_max_ms;
3097 1481 }
3098
3099
3100 629 void DownloadManager::SetMaxIpaddrPerProxy(unsigned limit) {
3101 629 const MutexLockGuard m(lock_options_);
3102 629 resolver_->set_throttle(limit);
3103 629 }
3104
3105
3106 760 void DownloadManager::SetProxyTemplates(const std::string &direct,
3107 const std::string &forced) {
3108 760 const MutexLockGuard m(lock_options_);
3109
1/2
✓ Branch 1 taken 760 times.
✗ Branch 2 not taken.
 760 proxy_template_direct_ = direct;
3110
1/2
✓ Branch 1 taken 760 times.
✗ Branch 2 not taken.
 760 proxy_template_forced_ = forced;
3111 760 }
3112
3113
3114 ✗ void DownloadManager::EnableInfoHeader() { enable_info_header_ = true; }
3115
3116
3117 850 void DownloadManager::EnableRedirects() { follow_redirects_ = true; }
3118
3119 ✗ void DownloadManager::EnableIgnoreSignatureFailures() {
3120 ✗ ignore_signature_failures_ = true;
3121 }
3122
3123 ✗ void DownloadManager::EnableHTTPTracing() { enable_http_tracing_ = true; }
3124
3125 ✗ void DownloadManager::AddHTTPTracingHeader(const std::string &header) {
3126 ✗ http_tracing_headers_.push_back(header);
3127 }
3128
3129 721 void DownloadManager::UseSystemCertificatePath() {
3130 721 ssl_certificate_store_.UseSystemCertificatePath();
3131 721 }
3132
3133 ✗ bool DownloadManager::SetShardingPolicy(const ShardingPolicySelector type) {
3134 ✗ const bool success = false;
3135 ✗ switch (type) {
3136 default:
3137 ✗ LogCvmfs(
3138 kLogDownload, kLogDebug | kLogSyslogErr,
3139 "(manager '%s') "
3140 "Proposed sharding policy does not exist. Falling back to default",
3141 name_.c_str());
3142 }
3143 ✗ return success;
3144 }
3145
3146 ✗ void DownloadManager::SetFailoverIndefinitely() {
3147 ✗ failover_indefinitely_ = true;
3148 }
3149
3150 /**
3151 * Creates a copy of the existing download manager. Must only be called in
3152 * single-threaded stage because it calls curl_global_init().
3153 */
3154 629 DownloadManager *DownloadManager::Clone(
3155 const perf::StatisticsTemplate &statistics,
3156 const std::string &cloned_name) {
3157 DownloadManager *clone = new DownloadManager(pool_max_handles_, statistics,
3158
1/2
✓ Branch 2 taken 629 times.
✗ Branch 3 not taken.
 629 cloned_name);
3159
3160 629 clone->SetDnsParameters(resolver_->retries(), resolver_->timeout_ms());
3161 629 clone->SetDnsTtlLimits(resolver_->min_ttl(), resolver_->max_ttl());
3162 629 clone->SetMaxIpaddrPerProxy(resolver_->throttle());
3163
3164
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 629 times.
 629 if (!opt_dns_server_.empty())
3165 ✗ clone->SetDnsServer(opt_dns_server_);
3166 629 clone->opt_timeout_proxy_ = opt_timeout_proxy_;
3167 629 clone->opt_timeout_direct_ = opt_timeout_direct_;
3168 629 clone->opt_low_speed_limit_ = opt_low_speed_limit_;
3169 629 clone->opt_max_retries_ = opt_max_retries_;
3170 629 clone->opt_backoff_init_ms_ = opt_backoff_init_ms_;
3171 629 clone->opt_backoff_max_ms_ = opt_backoff_max_ms_;
3172 629 clone->enable_info_header_ = enable_info_header_;
3173 629 clone->enable_http_tracing_ = enable_http_tracing_;
3174 629 clone->http_tracing_headers_ = http_tracing_headers_;
3175 629 clone->follow_redirects_ = follow_redirects_;
3176 629 clone->ignore_signature_failures_ = ignore_signature_failures_;
3177
2/2
✓ Branch 0 taken 543 times.
✓ Branch 1 taken 86 times.
 629 if (opt_host_.chain) {
3178
1/2
✓ Branch 2 taken 543 times.
✗ Branch 3 not taken.
 543 clone->opt_host_.chain = new vector<string>(*opt_host_.chain);
3179
1/2
✓ Branch 2 taken 543 times.
✗ Branch 3 not taken.
 543 clone->opt_host_chain_rtt_ = new vector<int>(*opt_host_chain_rtt_);
3180 }
3181
3182 629 CloneProxyConfig(clone);
3183 629 clone->opt_ip_preference_ = opt_ip_preference_;
3184 629 clone->proxy_template_direct_ = proxy_template_direct_;
3185 629 clone->proxy_template_forced_ = proxy_template_forced_;
3186 629 clone->opt_proxy_groups_reset_after_ = opt_proxy_groups_reset_after_;
3187 629 clone->opt_metalink_.reset_after = opt_metalink_.reset_after;
3188 629 clone->opt_host_.reset_after = opt_host_.reset_after;
3189 629 clone->credentials_attachment_ = credentials_attachment_;
3190 629 clone->ssl_certificate_store_ = ssl_certificate_store_;
3191
3192 629 clone->health_check_ = health_check_;
3193 629 clone->sharding_policy_ = sharding_policy_;
3194 629 clone->failover_indefinitely_ = failover_indefinitely_;
3195 629 clone->fqrn_ = fqrn_;
3196
3197 629 return clone;
3198 }
3199
3200
3201 629 void DownloadManager::CloneProxyConfig(DownloadManager *clone) {
3202 629 clone->opt_proxy_groups_current_ = opt_proxy_groups_current_;
3203 629 clone->opt_proxy_groups_current_burned_ = opt_proxy_groups_current_burned_;
3204 629 clone->opt_proxy_groups_fallback_ = opt_proxy_groups_fallback_;
3205 629 clone->opt_num_proxies_ = opt_num_proxies_;
3206 629 clone->opt_proxy_shard_ = opt_proxy_shard_;
3207 629 clone->opt_proxy_list_ = opt_proxy_list_;
3208 629 clone->opt_proxy_fallback_list_ = opt_proxy_fallback_list_;
3209
2/2
✓ Branch 0 taken 86 times.
✓ Branch 1 taken 543 times.
 629 if (opt_proxy_groups_ == NULL)
3210 86 return;
3211
3212
1/2
✓ Branch 2 taken 543 times.
✗ Branch 3 not taken.
 543 clone->opt_proxy_groups_ = new vector<vector<ProxyInfo> >(*opt_proxy_groups_);
3213
2/4
✓ Branch 2 taken 543 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 543 times.
✗ Branch 6 not taken.
 543 clone->UpdateProxiesUnlocked("cloned");
3214 }
3215
3216 } // namespace download
3217