GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/network/download.cc
Date: 2026-03-22 02:40:38
Exec Total Coverage
Lines: 877 1719 51.0%
Branches: 666 2194 30.4%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * The download module provides an interface for fetching files via HTTP
5 * and file. It is internally using libcurl and the asynchronous DNS resolver
6 * c-ares. The JobInfo struct describes a single file/url to download and
7 * keeps the state during the several phases of downloading.
8 *
9 * The module starts in single-threaded mode and can be switched to multi-
10 * threaded mode by Spawn(). In multi-threaded mode, the Fetch() function still
11 * blocks but there is a separate I/O thread using asynchronous I/O, which
12 * maintains all concurrent connections simultaneously. As there might be more
13 * than 1024 file descriptors for the CernVM-FS process, the I/O thread uses
14 * poll and the libcurl multi socket interface.
15 *
16 * While downloading, files can be decompressed and the secure hash can be
17 * calculated on the fly.
18 *
19 * The module also implements failure handling. If corrupted data has been
20 * downloaded, the transfer is restarted using HTTP "no-cache" pragma.
21 * A "host chain" can be configured. When a host fails, there is automatic
22 * fail-over to the next host in the chain until all hosts are probed.
23 * Similarly a chain of proxy sets can be configured. Inside a proxy set,
24 * proxies are selected randomly (load-balancing set).
25 */
26
27 // TODO(jblomer): MS for time summing
28 // NOLINTNEXTLINE
29 #define __STDC_FORMAT_MACROS
30
31
32 #include "download.h"
33
34 #include <alloca.h>
35 #include <errno.h>
36 #include <inttypes.h>
37 #include <poll.h>
38 #include <pthread.h>
39 #include <signal.h>
40 #include <stdint.h>
41 #include <sys/time.h>
42 #include <unistd.h>
43
44 #include <algorithm>
45 #include <cassert>
46 #include <cstdio>
47 #include <cstdlib>
48 #include <cstring>
49 #include <map>
50 #include <set>
51 #include <utility>
52
53 #include "compression/compression.h"
54 #include "crypto/hash.h"
55 #include "duplex_curl.h"
56 #include "interrupt.h"
57 #include "sanitizer.h"
58 #include "ssl.h"
59 #include "util/algorithm.h"
60 #include "util/atomic.h"
61 #include "util/concurrency.h"
62 #include "util/exception.h"
63 #include "util/logging.h"
64 #include "util/posix.h"
65 #include "util/prng.h"
66 #include "util/smalloc.h"
67 #include "util/string.h"
68
69 using namespace std; // NOLINT
70
71 namespace download {
72
73 /**
74 * Returns the status if an interrupt happened for a given repository.
75 *
76 * Used only in case CVMFS_FAILOVER_INDEFINITELY (failover_indefinitely_) is set
77 * where failed downloads are retried indefinitely, unless an interrupt occurred
78 *
79 * @note If you use this functionality you need to change the source code of
80 * e.g. cvmfs_config reload to create a sentinel file. See comment below.
81 *
82 * @return true if an interrupt occurred
83 * false otherwise
84 */
85 ✗ bool Interrupted(const std::string &fqrn, JobInfo *info) {
86 ✗ if (info->allow_failure()) {
87 ✗ return true;
88 }
89
90 ✗ if (!fqrn.empty()) {
91 // it is up to the user the create this sentinel file ("pause_file") if
92 // CVMFS_FAILOVER_INDEFINITELY is used. It must be created during
93 // "cvmfs_config reload" and "cvmfs_config reload $fqrn"
94 ✗ const std::string pause_file = std::string("/var/run/cvmfs/interrupt.")
95 ✗ + fqrn;
96
97 ✗ LogCvmfs(kLogDownload, kLogDebug,
98 "(id %" PRId64 ") Interrupted(): checking for existence of %s",
99 info->id(), pause_file.c_str());
100 ✗ if (FileExists(pause_file)) {
101 ✗ LogCvmfs(kLogDownload, kLogDebug,
102 "(id %" PRId64 ") Interrupt marker found - "
103 "Interrupting current download, this will EIO outstanding IO.",
104 info->id());
105 ✗ if (0 != unlink(pause_file.c_str())) {
106 ✗ LogCvmfs(kLogDownload, kLogDebug,
107 "(id %" PRId64 ") Couldn't delete interrupt marker: errno=%d",
108 ✗ info->id(), errno);
109 }
110 ✗ return true;
111 }
112 }
113 ✗ return false;
114 }
115
116 5827 static Failures PrepareDownloadDestination(JobInfo *info) {
117
3/6
✓ Branch 1 taken 5827 times.
✗ Branch 2 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 5827 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 5827 times.
 5827 if (info->sink() != NULL && !info->sink()->IsValid()) {
118 ✗ cvmfs::PathSink *psink = dynamic_cast<cvmfs::PathSink *>(info->sink());
119 ✗ if (psink != NULL) {
120 ✗ LogCvmfs(kLogDownload, kLogDebug,
121 "(id %" PRId64 ") Failed to open path %s: %s (errno=%d).",
122 ✗ info->id(), psink->path().c_str(), strerror(errno), errno);
123 ✗ return kFailLocalIO;
124 } else {
125 ✗ LogCvmfs(kLogDownload, kLogDebug,
126 "(id %" PRId64 ") "
127 "Failed to create a valid sink: \n %s",
128 ✗ info->id(), info->sink()->Describe().c_str());
129 ✗ return kFailOther;
130 }
131 }
132
133 5827 return kFailOk;
134 }
135
136
137 /**
138 * Called by curl for every HTTP header. Not called for file:// transfers.
139 */
140 17071 static size_t CallbackCurlHeader(void *ptr, size_t size, size_t nmemb,
141 void *info_link) {
142 17071 const size_t num_bytes = size * nmemb;
143
1/2
✓ Branch 2 taken 17071 times.
✗ Branch 3 not taken.
 17071 const string header_line(static_cast<const char *>(ptr), num_bytes);
144 17071 JobInfo *info = static_cast<JobInfo *>(info_link);
145
146 // LogCvmfs(kLogDownload, kLogDebug, "REMOVE-ME: Header callback with %s",
147 // header_line.c_str());
148
149 // Check http status codes
150
4/6
✓ Branch 2 taken 17071 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 17071 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 204 times.
✓ Branch 10 taken 16867 times.
 17071 if (HasPrefix(header_line, "HTTP/1.", false)) {
151
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 204 times.
 204 if (header_line.length() < 10) {
152 ✗ return 0;
153 }
154
155 unsigned i;
156
5/6
✓ Branch 1 taken 408 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 204 times.
✓ Branch 5 taken 204 times.
✓ Branch 6 taken 204 times.
✓ Branch 7 taken 204 times.
 408 for (i = 8; (i < header_line.length()) && (header_line[i] == ' '); ++i) {
157 }
158
159 // Code is initialized to -1
160
1/2
✓ Branch 1 taken 204 times.
✗ Branch 2 not taken.
 204 if (header_line.length() > i + 2) {
161 204 info->SetHttpCode(DownloadManager::ParseHttpCode(&header_line[i]));
162 }
163
164
2/2
✓ Branch 1 taken 153 times.
✓ Branch 2 taken 51 times.
 204 if ((info->http_code() / 100) == 2) {
165 153 return num_bytes;
166
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 51 } else if ((info->http_code() == 301) || (info->http_code() == 302)
167
2/8
✗ Branch 0 not taken.
✓ Branch 1 taken 51 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 51 times.
✗ Branch 9 not taken.
 51 || (info->http_code() == 303) || (info->http_code() == 307)) {
168
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 51 times.
 51 if (!info->follow_redirects()) {
169 ✗ LogCvmfs(kLogDownload, kLogDebug,
170 "(id %" PRId64 ") redirect support not enabled: %s",
171 info->id(), header_line.c_str());
172 ✗ info->SetErrorCode(kFailHostHttp);
173 ✗ return 0;
174 }
175
1/2
✓ Branch 3 taken 51 times.
✗ Branch 4 not taken.
 51 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") http redirect: %s",
176 info->id(), header_line.c_str());
177 // libcurl will handle this because of CURLOPT_FOLLOWLOCATION
178 51 return num_bytes;
179 } else {
180 ✗ LogCvmfs(kLogDownload, kLogDebug,
181 "(id %" PRId64 ") http status error code: %s [%d]", info->id(),
182 header_line.c_str(), info->http_code());
183 ✗ if (((info->http_code() / 100) == 5) || (info->http_code() == 400)
184 ✗ || (info->http_code() == 404)) {
185 // 5XX returned by host
186 // 400: error from the GeoAPI module
187 // 404: the stratum 1 does not have the newest files
188 ✗ info->SetErrorCode(kFailHostHttp);
189 ✗ } else if (info->http_code() == 429) {
190 // 429: rate throttling (we ignore the backoff hint for the time being)
191 ✗ info->SetErrorCode(kFailHostConnection);
192 } else {
193 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostHttp
194 : kFailProxyHttp);
195 }
196 ✗ return 0;
197 }
198 }
199
200 // If needed: allocate space in sink
201
3/4
✓ Branch 2 taken 16867 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 5839 times.
✓ Branch 5 taken 11028 times.
 16867 if (info->sink() != NULL && info->sink()->RequiresReserve()
202
11/20
✓ Branch 1 taken 16867 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 5839 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 5839 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 1884 times.
✓ Branch 11 taken 3955 times.
✓ Branch 12 taken 5839 times.
✓ Branch 13 taken 11028 times.
✗ Branch 14 not taken.
✓ Branch 15 taken 5839 times.
✓ Branch 16 taken 11028 times.
✓ Branch 18 taken 1884 times.
✓ Branch 19 taken 14983 times.
✗ Branch 20 not taken.
✗ Branch 21 not taken.
✗ Branch 23 not taken.
✗ Branch 24 not taken.
 33734 && HasPrefix(header_line, "CONTENT-LENGTH:", true)) {
203 1884 char *tmp = reinterpret_cast<char *>(alloca(num_bytes + 1));
204 1884 uint64_t length = 0;
205 1884 sscanf(header_line.c_str(), "%s %" PRIu64, tmp, &length);
206
1/2
✓ Branch 0 taken 1884 times.
✗ Branch 1 not taken.
 1884 if (length > 0) {
207
2/4
✓ Branch 2 taken 1884 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 1884 times.
 1884 if (!info->sink()->Reserve(length)) {
208 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
209 "(id %" PRId64 ") "
210 "resource %s too large to store in memory (%" PRIu64 ")",
211 info->id(), info->url()->c_str(), length);
212 ✗ info->SetErrorCode(kFailTooBig);
213 ✗ return 0;
214 }
215 } else {
216 // Empty resource
217 ✗ info->sink()->Reserve(0);
218 }
219
4/6
✓ Branch 2 taken 14983 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 14983 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 51 times.
✓ Branch 10 taken 14932 times.
 14983 } else if (HasPrefix(header_line, "LOCATION:", true)) {
220 // This comes along with redirects
221
1/2
✓ Branch 3 taken 51 times.
✗ Branch 4 not taken.
 51 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") %s", info->id(),
222 header_line.c_str());
223
3/6
✓ Branch 2 taken 14932 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 14932 times.
✗ Branch 6 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 14932 times.
 14932 } else if (HasPrefix(header_line, "LINK:", true)) {
224 // This is metalink info
225 ✗ LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") %s", info->id(),
226 header_line.c_str());
227 ✗ std::string link = info->link();
228 ✗ if (link.size() != 0) {
229 // multiple LINK headers are allowed
230 ✗ link = link + ", " + header_line.substr(5);
231 } else {
232 ✗ link = header_line.substr(5);
233 }
234 ✗ info->SetLink(link);
235
3/6
✓ Branch 3 taken 14932 times.
✗ Branch 4 not taken.
✓ Branch 6 taken 14932 times.
✗ Branch 7 not taken.
✗ Branch 10 not taken.
✓ Branch 11 taken 14932 times.
 14932 } else if (HasPrefix(header_line, "X-SQUID-ERROR:", true)) {
236 // Reinterpret host error as proxy error
237 ✗ if (info->error_code() == kFailHostHttp) {
238 ✗ info->SetErrorCode(kFailProxyHttp);
239 }
240
3/6
✓ Branch 2 taken 14932 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 14932 times.
✗ Branch 6 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 14932 times.
 14932 } else if (HasPrefix(header_line, "PROXY-STATUS:", true)) {
241 // Reinterpret host error as proxy error if applicable
242 ✗ if ((info->error_code() == kFailHostHttp)
243 ✗ && (header_line.find("error=") != string::npos)) {
244 ✗ info->SetErrorCode(kFailProxyHttp);
245 }
246 }
247
248 16867 return num_bytes;
249 17071 }
250
251
252 /**
253 * Called by curl for every received data chunk.
254 */
255 6088 static size_t CallbackCurlData(void *ptr, size_t size, size_t nmemb,
256 void *info_link) {
257 6088 const size_t num_bytes = size * nmemb;
258 6088 JobInfo *info = static_cast<JobInfo *>(info_link);
259
260 // TODO(heretherebedragons) remove if no error comes up
261 // as this means only jobinfo data request (and not header only)
262 // come here
263
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6088 times.
 6088 assert(info->sink() != NULL);
264
265 // LogCvmfs(kLogDownload, kLogDebug, "Data callback, %d bytes", num_bytes);
266
267
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6088 times.
 6088 if (num_bytes == 0)
268 ✗ return 0;
269
270
2/2
✓ Branch 1 taken 3944 times.
✓ Branch 2 taken 2144 times.
 6088 if (info->expected_hash()) {
271 3944 shash::Update(reinterpret_cast<unsigned char *>(ptr), num_bytes,
272 info->hash_context());
273 }
274
275
2/2
✓ Branch 1 taken 3941 times.
✓ Branch 2 taken 2147 times.
 6088 if (info->compressed()) {
276 3941 const zlib::StreamStates retval = zlib::DecompressZStream2Sink(
277 ptr, static_cast<int64_t>(num_bytes), info->GetZstreamPtr(),
278 info->sink());
279
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3941 times.
 3941 if (retval == zlib::kStreamDataError) {
280 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
281 "(id %" PRId64 ") failed to decompress %s", info->id(),
282 info->url()->c_str());
283 ✗ info->SetErrorCode(kFailBadData);
284 ✗ return 0;
285
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3941 times.
 3941 } else if (retval == zlib::kStreamIOError) {
286 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
287 "(id %" PRId64 ") decompressing %s, local IO error", info->id(),
288 info->url()->c_str());
289 ✗ info->SetErrorCode(kFailLocalIO);
290 ✗ return 0;
291 }
292 } else {
293 2147 const int64_t written = info->sink()->Write(ptr, num_bytes);
294
2/4
✓ Branch 0 taken 2147 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 2147 times.
 2147 if (written < 0 || static_cast<uint64_t>(written) != num_bytes) {
295 ✗ LogCvmfs(kLogDownload, kLogDebug,
296 "(id %" PRId64 ") "
297 "Failed to perform write of %zu bytes to sink %s with errno %ld",
298 ✗ info->id(), num_bytes, info->sink()->Describe().c_str(),
299 written);
300 }
301 }
302
303 6088 return num_bytes;
304 }
305
306 #ifdef DEBUGMSG
307 2110 static int CallbackCurlDebug(CURL *handle,
308 curl_infotype type,
309 char *data,
310 size_t size,
311 void * /* clientp */) {
312 JobInfo *info;
313
1/2
✓ Branch 1 taken 2110 times.
✗ Branch 2 not taken.
 2110 curl_easy_getinfo(handle, CURLINFO_PRIVATE, &info);
314
315
3/6
✓ Branch 2 taken 2110 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2110 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 2110 times.
✗ Branch 9 not taken.
 4220 std::string prefix = "(id " + StringifyInt(info->id()) + ") ";
316
4/8
✓ Branch 0 taken 495 times.
✓ Branch 1 taken 799 times.
✓ Branch 2 taken 204 times.
✓ Branch 3 taken 612 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
 2110 switch (type) {
317 495 case CURLINFO_TEXT:
318
1/2
✓ Branch 1 taken 495 times.
✗ Branch 2 not taken.
 495 prefix += "{info} ";
319 495 break;
320 799 case CURLINFO_HEADER_IN:
321
1/2
✓ Branch 1 taken 799 times.
✗ Branch 2 not taken.
 799 prefix += "{header/recv} ";
322 799 break;
323 204 case CURLINFO_HEADER_OUT:
324
1/2
✓ Branch 1 taken 204 times.
✗ Branch 2 not taken.
 204 prefix += "{header/sent} ";
325 204 break;
326 612 case CURLINFO_DATA_IN:
327
2/2
✓ Branch 0 taken 68 times.
✓ Branch 1 taken 544 times.
 612 if (size < 50) {
328
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 prefix += "{data/recv} ";
329 68 break;
330 } else {
331
1/2
✓ Branch 2 taken 544 times.
✗ Branch 3 not taken.
 544 LogCvmfs(kLogCurl, kLogDebug, "%s{data/recv} <snip>", prefix.c_str());
332 544 return 0;
333 }
334 ✗ case CURLINFO_DATA_OUT:
335 ✗ if (size < 50) {
336 ✗ prefix += "{data/sent} ";
337 ✗ break;
338 } else {
339 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{data/sent} <snip>", prefix.c_str());
340 ✗ return 0;
341 }
342 ✗ case CURLINFO_SSL_DATA_IN:
343 ✗ if (size < 50) {
344 ✗ prefix += "{ssldata/recv} ";
345 ✗ break;
346 } else {
347 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{ssldata/recv} <snip>",
348 prefix.c_str());
349 ✗ return 0;
350 }
351 ✗ case CURLINFO_SSL_DATA_OUT:
352 ✗ if (size < 50) {
353 ✗ prefix += "{ssldata/sent} ";
354 ✗ break;
355 } else {
356 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{ssldata/sent} <snip>",
357 prefix.c_str());
358 ✗ return 0;
359 }
360 ✗ default:
361 // just log the message
362 ✗ break;
363 }
364
365 1566 bool valid_char = true;
366
1/2
✓ Branch 2 taken 1566 times.
✗ Branch 3 not taken.
 1566 std::string msg(data, size);
367
2/2
✓ Branch 1 taken 89142 times.
✓ Branch 2 taken 1566 times.
 90708 for (size_t i = 0; i < msg.length(); ++i) {
368
2/4
✓ Branch 1 taken 89142 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 89142 times.
 89142 if (msg[i] == '\0') {
369 ✗ msg[i] = '~';
370 }
371
372 // verify that char is a valid printable char
373
3/6
✓ Branch 1 taken 89142 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 84465 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 84465 times.
 173607 if ((msg[i] < ' ' || msg[i] > '~')
374
6/8
✓ Branch 0 taken 84465 times.
✓ Branch 1 taken 4677 times.
✓ Branch 3 taken 4677 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 2091 times.
✓ Branch 6 taken 2586 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 89142 times.
 175698 && (msg[i] != 10 /*line feed*/
375
2/4
✓ Branch 1 taken 2091 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 2091 times.
 2091 && msg[i] != 13 /*carriage return*/)) {
376 ✗ valid_char = false;
377 }
378 }
379
380
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1566 times.
 1566 if (!valid_char) {
381 ✗ msg = "<Non-plaintext sequence>";
382 }
383
384
1/2
✓ Branch 3 taken 1566 times.
✗ Branch 4 not taken.
 1566 LogCvmfs(kLogCurl, kLogDebug, "%s%s", prefix.c_str(),
385
1/2
✓ Branch 1 taken 1566 times.
✗ Branch 2 not taken.
 3132 Trim(msg, true /* trim_newline */).c_str());
386 1566 return 0;
387 2110 }
388 #endif
389
390 //------------------------------------------------------------------------------
391
392
393 const int DownloadManager::kProbeUnprobed = -1;
394 const int DownloadManager::kProbeDown = -2;
395 const int DownloadManager::kProbeGeo = -3;
396
397 /**
398 * Escape special chars from the URL, except for ':' and '/',
399 * which should keep their meaning.
400 */
401 5895 string DownloadManager::EscapeUrl(const int64_t jobinfo_id, const string &url) {
402 5895 string escaped;
403
1/2
✓ Branch 2 taken 5895 times.
✗ Branch 3 not taken.
 5895 escaped.reserve(url.length());
404
405 char escaped_char[3];
406
2/2
✓ Branch 1 taken 516106 times.
✓ Branch 2 taken 5895 times.
 522001 for (unsigned i = 0, s = url.length(); i < s; ++i) {
407
3/4
✓ Branch 2 taken 516106 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 153 times.
✓ Branch 5 taken 515953 times.
 516106 if (JobInfo::EscapeUrlChar(url[i], escaped_char)) {
408
1/2
✓ Branch 1 taken 153 times.
✗ Branch 2 not taken.
 153 escaped.append(escaped_char, 3);
409 } else {
410
1/2
✓ Branch 1 taken 515953 times.
✗ Branch 2 not taken.
 515953 escaped.push_back(escaped_char[0]);
411 }
412 }
413
1/2
✓ Branch 3 taken 5895 times.
✗ Branch 4 not taken.
 5895 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") escaped %s to %s",
414 jobinfo_id, url.c_str(), escaped.c_str());
415
416 11790 return escaped;
417 }
418
419 /**
420 * -1 of digits is not a valid Http return code
421 */
422 289 int DownloadManager::ParseHttpCode(const char digits[3]) {
423 289 int result = 0;
424 289 int factor = 100;
425
2/2
✓ Branch 0 taken 867 times.
✓ Branch 1 taken 272 times.
 1139 for (int i = 0; i < 3; ++i) {
426
3/4
✓ Branch 0 taken 867 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 17 times.
✓ Branch 3 taken 850 times.
 867 if ((digits[i] < '0') || (digits[i] > '9'))
427 17 return -1;
428 850 result += (digits[i] - '0') * factor;
429 850 factor /= 10;
430 }
431 272 return result;
432 }
433
434
435 /**
436 * Called when new curl sockets arrive or existing curl sockets depart.
437 */
438 ✗ int DownloadManager::CallbackCurlSocket(CURL * /* easy */,
439 curl_socket_t s,
440 int action,
441 void *userp,
442 void * /* socketp */) {
443 // LogCvmfs(kLogDownload, kLogDebug, "CallbackCurlSocket called with easy "
444 // "handle %p, socket %d, action %d", easy, s, action);
445 ✗ DownloadManager *download_mgr = static_cast<DownloadManager *>(userp);
446 ✗ if (action == CURL_POLL_NONE)
447 ✗ return 0;
448
449 // Find s in watch_fds_
450 unsigned index;
451
452 // TODO(heretherebedragons) why start at index = 0 and not 2?
453 // fd[0] and fd[1] are fixed?
454 ✗ for (index = 0; index < download_mgr->watch_fds_inuse_; ++index) {
455 ✗ if (download_mgr->watch_fds_[index].fd == s)
456 ✗ break;
457 }
458 // Or create newly
459 ✗ if (index == download_mgr->watch_fds_inuse_) {
460 // Extend array if necessary
461 ✗ if (download_mgr->watch_fds_inuse_ == download_mgr->watch_fds_size_) {
462 ✗ assert(download_mgr->watch_fds_size_ > 0);
463 ✗ download_mgr->watch_fds_size_ *= 2;
464 ✗ download_mgr->watch_fds_ = static_cast<struct pollfd *>(
465 ✗ srealloc(download_mgr->watch_fds_,
466 ✗ download_mgr->watch_fds_size_ * sizeof(struct pollfd)));
467 }
468 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].fd = s;
469 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].events = 0;
470 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].revents = 0;
471 ✗ download_mgr->watch_fds_inuse_++;
472 }
473
474 ✗ switch (action) {
475 ✗ case CURL_POLL_IN:
476 ✗ download_mgr->watch_fds_[index].events = POLLIN | POLLPRI;
477 ✗ break;
478 ✗ case CURL_POLL_OUT:
479 ✗ download_mgr->watch_fds_[index].events = POLLOUT | POLLWRBAND;
480 ✗ break;
481 ✗ case CURL_POLL_INOUT:
482 ✗ download_mgr->watch_fds_[index].events = POLLIN | POLLPRI | POLLOUT
483 | POLLWRBAND;
484 ✗ break;
485 ✗ case CURL_POLL_REMOVE:
486 ✗ if (index < download_mgr->watch_fds_inuse_ - 1) {
487 download_mgr
488 ✗ ->watch_fds_[index] = download_mgr->watch_fds_
489 ✗ [download_mgr->watch_fds_inuse_ - 1];
490 }
491 ✗ download_mgr->watch_fds_inuse_--;
492 // Shrink array if necessary
493 ✗ if ((download_mgr->watch_fds_inuse_ > download_mgr->watch_fds_max_)
494 ✗ && (download_mgr->watch_fds_inuse_
495 ✗ < download_mgr->watch_fds_size_ / 2)) {
496 ✗ download_mgr->watch_fds_size_ /= 2;
497 // LogCvmfs(kLogDownload, kLogDebug, "shrinking watch_fds_ (%d)",
498 // watch_fds_size_);
499 ✗ download_mgr->watch_fds_ = static_cast<struct pollfd *>(
500 ✗ srealloc(download_mgr->watch_fds_,
501 ✗ download_mgr->watch_fds_size_ * sizeof(struct pollfd)));
502 // LogCvmfs(kLogDownload, kLogDebug, "shrinking watch_fds_ done",
503 // watch_fds_size_);
504 }
505 ✗ break;
506 ✗ default:
507 ✗ break;
508 }
509
510 ✗ return 0;
511 }
512
513
514 /**
515 * Worker thread event loop. Waits on new JobInfo structs on a pipe.
516 */
517 ✗ void *DownloadManager::MainDownload(void *data) {
518 ✗ DownloadManager *download_mgr = static_cast<DownloadManager *>(data);
519 ✗ LogCvmfs(kLogDownload, kLogDebug,
520 "download I/O thread of DownloadManager '%s' started",
521 download_mgr->name_.c_str());
522
523 ✗ const int kIdxPipeTerminate = 0;
524 ✗ const int kIdxPipeJobs = 1;
525
526 ✗ download_mgr->watch_fds_ = static_cast<struct pollfd *>(
527 ✗ smalloc(2 * sizeof(struct pollfd)));
528 ✗ download_mgr->watch_fds_size_ = 2;
529 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].fd = download_mgr->pipe_terminate_
530 ✗ ->GetReadFd();
531 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].events = POLLIN | POLLPRI;
532 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].revents = 0;
533 ✗ download_mgr->watch_fds_[kIdxPipeJobs].fd = download_mgr->pipe_jobs_
534 ✗ ->GetReadFd();
535 ✗ download_mgr->watch_fds_[kIdxPipeJobs].events = POLLIN | POLLPRI;
536 ✗ download_mgr->watch_fds_[kIdxPipeJobs].revents = 0;
537 ✗ download_mgr->watch_fds_inuse_ = 2;
538
539 ✗ int still_running = 0;
540 struct timeval timeval_start, timeval_stop;
541 ✗ gettimeofday(&timeval_start, NULL);
542 while (true) {
543 int timeout;
544 ✗ if (still_running) {
545 /* NOTE: The following might degrade the performance for many small files
546 * use case. TODO(jblomer): look into it.
547 // Specify a timeout for polling in ms; this allows us to return
548 // to libcurl once a second so it can look for internal operations
549 // which timed out. libcurl has a more elaborate mechanism
550 // (CURLMOPT_TIMERFUNCTION) that would inform us of the next potential
551 // timeout. TODO(bbockelm) we should switch to that in the future.
552 timeout = 100;
553 */
554 ✗ timeout = 1;
555 } else {
556 ✗ timeout = -1;
557 ✗ gettimeofday(&timeval_stop, NULL);
558 const int64_t delta = static_cast<int64_t>(
559 ✗ 1000 * DiffTimeSeconds(timeval_start, timeval_stop));
560 ✗ perf::Xadd(download_mgr->counters_->sz_transfer_time, delta);
561 }
562 ✗ const int retval = poll(download_mgr->watch_fds_,
563 ✗ download_mgr->watch_fds_inuse_, timeout);
564 ✗ if (retval < 0) {
565 ✗ continue;
566 }
567
568 // Handle timeout
569 ✗ if (retval == 0) {
570 ✗ curl_multi_socket_action(download_mgr->curl_multi_, CURL_SOCKET_TIMEOUT,
571 0, &still_running);
572 }
573
574 // Terminate I/O thread
575 ✗ if (download_mgr->watch_fds_[kIdxPipeTerminate].revents)
576 ✗ break;
577
578 // New job arrives
579 ✗ if (download_mgr->watch_fds_[kIdxPipeJobs].revents) {
580 ✗ download_mgr->watch_fds_[kIdxPipeJobs].revents = 0;
581 JobInfo *info;
582 ✗ download_mgr->pipe_jobs_->Read<JobInfo *>(&info);
583 ✗ if (!still_running) {
584 ✗ gettimeofday(&timeval_start, NULL);
585 }
586 ✗ CURL *handle = download_mgr->AcquireCurlHandle();
587 ✗ download_mgr->InitializeRequest(info, handle);
588 ✗ download_mgr->SetUrlOptions(info);
589 ✗ curl_multi_add_handle(download_mgr->curl_multi_, handle);
590 ✗ curl_multi_socket_action(download_mgr->curl_multi_, CURL_SOCKET_TIMEOUT,
591 0, &still_running);
592 }
593
594 // Activity on curl sockets
595 // Within this loop the curl_multi_socket_action() may cause socket(s)
596 // to be removed from watch_fds_. If a socket is removed it is replaced
597 // by the socket at the end of the array and the inuse count is decreased.
598 // Therefore loop over the array in reverse order.
599 ✗ for (int64_t i = download_mgr->watch_fds_inuse_ - 1; i >= 2; --i) {
600 ✗ if (i >= download_mgr->watch_fds_inuse_) {
601 ✗ continue;
602 }
603 ✗ if (download_mgr->watch_fds_[i].revents) {
604 ✗ int ev_bitmask = 0;
605 ✗ if (download_mgr->watch_fds_[i].revents & (POLLIN | POLLPRI))
606 ✗ ev_bitmask |= CURL_CSELECT_IN;
607 ✗ if (download_mgr->watch_fds_[i].revents & (POLLOUT | POLLWRBAND))
608 ✗ ev_bitmask |= CURL_CSELECT_OUT;
609 ✗ if (download_mgr->watch_fds_[i].revents
610 ✗ & (POLLERR | POLLHUP | POLLNVAL)) {
611 ✗ ev_bitmask |= CURL_CSELECT_ERR;
612 }
613 ✗ download_mgr->watch_fds_[i].revents = 0;
614
615 ✗ curl_multi_socket_action(download_mgr->curl_multi_,
616 ✗ download_mgr->watch_fds_[i].fd,
617 ev_bitmask,
618 &still_running);
619 }
620 }
621
622 // Check if transfers are completed
623 CURLMsg *curl_msg;
624 int msgs_in_queue;
625 ✗ while ((curl_msg = curl_multi_info_read(download_mgr->curl_multi_,
626 &msgs_in_queue))) {
627 ✗ if (curl_msg->msg == CURLMSG_DONE) {
628 ✗ perf::Inc(download_mgr->counters_->n_requests);
629 JobInfo *info;
630 ✗ CURL *easy_handle = curl_msg->easy_handle;
631 ✗ const int curl_error = curl_msg->data.result;
632 ✗ curl_easy_getinfo(easy_handle, CURLINFO_PRIVATE, &info);
633
634 int64_t redir_count;
635 ✗ curl_easy_getinfo(easy_handle, CURLINFO_REDIRECT_COUNT, &redir_count);
636 ✗ LogCvmfs(kLogDownload, kLogDebug,
637 "(manager '%s' - id %" PRId64 ") "
638 "Number of CURL redirects %" PRId64,
639 download_mgr->name_.c_str(), info->id(), redir_count);
640
641 ✗ curl_multi_remove_handle(download_mgr->curl_multi_, easy_handle);
642 ✗ if (download_mgr->VerifyAndFinalize(curl_error, info)) {
643 ✗ curl_multi_add_handle(download_mgr->curl_multi_, easy_handle);
644 ✗ curl_multi_socket_action(download_mgr->curl_multi_,
645 CURL_SOCKET_TIMEOUT,
646 0,
647 &still_running);
648 } else {
649 // Return easy handle into pool and write result back
650 ✗ download_mgr->ReleaseCurlHandle(easy_handle);
651
652 ✗ DataTubeElement *ele = new DataTubeElement(kActionStop);
653 ✗ info->GetDataTubePtr()->EnqueueBack(ele);
654 ✗ info->GetPipeJobResultPtr()->Write<download::Failures>(
655 ✗ info->error_code());
656 }
657 }
658 }
659 }
660
661 ✗ for (set<CURL *>::iterator i = download_mgr->pool_handles_inuse_->begin(),
662 ✗ iEnd = download_mgr->pool_handles_inuse_->end();
663 ✗ i != iEnd;
664 ✗ ++i) {
665 ✗ curl_multi_remove_handle(download_mgr->curl_multi_, *i);
666 ✗ curl_easy_cleanup(*i);
667 }
668 ✗ download_mgr->pool_handles_inuse_->clear();
669 ✗ free(download_mgr->watch_fds_);
670
671 ✗ LogCvmfs(kLogDownload, kLogDebug,
672 "download I/O thread of DownloadManager '%s' terminated",
673 download_mgr->name_.c_str());
674 ✗ return NULL;
675 }
676
677
678 //------------------------------------------------------------------------------
679
680
681 3777 HeaderLists::~HeaderLists() {
682
2/2
✓ Branch 1 taken 3778 times.
✓ Branch 2 taken 3777 times.
 7555 for (unsigned i = 0; i < blocks_.size(); ++i) {
683
1/2
✓ Branch 1 taken 3778 times.
✗ Branch 2 not taken.
 3778 delete[] blocks_[i];
684 }
685 3777 blocks_.clear();
686 3777 }
687
688
689 10122 curl_slist *HeaderLists::GetList(const char *header) { return Get(header); }
690
691
692 5827 curl_slist *HeaderLists::DuplicateList(curl_slist *slist) {
693
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5827 times.
 5827 assert(slist);
694 5827 curl_slist *copy = GetList(slist->data);
695 5827 copy->next = slist->next;
696 5827 curl_slist *prev = copy;
697 5827 slist = slist->next;
698
2/2
✓ Branch 0 taken 11654 times.
✓ Branch 1 taken 5827 times.
 17481 while (slist) {
699 11654 curl_slist *new_link = Get(slist->data);
700 11654 new_link->next = slist->next;
701 11654 prev->next = new_link;
702 11654 prev = new_link;
703 11654 slist = slist->next;
704 }
705 5827 return copy;
706 }
707
708
709 7944 void HeaderLists::AppendHeader(curl_slist *slist, const char *header) {
710
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7944 times.
 7944 assert(slist);
711 7944 curl_slist *new_link = Get(header);
712 7944 new_link->next = NULL;
713
714
2/2
✓ Branch 0 taken 4747 times.
✓ Branch 1 taken 7944 times.
 12691 while (slist->next)
715 4747 slist = slist->next;
716 7944 slist->next = new_link;
717 7944 }
718
719
720 /**
721 * Ensures that a certain header string is _not_ part of slist on return.
722 * Note that if the first header element matches, the returned slist points
723 * to a different value.
724 */
725 5 void HeaderLists::CutHeader(const char *header, curl_slist **slist) {
726
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5 times.
 5 assert(slist);
727 curl_slist head;
728 5 head.next = *slist;
729 5 curl_slist *prev = &head;
730 5 curl_slist *rover = *slist;
731
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 5 times.
 14 while (rover) {
732
2/2
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 5 times.
 9 if (strcmp(rover->data, header) == 0) {
733 4 prev->next = rover->next;
734 4 Put(rover);
735 4 rover = prev;
736 }
737 9 prev = rover;
738 9 rover = rover->next;
739 }
740 5 *slist = head.next;
741 5 }
742
743
744 6087 void HeaderLists::PutList(curl_slist *slist) {
745
2/2
✓ Branch 0 taken 18131 times.
✓ Branch 1 taken 6087 times.
 24218 while (slist) {
746 18131 curl_slist *next = slist->next;
747 18131 Put(slist);
748 18131 slist = next;
749 }
750 6087 }
751
752
753 4 string HeaderLists::Print(curl_slist *slist) {
754 4 string verbose;
755
2/2
✓ Branch 0 taken 8 times.
✓ Branch 1 taken 4 times.
 12 while (slist) {
756
3/6
✓ Branch 2 taken 8 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 8 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 8 times.
✗ Branch 9 not taken.
 8 verbose += string(slist->data) + "\n";
757 8 slist = slist->next;
758 }
759 4 return verbose;
760 }
761
762
763 29720 curl_slist *HeaderLists::Get(const char *header) {
764
2/2
✓ Branch 1 taken 25941 times.
✓ Branch 2 taken 3780 times.
 29721 for (unsigned i = 0; i < blocks_.size(); ++i) {
765
2/2
✓ Branch 0 taken 175504 times.
✓ Branch 1 taken 1 times.
 175505 for (unsigned j = 0; j < kBlockSize; ++j) {
766
2/2
✓ Branch 2 taken 25940 times.
✓ Branch 3 taken 149564 times.
 175504 if (!IsUsed(&(blocks_[i][j]))) {
767 25940 blocks_[i][j].data = const_cast<char *>(header);
768 25940 return &(blocks_[i][j]);
769 }
770 }
771 }
772
773 // All used, new block
774 3780 AddBlock();
775 3780 blocks_[blocks_.size() - 1][0].data = const_cast<char *>(header);
776 3780 return &(blocks_[blocks_.size() - 1][0]);
777 }
778
779
780 985815 void HeaderLists::Put(curl_slist *slist) {
781 985815 slist->data = NULL;
782 985815 slist->next = NULL;
783 985815 }
784
785
786 3780 void HeaderLists::AddBlock() {
787
1/2
✓ Branch 1 taken 3780 times.
✗ Branch 2 not taken.
 3780 curl_slist *new_block = new curl_slist[kBlockSize];
788
2/2
✓ Branch 0 taken 967680 times.
✓ Branch 1 taken 3780 times.
 971460 for (unsigned i = 0; i < kBlockSize; ++i) {
789 967680 Put(&new_block[i]);
790 }
791
1/2
✓ Branch 1 taken 3780 times.
✗ Branch 2 not taken.
 3780 blocks_.push_back(new_block);
792 3780 }
793
794
795 //------------------------------------------------------------------------------
796
797
798 ✗ string DownloadManager::ProxyInfo::Print() {
799 ✗ if (url == "DIRECT")
800 ✗ return url;
801
802 ✗ string result = url;
803 ✗ const int remaining = static_cast<int>(host.deadline())
804 ✗ - static_cast<int>(time(NULL));
805 ✗ string expinfo = (remaining >= 0) ? "+" : "";
806 ✗ if (abs(remaining) >= 3600) {
807 ✗ expinfo += StringifyInt(remaining / 3600) + "h";
808 ✗ } else if (abs(remaining) >= 60) {
809 ✗ expinfo += StringifyInt(remaining / 60) + "m";
810 } else {
811 ✗ expinfo += StringifyInt(remaining) + "s";
812 }
813 ✗ if (host.status() == dns::kFailOk) {
814 ✗ result += " (" + host.name() + ", " + expinfo + ")";
815 } else {
816 ✗ result += " (:unresolved:, " + expinfo + ")";
817 }
818 ✗ return result;
819 }
820
821
822 /**
823 * Gets an idle CURL handle from the pool. Creates a new one and adds it to
824 * the pool if necessary.
825 */
826 5827 CURL *DownloadManager::AcquireCurlHandle() {
827 CURL *handle;
828
829
2/2
✓ Branch 1 taken 2092 times.
✓ Branch 2 taken 3735 times.
 5827 if (pool_handles_idle_->empty()) {
830 // Create a new handle
831
1/2
✓ Branch 1 taken 2092 times.
✗ Branch 2 not taken.
 2092 handle = curl_easy_init();
832
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2092 times.
 2092 assert(handle != NULL);
833
834
1/2
✓ Branch 1 taken 2092 times.
✗ Branch 2 not taken.
 2092 curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1);
835 // curl_easy_setopt(curl_default, CURLOPT_FAILONERROR, 1);
836
1/2
✓ Branch 1 taken 2092 times.
✗ Branch 2 not taken.
 2092 curl_easy_setopt(handle, CURLOPT_HEADERFUNCTION, CallbackCurlHeader);
837
1/2
✓ Branch 1 taken 2092 times.
✗ Branch 2 not taken.
 2092 curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, CallbackCurlData);
838 } else {
839 3735 handle = *(pool_handles_idle_->begin());
840
1/2
✓ Branch 2 taken 3735 times.
✗ Branch 3 not taken.
 3735 pool_handles_idle_->erase(pool_handles_idle_->begin());
841 }
842
843
1/2
✓ Branch 1 taken 5827 times.
✗ Branch 2 not taken.
 5827 pool_handles_inuse_->insert(handle);
844
845 5827 return handle;
846 }
847
848
849 5827 void DownloadManager::ReleaseCurlHandle(CURL *handle) {
850
1/2
✓ Branch 1 taken 5827 times.
✗ Branch 2 not taken.
 5827 const set<CURL *>::iterator elem = pool_handles_inuse_->find(handle);
851
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 5827 times.
 5827 assert(elem != pool_handles_inuse_->end());
852
853
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5827 times.
 5827 if (pool_handles_idle_->size() > pool_max_handles_) {
854 ✗ curl_easy_cleanup(*elem);
855 } else {
856
1/2
✓ Branch 2 taken 5827 times.
✗ Branch 3 not taken.
 5827 pool_handles_idle_->insert(*elem);
857 }
858
859
1/2
✓ Branch 1 taken 5827 times.
✗ Branch 2 not taken.
 5827 pool_handles_inuse_->erase(elem);
860 5827 }
861
862
863 /**
864 * HTTP request options: set the URL and other options such as timeout and
865 * proxy.
866 */
867 5827 void DownloadManager::InitializeRequest(JobInfo *info, CURL *handle) {
868 // Initialize internal download state
869 5827 info->SetCurlHandle(handle);
870 5827 info->SetErrorCode(kFailOk);
871 5827 info->SetHttpCode(-1);
872 5827 info->SetFollowRedirects(follow_redirects_);
873 5827 info->SetNumUsedProxies(1);
874 5827 info->SetNumUsedMetalinks(1);
875 5827 info->SetNumUsedHosts(1);
876 5827 info->SetNumRetries(0);
877 5827 info->SetBackoffMs(0);
878 5827 info->SetHeaders(header_lists_->DuplicateList(default_headers_));
879
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5827 times.
 5827 if (info->info_header()) {
880 ✗ header_lists_->AppendHeader(info->headers(), info->info_header());
881 }
882
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5827 times.
 5827 if (enable_http_tracing_) {
883 ✗ for (unsigned int i = 0; i < http_tracing_headers_.size(); i++) {
884 ✗ header_lists_->AppendHeader(info->headers(),
885 ✗ (http_tracing_headers_)[i].c_str());
886 }
887
888 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_pid());
889 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_gid());
890 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_uid());
891
892 ✗ LogCvmfs(kLogDownload, kLogDebug,
893 "(manager '%s' - id %" PRId64 ") "
894 "CURL Header for URL: %s is:\n %s",
895 name_.c_str(), info->id(), info->url()->c_str(),
896 ✗ header_lists_->Print(info->headers()).c_str());
897 }
898
899
2/2
✓ Branch 1 taken 114 times.
✓ Branch 2 taken 5713 times.
 5827 if (info->force_nocache()) {
900 114 SetNocache(info);
901 } else {
902 5713 info->SetNocache(false);
903 }
904
2/2
✓ Branch 1 taken 4146 times.
✓ Branch 2 taken 1681 times.
 5827 if (info->compressed()) {
905 4146 zlib::DecompressInit(info->GetZstreamPtr());
906 }
907
2/2
✓ Branch 1 taken 4149 times.
✓ Branch 2 taken 1678 times.
 5827 if (info->expected_hash()) {
908
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4149 times.
 4149 assert(info->hash_context().buffer != NULL);
909 4149 shash::Init(info->hash_context());
910 }
911
912
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 5827 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 5827 times.
 5827 if ((info->range_offset() != -1) && (info->range_size())) {
913 char byte_range_array[100];
914 ✗ const int64_t range_lower = static_cast<int64_t>(info->range_offset());
915 ✗ const int64_t range_upper = static_cast<int64_t>(info->range_offset()
916 ✗ + info->range_size() - 1);
917 ✗ if (snprintf(byte_range_array, sizeof(byte_range_array),
918 "%" PRId64 "-%" PRId64, range_lower, range_upper)
919 ✗ == 100) {
920 ✗ PANIC(NULL); // Should be impossible given limits on offset size.
921 }
922 ✗ curl_easy_setopt(handle, CURLOPT_RANGE, byte_range_array);
923 } else {
924 5827 curl_easy_setopt(handle, CURLOPT_RANGE, NULL);
925 }
926
927 // Set curl parameters
928 5827 curl_easy_setopt(handle, CURLOPT_PRIVATE, static_cast<void *>(info));
929 5827 curl_easy_setopt(handle, CURLOPT_WRITEHEADER, static_cast<void *>(info));
930 5827 curl_easy_setopt(handle, CURLOPT_WRITEDATA, static_cast<void *>(info));
931 5827 curl_easy_setopt(handle, CURLOPT_HTTPHEADER, info->headers());
932
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5827 times.
 5827 if (info->head_request()) {
933 ✗ curl_easy_setopt(handle, CURLOPT_NOBODY, 1);
934 } else {
935 5827 curl_easy_setopt(handle, CURLOPT_HTTPGET, 1);
936 }
937
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5827 times.
 5827 if (opt_ipv4_only_) {
938 ✗ curl_easy_setopt(handle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
939 }
940
2/2
✓ Branch 0 taken 2334 times.
✓ Branch 1 taken 3493 times.
 5827 if (follow_redirects_) {
941 2334 curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1);
942 2334 curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 4);
943 }
944 #ifdef DEBUGMSG
945 5827 curl_easy_setopt(handle, CURLOPT_VERBOSE, 1);
946 5827 curl_easy_setopt(handle, CURLOPT_DEBUGFUNCTION, CallbackCurlDebug);
947 #endif
948 5827 }
949
950 11756 void DownloadManager::CheckHostInfoReset(const std::string &typ,
951 HostInfo &info,
952 JobInfo *jobinfo,
953 time_t &now) {
954
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11756 times.
 11756 if (info.timestamp_backup > 0) {
955 ✗ if (now == 0)
956 ✗ now = time(NULL);
957 ✗ if (static_cast<int64_t>(now)
958 ✗ > static_cast<int64_t>(info.timestamp_backup + info.reset_after)) {
959 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
960 "(manager %s - id %" PRId64 ") "
961 "switching %s from %s to %s (reset %s)",
962 name_.c_str(), jobinfo->id(), typ.c_str(),
963 ✗ (*info.chain)[info.current].c_str(), (*info.chain)[0].c_str(),
964 typ.c_str());
965 ✗ info.current = 0;
966 ✗ info.timestamp_backup = 0;
967 }
968 }
969 11756 }
970
971
972 /**
973 * Sets the URL specific options such as host to use and timeout. It might also
974 * set an error code, in which case the further processing should react on.
975 */
976 5878 void DownloadManager::SetUrlOptions(JobInfo *info) {
977 5878 CURL *curl_handle = info->curl_handle();
978 5878 string url_prefix;
979 5878 time_t now = 0;
980
981 5878 const MutexLockGuard m(lock_options_);
982
983 // sharding policy
984
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5878 times.
 5878 if (sharding_policy_.UseCount() > 0) {
985 ✗ if (info->proxy() != "") {
986 // proxy already set, so this is a failover event
987 ✗ perf::Inc(counters_->n_proxy_failover);
988 }
989 ✗ info->SetProxy(sharding_policy_->GetNextProxy(
990 ✗ info->url(), info->proxy(),
991 ✗ info->range_offset() == -1 ? 0 : info->range_offset()));
992
993 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, info->proxy().c_str());
994 } else { // no sharding policy
995 // Check if proxy group needs to be reset from backup to primary
996
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5878 times.
 5878 if (opt_timestamp_backup_proxies_ > 0) {
997 ✗ now = time(NULL);
998 ✗ if (static_cast<int64_t>(now) > static_cast<int64_t>(
999 ✗ opt_timestamp_backup_proxies_ + opt_proxy_groups_reset_after_)) {
1000 ✗ opt_proxy_groups_current_ = 0;
1001 ✗ opt_timestamp_backup_proxies_ = 0;
1002 ✗ RebalanceProxiesUnlocked("Reset proxy group from backup to primary");
1003 }
1004 }
1005 // Check if load-balanced proxies within the group need to be reset
1006
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5878 times.
 5878 if (opt_timestamp_failover_proxies_ > 0) {
1007 ✗ if (now == 0)
1008 ✗ now = time(NULL);
1009 ✗ if (static_cast<int64_t>(now)
1010 ✗ > static_cast<int64_t>(opt_timestamp_failover_proxies_
1011 ✗ + opt_proxy_groups_reset_after_)) {
1012 ✗ RebalanceProxiesUnlocked(
1013 "Reset load-balanced proxies within the active group");
1014 }
1015 }
1016
1017
1/2
✓ Branch 2 taken 5878 times.
✗ Branch 3 not taken.
 5878 ProxyInfo *proxy = ChooseProxyUnlocked(info->expected_hash());
1018
6/6
✓ Branch 0 taken 1866 times.
✓ Branch 1 taken 4012 times.
✓ Branch 3 taken 1798 times.
✓ Branch 4 taken 68 times.
✓ Branch 5 taken 5810 times.
✓ Branch 6 taken 68 times.
 5878 if (!proxy || (proxy->url == "DIRECT")) {
1019
2/4
✓ Branch 2 taken 5810 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 5810 times.
✗ Branch 6 not taken.
 5810 info->SetProxy("DIRECT");
1020
1/2
✓ Branch 2 taken 5810 times.
✗ Branch 3 not taken.
 5810 curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "");
1021 } else {
1022 // Note: inside ValidateProxyIpsUnlocked() we may change the proxy data
1023 // structure, so we must not pass proxy->... (== current_proxy())
1024 // parameters directly
1025
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 const std::string purl = proxy->url;
1026
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 const dns::Host phost = proxy->host;
1027
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 const bool changed = ValidateProxyIpsUnlocked(purl, phost);
1028 // Current proxy may have changed
1029
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 68 times.
 68 if (changed) {
1030 ✗ proxy = ChooseProxyUnlocked(info->expected_hash());
1031 }
1032
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 info->SetProxy(proxy->url);
1033
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 if (proxy->host.status() == dns::kFailOk) {
1034
2/4
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
✓ Branch 6 taken 68 times.
✗ Branch 7 not taken.
 68 curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY,
1035 info->proxy().c_str());
1036 } else {
1037 // We know it can't work, don't even try to download
1038 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "0.0.0.0");
1039 }
1040 68 }
1041 } // end !sharding
1042
1043 // Check if metalink and host chains need to be reset
1044
2/4
✓ Branch 2 taken 5878 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 5878 times.
✗ Branch 6 not taken.
 5878 CheckHostInfoReset("metalink", opt_metalink_, info, now);
1045
2/4
✓ Branch 2 taken 5878 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 5878 times.
✗ Branch 6 not taken.
 5878 CheckHostInfoReset("host", opt_host_, info, now);
1046
1047
1/2
✓ Branch 1 taken 5878 times.
✗ Branch 2 not taken.
 5878 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_LIMIT, opt_low_speed_limit_);
1048
3/6
✓ Branch 1 taken 5878 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 68 times.
✓ Branch 6 taken 5810 times.
 5878 if (info->proxy() != "DIRECT") {
1049
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT, opt_timeout_proxy_);
1050
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_TIME, opt_timeout_proxy_);
1051 } else {
1052
1/2
✓ Branch 1 taken 5810 times.
✗ Branch 2 not taken.
 5810 curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT, opt_timeout_direct_);
1053
1/2
✓ Branch 1 taken 5810 times.
✗ Branch 2 not taken.
 5810 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_TIME, opt_timeout_direct_);
1054 }
1055
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5878 times.
 5878 if (!opt_dns_server_.empty())
1056 ✗ curl_easy_setopt(curl_handle, CURLOPT_DNS_SERVERS, opt_dns_server_.c_str());
1057
1058
2/2
✓ Branch 1 taken 1934 times.
✓ Branch 2 taken 3944 times.
 5878 if (info->probe_hosts()) {
1059
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1934 times.
 1934 if (CheckMetalinkChain(now)) {
1060 ✗ url_prefix = (*opt_metalink_.chain)[opt_metalink_.current];
1061 ✗ info->SetCurrentMetalinkChainIndex(opt_metalink_.current);
1062 ✗ LogCvmfs(kLogDownload, kLogDebug,
1063 "(manager %s - id %" PRId64 ") "
1064 "reading from metalink %d",
1065 name_.c_str(), info->id(), opt_metalink_.current);
1066
1/2
✓ Branch 0 taken 1934 times.
✗ Branch 1 not taken.
 1934 } else if (opt_host_.chain) {
1067
1/2
✓ Branch 2 taken 1934 times.
✗ Branch 3 not taken.
 1934 url_prefix = (*opt_host_.chain)[opt_host_.current];
1068 1934 info->SetCurrentHostChainIndex(opt_host_.current);
1069
1/2
✓ Branch 3 taken 1934 times.
✗ Branch 4 not taken.
 1934 LogCvmfs(kLogDownload, kLogDebug,
1070 "(manager %s - id %" PRId64 ") "
1071 "reading from host %d",
1072 name_.c_str(), info->id(), opt_host_.current);
1073 }
1074 }
1075
1076
1/2
✓ Branch 2 taken 5878 times.
✗ Branch 3 not taken.
 5878 string url = url_prefix + *(info->url());
1077
1078
1/2
✓ Branch 1 taken 5878 times.
✗ Branch 2 not taken.
 5878 curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, 1L);
1079
2/6
✓ Branch 1 taken 5878 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 5878 times.
 5878 if (url.substr(0, 5) == "https") {
1080 ✗ const bool rvb = ssl_certificate_store_.ApplySslCertificatePath(
1081 curl_handle);
1082 ✗ if (!rvb) {
1083 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1084 "(manager %s - id %" PRId64 ") "
1085 "Failed to set SSL certificate path %s",
1086 name_.c_str(), info->id(),
1087 ✗ ssl_certificate_store_.GetCaPath().c_str());
1088 }
1089 ✗ if (info->pid() != -1) {
1090 ✗ if (credentials_attachment_ == NULL) {
1091 ✗ LogCvmfs(kLogDownload, kLogDebug,
1092 "(manager %s - id %" PRId64 ") "
1093 "uses secure downloads but no credentials attachment set",
1094 name_.c_str(), info->id());
1095 } else {
1096 ✗ const bool retval = credentials_attachment_->ConfigureCurlHandle(
1097 curl_handle, info->pid(), info->GetCredDataPtr());
1098 ✗ if (!retval) {
1099 ✗ LogCvmfs(kLogDownload, kLogDebug,
1100 "(manager %s - id %" PRId64 ") "
1101 "failed attaching credentials",
1102 name_.c_str(), info->id());
1103 }
1104 }
1105 }
1106 // The download manager disables signal handling in the curl library;
1107 // as OpenSSL's implementation of TLS will generate a sigpipe in some
1108 // error paths, we must explicitly disable SIGPIPE here.
1109 // TODO(jblomer): it should be enough to do this once
1110 ✗ signal(SIGPIPE, SIG_IGN);
1111 }
1112
1113
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5878 times.
 5878 if (url.find("@proxy@") != string::npos) {
1114 // This is used in Geo-API requests (only), to replace a portion of the
1115 // URL with the current proxy name for the sake of caching the result.
1116 // Replace the @proxy@ either with a passed in "forced" template (which
1117 // is set from $CVMFS_PROXY_TEMPLATE) if there is one, or a "direct"
1118 // template (which is the uuid) if there's no proxy, or the name of the
1119 // proxy.
1120 ✗ string replacement;
1121 ✗ if (proxy_template_forced_ != "") {
1122 ✗ replacement = proxy_template_forced_;
1123 ✗ } else if (info->proxy() == "DIRECT") {
1124 ✗ replacement = proxy_template_direct_;
1125 } else {
1126 ✗ if (opt_proxy_groups_current_ >= opt_proxy_groups_fallback_) {
1127 // It doesn't make sense to use the fallback proxies in Geo-API requests
1128 // since the fallback proxies are supposed to get sorted, too.
1129 ✗ info->SetProxy("DIRECT");
1130 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "");
1131 ✗ replacement = proxy_template_direct_;
1132 } else {
1133 ✗ replacement = ChooseProxyUnlocked(info->expected_hash())->host.name();
1134 }
1135 }
1136 ✗ replacement = (replacement == "") ? proxy_template_direct_ : replacement;
1137 ✗ LogCvmfs(kLogDownload, kLogDebug,
1138 "(manager %s - id %" PRId64 ") "
1139 "replacing @proxy@ by %s",
1140 name_.c_str(), info->id(), replacement.c_str());
1141 ✗ url = ReplaceAll(url, "@proxy@", replacement);
1142 }
1143
1144 // TODO(heretherebedragons) before removing
1145 // static_cast<cvmfs::MemSink*>(info->sink)->size() == 0
1146 // and just always call info->sink->Reserve()
1147 // we should do a speed check
1148
3/4
✓ Branch 2 taken 5878 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2081 times.
✓ Branch 5 taken 3797 times.
 5878 if ((info->sink() != NULL) && info->sink()->RequiresReserve()
1149
1/2
✓ Branch 2 taken 2081 times.
✗ Branch 3 not taken.
 2081 && (static_cast<cvmfs::MemSink *>(info->sink())->size() == 0)
1150
11/20
✓ Branch 1 taken 5878 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 2081 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 2081 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 1840 times.
✓ Branch 11 taken 241 times.
✓ Branch 12 taken 2081 times.
✓ Branch 13 taken 3797 times.
✗ Branch 14 not taken.
✓ Branch 15 taken 2081 times.
✓ Branch 16 taken 3797 times.
✓ Branch 18 taken 1840 times.
✓ Branch 19 taken 4038 times.
✗ Branch 20 not taken.
✗ Branch 21 not taken.
✗ Branch 23 not taken.
✗ Branch 24 not taken.
 11756 && HasPrefix(url, "file://", false)) {
1151 platform_stat64 stat_buf;
1152 1840 const int retval = platform_stat(url.c_str(), &stat_buf);
1153
1/2
✓ Branch 0 taken 1840 times.
✗ Branch 1 not taken.
 1840 if (retval != 0) {
1154 // this is an error: file does not exist or out of memory
1155 // error is caught in other code section.
1156
1/2
✓ Branch 2 taken 1840 times.
✗ Branch 3 not taken.
 1840 info->sink()->Reserve(64ul * 1024ul);
1157 } else {
1158 ✗ info->sink()->Reserve(stat_buf.st_size);
1159 }
1160 }
1161
1162
2/4
✓ Branch 2 taken 5878 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 5878 times.
✗ Branch 7 not taken.
 5878 curl_easy_setopt(curl_handle, CURLOPT_URL,
1163 EscapeUrl(info->id(), url).c_str());
1164 5878 }
1165
1166
1167 /**
1168 * Checks if the name resolving information is still up to date. The host
1169 * object should be one from the current load-balance group. If the information
1170 * changed, gather new set of resolved IPs and, if different, exchange them in
1171 * the load-balance group on the fly. In the latter case, also rebalance the
1172 * proxies. The options mutex needs to be open.
1173 *
1174 * Returns true if proxies may have changed.
1175 */
1176 68 bool DownloadManager::ValidateProxyIpsUnlocked(const string &url,
1177 const dns::Host &host) {
1178
2/4
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 68 times.
✗ Branch 4 not taken.
 68 if (!host.IsExpired())
1179 68 return false;
1180 ✗ LogCvmfs(kLogDownload, kLogDebug, "(manager '%s') validate DNS entry for %s",
1181 ✗ name_.c_str(), host.name().c_str());
1182
1183 ✗ const unsigned group_idx = opt_proxy_groups_current_;
1184 ✗ dns::Host new_host = resolver_->Resolve(host.name());
1185
1186 ✗ bool update_only = true; // No changes to the list of IP addresses.
1187 ✗ if (new_host.status() != dns::kFailOk) {
1188 // Try again later in case resolving fails.
1189 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1190 "(manager '%s') failed to resolve IP addresses for %s (%d - %s)",
1191 ✗ name_.c_str(), host.name().c_str(), new_host.status(),
1192 dns::Code2Ascii(new_host.status()));
1193 ✗ new_host = dns::Host::ExtendDeadline(host, resolver_->min_ttl());
1194 ✗ } else if (!host.IsEquivalent(new_host)) {
1195 ✗ update_only = false;
1196 }
1197
1198 ✗ if (update_only) {
1199 ✗ for (unsigned i = 0; i < (*opt_proxy_groups_)[group_idx].size(); ++i) {
1200 ✗ if ((*opt_proxy_groups_)[group_idx][i].host.id() == host.id())
1201 ✗ (*opt_proxy_groups_)[group_idx][i].host = new_host;
1202 }
1203 ✗ return false;
1204 }
1205
1206 ✗ assert(new_host.status() == dns::kFailOk);
1207
1208 // Remove old host objects, insert new objects, and rebalance.
1209 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslog,
1210 "(manager '%s') DNS entries for proxy %s changed, adjusting",
1211 ✗ name_.c_str(), host.name().c_str());
1212 ✗ vector<ProxyInfo> *group = current_proxy_group();
1213 ✗ opt_num_proxies_ -= group->size();
1214 ✗ for (unsigned i = 0; i < group->size();) {
1215 ✗ if ((*group)[i].host.id() == host.id()) {
1216 ✗ group->erase(group->begin() + i);
1217 } else {
1218 ✗ i++;
1219 }
1220 }
1221 ✗ vector<ProxyInfo> new_infos;
1222 ✗ set<string> best_addresses = new_host.ViewBestAddresses(opt_ip_preference_);
1223 ✗ set<string>::const_iterator iter_ips = best_addresses.begin();
1224 ✗ for (; iter_ips != best_addresses.end(); ++iter_ips) {
1225 ✗ const string url_ip = dns::RewriteUrl(url, *iter_ips);
1226 ✗ new_infos.push_back(ProxyInfo(new_host, url_ip));
1227 }
1228 ✗ group->insert(group->end(), new_infos.begin(), new_infos.end());
1229 ✗ opt_num_proxies_ += new_infos.size();
1230
1231 ✗ const std::string msg = "DNS entries for proxy " + host.name() + " changed";
1232
1233 ✗ RebalanceProxiesUnlocked(msg);
1234 ✗ return true;
1235 }
1236
1237
1238 /**
1239 * Adds transfer time and downloaded bytes to the global counters.
1240 */
1241 6148 void DownloadManager::UpdateStatistics(CURL *handle) {
1242 double val;
1243 int retval;
1244 6148 int64_t sum = 0;
1245
1246
1/2
✓ Branch 1 taken 6148 times.
✗ Branch 2 not taken.
 6148 retval = curl_easy_getinfo(handle, CURLINFO_SIZE_DOWNLOAD, &val);
1247
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6148 times.
 6148 assert(retval == CURLE_OK);
1248 6148 sum += static_cast<int64_t>(val);
1249 /*retval = curl_easy_getinfo(handle, CURLINFO_HEADER_SIZE, &val);
1250 assert(retval == CURLE_OK);
1251 sum += static_cast<int64_t>(val);*/
1252 6148 perf::Xadd(counters_->sz_transferred_bytes, sum);
1253 6148 }
1254
1255
1256 /**
1257 * Retry if possible if not on no-cache and if not already done too often.
1258 */
1259 6148 bool DownloadManager::CanRetry(const JobInfo *info) {
1260 6148 const MutexLockGuard m(lock_options_);
1261 6148 const unsigned max_retries = opt_max_retries_;
1262
1263
2/2
✓ Branch 2 taken 2816 times.
✓ Branch 3 taken 3139 times.
 12103 return !(info->nocache()) && (info->num_retries() < max_retries)
1264
3/4
✓ Branch 0 taken 5955 times.
✓ Branch 1 taken 193 times.
✓ Branch 4 taken 2816 times.
✗ Branch 5 not taken.
 14919 && (IsProxyTransferError(info->error_code())
1265
2/2
✓ Branch 2 taken 191 times.
✓ Branch 3 taken 2625 times.
 2816 || IsHostTransferError(info->error_code()));
1266 6148 }
1267
1268 /**
1269 * Backoff for retry to introduce a jitter into a cluster of requesting
1270 * cvmfs nodes.
1271 * Retry only when HTTP caching is on.
1272 *
1273 * \return true if backoff has been performed, false otherwise
1274 */
1275 191 void DownloadManager::Backoff(JobInfo *info) {
1276 191 unsigned backoff_init_ms = 0;
1277 191 unsigned backoff_max_ms = 0;
1278 {
1279 191 const MutexLockGuard m(lock_options_);
1280 191 backoff_init_ms = opt_backoff_init_ms_;
1281 191 backoff_max_ms = opt_backoff_max_ms_;
1282 191 }
1283
1284 191 info->SetNumRetries(info->num_retries() + 1);
1285 191 perf::Inc(counters_->n_retries);
1286
2/2
✓ Branch 1 taken 76 times.
✓ Branch 2 taken 115 times.
 191 if (info->backoff_ms() == 0) {
1287 76 info->SetBackoffMs(prng_.Next(backoff_init_ms + 1)); // Must be != 0
1288 } else {
1289 115 info->SetBackoffMs(info->backoff_ms() * 2);
1290 }
1291
2/2
✓ Branch 1 taken 39 times.
✓ Branch 2 taken 152 times.
 191 if (info->backoff_ms() > backoff_max_ms) {
1292 39 info->SetBackoffMs(backoff_max_ms);
1293 }
1294
1295 191 LogCvmfs(kLogDownload, kLogDebug,
1296 "(manager '%s' - id %" PRId64 ") backing off for %d ms",
1297 name_.c_str(), info->id(), info->backoff_ms());
1298 191 SafeSleepMs(info->backoff_ms());
1299 191 }
1300
1301 193 void DownloadManager::SetNocache(JobInfo *info) {
1302
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 193 times.
 193 if (info->nocache())
1303 ✗ return;
1304 193 header_lists_->AppendHeader(info->headers(), "Pragma: no-cache");
1305 193 header_lists_->AppendHeader(info->headers(), "Cache-Control: no-cache");
1306 193 curl_easy_setopt(info->curl_handle(), CURLOPT_HTTPHEADER, info->headers());
1307 193 info->SetNocache(true);
1308 }
1309
1310
1311 /**
1312 * Reverse operation of SetNocache. Makes sure that "no-cache" header
1313 * disappears from the list of headers to let proxies work normally.
1314 */
1315 321 void DownloadManager::SetRegularCache(JobInfo *info) {
1316
1/2
✓ Branch 1 taken 321 times.
✗ Branch 2 not taken.
 321 if (info->nocache() == false)
1317 321 return;
1318 ✗ header_lists_->CutHeader("Pragma: no-cache", info->GetHeadersPtr());
1319 ✗ header_lists_->CutHeader("Cache-Control: no-cache", info->GetHeadersPtr());
1320 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_HTTPHEADER, info->headers());
1321 ✗ info->SetNocache(false);
1322 }
1323
1324
1325 /**
1326 * Frees the storage associated with the authz attachment from the job
1327 */
1328 5878 void DownloadManager::ReleaseCredential(JobInfo *info) {
1329
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5878 times.
 5878 if (info->cred_data()) {
1330 ✗ assert(credentials_attachment_ != NULL); // Someone must have set it
1331 ✗ credentials_attachment_->ReleaseCurlHandle(info->curl_handle(),
1332 info->cred_data());
1333 ✗ info->SetCredData(NULL);
1334 }
1335 5878 }
1336
1337
1338 /* Sort links based on the "pri=" parameter */
1339 ✗ static bool sortlinks(const std::string &s1, const std::string &s2) {
1340 ✗ const size_t pos1 = s1.find("; pri=");
1341 ✗ const size_t pos2 = s2.find("; pri=");
1342 int pri1, pri2;
1343 ✗ if ((pos1 != std::string::npos) && (pos2 != std::string::npos)
1344 ✗ && (sscanf(s1.substr(pos1 + 6).c_str(), "%d", &pri1) == 1)
1345 ✗ && (sscanf(s2.substr(pos2 + 6).c_str(), "%d", &pri2) == 1)) {
1346 ✗ return pri1 < pri2;
1347 }
1348 ✗ return false;
1349 }
1350
1351 /**
1352 * Parses Link header and uses it to set a new host chain.
1353 * See rfc6249.
1354 */
1355 ✗ void DownloadManager::ProcessLink(JobInfo *info) {
1356 ✗ std::vector<std::string> links = SplitString(info->link(), ',');
1357 ✗ if (info->link().find("; pri=") != std::string::npos)
1358 ✗ std::sort(links.begin(), links.end(), sortlinks);
1359
1360 ✗ std::vector<std::string> host_list;
1361
1362 ✗ std::vector<std::string>::const_iterator il = links.begin();
1363 ✗ for (; il != links.end(); ++il) {
1364 ✗ const std::string &link = *il;
1365 ✗ if ((link.find("; rel=duplicate") == std::string::npos)
1366 ✗ && (link.find("; rel=\"duplicate\"") == std::string::npos)) {
1367 ✗ LogCvmfs(kLogDownload, kLogDebug,
1368 "skipping link '%s' because it does not contain rel=duplicate",
1369 link.c_str());
1370 ✗ continue;
1371 }
1372 // ignore depth= field since there's nothing useful we can do with it
1373
1374 ✗ size_t start = link.find('<');
1375 ✗ if (start == std::string::npos) {
1376 ✗ LogCvmfs(
1377 kLogDownload, kLogDebug,
1378 "skipping link '%s' because it does not have a left angle bracket",
1379 link.c_str());
1380 ✗ continue;
1381 }
1382
1383 ✗ start++;
1384 ✗ if ((link.substr(start, 7) != "http://")
1385 ✗ && (link.substr(start, 8) != "https://")) {
1386 ✗ LogCvmfs(kLogDownload, kLogDebug,
1387 "skipping link '%s' of unrecognized url protocol", link.c_str());
1388 ✗ continue;
1389 }
1390
1391 ✗ size_t end = link.find('/', start + 8);
1392 ✗ if (end == std::string::npos)
1393 ✗ end = link.find('>');
1394 ✗ if (end == std::string::npos) {
1395 ✗ LogCvmfs(kLogDownload, kLogDebug,
1396 "skipping link '%s' because no slash in url and no right angle "
1397 "bracket",
1398 link.c_str());
1399 ✗ continue;
1400 }
1401 ✗ const std::string host = link.substr(start, end - start);
1402 ✗ LogCvmfs(kLogDownload, kLogDebug, "adding linked host '%s'", host.c_str());
1403 ✗ host_list.push_back(host);
1404 }
1405
1406 ✗ if (host_list.size() > 0) {
1407 ✗ SetHostChain(host_list);
1408 ✗ opt_metalink_timestamp_link_ = time(NULL);
1409 }
1410 }
1411
1412
1413 /**
1414 * Checks the result of a curl download and implements the failure logic, such
1415 * as changing the proxy server. Takes care of cleanup.
1416 *
1417 * \return true if another download should be performed, false otherwise
1418 */
1419 6148 bool DownloadManager::VerifyAndFinalize(const int curl_error, JobInfo *info) {
1420
1/2
✓ Branch 6 taken 6148 times.
✗ Branch 7 not taken.
 6148 LogCvmfs(kLogDownload, kLogDebug,
1421 "(manager '%s' - id %" PRId64 ") "
1422 "Verify downloaded url %s, proxy %s (curl error %d)",
1423 name_.c_str(), info->id(), info->url()->c_str(),
1424
1/2
✓ Branch 1 taken 6148 times.
✗ Branch 2 not taken.
 12296 info->proxy().c_str(), curl_error);
1425
1/2
✓ Branch 2 taken 6148 times.
✗ Branch 3 not taken.
 6148 UpdateStatistics(info->curl_handle());
1426
1427 bool was_metalink;
1428 6148 std::string typ;
1429
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6148 times.
 6148 if (info->current_metalink_chain_index() >= 0) {
1430 ✗ was_metalink = true;
1431 ✗ typ = "metalink";
1432 ✗ if (info->link() != "") {
1433 // process Link header whether or not the redirected URL got an error
1434 ✗ ProcessLink(info);
1435 }
1436 } else {
1437 6148 was_metalink = false;
1438
1/2
✓ Branch 1 taken 6148 times.
✗ Branch 2 not taken.
 6148 typ = "host";
1439 }
1440
1441
1442 // Verification and error classification
1443
3/14
✓ Branch 0 taken 5577 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 76 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 495 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
 6148 switch (curl_error) {
1444 5577 case CURLE_OK:
1445 // Verify content hash
1446
2/2
✓ Branch 1 taken 4028 times.
✓ Branch 2 taken 1549 times.
 5577 if (info->expected_hash()) {
1447
1/2
✓ Branch 1 taken 4028 times.
✗ Branch 2 not taken.
 4028 shash::Any match_hash;
1448
1/2
✓ Branch 2 taken 4028 times.
✗ Branch 3 not taken.
 4028 shash::Final(info->hash_context(), &match_hash);
1449
3/4
✓ Branch 2 taken 4028 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 158 times.
✓ Branch 5 taken 3870 times.
 4028 if (match_hash != *(info->expected_hash())) {
1450
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 158 times.
 158 if (ignore_signature_failures_) {
1451 ✗ LogCvmfs(
1452 kLogDownload, kLogDebug | kLogSyslogErr,
1453 "(manager '%s' - id %" PRId64 ") "
1454 "ignoring failed hash verification of %s (expected %s, got %s)",
1455 name_.c_str(), info->id(), info->url()->c_str(),
1456 ✗ info->expected_hash()->ToString().c_str(),
1457 ✗ match_hash.ToString().c_str());
1458 } else {
1459
1/2
✓ Branch 7 taken 158 times.
✗ Branch 8 not taken.
 316 LogCvmfs(kLogDownload, kLogDebug,
1460 "(manager '%s' - id %" PRId64 ") "
1461 "hash verification of %s failed (expected %s, got %s)",
1462 name_.c_str(), info->id(), info->url()->c_str(),
1463
1/2
✓ Branch 2 taken 158 times.
✗ Branch 3 not taken.
 316 info->expected_hash()->ToString().c_str(),
1464
1/2
✓ Branch 1 taken 158 times.
✗ Branch 2 not taken.
 316 match_hash.ToString().c_str());
1465 158 info->SetErrorCode(kFailBadData);
1466 158 break;
1467 }
1468 }
1469 }
1470
1471 5419 info->SetErrorCode(kFailOk);
1472 5419 break;
1473 ✗ case CURLE_UNSUPPORTED_PROTOCOL:
1474 ✗ info->SetErrorCode(kFailUnsupportedProtocol);
1475 ✗ break;
1476 76 case CURLE_URL_MALFORMAT:
1477 76 info->SetErrorCode(kFailBadUrl);
1478 76 break;
1479 ✗ case CURLE_COULDNT_RESOLVE_PROXY:
1480 ✗ info->SetErrorCode(kFailProxyResolve);
1481 ✗ break;
1482 ✗ case CURLE_COULDNT_RESOLVE_HOST:
1483 ✗ info->SetErrorCode(kFailHostResolve);
1484 ✗ break;
1485 ✗ case CURLE_OPERATION_TIMEDOUT:
1486 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostTooSlow
1487 : kFailProxyTooSlow);
1488 ✗ break;
1489 ✗ case CURLE_PARTIAL_FILE:
1490 case CURLE_GOT_NOTHING:
1491 case CURLE_RECV_ERROR:
1492 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostShortTransfer
1493 : kFailProxyShortTransfer);
1494 ✗ break;
1495 495 case CURLE_FILE_COULDNT_READ_FILE:
1496 case CURLE_COULDNT_CONNECT:
1497
3/6
✓ Branch 1 taken 495 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 17 times.
✓ Branch 6 taken 478 times.
 495 if (info->proxy() != "DIRECT") {
1498 // This is a guess. Fail-over can still change to switching host
1499 17 info->SetErrorCode(kFailProxyConnection);
1500 } else {
1501 478 info->SetErrorCode(kFailHostConnection);
1502 }
1503 495 break;
1504 ✗ case CURLE_TOO_MANY_REDIRECTS:
1505 ✗ info->SetErrorCode(kFailHostConnection);
1506 ✗ break;
1507 ✗ case CURLE_SSL_CACERT_BADFILE:
1508 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
1509 "(manager '%s' -id %" PRId64 ") "
1510 "Failed to load certificate bundle. "
1511 "X509_CERT_BUNDLE might point to the wrong location.",
1512 name_.c_str(), info->id());
1513 ✗ info->SetErrorCode(kFailHostConnection);
1514 ✗ break;
1515 // As of curl 7.62.0, CURLE_SSL_CACERT is the same as
1516 // CURLE_PEER_FAILED_VERIFICATION
1517 ✗ case CURLE_PEER_FAILED_VERIFICATION:
1518 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
1519 "(manager '%s' - id %" PRId64 ") "
1520 "invalid SSL certificate of remote host. "
1521 "X509_CERT_DIR and/or X509_CERT_BUNDLE might point to the wrong "
1522 "location.",
1523 name_.c_str(), info->id());
1524 ✗ info->SetErrorCode(kFailHostConnection);
1525 ✗ break;
1526 ✗ case CURLE_ABORTED_BY_CALLBACK:
1527 case CURLE_WRITE_ERROR:
1528 // Error set by callback
1529 ✗ break;
1530 ✗ case CURLE_SEND_ERROR:
1531 // The curl error CURLE_SEND_ERROR can be seen when a cache is misbehaving
1532 // and closing connections before the http request send is completed.
1533 // Handle this error, treating it as a short transfer error.
1534 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostShortTransfer
1535 : kFailProxyShortTransfer);
1536 ✗ break;
1537 ✗ default:
1538 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
1539 "(manager '%s' - id %" PRId64 ") "
1540 "unexpected curl error (%d) while trying to fetch %s",
1541 name_.c_str(), info->id(), curl_error, info->url()->c_str());
1542 ✗ info->SetErrorCode(kFailOther);
1543 ✗ break;
1544 }
1545
1546 std::vector<std::string> *host_chain;
1547 unsigned char num_used_hosts;
1548
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6148 times.
 6148 if (was_metalink) {
1549 ✗ host_chain = opt_metalink_.chain;
1550 ✗ num_used_hosts = info->num_used_metalinks();
1551 } else {
1552 6148 host_chain = opt_host_.chain;
1553 6148 num_used_hosts = info->num_used_hosts();
1554 }
1555
1556 // Determination if download should be repeated
1557 6148 bool try_again = false;
1558
1/2
✓ Branch 1 taken 6148 times.
✗ Branch 2 not taken.
 6148 bool same_url_retry = CanRetry(info);
1559
2/2
✓ Branch 1 taken 729 times.
✓ Branch 2 taken 5419 times.
 6148 if (info->error_code() != kFailOk) {
1560 729 const MutexLockGuard m(lock_options_);
1561
2/2
✓ Branch 1 taken 158 times.
✓ Branch 2 taken 571 times.
 729 if (info->error_code() == kFailBadData) {
1562
2/2
✓ Branch 1 taken 79 times.
✓ Branch 2 taken 79 times.
 158 if (!info->nocache()) {
1563 79 try_again = true;
1564 } else {
1565 // Make it a host failure
1566
1/2
✓ Branch 4 taken 79 times.
✗ Branch 5 not taken.
 79 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1567 "(manager '%s' - id %" PRId64 ") "
1568 "data corruption with no-cache header, try another %s",
1569 name_.c_str(), info->id(), typ.c_str());
1570
1571 79 info->SetErrorCode(kFailHostHttp);
1572 }
1573 }
1574 729 if (same_url_retry
1575
5/6
✓ Branch 0 taken 538 times.
✓ Branch 1 taken 191 times.
✓ Branch 3 taken 538 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 242 times.
✓ Branch 6 taken 487 times.
 1267 || (((info->error_code() == kFailHostResolve)
1576
2/2
✓ Branch 2 taken 251 times.
✓ Branch 3 taken 287 times.
 538 || IsHostTransferError(info->error_code())
1577
2/2
✓ Branch 1 taken 79 times.
✓ Branch 2 taken 172 times.
 251 || (info->error_code() == kFailHostHttp))
1578
3/4
✓ Branch 1 taken 137 times.
✓ Branch 2 taken 229 times.
✓ Branch 3 taken 137 times.
✗ Branch 4 not taken.
 366 && info->probe_hosts() && host_chain
1579
2/2
✓ Branch 1 taken 51 times.
✓ Branch 2 taken 86 times.
 137 && (num_used_hosts < host_chain->size()))) {
1580 242 try_again = true;
1581 }
1582 729 if (same_url_retry
1583
5/6
✓ Branch 0 taken 538 times.
✓ Branch 1 taken 191 times.
✓ Branch 3 taken 538 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 208 times.
✓ Branch 6 taken 521 times.
 1267 || (((info->error_code() == kFailProxyResolve)
1584
2/2
✓ Branch 2 taken 521 times.
✓ Branch 3 taken 17 times.
 538 || IsProxyTransferError(info->error_code())
1585
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 521 times.
 521 || (info->error_code() == kFailProxyHttp)))) {
1586
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 208 times.
 208 if (sharding_policy_.UseCount() > 0) { // sharding policy
1587 ✗ try_again = true;
1588 ✗ same_url_retry = false;
1589 } else { // no sharding
1590 208 try_again = true;
1591 // If all proxies failed, do a next round with the next host
1592
4/6
✓ Branch 0 taken 17 times.
✓ Branch 1 taken 191 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 17 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 208 times.
 208 if (!same_url_retry && (info->num_used_proxies() >= opt_num_proxies_)) {
1593 // Check if this can be made a host fail-over
1594 ✗ if (info->probe_hosts() && host_chain
1595 ✗ && (num_used_hosts < host_chain->size())) {
1596 // reset proxy group if not already performed by other handle
1597 ✗ if (opt_proxy_groups_) {
1598 ✗ if ((opt_proxy_groups_current_ > 0)
1599 ✗ || (opt_proxy_groups_current_burned_ > 0)) {
1600 ✗ opt_proxy_groups_current_ = 0;
1601 ✗ opt_timestamp_backup_proxies_ = 0;
1602 ✗ const std::string msg = "reset proxies for " + typ
1603 ✗ + " failover";
1604 ✗ RebalanceProxiesUnlocked(msg);
1605 }
1606 }
1607
1608 // Make it a host failure
1609 ✗ LogCvmfs(kLogDownload, kLogDebug,
1610 "(manager '%s' - id %" PRId64 ") make it a %s failure",
1611 name_.c_str(), info->id(), typ.c_str());
1612 ✗ info->SetNumUsedProxies(1);
1613 ✗ info->SetErrorCode(kFailHostAfterProxy);
1614 } else {
1615 ✗ if (failover_indefinitely_) {
1616 // Instead of giving up, reset the num_used_proxies counter,
1617 // switch proxy and try again
1618 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1619 "(manager '%s' - id %" PRId64 ") "
1620 "VerifyAndFinalize() would fail the download here. "
1621 "Instead switch proxy and retry download. "
1622 "typ=%s "
1623 "info->probe_hosts=%d host_chain=%p num_used_hosts=%d "
1624 "host_chain->size()=%lu same_url_retry=%d "
1625 "info->num_used_proxies=%d opt_num_proxies_=%d",
1626 name_.c_str(), info->id(), typ.c_str(),
1627 ✗ static_cast<int>(info->probe_hosts()), host_chain,
1628 ✗ num_used_hosts, host_chain ? host_chain->size() : -1,
1629 static_cast<int>(same_url_retry),
1630 ✗ info->num_used_proxies(), opt_num_proxies_);
1631 ✗ info->SetNumUsedProxies(1);
1632 ✗ RebalanceProxiesUnlocked(
1633 "download failed - failover indefinitely");
1634 ✗ try_again = !Interrupted(fqrn_, info);
1635 } else {
1636 ✗ try_again = false;
1637 }
1638 }
1639 } // Make a proxy failure a host failure
1640 } // Proxy failure assumed
1641 } // end !sharding
1642 729 }
1643
1644
2/2
✓ Branch 0 taken 338 times.
✓ Branch 1 taken 5810 times.
 6148 if (try_again) {
1645
1/2
✓ Branch 3 taken 338 times.
✗ Branch 4 not taken.
 338 LogCvmfs(kLogDownload, kLogDebug,
1646 "(manager '%s' - id %" PRId64 ") "
1647 "Trying again on same curl handle, same url: %d, "
1648 "error code %d no-cache %d",
1649 338 name_.c_str(), info->id(), same_url_retry, info->error_code(),
1650 338 info->nocache());
1651 // Reset internal state and destination
1652
4/8
✓ Branch 1 taken 338 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 338 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 338 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 338 times.
 338 if (info->sink() != NULL && info->sink()->Reset() != 0) {
1653 ✗ info->SetErrorCode(kFailLocalIO);
1654 ✗ goto verify_and_finalize_stop;
1655 }
1656
6/8
✓ Branch 1 taken 17 times.
✓ Branch 2 taken 321 times.
✓ Branch 5 taken 17 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 17 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 17 times.
✓ Branch 10 taken 321 times.
 338 if (info->interrupt_cue() && info->interrupt_cue()->IsCanceled()) {
1657 17 info->SetErrorCode(kFailCanceled);
1658 17 goto verify_and_finalize_stop;
1659 }
1660
1661
2/2
✓ Branch 1 taken 196 times.
✓ Branch 2 taken 125 times.
 321 if (info->expected_hash()) {
1662
1/2
✓ Branch 2 taken 196 times.
✗ Branch 3 not taken.
 196 shash::Init(info->hash_context());
1663 }
1664
2/2
✓ Branch 1 taken 196 times.
✓ Branch 2 taken 125 times.
 321 if (info->compressed()) {
1665
1/2
✓ Branch 2 taken 196 times.
✗ Branch 3 not taken.
 196 zlib::DecompressInit(info->GetZstreamPtr());
1666 }
1667
1668
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 321 times.
 321 if (sharding_policy_.UseCount() > 0) { // sharding policy
1669 ✗ ReleaseCredential(info);
1670 ✗ SetUrlOptions(info);
1671 } else { // no sharding policy
1672
1/2
✓ Branch 1 taken 321 times.
✗ Branch 2 not taken.
 321 SetRegularCache(info);
1673
1674 // Failure handling
1675 321 bool switch_proxy = false;
1676 321 bool switch_host = false;
1677
2/4
✓ Branch 1 taken 79 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 242 times.
 321 switch (info->error_code()) {
1678 79 case kFailBadData:
1679
1/2
✓ Branch 1 taken 79 times.
✗ Branch 2 not taken.
 79 SetNocache(info);
1680 79 break;
1681 ✗ case kFailProxyResolve:
1682 case kFailProxyHttp:
1683 ✗ switch_proxy = true;
1684 ✗ break;
1685 ✗ case kFailHostResolve:
1686 case kFailHostHttp:
1687 case kFailHostAfterProxy:
1688 ✗ switch_host = true;
1689 ✗ break;
1690 242 default:
1691
2/2
✓ Branch 2 taken 17 times.
✓ Branch 3 taken 225 times.
 242 if (IsProxyTransferError(info->error_code())) {
1692
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 17 times.
 17 if (same_url_retry) {
1693 ✗ Backoff(info);
1694 } else {
1695 17 switch_proxy = true;
1696 }
1697
1/2
✓ Branch 2 taken 225 times.
✗ Branch 3 not taken.
 225 } else if (IsHostTransferError(info->error_code())) {
1698
2/2
✓ Branch 0 taken 191 times.
✓ Branch 1 taken 34 times.
 225 if (same_url_retry) {
1699
1/2
✓ Branch 1 taken 191 times.
✗ Branch 2 not taken.
 191 Backoff(info);
1700 } else {
1701 34 switch_host = true;
1702 }
1703 } else {
1704 // No other errors expected when retrying
1705 ✗ PANIC(NULL);
1706 }
1707 }
1708
2/2
✓ Branch 0 taken 17 times.
✓ Branch 1 taken 304 times.
 321 if (switch_proxy) {
1709
1/2
✓ Branch 1 taken 17 times.
✗ Branch 2 not taken.
 17 ReleaseCredential(info);
1710
1/2
✓ Branch 1 taken 17 times.
✗ Branch 2 not taken.
 17 SwitchProxy(info);
1711 17 info->SetNumUsedProxies(info->num_used_proxies() + 1);
1712
1/2
✓ Branch 1 taken 17 times.
✗ Branch 2 not taken.
 17 SetUrlOptions(info);
1713 }
1714
2/2
✓ Branch 0 taken 34 times.
✓ Branch 1 taken 287 times.
 321 if (switch_host) {
1715
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 ReleaseCredential(info);
1716
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 34 times.
 34 if (was_metalink) {
1717 ✗ SwitchMetalink(info);
1718 ✗ info->SetNumUsedMetalinks(num_used_hosts + 1);
1719 } else {
1720
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 SwitchHost(info);
1721 34 info->SetNumUsedHosts(num_used_hosts + 1);
1722 }
1723
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 SetUrlOptions(info);
1724 }
1725 } // end !sharding
1726
1727
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 321 times.
 321 if (failover_indefinitely_) {
1728 // try again, breaking if there's a cvmfs reload happening and we are in a
1729 // proxy failover. This will EIO the call application.
1730 ✗ return !Interrupted(fqrn_, info);
1731 }
1732 321 return true; // try again
1733 }
1734
1735 5810 verify_and_finalize_stop:
1736 // Finalize, flush destination file
1737
1/2
✓ Branch 1 taken 5827 times.
✗ Branch 2 not taken.
 5827 ReleaseCredential(info);
1738
4/8
✓ Branch 1 taken 5827 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 5827 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 5827 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 5827 times.
 5827 if (info->sink() != NULL && info->sink()->Flush() != 0) {
1739 ✗ info->SetErrorCode(kFailLocalIO);
1740 }
1741
1742
2/2
✓ Branch 1 taken 4146 times.
✓ Branch 2 taken 1681 times.
 5827 if (info->compressed())
1743
1/2
✓ Branch 2 taken 4146 times.
✗ Branch 3 not taken.
 4146 zlib::DecompressFini(info->GetZstreamPtr());
1744
1745
1/2
✓ Branch 1 taken 5827 times.
✗ Branch 2 not taken.
 5827 if (info->headers()) {
1746
1/2
✓ Branch 2 taken 5827 times.
✗ Branch 3 not taken.
 5827 header_lists_->PutList(info->headers());
1747 5827 info->SetHeaders(NULL);
1748 }
1749
1750 5827 return false; // stop transfer and return to Fetch()
1751 6148 }
1752
1753 3775 DownloadManager::~DownloadManager() {
1754 // cleaned up fini
1755
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 3775 times.
 3775 if (sharding_policy_.UseCount() > 0) {
1756 ✗ sharding_policy_.Reset();
1757 }
1758
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 3775 times.
 3775 if (health_check_.UseCount() > 0) {
1759 ✗ if (health_check_.Unique()) {
1760 ✗ LogCvmfs(kLogDownload, kLogDebug,
1761 "(manager '%s') Stopping healthcheck thread", name_.c_str());
1762 ✗ health_check_->StopHealthcheck();
1763 }
1764 ✗ health_check_.Reset();
1765 }
1766
1767
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 3775 times.
 3775 if (atomic_xadd32(&multi_threaded_, 0) == 1) {
1768 // Shutdown I/O thread
1769 ✗ pipe_terminate_->Write(kPipeTerminateSignal);
1770 ✗ pthread_join(thread_download_, NULL);
1771 // All handles are removed from the multi stack
1772 ✗ pipe_terminate_.Destroy();
1773 ✗ pipe_jobs_.Destroy();
1774 }
1775
1776 7550 for (set<CURL *>::iterator i = pool_handles_idle_->begin(),
1777 3775 iEnd = pool_handles_idle_->end();
1778
2/2
✓ Branch 1 taken 2091 times.
✓ Branch 2 taken 3775 times.
 5866 i != iEnd;
1779 2091 ++i) {
1780 2091 curl_easy_cleanup(*i);
1781 }
1782
1783
1/2
✓ Branch 0 taken 3775 times.
✗ Branch 1 not taken.
 3775 delete pool_handles_idle_;
1784
1/2
✓ Branch 0 taken 3775 times.
✗ Branch 1 not taken.
 3775 delete pool_handles_inuse_;
1785 3775 curl_multi_cleanup(curl_multi_);
1786
1787
1/2
✓ Branch 0 taken 3775 times.
✗ Branch 1 not taken.
 3775 delete header_lists_;
1788
1/2
✓ Branch 0 taken 3775 times.
✗ Branch 1 not taken.
 3775 if (user_agent_)
1789 3775 free(user_agent_);
1790
1791
1/2
✓ Branch 0 taken 3775 times.
✗ Branch 1 not taken.
 3775 delete counters_;
1792
2/2
✓ Branch 0 taken 1600 times.
✓ Branch 1 taken 2175 times.
 3775 delete opt_host_.chain;
1793
2/2
✓ Branch 0 taken 1600 times.
✓ Branch 1 taken 2175 times.
 3775 delete opt_host_chain_rtt_;
1794
2/2
✓ Branch 0 taken 1553 times.
✓ Branch 1 taken 2222 times.
 3775 delete opt_proxy_groups_;
1795
1796 3775 curl_global_cleanup();
1797
1/2
✓ Branch 0 taken 3775 times.
✗ Branch 1 not taken.
 3775 delete resolver_;
1798
1799 // old destructor
1800 3775 pthread_mutex_destroy(lock_options_);
1801 3775 pthread_mutex_destroy(lock_synchronous_mode_);
1802 3775 free(lock_options_);
1803 3775 free(lock_synchronous_mode_);
1804 3775 }
1805
1806 3776 void DownloadManager::InitHeaders() {
1807 // User-Agent
1808
1/2
✓ Branch 2 taken 3776 times.
✗ Branch 3 not taken.
 3776 string cernvm_id = "User-Agent: cvmfs ";
1809 #ifdef CVMFS_LIBCVMFS
1810
1/2
✓ Branch 1 taken 3776 times.
✗ Branch 2 not taken.
 3776 cernvm_id += "libcvmfs ";
1811 #else
1812 ✗ cernvm_id += "Fuse ";
1813 #endif
1814
2/4
✓ Branch 2 taken 3776 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 3776 times.
✗ Branch 6 not taken.
 3776 cernvm_id += string(CVMFS_VERSION);
1815
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 3776 times.
 3776 if (getenv("CERNVM_UUID") != NULL) {
1816 cernvm_id += " "
1817 ✗ + sanitizer::InputSanitizer("az AZ 09 -")
1818 ✗ .Filter(getenv("CERNVM_UUID"));
1819 }
1820 3776 user_agent_ = strdup(cernvm_id.c_str());
1821
1822
1/2
✓ Branch 1 taken 3776 times.
✗ Branch 2 not taken.
 3776 header_lists_ = new HeaderLists();
1823
1824
1/2
✓ Branch 1 taken 3776 times.
✗ Branch 2 not taken.
 3776 default_headers_ = header_lists_->GetList("Connection: Keep-Alive");
1825
1/2
✓ Branch 1 taken 3776 times.
✗ Branch 2 not taken.
 3776 header_lists_->AppendHeader(default_headers_, "Pragma:");
1826
1/2
✓ Branch 1 taken 3776 times.
✗ Branch 2 not taken.
 3776 header_lists_->AppendHeader(default_headers_, user_agent_);
1827 3776 }
1828
1829 3776 DownloadManager::DownloadManager(const unsigned max_pool_handles,
1830 const perf::StatisticsTemplate &statistics,
1831 3776 const std::string &name)
1832 3776 : prng_(Prng())
1833 3776 , pool_handles_idle_(new set<CURL *>)
1834 3776 , pool_handles_inuse_(new set<CURL *>)
1835 3776 , pool_max_handles_(max_pool_handles)
1836 3776 , pipe_terminate_(NULL)
1837
1/2
✓ Branch 1 taken 3776 times.
✗ Branch 2 not taken.
 3776 , pipe_jobs_(NULL)
1838 3776 , watch_fds_(NULL)
1839 3776 , watch_fds_size_(0)
1840 3776 , watch_fds_inuse_(0)
1841 3776 , watch_fds_max_(4 * max_pool_handles)
1842 3776 , opt_timeout_proxy_(5)
1843 3776 , opt_timeout_direct_(10)
1844 3776 , opt_low_speed_limit_(1024)
1845 3776 , opt_max_retries_(0)
1846 3776 , opt_backoff_init_ms_(0)
1847 3776 , opt_backoff_max_ms_(0)
1848 3776 , enable_info_header_(false)
1849 3776 , opt_ipv4_only_(false)
1850 3776 , follow_redirects_(false)
1851 3776 , ignore_signature_failures_(false)
1852 3776 , enable_http_tracing_(false)
1853 3776 , opt_metalink_(NULL, 0, 0, 0)
1854 3776 , opt_metalink_timestamp_link_(0)
1855 3776 , opt_host_(NULL, 0, 0, 0)
1856 3776 , opt_host_chain_rtt_(NULL)
1857 3776 , opt_proxy_groups_(NULL)
1858 3776 , opt_proxy_groups_current_(0)
1859 3776 , opt_proxy_groups_current_burned_(0)
1860 3776 , opt_proxy_groups_fallback_(0)
1861 3776 , opt_num_proxies_(0)
1862 3776 , opt_proxy_shard_(false)
1863 3776 , failover_indefinitely_(false)
1864
1/2
✓ Branch 1 taken 3776 times.
✗ Branch 2 not taken.
 3776 , name_(name)
1865 3776 , opt_ip_preference_(dns::kIpPreferSystem)
1866 3776 , opt_timestamp_backup_proxies_(0)
1867 3776 , opt_timestamp_failover_proxies_(0)
1868 3776 , opt_proxy_groups_reset_after_(0)
1869 3776 , credentials_attachment_(NULL)
1870
4/8
✓ Branch 13 taken 3776 times.
✗ Branch 14 not taken.
✓ Branch 16 taken 3776 times.
✗ Branch 17 not taken.
✓ Branch 19 taken 3776 times.
✗ Branch 20 not taken.
✓ Branch 23 taken 3776 times.
✗ Branch 24 not taken.
 11328 , counters_(new Counters(statistics)) {
1871 3776 atomic_init32(&multi_threaded_);
1872
1873 3776 lock_options_ = reinterpret_cast<pthread_mutex_t *>(
1874 3776 smalloc(sizeof(pthread_mutex_t)));
1875 3776 int retval = pthread_mutex_init(lock_options_, NULL);
1876
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3776 times.
 3776 assert(retval == 0);
1877 3776 lock_synchronous_mode_ = reinterpret_cast<pthread_mutex_t *>(
1878 3776 smalloc(sizeof(pthread_mutex_t)));
1879 3776 retval = pthread_mutex_init(lock_synchronous_mode_, NULL);
1880
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3776 times.
 3776 assert(retval == 0);
1881
1882
1/2
✓ Branch 1 taken 3776 times.
✗ Branch 2 not taken.
 3776 retval = curl_global_init(CURL_GLOBAL_ALL);
1883
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3776 times.
 3776 assert(retval == CURLE_OK);
1884
1885
1/2
✓ Branch 1 taken 3776 times.
✗ Branch 2 not taken.
 3776 InitHeaders();
1886
1887
1/2
✓ Branch 1 taken 3776 times.
✗ Branch 2 not taken.
 3776 curl_multi_ = curl_multi_init();
1888
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3776 times.
 3776 assert(curl_multi_ != NULL);
1889
1/2
✓ Branch 1 taken 3776 times.
✗ Branch 2 not taken.
 3776 curl_multi_setopt(curl_multi_, CURLMOPT_SOCKETFUNCTION, CallbackCurlSocket);
1890
1/2
✓ Branch 1 taken 3776 times.
✗ Branch 2 not taken.
 3776 curl_multi_setopt(curl_multi_, CURLMOPT_SOCKETDATA,
1891 static_cast<void *>(this));
1892
1/2
✓ Branch 1 taken 3776 times.
✗ Branch 2 not taken.
 3776 curl_multi_setopt(curl_multi_, CURLMOPT_MAXCONNECTS, watch_fds_max_);
1893
1/2
✓ Branch 1 taken 3776 times.
✗ Branch 2 not taken.
 3776 curl_multi_setopt(curl_multi_, CURLMOPT_MAX_TOTAL_CONNECTIONS,
1894 pool_max_handles_);
1895
1896 3776 prng_.InitLocaltime();
1897
1898 // Name resolving
1899 3776 if ((getenv("CVMFS_IPV4_ONLY") != NULL)
1900
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 3776 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 3776 times.
 3776 && (strlen(getenv("CVMFS_IPV4_ONLY")) > 0)) {
1901 ✗ opt_ipv4_only_ = true;
1902 }
1903
1/2
✓ Branch 1 taken 3776 times.
✗ Branch 2 not taken.
 3776 resolver_ = dns::NormalResolver::Create(opt_ipv4_only_, kDnsDefaultRetries,
1904 kDnsDefaultTimeoutMs);
1905
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3776 times.
 3776 assert(resolver_);
1906 3776 }
1907
1908 /**
1909 * Spawns the I/O worker thread and switches the module in multi-threaded mode.
1910 * No way back except Fini(); Init();
1911 */
1912 ✗ void DownloadManager::Spawn() {
1913 ✗ pipe_terminate_ = new Pipe<kPipeThreadTerminator>();
1914 ✗ pipe_jobs_ = new Pipe<kPipeDownloadJobs>();
1915
1916 ✗ const int retval = pthread_create(&thread_download_, NULL, MainDownload,
1917 static_cast<void *>(this));
1918 ✗ assert(retval == 0);
1919
1920 ✗ atomic_inc32(&multi_threaded_);
1921
1922 ✗ if (health_check_.UseCount() > 0) {
1923 ✗ LogCvmfs(kLogDownload, kLogDebug,
1924 "(manager '%s') Starting healthcheck thread", name_.c_str());
1925 ✗ health_check_->StartHealthcheck();
1926 }
1927 }
1928
1929
1930 /**
1931 * Downloads data from an insecure outside channel (currently HTTP or file).
1932 */
1933 5827 Failures DownloadManager::Fetch(JobInfo *info) {
1934
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5827 times.
 5827 assert(info != NULL);
1935
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5827 times.
 5827 assert(info->url() != NULL);
1936
1937 Failures result;
1938
1/2
✓ Branch 1 taken 5827 times.
✗ Branch 2 not taken.
 5827 result = PrepareDownloadDestination(info);
1939
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5827 times.
 5827 if (result != kFailOk)
1940 ✗ return result;
1941
1942
2/2
✓ Branch 1 taken 4149 times.
✓ Branch 2 taken 1678 times.
 5827 if (info->expected_hash()) {
1943 4149 const shash::Algorithms algorithm = info->expected_hash()->algorithm;
1944 4149 info->GetHashContextPtr()->algorithm = algorithm;
1945
1/2
✓ Branch 1 taken 4149 times.
✗ Branch 2 not taken.
 4149 info->GetHashContextPtr()->size = shash::GetContextSize(algorithm);
1946 4149 info->GetHashContextPtr()->buffer = alloca(info->hash_context().size);
1947 }
1948
1949 // In case JobInfo object is being reused
1950
2/4
✓ Branch 2 taken 5827 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 5827 times.
✗ Branch 6 not taken.
 5827 info->SetLink("");
1951
1952 // Prepare cvmfs-info: header, allocate string on the stack
1953 5827 info->SetInfoHeader(NULL);
1954
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5827 times.
 5827 if (enable_info_header_) {
1955 ✗ const string header_info = info->GetInfoHeaderContents(info_header_template_);
1956 ✗ if (header_info != "") {
1957 ✗ const char * const header_name = "cvmfs-info: ";
1958 ✗ const size_t header_name_len = strlen(header_name);
1959 ✗ const size_t info_len = header_info.length();
1960 ✗ char *buf = static_cast<char *>(alloca(header_name_len + info_len + 1));
1961 ✗ memcpy(buf, header_name, header_name_len);
1962 ✗ memcpy(buf + header_name_len, header_info.c_str(), info_len);
1963 ✗ buf[header_name_len + info_len] = '\0';
1964 ✗ info->SetInfoHeader(buf);
1965 }
1966 }
1967
1968
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5827 times.
 5827 if (enable_http_tracing_) {
1969 ✗ const std::string str_pid = "X-CVMFS-PID: " + StringifyInt(info->pid());
1970 ✗ const std::string str_gid = "X-CVMFS-GID: " + StringifyUint(info->gid());
1971 ✗ const std::string str_uid = "X-CVMFS-UID: " + StringifyUint(info->uid());
1972
1973 // will be auto freed at the end of this function Fetch(JobInfo *info)
1974 ✗ info->SetTracingHeaderPid(static_cast<char *>(alloca(str_pid.size() + 1)));
1975 ✗ info->SetTracingHeaderGid(static_cast<char *>(alloca(str_gid.size() + 1)));
1976 ✗ info->SetTracingHeaderUid(static_cast<char *>(alloca(str_uid.size() + 1)));
1977
1978 ✗ memcpy(info->tracing_header_pid(), str_pid.c_str(), str_pid.size() + 1);
1979 ✗ memcpy(info->tracing_header_gid(), str_gid.c_str(), str_gid.size() + 1);
1980 ✗ memcpy(info->tracing_header_uid(), str_uid.c_str(), str_uid.size() + 1);
1981 }
1982
1983
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5827 times.
 5827 if (atomic_xadd32(&multi_threaded_, 0) == 1) {
1984 ✗ if (!info->IsValidPipeJobResults()) {
1985 ✗ info->CreatePipeJobResults();
1986 }
1987 ✗ if (!info->IsValidDataTube()) {
1988 ✗ info->CreateDataTube();
1989 }
1990
1991 // LogCvmfs(kLogDownload, kLogDebug, "send job to thread, pipe %d %d",
1992 // info->wait_at[0], info->wait_at[1]);
1993 ✗ pipe_jobs_->Write<JobInfo *>(info);
1994
1995 do {
1996 ✗ DataTubeElement *ele = info->GetDataTubePtr()->PopFront();
1997
1998 ✗ if (ele->action == kActionStop) {
1999 ✗ delete ele;
2000 ✗ break;
2001 }
2002 // TODO(heretherebedragons) add compression
2003 ✗ } while (true);
2004
2005 ✗ info->GetPipeJobResultPtr()->Read<download::Failures>(&result);
2006 // LogCvmfs(kLogDownload, kLogDebug, "got result %d", result);
2007 } else {
2008 5827 const MutexLockGuard l(lock_synchronous_mode_);
2009
1/2
✓ Branch 1 taken 5827 times.
✗ Branch 2 not taken.
 5827 CURL *handle = AcquireCurlHandle();
2010
1/2
✓ Branch 1 taken 5827 times.
✗ Branch 2 not taken.
 5827 InitializeRequest(info, handle);
2011
1/2
✓ Branch 1 taken 5827 times.
✗ Branch 2 not taken.
 5827 SetUrlOptions(info);
2012 // curl_easy_setopt(handle, CURLOPT_VERBOSE, 1);
2013 int retval;
2014 do {
2015
1/2
✓ Branch 1 taken 6148 times.
✗ Branch 2 not taken.
 6148 retval = curl_easy_perform(handle);
2016 6148 perf::Inc(counters_->n_requests);
2017 double elapsed;
2018
1/2
✓ Branch 1 taken 6148 times.
✗ Branch 2 not taken.
 6148 if (curl_easy_getinfo(handle, CURLINFO_TOTAL_TIME, &elapsed)
2019
1/2
✓ Branch 0 taken 6148 times.
✗ Branch 1 not taken.
 6148 == CURLE_OK) {
2020 6148 perf::Xadd(counters_->sz_transfer_time,
2021 6148 static_cast<int64_t>(elapsed * 1000));
2022 }
2023
3/4
✓ Branch 1 taken 6148 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 321 times.
✓ Branch 4 taken 5827 times.
 6148 } while (VerifyAndFinalize(retval, info));
2024 5827 result = info->error_code();
2025
1/2
✓ Branch 2 taken 5827 times.
✗ Branch 3 not taken.
 5827 ReleaseCurlHandle(info->curl_handle());
2026 5827 }
2027
2028
2/2
✓ Branch 0 taken 408 times.
✓ Branch 1 taken 5419 times.
 5827 if (result != kFailOk) {
2029
1/2
✓ Branch 4 taken 408 times.
✗ Branch 5 not taken.
 408 LogCvmfs(kLogDownload, kLogDebug,
2030 "(manager '%s' - id %" PRId64 ") "
2031 "download failed (error %d - %s)",
2032 name_.c_str(), info->id(), result, Code2Ascii(result));
2033
2034
1/2
✓ Branch 1 taken 408 times.
✗ Branch 2 not taken.
 408 if (info->sink() != NULL) {
2035
1/2
✓ Branch 2 taken 408 times.
✗ Branch 3 not taken.
 408 info->sink()->Purge();
2036 }
2037 }
2038
2039 5827 return result;
2040 }
2041
2042
2043 /**
2044 * Used by the client to connect the authz session manager to the download
2045 * manager.
2046 */
2047 774 void DownloadManager::SetCredentialsAttachment(CredentialsAttachment *ca) {
2048 774 const MutexLockGuard m(lock_options_);
2049 774 credentials_attachment_ = ca;
2050 774 }
2051
2052 /**
2053 * Gets the DNS sever.
2054 */
2055 ✗ std::string DownloadManager::GetDnsServer() const { return opt_dns_server_; }
2056
2057 /**
2058 * Sets a DNS server. Only for testing as it cannot be reverted to the system
2059 * default.
2060 */
2061 ✗ void DownloadManager::SetDnsServer(const string &address) {
2062 ✗ if (!address.empty()) {
2063 ✗ const MutexLockGuard m(lock_options_);
2064 ✗ opt_dns_server_ = address;
2065 ✗ assert(!opt_dns_server_.empty());
2066
2067 ✗ vector<string> servers;
2068 ✗ servers.push_back(address);
2069 ✗ const bool retval = resolver_->SetResolvers(servers);
2070 ✗ assert(retval);
2071 }
2072 ✗ LogCvmfs(kLogDownload, kLogSyslog, "(manager '%s') set nameserver to %s",
2073 name_.c_str(), address.c_str());
2074 }
2075
2076
2077 /**
2078 * Sets the DNS query timeout parameters.
2079 */
2080 1559 void DownloadManager::SetDnsParameters(const unsigned retries,
2081 const unsigned timeout_ms) {
2082 1559 const MutexLockGuard m(lock_options_);
2083 1559 if ((resolver_->retries() == retries)
2084
3/6
✓ Branch 0 taken 1559 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 1559 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 1559 times.
✗ Branch 6 not taken.
 1559 && (resolver_->timeout_ms() == timeout_ms)) {
2085 1559 return;
2086 }
2087 ✗ delete resolver_;
2088 ✗ resolver_ = NULL;
2089 ✗ resolver_ = dns::NormalResolver::Create(opt_ipv4_only_, retries, timeout_ms);
2090 ✗ assert(resolver_);
2091
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1559 times.
 1559 }
2092
2093
2094 1559 void DownloadManager::SetDnsTtlLimits(const unsigned min_seconds,
2095 const unsigned max_seconds) {
2096 1559 const MutexLockGuard m(lock_options_);
2097 1559 resolver_->set_min_ttl(min_seconds);
2098 1559 resolver_->set_max_ttl(max_seconds);
2099 1559 }
2100
2101
2102 ✗ void DownloadManager::SetIpPreference(dns::IpPreference preference) {
2103 ✗ const MutexLockGuard m(lock_options_);
2104 ✗ opt_ip_preference_ = preference;
2105 }
2106
2107
2108 /**
2109 * Sets two timeout values for proxied and for direct connections, respectively.
2110 * The timeout counts for all sorts of connection phases,
2111 * DNS, HTTP connect, etc.
2112 */
2113 2415 void DownloadManager::SetTimeout(const unsigned seconds_proxy,
2114 const unsigned seconds_direct) {
2115 2415 const MutexLockGuard m(lock_options_);
2116 2415 opt_timeout_proxy_ = seconds_proxy;
2117 2415 opt_timeout_direct_ = seconds_direct;
2118 2415 }
2119
2120
2121 /**
2122 * Sets contains the average transfer speed in bytes per second that the
2123 * transfer should be below during CURLOPT_LOW_SPEED_TIME seconds for libcurl to
2124 * consider it to be too slow and abort. Only effective for new connections.
2125 */
2126 ✗ void DownloadManager::SetLowSpeedLimit(const unsigned low_speed_limit) {
2127 ✗ const MutexLockGuard m(lock_options_);
2128 ✗ opt_low_speed_limit_ = low_speed_limit;
2129 }
2130
2131
2132 /**
2133 * Receives the currently active timeout values.
2134 */
2135 751 void DownloadManager::GetTimeout(unsigned *seconds_proxy,
2136 unsigned *seconds_direct) {
2137 751 const MutexLockGuard m(lock_options_);
2138 751 *seconds_proxy = opt_timeout_proxy_;
2139 751 *seconds_direct = opt_timeout_direct_;
2140 751 }
2141
2142
2143 /**
2144 * Parses a list of ';'-separated hosts for the metalink chain. The empty
2145 * string removes the metalink list.
2146 */
2147 ✗ void DownloadManager::SetMetalinkChain(const string &metalink_list) {
2148 ✗ SetMetalinkChain(SplitString(metalink_list, ';'));
2149 }
2150
2151
2152 ✗ void DownloadManager::SetMetalinkChain(
2153 const std::vector<std::string> &metalink_list) {
2154 ✗ const MutexLockGuard m(lock_options_);
2155 ✗ opt_metalink_.timestamp_backup = 0;
2156 ✗ delete opt_metalink_.chain;
2157 ✗ opt_metalink_.current = 0;
2158
2159 ✗ if (metalink_list.empty()) {
2160 ✗ opt_metalink_.chain = NULL;
2161 ✗ return;
2162 }
2163
2164 ✗ opt_metalink_.chain = new vector<string>(metalink_list);
2165 }
2166
2167
2168 /**
2169 * Retrieves the currently set chain of metalink hosts and the currently
2170 * used metalink host.
2171 */
2172 ✗ void DownloadManager::GetMetalinkInfo(vector<string> *metalink_chain,
2173 unsigned *current_metalink) {
2174 ✗ const MutexLockGuard m(lock_options_);
2175 ✗ if (opt_metalink_.chain) {
2176 ✗ if (current_metalink) {
2177 ✗ *current_metalink = opt_metalink_.current;
2178 }
2179 ✗ if (metalink_chain) {
2180 ✗ *metalink_chain = *opt_metalink_.chain;
2181 }
2182 }
2183 }
2184
2185
2186 /**
2187 * Parses a list of ';'-separated hosts for the host chain. The empty string
2188 * removes the host list.
2189 */
2190 1638 void DownloadManager::SetHostChain(const string &host_list) {
2191
1/2
✓ Branch 2 taken 1638 times.
✗ Branch 3 not taken.
 1638 SetHostChain(SplitString(host_list, ';'));
2192 1638 }
2193
2194
2195 1672 void DownloadManager::SetHostChain(const std::vector<std::string> &host_list) {
2196 1672 const MutexLockGuard m(lock_options_);
2197 1672 opt_host_.timestamp_backup = 0;
2198
2/2
✓ Branch 0 taken 823 times.
✓ Branch 1 taken 849 times.
 1672 delete opt_host_.chain;
2199
2/2
✓ Branch 0 taken 823 times.
✓ Branch 1 taken 849 times.
 1672 delete opt_host_chain_rtt_;
2200 1672 opt_host_.current = 0;
2201
2202
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1672 times.
 1672 if (host_list.empty()) {
2203 ✗ opt_host_.chain = NULL;
2204 ✗ opt_host_chain_rtt_ = NULL;
2205 ✗ return;
2206 }
2207
2208
2/4
✓ Branch 1 taken 1672 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1672 times.
✗ Branch 5 not taken.
 1672 opt_host_.chain = new vector<string>(host_list);
2209 3344 opt_host_chain_rtt_ = new vector<int>(opt_host_.chain->size(),
2210
2/4
✓ Branch 1 taken 1672 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1672 times.
✗ Branch 5 not taken.
 1672 kProbeUnprobed);
2211 // LogCvmfs(kLogDownload, kLogSyslog, "using host %s",
2212 // (*opt_host_.chain)[0].c_str());
2213
1/2
✓ Branch 1 taken 1672 times.
✗ Branch 2 not taken.
 1672 }
2214
2215
2216 /**
2217 * Retrieves the currently set chain of hosts, their round trip times, and the
2218 * currently used host.
2219 */
2220 238 void DownloadManager::GetHostInfo(vector<string> *host_chain, vector<int> *rtt,
2221 unsigned *current_host) {
2222 238 const MutexLockGuard m(lock_options_);
2223
1/2
✓ Branch 0 taken 238 times.
✗ Branch 1 not taken.
 238 if (opt_host_.chain) {
2224
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 238 times.
 238 if (current_host) {
2225 ✗ *current_host = opt_host_.current;
2226 }
2227
1/2
✓ Branch 0 taken 238 times.
✗ Branch 1 not taken.
 238 if (host_chain) {
2228
1/2
✓ Branch 1 taken 238 times.
✗ Branch 2 not taken.
 238 *host_chain = *opt_host_.chain;
2229 }
2230
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 238 times.
 238 if (rtt) {
2231 ✗ *rtt = *opt_host_chain_rtt_;
2232 }
2233 }
2234 238 }
2235
2236
2237 /**
2238 * Jumps to the next proxy in the ring of forward proxy servers.
2239 * Selects one randomly from a load-balancing group.
2240 *
2241 * Allow for the fact that the proxy may have already been failed by
2242 * another transfer, or that the proxy may no longer be part of the
2243 * current load-balancing group.
2244 */
2245 17 void DownloadManager::SwitchProxy(JobInfo *info) {
2246 17 const MutexLockGuard m(lock_options_);
2247
2248
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 17 times.
 17 if (!opt_proxy_groups_) {
2249 ✗ return;
2250 }
2251
2252 // Fail any matching proxies within the current load-balancing group
2253 17 vector<ProxyInfo> *group = current_proxy_group();
2254 17 const unsigned group_size = group->size();
2255 17 unsigned failed = 0;
2256
2/2
✓ Branch 0 taken 17 times.
✓ Branch 1 taken 17 times.
 34 for (unsigned i = 0; i < group_size - opt_proxy_groups_current_burned_; ++i) {
2257
5/14
✓ Branch 0 taken 17 times.
✗ Branch 1 not taken.
✓ Branch 4 taken 17 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 17 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 17 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✓ Branch 12 taken 17 times.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
 17 if (info && (info->proxy() == (*group)[i].url)) {
2258 // Move to list of failed proxies
2259 17 opt_proxy_groups_current_burned_++;
2260
1/2
✓ Branch 2 taken 17 times.
✗ Branch 3 not taken.
 17 swap((*group)[i],
2261 17 (*group)[group_size - opt_proxy_groups_current_burned_]);
2262 17 perf::Inc(counters_->n_proxy_failover);
2263 17 failed++;
2264 }
2265 }
2266
2267 // Do nothing more unless at least one proxy was marked as failed
2268
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 17 times.
 17 if (!failed)
2269 ✗ return;
2270
2271 // If all proxies from the current load-balancing group are burned, switch to
2272 // another group
2273
1/2
✓ Branch 1 taken 17 times.
✗ Branch 2 not taken.
 17 if (opt_proxy_groups_current_burned_ == group->size()) {
2274 17 opt_proxy_groups_current_burned_ = 0;
2275
1/2
✓ Branch 1 taken 17 times.
✗ Branch 2 not taken.
 17 if (opt_proxy_groups_->size() > 1) {
2276 34 opt_proxy_groups_current_ = (opt_proxy_groups_current_ + 1)
2277 17 % opt_proxy_groups_->size();
2278 // Remember the timestamp of switching to backup proxies
2279
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 17 times.
 17 if (opt_proxy_groups_reset_after_ > 0) {
2280 ✗ if (opt_proxy_groups_current_ > 0) {
2281 ✗ if (opt_timestamp_backup_proxies_ == 0)
2282 ✗ opt_timestamp_backup_proxies_ = time(NULL);
2283 // LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2284 // "switched to (another) backup proxy group");
2285 } else {
2286 ✗ opt_timestamp_backup_proxies_ = 0;
2287 // LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2288 // "switched back to primary proxy group");
2289 }
2290 ✗ opt_timestamp_failover_proxies_ = 0;
2291 }
2292 }
2293 } else {
2294 // Record failover time
2295 ✗ if (opt_proxy_groups_reset_after_ > 0) {
2296 ✗ if (opt_timestamp_failover_proxies_ == 0)
2297 ✗ opt_timestamp_failover_proxies_ = time(NULL);
2298 }
2299 }
2300
2301
2/4
✓ Branch 2 taken 17 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 17 times.
✗ Branch 6 not taken.
 17 UpdateProxiesUnlocked("failed proxy");
2302
1/2
✓ Branch 2 taken 17 times.
✗ Branch 3 not taken.
 17 LogCvmfs(kLogDownload, kLogDebug,
2303 "(manager '%s' - id %" PRId64 ") "
2304 "%lu proxies remain in group",
2305 name_.c_str(), info->id(),
2306 17 current_proxy_group()->size() - opt_proxy_groups_current_burned_);
2307
1/2
✓ Branch 1 taken 17 times.
✗ Branch 2 not taken.
 17 }
2308
2309
2310 /**
2311 * Switches to the next host in the chain. If jobinfo is set, switch only if
2312 * the current host is identical to the one used by jobinfo, otherwise another
2313 * transfer has already done the switch.
2314 */
2315 72 void DownloadManager::SwitchHostInfo(const std::string &typ,
2316 HostInfo &info,
2317 JobInfo *jobinfo) {
2318 72 const MutexLockGuard m(lock_options_);
2319
2320
5/6
✓ Branch 0 taken 72 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 38 times.
✓ Branch 4 taken 34 times.
✓ Branch 5 taken 38 times.
✓ Branch 6 taken 34 times.
 72 if (!info.chain || (info.chain->size() == 1)) {
2321 38 return;
2322 }
2323
2324
1/2
✓ Branch 0 taken 34 times.
✗ Branch 1 not taken.
 34 if (jobinfo) {
2325 int lastused;
2326
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 if (typ == "host") {
2327 34 lastused = jobinfo->current_host_chain_index();
2328 } else {
2329 ✗ lastused = jobinfo->current_metalink_chain_index();
2330 }
2331
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 34 times.
 34 if (lastused != info.current) {
2332 ✗ LogCvmfs(kLogDownload, kLogDebug,
2333 "(manager '%s' - id %" PRId64 ")"
2334 "don't switch %s, "
2335 "last used %s: %s, current %s: %s",
2336 name_.c_str(), jobinfo->id(), typ.c_str(), typ.c_str(),
2337 ✗ (*info.chain)[lastused].c_str(), typ.c_str(),
2338 ✗ (*info.chain)[info.current].c_str());
2339 ✗ return;
2340 }
2341 }
2342
2343
1/2
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
 34 string reason = "manually triggered";
2344
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 string info_id = "(manager " + name_;
2345
1/2
✓ Branch 0 taken 34 times.
✗ Branch 1 not taken.
 34 if (jobinfo) {
2346
1/2
✓ Branch 3 taken 34 times.
✗ Branch 4 not taken.
 34 reason = download::Code2Ascii(jobinfo->error_code());
2347
2/4
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 34 times.
✗ Branch 6 not taken.
 34 info_id = " - id " + StringifyInt(jobinfo->id());
2348 }
2349
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 info_id += ")";
2350
2351
1/2
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
 34 const std::string old_host = (*info.chain)[info.current];
2352 34 info.current = (info.current + 1) % static_cast<int>(info.chain->size());
2353
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 if (typ == "host") {
2354 34 perf::Inc(counters_->n_host_failover);
2355 } else {
2356 ✗ perf::Inc(counters_->n_metalink_failover);
2357 }
2358
1/3
✓ Branch 6 taken 34 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
 68 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2359 "%s switching %s from %s to %s (%s)", info_id.c_str(), typ.c_str(),
2360 34 old_host.c_str(), (*info.chain)[info.current].c_str(),
2361 reason.c_str());
2362
2363 // Remember the timestamp of switching to backup host
2364
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 34 times.
 34 if (info.reset_after > 0) {
2365 ✗ if (info.current != 0) {
2366 ✗ if (info.timestamp_backup == 0)
2367 ✗ info.timestamp_backup = time(NULL);
2368 } else {
2369 ✗ info.timestamp_backup = 0;
2370 }
2371 }
2372
2/2
✓ Branch 4 taken 34 times.
✓ Branch 5 taken 38 times.
 72 }
2373
2374 72 void DownloadManager::SwitchHost(JobInfo *info) {
2375
2/4
✓ Branch 2 taken 72 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 72 times.
✗ Branch 6 not taken.
 72 SwitchHostInfo("host", opt_host_, info);
2376 72 }
2377
2378 38 void DownloadManager::SwitchHost() { SwitchHost(NULL); }
2379
2380
2381 ✗ void DownloadManager::SwitchMetalink(JobInfo *info) {
2382 ✗ SwitchHostInfo("metalink", opt_metalink_, info);
2383 }
2384
2385
2386 ✗ void DownloadManager::SwitchMetalink() { SwitchMetalink(NULL); }
2387
2388 1934 bool DownloadManager::CheckMetalinkChain(time_t now) {
2389 1934 return (opt_metalink_.chain
2390
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1934 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 1934 && ((opt_metalink_timestamp_link_ == 0)
2391 ✗ || (static_cast<int64_t>((now == 0) ? time(NULL) : now)
2392 ✗ > static_cast<int64_t>(opt_metalink_timestamp_link_
2393
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 1934 + opt_metalink_.reset_after))));
2394 }
2395
2396
2397 /**
2398 * Orders the hostlist according to RTT of downloading .cvmfschecksum.
2399 * Sets the current host to the best-responsive host.
2400 * If you change the host list in between by SetHostChain(), it will be
2401 * overwritten by this function.
2402 */
2403 ✗ void DownloadManager::ProbeHosts() {
2404 ✗ vector<string> host_chain;
2405 ✗ vector<int> host_rtt;
2406 unsigned current_host;
2407
2408 ✗ GetHostInfo(&host_chain, &host_rtt, &current_host);
2409
2410 // Stopwatch, two times to fill caches first
2411 unsigned i, retries;
2412 ✗ string url;
2413
2414 ✗ cvmfs::MemSink memsink;
2415 ✗ JobInfo info(&url, false, false, NULL, &memsink);
2416 ✗ for (retries = 0; retries < 2; ++retries) {
2417 ✗ for (i = 0; i < host_chain.size(); ++i) {
2418 ✗ url = host_chain[i] + "/.cvmfspublished";
2419
2420 struct timeval tv_start, tv_end;
2421 ✗ gettimeofday(&tv_start, NULL);
2422 ✗ const Failures result = Fetch(&info);
2423 ✗ gettimeofday(&tv_end, NULL);
2424 ✗ memsink.Reset();
2425 ✗ if (result == kFailOk) {
2426 ✗ host_rtt[i] = static_cast<int>(DiffTimeSeconds(tv_start, tv_end)
2427 ✗ * 1000);
2428 ✗ LogCvmfs(kLogDownload, kLogDebug,
2429 "(manager '%s' - id %" PRId64 ") "
2430 "probing host %s had %dms rtt",
2431 ✗ name_.c_str(), info.id(), url.c_str(), host_rtt[i]);
2432 } else {
2433 ✗ LogCvmfs(kLogDownload, kLogDebug,
2434 "(manager '%s' - id %" PRId64 ") "
2435 "error while probing host %s: %d %s",
2436 name_.c_str(), info.id(), url.c_str(), result,
2437 Code2Ascii(result));
2438 ✗ host_rtt[i] = INT_MAX;
2439 }
2440 }
2441 }
2442
2443 ✗ SortTeam(&host_rtt, &host_chain);
2444 ✗ for (i = 0; i < host_chain.size(); ++i) {
2445 ✗ if (host_rtt[i] == INT_MAX)
2446 ✗ host_rtt[i] = kProbeDown;
2447 }
2448
2449 ✗ const MutexLockGuard m(lock_options_);
2450 ✗ delete opt_host_.chain;
2451 ✗ delete opt_host_chain_rtt_;
2452 ✗ opt_host_.chain = new vector<string>(host_chain);
2453 ✗ opt_host_chain_rtt_ = new vector<int>(host_rtt);
2454 ✗ opt_host_.current = 0;
2455 }
2456
2457 ✗ bool DownloadManager::GeoSortServers(std::vector<std::string> *servers,
2458 std::vector<uint64_t> *output_order) {
2459 ✗ if (!servers) {
2460 ✗ return false;
2461 }
2462 ✗ if (servers->size() == 1) {
2463 ✗ if (output_order) {
2464 ✗ output_order->clear();
2465 ✗ output_order->push_back(0);
2466 }
2467 ✗ return true;
2468 }
2469
2470 ✗ std::vector<std::string> host_chain;
2471 ✗ GetHostInfo(&host_chain, NULL, NULL);
2472
2473 ✗ std::vector<std::string> server_dns_names;
2474 ✗ server_dns_names.reserve(servers->size());
2475 ✗ for (unsigned i = 0; i < servers->size(); ++i) {
2476 ✗ const std::string host = dns::ExtractHost((*servers)[i]);
2477 ✗ server_dns_names.push_back(host.empty() ? (*servers)[i] : host);
2478 }
2479 ✗ const std::string host_list = JoinStrings(server_dns_names, ",");
2480
2481 ✗ vector<string> host_chain_shuffled;
2482 {
2483 // Protect against concurrent access to prng_
2484 ✗ const MutexLockGuard m(lock_options_);
2485 // Determine random hosts for the Geo-API query
2486 ✗ host_chain_shuffled = Shuffle(host_chain, &prng_);
2487 }
2488 // Request ordered list via Geo-API
2489 ✗ bool success = false;
2490 ✗ const unsigned max_attempts = std::min(host_chain_shuffled.size(), size_t(3));
2491 ✗ vector<uint64_t> geo_order(servers->size());
2492 ✗ for (unsigned i = 0; i < max_attempts; ++i) {
2493 ✗ const string url = host_chain_shuffled[i] + "/api/v1.0/geo/@proxy@/"
2494 ✗ + host_list;
2495 ✗ LogCvmfs(kLogDownload, kLogDebug,
2496 "(manager '%s') requesting ordered server list from %s",
2497 name_.c_str(), url.c_str());
2498 ✗ cvmfs::MemSink memsink;
2499 ✗ JobInfo info(&url, false, false, NULL, &memsink);
2500 ✗ const Failures result = Fetch(&info);
2501 ✗ if (result == kFailOk) {
2502 ✗ const string order(reinterpret_cast<char *>(memsink.data()),
2503 ✗ memsink.pos());
2504 ✗ memsink.Reset();
2505 ✗ const bool retval = ValidateGeoReply(order, servers->size(), &geo_order);
2506 ✗ if (!retval) {
2507 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2508 "(manager '%s') retrieved invalid GeoAPI reply from %s [%s]",
2509 name_.c_str(), url.c_str(), order.c_str());
2510 } else {
2511 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslog,
2512 "(manager '%s') "
2513 "geographic order of servers retrieved from %s",
2514 name_.c_str(),
2515 ✗ dns::ExtractHost(host_chain_shuffled[i]).c_str());
2516 // remove new line at end of "order"
2517 ✗ LogCvmfs(kLogDownload, kLogDebug, "order is %s",
2518 ✗ Trim(order, true /* trim_newline */).c_str());
2519 ✗ success = true;
2520 ✗ break;
2521 }
2522 ✗ } else {
2523 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2524 "(manager '%s') GeoAPI request for %s failed with error %d [%s]",
2525 name_.c_str(), url.c_str(), result, Code2Ascii(result));
2526 }
2527 }
2528 ✗ if (!success) {
2529 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2530 "(manager '%s') "
2531 "failed to retrieve geographic order from stratum 1 servers",
2532 name_.c_str());
2533 ✗ return false;
2534 }
2535
2536 ✗ if (output_order) {
2537 ✗ output_order->swap(geo_order);
2538 } else {
2539 ✗ std::vector<std::string> sorted_servers;
2540 ✗ sorted_servers.reserve(geo_order.size());
2541 ✗ for (unsigned i = 0; i < geo_order.size(); ++i) {
2542 ✗ const uint64_t orderval = geo_order[i];
2543 ✗ sorted_servers.push_back((*servers)[orderval]);
2544 }
2545 ✗ servers->swap(sorted_servers);
2546 }
2547 ✗ return true;
2548 }
2549
2550
2551 /**
2552 * Uses the Geo-API of Stratum 1s to let any of them order the list of servers
2553 * and fallback proxies (if any).
2554 * Tries at most three random Stratum 1s before giving up.
2555 * If you change the host list in between by SetHostChain() or the fallback
2556 * proxy list by SetProxyChain(), they will be overwritten by this function.
2557 */
2558 ✗ bool DownloadManager::ProbeGeo() {
2559 ✗ vector<string> host_chain;
2560 ✗ vector<int> host_rtt;
2561 unsigned current_host;
2562 ✗ vector<vector<ProxyInfo> > proxy_chain;
2563 unsigned fallback_group;
2564
2565 ✗ GetHostInfo(&host_chain, &host_rtt, &current_host);
2566 ✗ GetProxyInfo(&proxy_chain, NULL, &fallback_group);
2567 ✗ if ((host_chain.size() < 2) && ((proxy_chain.size() - fallback_group) < 2))
2568 ✗ return true;
2569
2570 ✗ vector<string> host_names;
2571 ✗ for (unsigned i = 0; i < host_chain.size(); ++i)
2572 ✗ host_names.push_back(dns::ExtractHost(host_chain[i]));
2573 ✗ SortTeam(&host_names, &host_chain);
2574 ✗ const unsigned last_geo_host = host_names.size();
2575
2576 ✗ if ((fallback_group == 0) && (last_geo_host > 1)) {
2577 // There are no non-fallback proxies, which means that the client
2578 // will always use the fallback proxies. Add a keyword separator
2579 // between the hosts and fallback proxies so the geosorting service
2580 // will know to sort the hosts based on the distance from the
2581 // closest fallback proxy rather than the distance from the client.
2582 ✗ host_names.push_back("+PXYSEP+");
2583 }
2584
2585 // Add fallback proxy names to the end of the host list
2586 ✗ const unsigned first_geo_fallback = host_names.size();
2587 ✗ for (unsigned i = fallback_group; i < proxy_chain.size(); ++i) {
2588 // We only take the first fallback proxy name from every group under the
2589 // assumption that load-balanced servers are at the same location
2590 ✗ host_names.push_back(proxy_chain[i][0].host.name());
2591 }
2592
2593 ✗ std::vector<uint64_t> geo_order;
2594 ✗ const bool success = GeoSortServers(&host_names, &geo_order);
2595 ✗ if (!success) {
2596 // GeoSortServers already logged a failure message.
2597 ✗ return false;
2598 }
2599
2600 // Re-install host chain and proxy chain
2601 ✗ const MutexLockGuard m(lock_options_);
2602 ✗ delete opt_host_.chain;
2603 ✗ opt_num_proxies_ = 0;
2604 ✗ opt_host_.chain = new vector<string>(host_chain.size());
2605
2606 // It's possible that opt_proxy_groups_fallback_ might have changed while
2607 // the lock wasn't held
2608 vector<vector<ProxyInfo> > *proxy_groups = new vector<vector<ProxyInfo> >(
2609 ✗ opt_proxy_groups_fallback_ + proxy_chain.size() - fallback_group);
2610 // First copy the non-fallback part of the current proxy chain
2611 ✗ for (unsigned i = 0; i < opt_proxy_groups_fallback_; ++i) {
2612 ✗ (*proxy_groups)[i] = (*opt_proxy_groups_)[i];
2613 ✗ opt_num_proxies_ += (*opt_proxy_groups_)[i].size();
2614 }
2615
2616 // Copy the host chain and fallback proxies by geo order. Array indices
2617 // in geo_order that are smaller than last_geo_host refer to a stratum 1,
2618 // and those indices greater than or equal to first_geo_fallback refer to
2619 // a fallback proxy.
2620 ✗ unsigned hosti = 0;
2621 ✗ unsigned proxyi = opt_proxy_groups_fallback_;
2622 ✗ for (unsigned i = 0; i < geo_order.size(); ++i) {
2623 ✗ const uint64_t orderval = geo_order[i];
2624 ✗ if (orderval < static_cast<uint64_t>(last_geo_host)) {
2625 // LogCvmfs(kLogCvmfs, kLogSyslog, "this is orderval %u at host index
2626 // %u", orderval, hosti);
2627 ✗ (*opt_host_.chain)[hosti++] = host_chain[orderval];
2628 ✗ } else if (orderval >= static_cast<uint64_t>(first_geo_fallback)) {
2629 // LogCvmfs(kLogCvmfs, kLogSyslog,
2630 // "this is orderval %u at proxy index %u, using proxy_chain index %u",
2631 // orderval, proxyi, fallback_group + orderval - first_geo_fallback);
2632 ✗ (*proxy_groups)[proxyi] = proxy_chain[fallback_group + orderval
2633 ✗ - first_geo_fallback];
2634 ✗ opt_num_proxies_ += (*proxy_groups)[proxyi].size();
2635 ✗ proxyi++;
2636 }
2637 }
2638
2639 ✗ opt_proxy_map_.clear();
2640 ✗ delete opt_proxy_groups_;
2641 ✗ opt_proxy_groups_ = proxy_groups;
2642 // In pathological cases, opt_proxy_groups_current_ can be larger now when
2643 // proxies changed in-between.
2644 ✗ if (opt_proxy_groups_current_ > opt_proxy_groups_->size()) {
2645 ✗ if (opt_proxy_groups_->size() == 0) {
2646 ✗ opt_proxy_groups_current_ = 0;
2647 } else {
2648 ✗ opt_proxy_groups_current_ = opt_proxy_groups_->size() - 1;
2649 }
2650 ✗ opt_proxy_groups_current_burned_ = 0;
2651 }
2652
2653 ✗ UpdateProxiesUnlocked("geosort");
2654
2655 ✗ delete opt_host_chain_rtt_;
2656 ✗ opt_host_chain_rtt_ = new vector<int>(host_chain.size(), kProbeGeo);
2657 ✗ opt_host_.current = 0;
2658
2659 ✗ return true;
2660 }
2661
2662
2663 /**
2664 * Validates a string of the form "1,4,2,3" representing in which order the
2665 * the expected_size number of hosts should be put for optimal geographic
2666 * proximity. Returns false if the reply_order string is invalid, otherwise
2667 * fills in the reply_vals array with zero-based order indexes (e.g.
2668 * [0,3,1,2]) and returns true.
2669 */
2670 238 bool DownloadManager::ValidateGeoReply(const string &reply_order,
2671 const unsigned expected_size,
2672 vector<uint64_t> *reply_vals) {
2673
2/2
✓ Branch 1 taken 17 times.
✓ Branch 2 taken 221 times.
 238 if (reply_order.empty())
2674 17 return false;
2675
2/4
✓ Branch 2 taken 221 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 221 times.
✗ Branch 6 not taken.
 442 const sanitizer::InputSanitizer sanitizer("09 , \n");
2676
3/4
✓ Branch 1 taken 221 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 17 times.
✓ Branch 4 taken 204 times.
 221 if (!sanitizer.IsValid(reply_order))
2677 17 return false;
2678
2/4
✓ Branch 2 taken 204 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 204 times.
✗ Branch 6 not taken.
 408 const sanitizer::InputSanitizer strip_newline("09 ,");
2679
1/2
✓ Branch 1 taken 204 times.
✗ Branch 2 not taken.
 204 vector<string> reply_strings = SplitString(strip_newline.Filter(reply_order),
2680
1/2
✓ Branch 1 taken 204 times.
✗ Branch 2 not taken.
 204 ',');
2681 204 vector<uint64_t> tmp_vals;
2682
2/2
✓ Branch 1 taken 391 times.
✓ Branch 2 taken 170 times.
 561 for (unsigned i = 0; i < reply_strings.size(); ++i) {
2683
2/2
✓ Branch 2 taken 34 times.
✓ Branch 3 taken 357 times.
 391 if (reply_strings[i].empty())
2684 34 return false;
2685
2/4
✓ Branch 2 taken 357 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 357 times.
✗ Branch 6 not taken.
 357 tmp_vals.push_back(String2Uint64(reply_strings[i]));
2686 }
2687
2/2
✓ Branch 1 taken 85 times.
✓ Branch 2 taken 85 times.
 170 if (tmp_vals.size() != expected_size)
2688 85 return false;
2689
2690 // Check if tmp_vals contains the number 1..n
2691
1/2
✓ Branch 3 taken 85 times.
✗ Branch 4 not taken.
 85 set<uint64_t> coverage(tmp_vals.begin(), tmp_vals.end());
2692
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 85 times.
 85 if (coverage.size() != tmp_vals.size())
2693 ✗ return false;
2694
5/6
✓ Branch 2 taken 68 times.
✓ Branch 3 taken 17 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 68 times.
✓ Branch 9 taken 17 times.
✓ Branch 10 taken 68 times.
 85 if ((*coverage.begin() != 1) || (*coverage.rbegin() != coverage.size()))
2695 17 return false;
2696
2697
2/2
✓ Branch 0 taken 153 times.
✓ Branch 1 taken 68 times.
 221 for (unsigned i = 0; i < expected_size; ++i) {
2698 153 (*reply_vals)[i] = tmp_vals[i] - 1;
2699 }
2700 68 return true;
2701 221 }
2702
2703
2704 /**
2705 * Removes DIRECT from a list of ';' and '|' separated proxies.
2706 * \return true if DIRECT was present, false otherwise
2707 */
2708 1723 bool DownloadManager::StripDirect(const string &proxy_list,
2709 string *cleaned_list) {
2710
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1723 times.
 1723 assert(cleaned_list);
2711
2/2
✓ Branch 1 taken 1570 times.
✓ Branch 2 taken 153 times.
 1723 if (proxy_list == "") {
2712
1/2
✓ Branch 1 taken 1570 times.
✗ Branch 2 not taken.
 1570 *cleaned_list = "";
2713 1570 return false;
2714 }
2715 153 bool result = false;
2716
2717
1/2
✓ Branch 1 taken 153 times.
✗ Branch 2 not taken.
 153 vector<string> proxy_groups = SplitString(proxy_list, ';');
2718 153 vector<string> cleaned_groups;
2719
2/2
✓ Branch 1 taken 374 times.
✓ Branch 2 taken 153 times.
 527 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2720
1/2
✓ Branch 2 taken 374 times.
✗ Branch 3 not taken.
 374 vector<string> group = SplitString(proxy_groups[i], '|');
2721 374 vector<string> cleaned;
2722
2/2
✓ Branch 1 taken 629 times.
✓ Branch 2 taken 374 times.
 1003 for (unsigned j = 0; j < group.size(); ++j) {
2723
6/6
✓ Branch 2 taken 459 times.
✓ Branch 3 taken 170 times.
✓ Branch 6 taken 221 times.
✓ Branch 7 taken 238 times.
✓ Branch 8 taken 391 times.
✓ Branch 9 taken 238 times.
 629 if ((group[j] == "DIRECT") || (group[j] == "")) {
2724 391 result = true;
2725 } else {
2726
1/2
✓ Branch 2 taken 238 times.
✗ Branch 3 not taken.
 238 cleaned.push_back(group[j]);
2727 }
2728 }
2729
2/2
✓ Branch 1 taken 119 times.
✓ Branch 2 taken 255 times.
 374 if (!cleaned.empty())
2730
3/6
✓ Branch 2 taken 119 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 119 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 119 times.
✗ Branch 9 not taken.
 119 cleaned_groups.push_back(JoinStrings(cleaned, "|"));
2731 374 }
2732
2733
2/4
✓ Branch 2 taken 153 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 153 times.
✗ Branch 6 not taken.
 153 *cleaned_list = JoinStrings(cleaned_groups, ";");
2734 153 return result;
2735 153 }
2736
2737
2738 /**
2739 * Parses a list of ';'- and '|'-separated proxy servers and fallback proxy
2740 * servers for the proxy groups.
2741 * The empty string for both removes the proxy chain.
2742 * The set_mode parameter can be used to set either proxies (leaving fallback
2743 * proxies unchanged) or fallback proxies (leaving regular proxies unchanged)
2744 * or both.
2745 */
2746 1553 void DownloadManager::SetProxyChain(const string &proxy_list,
2747 const string &fallback_proxy_list,
2748 const ProxySetModes set_mode) {
2749 1553 const MutexLockGuard m(lock_options_);
2750
2751 1553 opt_timestamp_backup_proxies_ = 0;
2752 1553 opt_timestamp_failover_proxies_ = 0;
2753
1/2
✓ Branch 1 taken 1553 times.
✗ Branch 2 not taken.
 1553 string set_proxy_list = opt_proxy_list_;
2754
1/2
✓ Branch 1 taken 1553 times.
✗ Branch 2 not taken.
 1553 string set_proxy_fallback_list = opt_proxy_fallback_list_;
2755 bool contains_direct;
2756
3/4
✓ Branch 0 taken 1553 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1502 times.
✓ Branch 3 taken 51 times.
 1553 if ((set_mode == kSetProxyFallback) || (set_mode == kSetProxyBoth)) {
2757
1/2
✓ Branch 1 taken 1502 times.
✗ Branch 2 not taken.
 1502 opt_proxy_fallback_list_ = fallback_proxy_list;
2758 }
2759
3/4
✓ Branch 0 taken 1502 times.
✓ Branch 1 taken 51 times.
✓ Branch 2 taken 1502 times.
✗ Branch 3 not taken.
 1553 if ((set_mode == kSetProxyRegular) || (set_mode == kSetProxyBoth)) {
2760
1/2
✓ Branch 1 taken 1553 times.
✗ Branch 2 not taken.
 1553 opt_proxy_list_ = proxy_list;
2761 }
2762
1/2
✓ Branch 1 taken 1553 times.
✗ Branch 2 not taken.
 1553 contains_direct = StripDirect(opt_proxy_fallback_list_,
2763 &set_proxy_fallback_list);
2764
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1553 times.
 1553 if (contains_direct) {
2765 ✗ LogCvmfs(kLogDownload, kLogSyslogWarn | kLogDebug,
2766 "(manager '%s') fallback proxies do not support DIRECT, removing",
2767 name_.c_str());
2768 }
2769
1/2
✓ Branch 1 taken 1553 times.
✗ Branch 2 not taken.
 1553 if (set_proxy_fallback_list == "") {
2770
1/2
✓ Branch 1 taken 1553 times.
✗ Branch 2 not taken.
 1553 set_proxy_list = opt_proxy_list_;
2771 } else {
2772 ✗ const bool contains_direct = StripDirect(opt_proxy_list_, &set_proxy_list);
2773 ✗ if (contains_direct) {
2774 ✗ LogCvmfs(kLogDownload, kLogSyslog | kLogDebug,
2775 "(manager '%s') skipping DIRECT proxy to use fallback proxy",
2776 name_.c_str());
2777 }
2778 }
2779
2780 // From this point on, use set_proxy_list and set_fallback_proxy_list as
2781 // effective proxy lists!
2782
2783 1553 opt_proxy_map_.clear();
2784
2/2
✓ Branch 0 taken 751 times.
✓ Branch 1 taken 802 times.
 1553 delete opt_proxy_groups_;
2785
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 1553 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 1553 times.
 1553 if ((set_proxy_list == "") && (set_proxy_fallback_list == "")) {
2786 ✗ opt_proxy_groups_ = NULL;
2787 ✗ opt_proxy_groups_current_ = 0;
2788 ✗ opt_proxy_groups_current_burned_ = 0;
2789 ✗ opt_proxy_groups_fallback_ = 0;
2790 ✗ opt_num_proxies_ = 0;
2791 ✗ return;
2792 }
2793
2794 // Determine number of regular proxy groups (== first fallback proxy group)
2795 1553 opt_proxy_groups_fallback_ = 0;
2796
1/2
✓ Branch 1 taken 1553 times.
✗ Branch 2 not taken.
 1553 if (set_proxy_list != "") {
2797
1/2
✓ Branch 1 taken 1553 times.
✗ Branch 2 not taken.
 1553 opt_proxy_groups_fallback_ = SplitString(set_proxy_list, ';').size();
2798 }
2799
1/2
✓ Branch 2 taken 1553 times.
✗ Branch 3 not taken.
 1553 LogCvmfs(kLogDownload, kLogDebug,
2800 "(manager '%s') "
2801 "first fallback proxy group %u",
2802 name_.c_str(), opt_proxy_groups_fallback_);
2803
2804 // Concatenate regular proxies and fallback proxies, both of which can be
2805 // empty.
2806
1/2
✓ Branch 1 taken 1553 times.
✗ Branch 2 not taken.
 1553 string all_proxy_list = set_proxy_list;
2807
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1553 times.
 1553 if (set_proxy_fallback_list != "") {
2808 ✗ if (all_proxy_list != "")
2809 ✗ all_proxy_list += ";";
2810 ✗ all_proxy_list += set_proxy_fallback_list;
2811 }
2812
1/2
✓ Branch 3 taken 1553 times.
✗ Branch 4 not taken.
 1553 LogCvmfs(kLogDownload, kLogDebug, "(manager '%s') full proxy list %s",
2813 name_.c_str(), all_proxy_list.c_str());
2814
2815 // Resolve server names in provided urls
2816 1553 vector<string> hostnames; // All encountered hostnames
2817 1553 vector<string> proxy_groups;
2818
1/2
✓ Branch 1 taken 1553 times.
✗ Branch 2 not taken.
 1553 if (all_proxy_list != "")
2819
1/2
✓ Branch 1 taken 1553 times.
✗ Branch 2 not taken.
 1553 proxy_groups = SplitString(all_proxy_list, ';');
2820
2/2
✓ Branch 1 taken 1570 times.
✓ Branch 2 taken 1553 times.
 3123 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2821
1/2
✓ Branch 2 taken 1570 times.
✗ Branch 3 not taken.
 1570 vector<string> this_group = SplitString(proxy_groups[i], '|');
2822
2/2
✓ Branch 1 taken 1570 times.
✓ Branch 2 taken 1570 times.
 3140 for (unsigned j = 0; j < this_group.size(); ++j) {
2823
1/2
✓ Branch 2 taken 1570 times.
✗ Branch 3 not taken.
 1570 this_group[j] = dns::AddDefaultScheme(this_group[j]);
2824 // Note: DIRECT strings will be "extracted" to an empty string.
2825
1/2
✓ Branch 2 taken 1570 times.
✗ Branch 3 not taken.
 1570 const string hostname = dns::ExtractHost(this_group[j]);
2826 // Save the hostname. Leave empty (DIRECT) names so indexes will
2827 // match later.
2828
1/2
✓ Branch 1 taken 1570 times.
✗ Branch 2 not taken.
 1570 hostnames.push_back(hostname);
2829 1570 }
2830 1570 }
2831 1553 vector<dns::Host> hosts;
2832
1/2
✓ Branch 3 taken 1553 times.
✗ Branch 4 not taken.
 1553 LogCvmfs(kLogDownload, kLogDebug,
2833 "(manager '%s') "
2834 "resolving %lu proxy addresses",
2835 name_.c_str(), hostnames.size());
2836
1/2
✓ Branch 1 taken 1553 times.
✗ Branch 2 not taken.
 1553 resolver_->ResolveMany(hostnames, &hosts);
2837
2838 // Construct opt_proxy_groups_: traverse proxy list in same order and expand
2839 // names to resolved IP addresses.
2840
1/2
✓ Branch 1 taken 1553 times.
✗ Branch 2 not taken.
 1553 opt_proxy_groups_ = new vector<vector<ProxyInfo> >();
2841 1553 opt_num_proxies_ = 0;
2842 1553 unsigned num_proxy = 0; // Combined i, j counter
2843
2/2
✓ Branch 1 taken 1570 times.
✓ Branch 2 taken 1553 times.
 3123 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2844
1/2
✓ Branch 2 taken 1570 times.
✗ Branch 3 not taken.
 1570 vector<string> this_group = SplitString(proxy_groups[i], '|');
2845 // Construct ProxyInfo objects from proxy string and DNS resolver result for
2846 // every proxy in this_group. One URL can result in multiple ProxyInfo
2847 // objects, one for each IP address.
2848 1570 vector<ProxyInfo> infos;
2849
2/2
✓ Branch 1 taken 1570 times.
✓ Branch 2 taken 1570 times.
 3140 for (unsigned j = 0; j < this_group.size(); ++j, ++num_proxy) {
2850
1/2
✓ Branch 2 taken 1570 times.
✗ Branch 3 not taken.
 1570 this_group[j] = dns::AddDefaultScheme(this_group[j]);
2851
2/2
✓ Branch 2 taken 1502 times.
✓ Branch 3 taken 68 times.
 1570 if (this_group[j] == "DIRECT") {
2852
3/6
✓ Branch 2 taken 1502 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1502 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1502 times.
✗ Branch 9 not taken.
 1502 infos.push_back(ProxyInfo("DIRECT"));
2853 1502 continue;
2854 }
2855
2856
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 68 times.
 68 if (hosts[num_proxy].status() != dns::kFailOk) {
2857 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2858 "(manager '%s') "
2859 "failed to resolve IP addresses for %s (%d - %s)",
2860 ✗ name_.c_str(), hosts[num_proxy].name().c_str(),
2861 ✗ hosts[num_proxy].status(),
2862 ✗ dns::Code2Ascii(hosts[num_proxy].status()));
2863 const dns::Host failed_host = dns::Host::ExtendDeadline(
2864 ✗ hosts[num_proxy], resolver_->min_ttl());
2865 ✗ infos.push_back(ProxyInfo(failed_host, this_group[j]));
2866 ✗ continue;
2867 }
2868
2869 // IPv4 addresses have precedence
2870 68 set<string> best_addresses = hosts[num_proxy].ViewBestAddresses(
2871
2/4
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 68 times.
✗ Branch 5 not taken.
 68 opt_ip_preference_);
2872 68 set<string>::const_iterator iter_ips = best_addresses.begin();
2873
2/2
✓ Branch 3 taken 68 times.
✓ Branch 4 taken 68 times.
 136 for (; iter_ips != best_addresses.end(); ++iter_ips) {
2874
1/2
✓ Branch 3 taken 68 times.
✗ Branch 4 not taken.
 68 const string url_ip = dns::RewriteUrl(this_group[j], *iter_ips);
2875
2/4
✓ Branch 2 taken 68 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 68 times.
✗ Branch 6 not taken.
 68 infos.push_back(ProxyInfo(hosts[num_proxy], url_ip));
2876
2877
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 68 times.
 68 if (sharding_policy_.UseCount() > 0) {
2878 ✗ sharding_policy_->AddProxy(url_ip);
2879 }
2880 68 }
2881 68 }
2882
1/2
✓ Branch 1 taken 1570 times.
✗ Branch 2 not taken.
 1570 opt_proxy_groups_->push_back(infos);
2883 1570 opt_num_proxies_ += infos.size();
2884 1570 }
2885
1/2
✓ Branch 2 taken 1553 times.
✗ Branch 3 not taken.
 1553 LogCvmfs(kLogDownload, kLogDebug,
2886 "(manager '%s') installed %u proxies in %lu load-balance groups",
2887 1553 name_.c_str(), opt_num_proxies_, opt_proxy_groups_->size());
2888 1553 opt_proxy_groups_current_ = 0;
2889 1553 opt_proxy_groups_current_burned_ = 0;
2890
2891 // Select random start proxy from the first group.
2892
1/2
✓ Branch 1 taken 1553 times.
✗ Branch 2 not taken.
 1553 if (opt_proxy_groups_->size() > 0) {
2893 // Select random start proxy from the first group.
2894
2/4
✓ Branch 2 taken 1553 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1553 times.
✗ Branch 6 not taken.
 1553 UpdateProxiesUnlocked("set random start proxy from the first proxy group");
2895 }
2896
3/6
✓ Branch 5 taken 1553 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1553 times.
✗ Branch 9 not taken.
✓ Branch 11 taken 1553 times.
✗ Branch 12 not taken.
 1553 }
2897
2898
2899 /**
2900 * Retrieves the proxy chain, optionally the currently active load-balancing
2901 * group, and optionally the index of the first fallback proxy group.
2902 * If there are no fallback proxies, the index will equal the size of
2903 * the proxy chain.
2904 */
2905 ✗ void DownloadManager::GetProxyInfo(vector<vector<ProxyInfo> > *proxy_chain,
2906 unsigned *current_group,
2907 unsigned *fallback_group) {
2908 ✗ assert(proxy_chain != NULL);
2909 ✗ const MutexLockGuard m(lock_options_);
2910
2911 ✗ if (!opt_proxy_groups_) {
2912 ✗ const vector<vector<ProxyInfo> > empty_chain;
2913 ✗ *proxy_chain = empty_chain;
2914 ✗ if (current_group != NULL)
2915 ✗ *current_group = 0;
2916 ✗ if (fallback_group != NULL)
2917 ✗ *fallback_group = 0;
2918 ✗ return;
2919 }
2920
2921 ✗ *proxy_chain = *opt_proxy_groups_;
2922 ✗ if (current_group != NULL)
2923 ✗ *current_group = opt_proxy_groups_current_;
2924 ✗ if (fallback_group != NULL)
2925 ✗ *fallback_group = opt_proxy_groups_fallback_;
2926 }
2927
2928 ✗ string DownloadManager::GetProxyList() { return opt_proxy_list_; }
2929
2930 ✗ string DownloadManager::GetFallbackProxyList() {
2931 ✗ return opt_proxy_fallback_list_;
2932 }
2933
2934 /**
2935 * Choose proxy
2936 */
2937 5878 DownloadManager::ProxyInfo *DownloadManager::ChooseProxyUnlocked(
2938 const shash::Any *hash) {
2939
2/2
✓ Branch 0 taken 4012 times.
✓ Branch 1 taken 1866 times.
 5878 if (!opt_proxy_groups_)
2940 4012 return NULL;
2941
2942
2/2
✓ Branch 0 taken 1038 times.
✓ Branch 1 taken 828 times.
 1866 const uint32_t key = (hash ? hash->Partial32() : 0);
2943
1/2
✓ Branch 1 taken 1866 times.
✗ Branch 2 not taken.
 1866 const map<uint32_t, ProxyInfo *>::iterator it = opt_proxy_map_.lower_bound(
2944 key);
2945 1866 ProxyInfo *proxy = it->second;
2946
2947 1866 return proxy;
2948 }
2949
2950 /**
2951 * Update currently selected proxy
2952 */
2953 2321 void DownloadManager::UpdateProxiesUnlocked(const string &reason) {
2954
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2321 times.
 2321 if (!opt_proxy_groups_)
2955 ✗ return;
2956
2957 // Identify number of non-burned proxies within the current group
2958 2321 vector<ProxyInfo> *group = current_proxy_group();
2959 2321 const unsigned num_alive = (group->size() - opt_proxy_groups_current_burned_);
2960
2/4
✓ Branch 2 taken 2321 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2321 times.
✗ Branch 6 not taken.
 4642 const string old_proxy = JoinStrings(opt_proxies_, "|");
2961
2962 // Rebuild proxy map and URL list
2963 2321 opt_proxy_map_.clear();
2964 2321 opt_proxies_.clear();
2965 2321 const uint32_t max_key = 0xffffffffUL;
2966
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2321 times.
 2321 if (opt_proxy_shard_) {
2967 // Build a consistent map with multiple entries for each proxy
2968 ✗ for (unsigned i = 0; i < num_alive; ++i) {
2969 ✗ ProxyInfo *proxy = &(*group)[i];
2970 ✗ shash::Any proxy_hash(shash::kSha1);
2971 ✗ HashString(proxy->url, &proxy_hash);
2972 ✗ Prng prng;
2973 ✗ prng.InitSeed(proxy_hash.Partial32());
2974 ✗ for (unsigned j = 0; j < kProxyMapScale; ++j) {
2975 ✗ const std::pair<uint32_t, ProxyInfo *> entry(prng.Next(max_key), proxy);
2976 ✗ opt_proxy_map_.insert(entry);
2977 }
2978 ✗ const std::string proxy_name = proxy->host.name().empty()
2979 ? ""
2980 ✗ : " (" + proxy->host.name() + ")";
2981 ✗ opt_proxies_.push_back(proxy->url + proxy_name);
2982 }
2983 // Ensure lower_bound() finds a value for all keys
2984 ✗ ProxyInfo *first_proxy = opt_proxy_map_.begin()->second;
2985 ✗ const std::pair<uint32_t, ProxyInfo *> last_entry(max_key, first_proxy);
2986 ✗ opt_proxy_map_.insert(last_entry);
2987 } else {
2988 // Build a map with a single entry for one randomly selected proxy
2989 2321 const unsigned select = prng_.Next(num_alive);
2990 2321 ProxyInfo *proxy = &(*group)[select];
2991 2321 const std::pair<uint32_t, ProxyInfo *> entry(max_key, proxy);
2992
1/2
✓ Branch 1 taken 2321 times.
✗ Branch 2 not taken.
 2321 opt_proxy_map_.insert(entry);
2993 2321 const std::string proxy_name = proxy->host.name().empty()
2994 ? ""
2995
9/18
✓ Branch 0 taken 2253 times.
✓ Branch 1 taken 68 times.
✓ Branch 4 taken 2253 times.
✗ Branch 5 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 68 times.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✓ Branch 11 taken 68 times.
✗ Branch 12 not taken.
✓ Branch 13 taken 68 times.
✓ Branch 14 taken 2253 times.
✓ Branch 15 taken 2253 times.
✓ Branch 16 taken 68 times.
✗ Branch 17 not taken.
✗ Branch 18 not taken.
✗ Branch 19 not taken.
✗ Branch 20 not taken.
 2389 : " (" + proxy->host.name() + ")";
2996
2/4
✓ Branch 1 taken 2321 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2321 times.
✗ Branch 5 not taken.
 2321 opt_proxies_.push_back(proxy->url + proxy_name);
2997 2321 }
2998
1/2
✓ Branch 3 taken 2321 times.
✗ Branch 4 not taken.
 2321 sort(opt_proxies_.begin(), opt_proxies_.end());
2999
3000 // Report any change in proxy usage
3001
2/4
✓ Branch 2 taken 2321 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2321 times.
✗ Branch 6 not taken.
 4642 const string new_proxy = JoinStrings(opt_proxies_, "|");
3002 const string curr_host = "Current host: "
3003 2321 + (opt_host_.chain
3004
6/12
✓ Branch 0 taken 2253 times.
✓ Branch 1 taken 68 times.
✓ Branch 4 taken 2253 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 68 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 68 times.
✓ Branch 11 taken 2253 times.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
 4642 ? (*opt_host_.chain)[opt_host_.current]
3005
1/2
✓ Branch 1 taken 2321 times.
✗ Branch 2 not taken.
 2321 : "");
3006
2/2
✓ Branch 1 taken 1570 times.
✓ Branch 2 taken 751 times.
 2321 if (new_proxy != old_proxy) {
3007
4/9
✗ Branch 2 not taken.
✓ Branch 3 taken 1570 times.
✓ Branch 4 taken 1553 times.
✓ Branch 5 taken 17 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 1570 times.
✗ Branch 9 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
 4727 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
3008 "(manager '%s') switching proxy from %s to %s. Reason: %s [%s]",
3009 1587 name_.c_str(), (old_proxy.empty() ? "(none)" : old_proxy.c_str()),
3010 3140 (new_proxy.empty() ? "(none)" : new_proxy.c_str()), reason.c_str(),
3011 curr_host.c_str());
3012 }
3013 2321 }
3014
3015 /**
3016 * Enable proxy sharding
3017 */
3018 ✗ void DownloadManager::ShardProxies() {
3019 ✗ opt_proxy_shard_ = true;
3020 ✗ RebalanceProxiesUnlocked("enable sharding");
3021 }
3022
3023 /**
3024 * Selects a new random proxy in the current load-balancing group. Resets the
3025 * "burned" counter.
3026 */
3027 ✗ void DownloadManager::RebalanceProxiesUnlocked(const string &reason) {
3028 ✗ if (!opt_proxy_groups_)
3029 ✗ return;
3030
3031 ✗ opt_timestamp_failover_proxies_ = 0;
3032 ✗ opt_proxy_groups_current_burned_ = 0;
3033 ✗ UpdateProxiesUnlocked(reason);
3034 }
3035
3036
3037 ✗ void DownloadManager::RebalanceProxies() {
3038 ✗ const MutexLockGuard m(lock_options_);
3039 ✗ RebalanceProxiesUnlocked("rebalance invoked manually");
3040 }
3041
3042
3043 /**
3044 * Switches to the next load-balancing group of proxy servers.
3045 */
3046 ✗ void DownloadManager::SwitchProxyGroup() {
3047 ✗ const MutexLockGuard m(lock_options_);
3048
3049 ✗ if (!opt_proxy_groups_ || (opt_proxy_groups_->size() < 2)) {
3050 ✗ return;
3051 }
3052
3053 ✗ opt_proxy_groups_current_ = (opt_proxy_groups_current_ + 1)
3054 ✗ % opt_proxy_groups_->size();
3055 ✗ opt_timestamp_backup_proxies_ = time(NULL);
3056
3057 const std::string msg = "switch to proxy group "
3058 ✗ + StringifyUint(opt_proxy_groups_current_);
3059 ✗ RebalanceProxiesUnlocked(msg);
3060 }
3061
3062
3063 ✗ void DownloadManager::SetProxyGroupResetDelay(const unsigned seconds) {
3064 ✗ const MutexLockGuard m(lock_options_);
3065 ✗ opt_proxy_groups_reset_after_ = seconds;
3066 ✗ if (opt_proxy_groups_reset_after_ == 0) {
3067 ✗ opt_timestamp_backup_proxies_ = 0;
3068 ✗ opt_timestamp_failover_proxies_ = 0;
3069 }
3070 }
3071
3072
3073 ✗ void DownloadManager::SetMetalinkResetDelay(const unsigned seconds) {
3074 ✗ const MutexLockGuard m(lock_options_);
3075 ✗ opt_metalink_.reset_after = seconds;
3076 ✗ if (opt_metalink_.reset_after == 0)
3077 ✗ opt_metalink_.timestamp_backup = 0;
3078 }
3079
3080
3081 ✗ void DownloadManager::SetHostResetDelay(const unsigned seconds) {
3082 ✗ const MutexLockGuard m(lock_options_);
3083 ✗ opt_host_.reset_after = seconds;
3084 ✗ if (opt_host_.reset_after == 0)
3085 ✗ opt_host_.timestamp_backup = 0;
3086 }
3087
3088
3089 1664 void DownloadManager::SetRetryParameters(const unsigned max_retries,
3090 const unsigned backoff_init_ms,
3091 const unsigned backoff_max_ms) {
3092 1664 const MutexLockGuard m(lock_options_);
3093 1664 opt_max_retries_ = max_retries;
3094 1664 opt_backoff_init_ms_ = backoff_init_ms;
3095 1664 opt_backoff_max_ms_ = backoff_max_ms;
3096 1664 }
3097
3098
3099 785 void DownloadManager::SetMaxIpaddrPerProxy(unsigned limit) {
3100 785 const MutexLockGuard m(lock_options_);
3101 785 resolver_->set_throttle(limit);
3102 785 }
3103
3104
3105 774 void DownloadManager::SetProxyTemplates(const std::string &direct,
3106 const std::string &forced) {
3107 774 const MutexLockGuard m(lock_options_);
3108
1/2
✓ Branch 1 taken 774 times.
✗ Branch 2 not taken.
 774 proxy_template_direct_ = direct;
3109
1/2
✓ Branch 1 taken 774 times.
✗ Branch 2 not taken.
 774 proxy_template_forced_ = forced;
3110 774 }
3111
3112
3113 ✗ void DownloadManager::EnableInfoHeader() { enable_info_header_ = true; }
3114
3115
3116 941 void DownloadManager::EnableRedirects() { follow_redirects_ = true; }
3117
3118 ✗ void DownloadManager::EnableIgnoreSignatureFailures() {
3119 ✗ ignore_signature_failures_ = true;
3120 }
3121
3122 ✗ void DownloadManager::EnableHTTPTracing() { enable_http_tracing_ = true; }
3123
3124 ✗ void DownloadManager::AddHTTPTracingHeader(const std::string &header) {
3125 ✗ http_tracing_headers_.push_back(header);
3126 }
3127
3128 890 void DownloadManager::UseSystemCertificatePath() {
3129 890 ssl_certificate_store_.UseSystemCertificatePath();
3130 890 }
3131
3132 ✗ bool DownloadManager::SetShardingPolicy(const ShardingPolicySelector type) {
3133 ✗ const bool success = false;
3134 ✗ switch (type) {
3135 default:
3136 ✗ LogCvmfs(
3137 kLogDownload, kLogDebug | kLogSyslogErr,
3138 "(manager '%s') "
3139 "Proposed sharding policy does not exist. Falling back to default",
3140 name_.c_str());
3141 }
3142 ✗ return success;
3143 }
3144
3145 ✗ void DownloadManager::SetFailoverIndefinitely() {
3146 ✗ failover_indefinitely_ = true;
3147 }
3148
3149 /**
3150 * Creates a copy of the existing download manager. Must only be called in
3151 * single-threaded stage because it calls curl_global_init().
3152 */
3153 785 DownloadManager *DownloadManager::Clone(
3154 const perf::StatisticsTemplate &statistics,
3155 const std::string &cloned_name) {
3156 DownloadManager *clone = new DownloadManager(pool_max_handles_, statistics,
3157
1/2
✓ Branch 2 taken 785 times.
✗ Branch 3 not taken.
 785 cloned_name);
3158
3159 785 clone->SetDnsParameters(resolver_->retries(), resolver_->timeout_ms());
3160 785 clone->SetDnsTtlLimits(resolver_->min_ttl(), resolver_->max_ttl());
3161 785 clone->SetMaxIpaddrPerProxy(resolver_->throttle());
3162
3163
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 785 times.
 785 if (!opt_dns_server_.empty())
3164 ✗ clone->SetDnsServer(opt_dns_server_);
3165 785 clone->opt_timeout_proxy_ = opt_timeout_proxy_;
3166 785 clone->opt_timeout_direct_ = opt_timeout_direct_;
3167 785 clone->opt_low_speed_limit_ = opt_low_speed_limit_;
3168 785 clone->opt_max_retries_ = opt_max_retries_;
3169 785 clone->opt_backoff_init_ms_ = opt_backoff_init_ms_;
3170 785 clone->opt_backoff_max_ms_ = opt_backoff_max_ms_;
3171 785 clone->enable_info_header_ = enable_info_header_;
3172 785 clone->enable_http_tracing_ = enable_http_tracing_;
3173 785 clone->http_tracing_headers_ = http_tracing_headers_;
3174 785 clone->follow_redirects_ = follow_redirects_;
3175 785 clone->ignore_signature_failures_ = ignore_signature_failures_;
3176
2/2
✓ Branch 0 taken 751 times.
✓ Branch 1 taken 34 times.
 785 if (opt_host_.chain) {
3177
1/2
✓ Branch 2 taken 751 times.
✗ Branch 3 not taken.
 751 clone->opt_host_.chain = new vector<string>(*opt_host_.chain);
3178
1/2
✓ Branch 2 taken 751 times.
✗ Branch 3 not taken.
 751 clone->opt_host_chain_rtt_ = new vector<int>(*opt_host_chain_rtt_);
3179 }
3180
3181 785 CloneProxyConfig(clone);
3182 785 clone->opt_ip_preference_ = opt_ip_preference_;
3183 785 clone->proxy_template_direct_ = proxy_template_direct_;
3184 785 clone->proxy_template_forced_ = proxy_template_forced_;
3185 785 clone->opt_proxy_groups_reset_after_ = opt_proxy_groups_reset_after_;
3186 785 clone->opt_metalink_.reset_after = opt_metalink_.reset_after;
3187 785 clone->opt_host_.reset_after = opt_host_.reset_after;
3188 785 clone->credentials_attachment_ = credentials_attachment_;
3189 785 clone->ssl_certificate_store_ = ssl_certificate_store_;
3190
3191 785 clone->health_check_ = health_check_;
3192 785 clone->sharding_policy_ = sharding_policy_;
3193 785 clone->failover_indefinitely_ = failover_indefinitely_;
3194 785 clone->fqrn_ = fqrn_;
3195
3196 785 return clone;
3197 }
3198
3199
3200 785 void DownloadManager::CloneProxyConfig(DownloadManager *clone) {
3201 785 clone->opt_proxy_groups_current_ = opt_proxy_groups_current_;
3202 785 clone->opt_proxy_groups_current_burned_ = opt_proxy_groups_current_burned_;
3203 785 clone->opt_proxy_groups_fallback_ = opt_proxy_groups_fallback_;
3204 785 clone->opt_num_proxies_ = opt_num_proxies_;
3205 785 clone->opt_proxy_shard_ = opt_proxy_shard_;
3206 785 clone->opt_proxy_list_ = opt_proxy_list_;
3207 785 clone->opt_proxy_fallback_list_ = opt_proxy_fallback_list_;
3208
2/2
✓ Branch 0 taken 34 times.
✓ Branch 1 taken 751 times.
 785 if (opt_proxy_groups_ == NULL)
3209 34 return;
3210
3211
1/2
✓ Branch 2 taken 751 times.
✗ Branch 3 not taken.
 751 clone->opt_proxy_groups_ = new vector<vector<ProxyInfo> >(*opt_proxy_groups_);
3212
2/4
✓ Branch 2 taken 751 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 751 times.
✗ Branch 6 not taken.
 751 clone->UpdateProxiesUnlocked("cloned");
3213 }
3214
3215 } // namespace download
3216