GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/network/download.cc
Date: 2024-04-21 02:33:16
Exec Total Coverage
Lines: 859 1616 53.2%
Branches: 643 1928 33.4%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * The download module provides an interface for fetching files via HTTP
5 * and file. It is internally using libcurl and the asynchronous DNS resolver
6 * c-ares. The JobInfo struct describes a single file/url to download and
7 * keeps the state during the several phases of downloading.
8 *
9 * The module starts in single-threaded mode and can be switched to multi-
10 * threaded mode by Spawn(). In multi-threaded mode, the Fetch() function still
11 * blocks but there is a separate I/O thread using asynchronous I/O, which
12 * maintains all concurrent connections simultaneously. As there might be more
13 * than 1024 file descriptors for the CernVM-FS process, the I/O thread uses
14 * poll and the libcurl multi socket interface.
15 *
16 * While downloading, files can be decompressed and the secure hash can be
17 * calculated on the fly.
18 *
19 * The module also implements failure handling. If corrupted data has been
20 * downloaded, the transfer is restarted using HTTP "no-cache" pragma.
21 * A "host chain" can be configured. When a host fails, there is automatic
22 * fail-over to the next host in the chain until all hosts are probed.
23 * Similarly a chain of proxy sets can be configured. Inside a proxy set,
24 * proxies are selected randomly (load-balancing set).
25 */
26
27 // TODO(jblomer): MS for time summing
28 // NOLINTNEXTLINE
29 #define __STDC_FORMAT_MACROS
30
31 #include "cvmfs_config.h"
32 #include "download.h"
33
34 #include <alloca.h>
35 #include <errno.h>
36 #include <inttypes.h>
37 #include <poll.h>
38 #include <pthread.h>
39 #include <signal.h>
40 #include <stdint.h>
41 #include <sys/time.h>
42 #include <unistd.h>
43
44 #include <algorithm>
45 #include <cassert>
46 #include <cstdio>
47 #include <cstdlib>
48 #include <cstring>
49 #include <map>
50 #include <set>
51 #include <utility>
52
53 #include "compression.h"
54 #include "crypto/hash.h"
55 #include "duplex_curl.h"
56 #include "interrupt.h"
57 #include "sanitizer.h"
58 #include "ssl.h"
59 #include "util/algorithm.h"
60 #include "util/atomic.h"
61 #include "util/concurrency.h"
62 #include "util/exception.h"
63 #include "util/logging.h"
64 #include "util/posix.h"
65 #include "util/prng.h"
66 #include "util/smalloc.h"
67 #include "util/string.h"
68
69 using namespace std; // NOLINT
70
71 namespace download {
72
73 /**
74 * Returns the status if an interrupt happened for a given repository.
75 *
76 * Used only in case CVMFS_FAILOVER_INDEFINITELY (failover_indefinitely_) is set
77 * where failed downloads are retried indefinitely, unless an interrupt occurred
78 *
79 * @note If you use this functionality you need to change the source code of
80 * e.g. cvmfs_config reload to create a sentinel file. See comment below.
81 *
82 * @return true if an interrupt occurred
83 * false otherwise
84 */
85 ✗ bool Interrupted(const std::string &fqrn, JobInfo *info) {
86 ✗ if (info->allow_failure()) {
87 ✗ return true;
88 }
89
90 ✗ if (!fqrn.empty()) {
91 // it is up to the user the create this sentinel file ("pause_file") if
92 // CVMFS_FAILOVER_INDEFINITELY is used. It must be created during
93 // "cvmfs_config reload" and "cvmfs_config reload $fqrn"
94 ✗ std::string pause_file = std::string("/var/run/cvmfs/interrupt.") + fqrn;
95
96 ✗ LogCvmfs(kLogDownload, kLogDebug,
97 "(id %" PRId64 ") Interrupted(): checking for existence of %s",
98 info->id(), pause_file.c_str());
99 ✗ if (FileExists(pause_file)) {
100 ✗ LogCvmfs(kLogDownload, kLogDebug,
101 "(id %" PRId64 ") Interrupt marker found - "
102 "Interrupting current download, this will EIO outstanding IO.",
103 info->id());
104 ✗ if (0 != unlink(pause_file.c_str())) {
105 ✗ LogCvmfs(kLogDownload, kLogDebug,
106 "(id %" PRId64 ") Couldn't delete interrupt marker: errno=%d",
107 ✗ info->id(), errno);
108 }
109 ✗ return true;
110 }
111 }
112 ✗ return false;
113 }
114
115 203 static Failures PrepareDownloadDestination(JobInfo *info) {
116
3/6
✓ Branch 1 taken 203 times.
✗ Branch 2 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 203 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 203 times.
 203 if (info->sink() != NULL && !info->sink()->IsValid()) {
117 ✗ cvmfs::PathSink* psink = dynamic_cast<cvmfs::PathSink*>(info->sink());
118 ✗ if (psink != NULL) {
119 ✗ LogCvmfs(kLogDownload, kLogDebug,
120 "(id %" PRId64 ") Failed to open path %s: %s (errno=%d).",
121 ✗ info->id(), psink->path().c_str(), strerror(errno), errno);
122 ✗ return kFailLocalIO;
123 } else {
124 ✗ LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") "
125 "Failed to create a valid sink: \n %s",
126 ✗ info->id(), info->sink()->Describe().c_str());
127 ✗ return kFailOther;
128 }
129 }
130
131 203 return kFailOk;
132 }
133
134
135 /**
136 * Called by curl for every HTTP header. Not called for file:// transfers.
137 */
138 602 static size_t CallbackCurlHeader(void *ptr, size_t size, size_t nmemb,
139 void *info_link)
140 {
141 602 const size_t num_bytes = size*nmemb;
142
1/2
✓ Branch 2 taken 602 times.
✗ Branch 3 not taken.
 602 const string header_line(static_cast<const char *>(ptr), num_bytes);
143 602 JobInfo *info = static_cast<JobInfo *>(info_link);
144
145 // LogCvmfs(kLogDownload, kLogDebug, "REMOVE-ME: Header callback with %s",
146 // header_line.c_str());
147
148 // Check http status codes
149
4/6
✓ Branch 2 taken 602 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 602 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 12 times.
✓ Branch 10 taken 590 times.
 602 if (HasPrefix(header_line, "HTTP/1.", false)) {
150
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 12 times.
 12 if (header_line.length() < 10) {
151 ✗ return 0;
152 }
153
154 unsigned i;
155
5/6
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 12 times.
✓ Branch 5 taken 12 times.
✓ Branch 6 taken 12 times.
✓ Branch 7 taken 12 times.
 24 for (i = 8; (i < header_line.length()) && (header_line[i] == ' '); ++i) {}
156
157 // Code is initialized to -1
158
1/2
✓ Branch 1 taken 12 times.
✗ Branch 2 not taken.
 12 if (header_line.length() > i+2) {
159 12 info->SetHttpCode(DownloadManager::ParseHttpCode(&header_line[i]));
160 }
161
162
2/2
✓ Branch 1 taken 9 times.
✓ Branch 2 taken 3 times.
 12 if ((info->http_code() / 100) == 2) {
163 9 return num_bytes;
164
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
 3 } else if ((info->http_code() == 301) ||
165 ✗ (info->http_code() == 302) ||
166
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 3 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 3 times.
✗ Branch 6 not taken.
 3 (info->http_code() == 303) ||
167 ✗ (info->http_code() == 307))
168 {
169
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 3 times.
 3 if (!info->follow_redirects()) {
170 ✗ LogCvmfs(kLogDownload, kLogDebug,
171 "(id %" PRId64 ") redirect support not enabled: %s",
172 info->id(), header_line.c_str());
173 ✗ info->SetErrorCode(kFailHostHttp);
174 ✗ return 0;
175 }
176
1/2
✓ Branch 3 taken 3 times.
✗ Branch 4 not taken.
 3 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") http redirect: %s",
177 info->id(), header_line.c_str());
178 // libcurl will handle this because of CURLOPT_FOLLOWLOCATION
179 3 return num_bytes;
180 } else {
181 ✗ LogCvmfs(kLogDownload, kLogDebug,
182 "(id %" PRId64 ") http status error code: %s [%d]",
183 info->id(), header_line.c_str(), info->http_code());
184 ✗ if (((info->http_code() / 100) == 5) ||
185 ✗ (info->http_code() == 400) || (info->http_code() == 404))
186 {
187 // 5XX returned by host
188 // 400: error from the GeoAPI module
189 // 404: the stratum 1 does not have the newest files
190 ✗ info->SetErrorCode(kFailHostHttp);
191 ✗ } else if (info->http_code() == 429) {
192 // 429: rate throttling (we ignore the backoff hint for the time being)
193 ✗ info->SetErrorCode(kFailHostConnection);
194 } else {
195 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ? kFailHostHttp :
196 kFailProxyHttp);
197 }
198 ✗ return 0;
199 }
200 }
201
202 // If needed: allocate space in sink
203
4/6
✓ Branch 1 taken 590 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 590 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 260 times.
✓ Branch 8 taken 330 times.
 850 if (info->sink() != NULL && info->sink()->RequiresReserve() &&
204
10/17
✓ Branch 2 taken 260 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 260 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 83 times.
✓ Branch 8 taken 177 times.
✓ Branch 9 taken 260 times.
✓ Branch 10 taken 330 times.
✓ Branch 12 taken 260 times.
✓ Branch 13 taken 330 times.
✓ Branch 15 taken 83 times.
✓ Branch 16 taken 507 times.
✗ Branch 17 not taken.
✗ Branch 18 not taken.
✗ Branch 20 not taken.
✗ Branch 21 not taken.
 850 HasPrefix(header_line, "CONTENT-LENGTH:", true))
205 {
206 83 char *tmp = reinterpret_cast<char *>(alloca(num_bytes+1));
207 83 uint64_t length = 0;
208 83 sscanf(header_line.c_str(), "%s %" PRIu64, tmp, &length);
209
1/2
✓ Branch 0 taken 83 times.
✗ Branch 1 not taken.
 83 if (length > 0) {
210
2/4
✓ Branch 2 taken 83 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 83 times.
 83 if (!info->sink()->Reserve(length)) {
211 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr, "(id %" PRId64 ") "
212 "resource %s too large to store in memory (%" PRIu64 ")",
213 info->id(), info->url()->c_str(), length);
214 ✗ info->SetErrorCode(kFailTooBig);
215 ✗ return 0;
216 }
217 } else {
218 // Empty resource
219 ✗ info->sink()->Reserve(0);
220 }
221
4/6
✓ Branch 2 taken 507 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 507 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 3 times.
✓ Branch 10 taken 504 times.
 507 } else if (HasPrefix(header_line, "LOCATION:", true)) {
222 // This comes along with redirects
223
1/2
✓ Branch 3 taken 3 times.
✗ Branch 4 not taken.
 3 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") %s",
224 info->id(), header_line.c_str());
225
3/6
✓ Branch 2 taken 504 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 504 times.
✗ Branch 6 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 504 times.
 504 } else if (HasPrefix(header_line, "X-SQUID-ERROR:", true)) {
226 // Reinterpret host error as proxy error
227 ✗ if (info->error_code() == kFailHostHttp) {
228 ✗ info->SetErrorCode(kFailProxyHttp);
229 }
230
3/6
✓ Branch 2 taken 504 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 504 times.
✗ Branch 6 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 504 times.
 504 } else if (HasPrefix(header_line, "PROXY-STATUS:", true)) {
231 // Reinterpret host error as proxy error if applicable
232 ✗ if ((info->error_code() == kFailHostHttp) &&
233 ✗ (header_line.find("error=") != string::npos)) {
234 ✗ info->SetErrorCode(kFailProxyHttp);
235 }
236 }
237
238 590 return num_bytes;
239 602 }
240
241
242 /**
243 * Called by curl for every received data chunk.
244 */
245 229 static size_t CallbackCurlData(void *ptr, size_t size, size_t nmemb,
246 void *info_link)
247 {
248 229 const size_t num_bytes = size*nmemb;
249 229 JobInfo *info = static_cast<JobInfo *>(info_link);
250
251 // TODO(heretherebedragons) remove if no error comes up
252 // as this means only jobinfo data request (and not header only)
253 // come here
254
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 229 times.
 229 assert(info->sink() != NULL);
255
256 // LogCvmfs(kLogDownload, kLogDebug, "Data callback, %d bytes", num_bytes);
257
258
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 229 times.
 229 if (num_bytes == 0)
259 ✗ return 0;
260
261
2/2
✓ Branch 1 taken 125 times.
✓ Branch 2 taken 104 times.
 229 if (info->expected_hash()) {
262 125 shash::Update(reinterpret_cast<unsigned char *>(ptr),
263 num_bytes, info->hash_context());
264 }
265
266
2/2
✓ Branch 1 taken 124 times.
✓ Branch 2 taken 105 times.
 229 if (info->compressed()) {
267 zlib::StreamStates retval =
268 124 zlib::DecompressZStream2Sink(ptr, static_cast<int64_t>(num_bytes),
269 info->GetZstreamPtr(), info->sink());
270
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 124 times.
 124 if (retval == zlib::kStreamDataError) {
271 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
272 "(id %" PRId64 ") failed to decompress %s",
273 info->id(), info->url()->c_str());
274 ✗ info->SetErrorCode(kFailBadData);
275 ✗ return 0;
276
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 124 times.
 124 } else if (retval == zlib::kStreamIOError) {
277 ✗ LogCvmfs(kLogDownload, kLogSyslogErr,
278 "(id %" PRId64 ") decompressing %s, local IO error",
279 info->id(), info->url()->c_str());
280 ✗ info->SetErrorCode(kFailLocalIO);
281 ✗ return 0;
282 }
283 } else {
284 105 int64_t written = info->sink()->Write(ptr, num_bytes);
285
2/4
✓ Branch 0 taken 105 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 105 times.
 105 if (written < 0 || static_cast<uint64_t>(written) != num_bytes) {
286 ✗ LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") "
287 "Failed to perform write of %zu bytes to sink %s with errno %ld",
288 ✗ info->id(), num_bytes, info->sink()->Describe().c_str(), written);
289 }
290 }
291
292 229 return num_bytes;
293 }
294
295 #ifdef DEBUGMSG
296 113 static int CallbackCurlDebug(
297 CURL * handle,
298 curl_infotype type,
299 char *data,
300 size_t size,
301 void * /* clientp */)
302 {
303 JobInfo *info;
304
1/2
✓ Branch 1 taken 113 times.
✗ Branch 2 not taken.
 113 curl_easy_getinfo(handle, CURLINFO_PRIVATE, &info);
305
306
3/6
✓ Branch 2 taken 113 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 113 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 113 times.
✗ Branch 9 not taken.
 226 std::string prefix = "(id " + StringifyInt(info->id()) + ") ";
307
4/8
✓ Branch 0 taken 18 times.
✓ Branch 1 taken 47 times.
✓ Branch 2 taken 12 times.
✓ Branch 3 taken 36 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
 113 switch (type) {
308 18 case CURLINFO_TEXT:
309
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 18 prefix += "{info} ";
310 18 break;
311 47 case CURLINFO_HEADER_IN:
312
1/2
✓ Branch 1 taken 47 times.
✗ Branch 2 not taken.
 47 prefix += "{header/recv} ";
313 47 break;
314 12 case CURLINFO_HEADER_OUT:
315
1/2
✓ Branch 1 taken 12 times.
✗ Branch 2 not taken.
 12 prefix += "{header/sent} ";
316 12 break;
317 36 case CURLINFO_DATA_IN:
318
2/2
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 32 times.
 36 if (size < 50) {
319
1/2
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
 4 prefix += "{data/recv} ";
320 4 break;
321 } else {
322
1/2
✓ Branch 2 taken 32 times.
✗ Branch 3 not taken.
 32 LogCvmfs(kLogCurl, kLogDebug, "%s{data/recv} <snip>", prefix.c_str());
323 32 return 0;
324 }
325 ✗ case CURLINFO_DATA_OUT:
326 ✗ if (size < 50) {
327 ✗ prefix += "{data/sent} ";
328 ✗ break;
329 } else {
330 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{data/sent} <snip>", prefix.c_str());
331 ✗ return 0;
332 }
333 ✗ case CURLINFO_SSL_DATA_IN:
334 ✗ if (size < 50) {
335 ✗ prefix += "{ssldata/recv} ";
336 ✗ break;
337 } else {
338 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{ssldata/recv} <snip>",
339 prefix.c_str());
340 ✗ return 0;
341 }
342 ✗ case CURLINFO_SSL_DATA_OUT:
343 ✗ if (size < 50) {
344 ✗ prefix += "{ssldata/sent} ";
345 ✗ break;
346 } else {
347 ✗ LogCvmfs(kLogCurl, kLogDebug, "%s{ssldata/sent} <snip>",
348 prefix.c_str());
349 ✗ return 0;
350 }
351 ✗ default:
352 // just log the message
353 ✗ break;
354 }
355
356 81 bool valid_char = true;
357
1/2
✓ Branch 2 taken 81 times.
✗ Branch 3 not taken.
 81 std::string msg(data, size);
358
2/2
✓ Branch 1 taken 4067 times.
✓ Branch 2 taken 81 times.
 4148 for (size_t i = 0; i < msg.length(); ++i) {
359
2/4
✓ Branch 1 taken 4067 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 4067 times.
 4067 if (msg[i] == '\0') {
360 ✗ msg[i] = '~';
361 }
362
363 // verify that char is a valid printable char
364
3/6
✓ Branch 1 taken 4067 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3803 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 3803 times.
 7870 if ((msg[i] < ' ' || msg[i] > '~')
365
6/8
✓ Branch 0 taken 3803 times.
✓ Branch 1 taken 264 times.
✓ Branch 3 taken 264 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 123 times.
✓ Branch 6 taken 141 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 4067 times.
 7993 && (msg[i] != 10 /*line feed*/
366
2/4
✓ Branch 1 taken 123 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 123 times.
 123 && msg[i] != 13 /*carriage return*/)) {
367 ✗ valid_char = false;
368 }
369 }
370
371
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 81 times.
 81 if (!valid_char) {
372 ✗ msg = "<Non-plaintext sequence>";
373 }
374
375
1/2
✓ Branch 3 taken 81 times.
✗ Branch 4 not taken.
 81 LogCvmfs(kLogCurl, kLogDebug, "%s%s",
376
1/2
✓ Branch 1 taken 81 times.
✗ Branch 2 not taken.
 162 prefix.c_str(), Trim(msg, true /* trim_newline */).c_str());
377 81 return 0;
378 113 }
379 #endif
380
381 //------------------------------------------------------------------------------
382
383
384 const int DownloadManager::kProbeUnprobed = -1;
385 const int DownloadManager::kProbeDown = -2;
386 const int DownloadManager::kProbeGeo = -3;
387
388 16965 bool DownloadManager::EscapeUrlChar(unsigned char input, char output[3]) {
389
6/6
✓ Branch 0 taken 15158 times.
✓ Branch 1 taken 1807 times.
✓ Branch 2 taken 11793 times.
✓ Branch 3 taken 3365 times.
✓ Branch 4 taken 11574 times.
✓ Branch 5 taken 2026 times.
 16965 if (((input >= '0') && (input <= '9')) ||
390
4/4
✓ Branch 0 taken 10787 times.
✓ Branch 1 taken 787 times.
✓ Branch 2 taken 10179 times.
✓ Branch 3 taken 2634 times.
 13600 ((input >= 'A') && (input <= 'Z')) ||
391
4/4
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 10170 times.
✓ Branch 2 taken 1165 times.
✓ Branch 3 taken 1478 times.
 12813 ((input >= 'a') && (input <= 'z')) ||
392
5/6
✓ Branch 0 taken 946 times.
✓ Branch 1 taken 219 times.
✓ Branch 2 taken 620 times.
✓ Branch 3 taken 326 times.
✓ Branch 4 taken 620 times.
✗ Branch 5 not taken.
 1165 (input == '/') || (input == ':') || (input == '.') ||
393
1/2
✓ Branch 0 taken 620 times.
✗ Branch 1 not taken.
 620 (input == '@') ||
394
4/4
✓ Branch 0 taken 617 times.
✓ Branch 1 taken 3 times.
✓ Branch 2 taken 9 times.
✓ Branch 3 taken 608 times.
 620 (input == '+') || (input == '-') ||
395
2/4
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
 9 (input == '_') || (input == '~') ||
396
2/4
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 9 times.
 9 (input == '[') || (input == ']') || (input == ','))
397 {
398 16956 output[0] = static_cast<char>(input);
399 16956 return false;
400 }
401
402 9 output[0] = '%';
403 9 output[1] = static_cast<char>(
404
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 7 times.
 9 (input / 16) + ((input / 16 <= 9) ? '0' : 'A'-10));
405 9 output[2] = static_cast<char>(
406
2/2
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 5 times.
 9 (input % 16) + ((input % 16 <= 9) ? '0' : 'A'-10));
407 9 return true;
408 }
409
410
411 /**
412 * Escape special chars from the URL, except for ':' and '/',
413 * which should keep their meaning.
414 */
415 207 string DownloadManager::EscapeUrl(const int64_t jobinfo_id, const string &url) {
416 207 string escaped;
417
1/2
✓ Branch 2 taken 207 times.
✗ Branch 3 not taken.
 207 escaped.reserve(url.length());
418
419 char escaped_char[3];
420
2/2
✓ Branch 1 taken 16965 times.
✓ Branch 2 taken 207 times.
 17172 for (unsigned i = 0, s = url.length(); i < s; ++i) {
421
2/2
✓ Branch 2 taken 9 times.
✓ Branch 3 taken 16956 times.
 16965 if (EscapeUrlChar(url[i], escaped_char)) {
422
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 escaped.append(escaped_char, 3);
423 } else {
424
1/2
✓ Branch 1 taken 16956 times.
✗ Branch 2 not taken.
 16956 escaped.push_back(escaped_char[0]);
425 }
426 }
427
1/2
✓ Branch 3 taken 207 times.
✗ Branch 4 not taken.
 207 LogCvmfs(kLogDownload, kLogDebug, "(id %" PRId64 ") escaped %s to %s",
428 jobinfo_id, url.c_str(), escaped.c_str());
429
430 414 return escaped;
431 }
432
433 /**
434 * escaped array needs to be sufficiently large. Its size is calculated by
435 * passing NULL to EscapeHeader.
436 */
437 ✗ unsigned DownloadManager::EscapeHeader(const string &header,
438 char *escaped_buf,
439 size_t buf_size)
440 {
441 ✗ unsigned esc_pos = 0;
442 char escaped_char[3];
443 ✗ for (unsigned i = 0, s = header.size(); i < s; ++i) {
444 ✗ if (EscapeUrlChar(header[i], escaped_char)) {
445 ✗ for (unsigned j = 0; j < 3; ++j) {
446 ✗ if (escaped_buf) {
447 ✗ if (esc_pos >= buf_size)
448 ✗ return esc_pos;
449 ✗ escaped_buf[esc_pos] = escaped_char[j];
450 }
451 ✗ esc_pos++;
452 }
453 } else {
454 ✗ if (escaped_buf) {
455 ✗ if (esc_pos >= buf_size)
456 ✗ return esc_pos;
457 ✗ escaped_buf[esc_pos] = escaped_char[0];
458 }
459 ✗ esc_pos++;
460 }
461 }
462
463 ✗ return esc_pos;
464 }
465
466 /**
467 * -1 of digits is not a valid Http return code
468 */
469 17 int DownloadManager::ParseHttpCode(const char digits[3]) {
470 17 int result = 0;
471 17 int factor = 100;
472
2/2
✓ Branch 0 taken 51 times.
✓ Branch 1 taken 16 times.
 67 for (int i = 0; i < 3; ++i) {
473
3/4
✓ Branch 0 taken 51 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 times.
✓ Branch 3 taken 50 times.
 51 if ((digits[i] < '0') || (digits[i] > '9'))
474 1 return -1;
475 50 result += (digits[i] - '0') * factor;
476 50 factor /= 10;
477 }
478 16 return result;
479 }
480
481
482 /**
483 * Called when new curl sockets arrive or existing curl sockets depart.
484 */
485 ✗ int DownloadManager::CallbackCurlSocket(CURL * /* easy */,
486 curl_socket_t s,
487 int action,
488 void *userp,
489 void * /* socketp */)
490 {
491 // LogCvmfs(kLogDownload, kLogDebug, "CallbackCurlSocket called with easy "
492 // "handle %p, socket %d, action %d", easy, s, action);
493 ✗ DownloadManager *download_mgr = static_cast<DownloadManager *>(userp);
494 ✗ if (action == CURL_POLL_NONE)
495 ✗ return 0;
496
497 // Find s in watch_fds_
498 unsigned index;
499
500 // TODO(heretherebedragons) why start at index = 0 and not 2?
501 // fd[0] and fd[1] are fixed?
502 ✗ for (index = 0; index < download_mgr->watch_fds_inuse_; ++index) {
503 ✗ if (download_mgr->watch_fds_[index].fd == s)
504 ✗ break;
505 }
506 // Or create newly
507 ✗ if (index == download_mgr->watch_fds_inuse_) {
508 // Extend array if necessary
509 ✗ if (download_mgr->watch_fds_inuse_ == download_mgr->watch_fds_size_)
510 {
511 ✗ assert(download_mgr->watch_fds_size_ > 0);
512 ✗ download_mgr->watch_fds_size_ *= 2;
513 ✗ download_mgr->watch_fds_ = static_cast<struct pollfd *>(
514 ✗ srealloc(download_mgr->watch_fds_,
515 ✗ download_mgr->watch_fds_size_ * sizeof(struct pollfd)));
516 }
517 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].fd = s;
518 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].events = 0;
519 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_].revents = 0;
520 ✗ download_mgr->watch_fds_inuse_++;
521 }
522
523 ✗ switch (action) {
524 ✗ case CURL_POLL_IN:
525 ✗ download_mgr->watch_fds_[index].events = POLLIN | POLLPRI;
526 ✗ break;
527 ✗ case CURL_POLL_OUT:
528 ✗ download_mgr->watch_fds_[index].events = POLLOUT | POLLWRBAND;
529 ✗ break;
530 ✗ case CURL_POLL_INOUT:
531 ✗ download_mgr->watch_fds_[index].events =
532 POLLIN | POLLPRI | POLLOUT | POLLWRBAND;
533 ✗ break;
534 ✗ case CURL_POLL_REMOVE:
535 ✗ if (index < download_mgr->watch_fds_inuse_-1) {
536 ✗ download_mgr->watch_fds_[index] =
537 ✗ download_mgr->watch_fds_[download_mgr->watch_fds_inuse_-1];
538 }
539 ✗ download_mgr->watch_fds_inuse_--;
540 // Shrink array if necessary
541 ✗ if ((download_mgr->watch_fds_inuse_ > download_mgr->watch_fds_max_) &&
542 ✗ (download_mgr->watch_fds_inuse_ < download_mgr->watch_fds_size_/2))
543 {
544 ✗ download_mgr->watch_fds_size_ /= 2;
545 // LogCvmfs(kLogDownload, kLogDebug, "shrinking watch_fds_ (%d)",
546 // watch_fds_size_);
547 ✗ download_mgr->watch_fds_ = static_cast<struct pollfd *>(
548 ✗ srealloc(download_mgr->watch_fds_,
549 ✗ download_mgr->watch_fds_size_*sizeof(struct pollfd)));
550 // LogCvmfs(kLogDownload, kLogDebug, "shrinking watch_fds_ done",
551 // watch_fds_size_);
552 }
553 ✗ break;
554 ✗ default:
555 ✗ break;
556 }
557
558 ✗ return 0;
559 }
560
561
562 /**
563 * Worker thread event loop. Waits on new JobInfo structs on a pipe.
564 */
565 ✗ void *DownloadManager::MainDownload(void *data) {
566 ✗ DownloadManager *download_mgr = static_cast<DownloadManager *>(data);
567 ✗ LogCvmfs(kLogDownload, kLogDebug,
568 "download I/O thread of DownloadManager '%s' started",
569 download_mgr->name_.c_str());
570
571 ✗ const int kIdxPipeTerminate = 0;
572 ✗ const int kIdxPipeJobs = 1;
573
574 ✗ download_mgr->watch_fds_ =
575 ✗ static_cast<struct pollfd *>(smalloc(2 * sizeof(struct pollfd)));
576 ✗ download_mgr->watch_fds_size_ = 2;
577 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].fd =
578 ✗ download_mgr->pipe_terminate_->GetReadFd();
579 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].events = POLLIN | POLLPRI;
580 ✗ download_mgr->watch_fds_[kIdxPipeTerminate].revents = 0;
581 ✗ download_mgr->watch_fds_[kIdxPipeJobs].fd =
582 ✗ download_mgr->pipe_jobs_->GetReadFd();
583 ✗ download_mgr->watch_fds_[kIdxPipeJobs].events = POLLIN | POLLPRI;
584 ✗ download_mgr->watch_fds_[kIdxPipeJobs].revents = 0;
585 ✗ download_mgr->watch_fds_inuse_ = 2;
586
587 ✗ int still_running = 0;
588 struct timeval timeval_start, timeval_stop;
589 ✗ gettimeofday(&timeval_start, NULL);
590 while (true) {
591 int timeout;
592 ✗ if (still_running) {
593 /* NOTE: The following might degrade the performance for many small files
594 * use case. TODO(jblomer): look into it.
595 // Specify a timeout for polling in ms; this allows us to return
596 // to libcurl once a second so it can look for internal operations
597 // which timed out. libcurl has a more elaborate mechanism
598 // (CURLMOPT_TIMERFUNCTION) that would inform us of the next potential
599 // timeout. TODO(bbockelm) we should switch to that in the future.
600 timeout = 100;
601 */
602 ✗ timeout = 1;
603 } else {
604 ✗ timeout = -1;
605 ✗ gettimeofday(&timeval_stop, NULL);
606 int64_t delta = static_cast<int64_t>(
607 ✗ 1000 * DiffTimeSeconds(timeval_start, timeval_stop));
608 ✗ perf::Xadd(download_mgr->counters_->sz_transfer_time, delta);
609 }
610 ✗ int retval = poll(download_mgr->watch_fds_, download_mgr->watch_fds_inuse_,
611 timeout);
612 ✗ if (retval < 0) {
613 ✗ continue;
614 }
615
616 // Handle timeout
617 ✗ if (retval == 0) {
618 ✗ curl_multi_socket_action(download_mgr->curl_multi_,
619 CURL_SOCKET_TIMEOUT,
620 0,
621 &still_running);
622 }
623
624 // Terminate I/O thread
625 ✗ if (download_mgr->watch_fds_[kIdxPipeTerminate].revents)
626 ✗ break;
627
628 // New job arrives
629 ✗ if (download_mgr->watch_fds_[kIdxPipeJobs].revents) {
630 ✗ download_mgr->watch_fds_[kIdxPipeJobs].revents = 0;
631 JobInfo *info;
632 ✗ download_mgr->pipe_jobs_->Read<JobInfo*>(&info);
633 ✗ if (!still_running) {
634 ✗ gettimeofday(&timeval_start, NULL);
635 }
636 ✗ CURL *handle = download_mgr->AcquireCurlHandle();
637 ✗ download_mgr->InitializeRequest(info, handle);
638 ✗ download_mgr->SetUrlOptions(info);
639 ✗ curl_multi_add_handle(download_mgr->curl_multi_, handle);
640 ✗ curl_multi_socket_action(download_mgr->curl_multi_,
641 CURL_SOCKET_TIMEOUT,
642 0,
643 &still_running);
644 }
645
646 // Activity on curl sockets
647 // Within this loop the curl_multi_socket_action() may cause socket(s)
648 // to be removed from watch_fds_. If a socket is removed it is replaced
649 // by the socket at the end of the array and the inuse count is decreased.
650 // Therefore loop over the array in reverse order.
651 ✗ for (int64_t i = download_mgr->watch_fds_inuse_-1; i >= 2; --i) {
652 ✗ if (i >= download_mgr->watch_fds_inuse_) {
653 ✗ continue;
654 }
655 ✗ if (download_mgr->watch_fds_[i].revents) {
656 ✗ int ev_bitmask = 0;
657 ✗ if (download_mgr->watch_fds_[i].revents & (POLLIN | POLLPRI))
658 ✗ ev_bitmask |= CURL_CSELECT_IN;
659 ✗ if (download_mgr->watch_fds_[i].revents & (POLLOUT | POLLWRBAND))
660 ✗ ev_bitmask |= CURL_CSELECT_OUT;
661 ✗ if (download_mgr->watch_fds_[i].revents &
662 (POLLERR | POLLHUP | POLLNVAL))
663 {
664 ✗ ev_bitmask |= CURL_CSELECT_ERR;
665 }
666 ✗ download_mgr->watch_fds_[i].revents = 0;
667
668 ✗ curl_multi_socket_action(download_mgr->curl_multi_,
669 ✗ download_mgr->watch_fds_[i].fd,
670 ev_bitmask,
671 &still_running);
672 }
673 }
674
675 // Check if transfers are completed
676 CURLMsg *curl_msg;
677 int msgs_in_queue;
678 ✗ while ((curl_msg = curl_multi_info_read(download_mgr->curl_multi_,
679 &msgs_in_queue)))
680 {
681 ✗ if (curl_msg->msg == CURLMSG_DONE) {
682 ✗ perf::Inc(download_mgr->counters_->n_requests);
683 JobInfo *info;
684 ✗ CURL *easy_handle = curl_msg->easy_handle;
685 ✗ int curl_error = curl_msg->data.result;
686 ✗ curl_easy_getinfo(easy_handle, CURLINFO_PRIVATE, &info);
687
688 int64_t redir_count;
689 ✗ curl_easy_getinfo(easy_handle, CURLINFO_REDIRECT_COUNT, &redir_count);
690 ✗ LogCvmfs(kLogDownload, kLogDebug, "(manager '%s' - id %" PRId64 ") "
691 "Number of CURL redirects %" PRId64 ,
692 download_mgr->name_.c_str(), info->id(),
693 redir_count);
694
695 ✗ curl_multi_remove_handle(download_mgr->curl_multi_, easy_handle);
696 ✗ if (download_mgr->VerifyAndFinalize(curl_error, info)) {
697 ✗ curl_multi_add_handle(download_mgr->curl_multi_, easy_handle);
698 ✗ curl_multi_socket_action(download_mgr->curl_multi_,
699 CURL_SOCKET_TIMEOUT,
700 0,
701 &still_running);
702 } else {
703 // Return easy handle into pool and write result back
704 ✗ download_mgr->ReleaseCurlHandle(easy_handle);
705
706 ✗ DataTubeElement *ele = new DataTubeElement(kActionStop);
707 ✗ info->GetDataTubePtr()->EnqueueBack(ele);
708 info->GetPipeJobResultPtr()->
709 ✗ Write<download::Failures>(info->error_code());
710 }
711 }
712 }
713 }
714
715 ✗ for (set<CURL *>::iterator i = download_mgr->pool_handles_inuse_->begin(),
716 ✗ iEnd = download_mgr->pool_handles_inuse_->end(); i != iEnd; ++i)
717 {
718 ✗ curl_multi_remove_handle(download_mgr->curl_multi_, *i);
719 ✗ curl_easy_cleanup(*i);
720 }
721 ✗ download_mgr->pool_handles_inuse_->clear();
722 ✗ free(download_mgr->watch_fds_);
723
724 ✗ LogCvmfs(kLogDownload, kLogDebug,
725 "download I/O thread of DownloadManager '%s' terminated",
726 download_mgr->name_.c_str());
727 ✗ return NULL;
728 }
729
730
731 //------------------------------------------------------------------------------
732
733
734 159 HeaderLists::~HeaderLists() {
735
2/2
✓ Branch 1 taken 160 times.
✓ Branch 2 taken 159 times.
 319 for (unsigned i = 0; i < blocks_.size(); ++i) {
736
1/2
✓ Branch 1 taken 160 times.
✗ Branch 2 not taken.
 160 delete[] blocks_[i];
737 }
738 159 blocks_.clear();
739 159 }
740
741
742 879 curl_slist *HeaderLists::GetList(const char *header) {
743 879 return Get(header);
744 }
745
746
747 203 curl_slist *HeaderLists::DuplicateList(curl_slist *slist) {
748
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 203 times.
 203 assert(slist);
749 203 curl_slist *copy = GetList(slist->data);
750 203 copy->next = slist->next;
751 203 curl_slist *prev = copy;
752 203 slist = slist->next;
753
2/2
✓ Branch 0 taken 406 times.
✓ Branch 1 taken 203 times.
 609 while (slist) {
754 406 curl_slist *new_link = Get(slist->data);
755 406 new_link->next = slist->next;
756 406 prev->next = new_link;
757 406 prev = new_link;
758 406 slist = slist->next;
759 }
760 203 return copy;
761 }
762
763
764 332 void HeaderLists::AppendHeader(curl_slist *slist, const char *header) {
765
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 332 times.
 332 assert(slist);
766 332 curl_slist *new_link = Get(header);
767 332 new_link->next = NULL;
768
769
2/2
✓ Branch 0 taken 193 times.
✓ Branch 1 taken 332 times.
 525 while (slist->next)
770 193 slist = slist->next;
771 332 slist->next = new_link;
772 332 }
773
774
775 /**
776 * Ensures that a certain header string is _not_ part of slist on return.
777 * Note that if the first header element matches, the returned slist points
778 * to a different value.
779 */
780 5 void HeaderLists::CutHeader(const char *header, curl_slist **slist) {
781
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5 times.
 5 assert(slist);
782 curl_slist head;
783 5 head.next = *slist;
784 5 curl_slist *prev = &head;
785 5 curl_slist *rover = *slist;
786
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 5 times.
 14 while (rover) {
787
2/2
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 5 times.
 9 if (strcmp(rover->data, header) == 0) {
788 4 prev->next = rover->next;
789 4 Put(rover);
790 4 rover = prev;
791 }
792 9 prev = rover;
793 9 rover = rover->next;
794 }
795 5 *slist = head.next;
796 5 }
797
798
799 463 void HeaderLists::PutList(curl_slist *slist) {
800
2/2
✓ Branch 0 taken 885 times.
✓ Branch 1 taken 463 times.
 1348 while (slist) {
801 885 curl_slist *next = slist->next;
802 885 Put(slist);
803 885 slist = next;
804 }
805 463 }
806
807
808 4 string HeaderLists::Print(curl_slist *slist) {
809 4 string verbose;
810
2/2
✓ Branch 0 taken 8 times.
✓ Branch 1 taken 4 times.
 12 while (slist) {
811
3/6
✓ Branch 2 taken 8 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 8 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 8 times.
✗ Branch 9 not taken.
 8 verbose += string(slist->data) + "\n";
812 8 slist = slist->next;
813 }
814 4 return verbose;
815 }
816
817
818 1617 curl_slist *HeaderLists::Get(const char *header) {
819
2/2
✓ Branch 1 taken 1457 times.
✓ Branch 2 taken 161 times.
 1618 for (unsigned i = 0; i < blocks_.size(); ++i) {
820
2/2
✓ Branch 0 taken 70244 times.
✓ Branch 1 taken 1 times.
 70245 for (unsigned j = 0; j < kBlockSize; ++j) {
821
2/2
✓ Branch 2 taken 1456 times.
✓ Branch 3 taken 68788 times.
 70244 if (!IsUsed(&(blocks_[i][j]))) {
822 1456 blocks_[i][j].data = const_cast<char *>(header);
823 1456 return &(blocks_[i][j]);
824 }
825 }
826 }
827
828 // All used, new block
829 161 AddBlock();
830 161 blocks_[blocks_.size()-1][0].data = const_cast<char *>(header);
831 161 return &(blocks_[blocks_.size()-1][0]);
832 }
833
834
835 42105 void HeaderLists::Put(curl_slist *slist) {
836 42105 slist->data = NULL;
837 42105 slist->next = NULL;
838 42105 }
839
840
841 161 void HeaderLists::AddBlock() {
842
1/2
✓ Branch 1 taken 161 times.
✗ Branch 2 not taken.
 161 curl_slist *new_block = new curl_slist[kBlockSize];
843
2/2
✓ Branch 0 taken 41216 times.
✓ Branch 1 taken 161 times.
 41377 for (unsigned i = 0; i < kBlockSize; ++i) {
844 41216 Put(&new_block[i]);
845 }
846
1/2
✓ Branch 1 taken 161 times.
✗ Branch 2 not taken.
 161 blocks_.push_back(new_block);
847 161 }
848
849
850 //------------------------------------------------------------------------------
851
852
853 ✗ string DownloadManager::ProxyInfo::Print() {
854 ✗ if (url == "DIRECT")
855 ✗ return url;
856
857 ✗ string result = url;
858 int remaining =
859 ✗ static_cast<int>(host.deadline()) - static_cast<int>(time(NULL));
860 ✗ string expinfo = (remaining >= 0) ? "+" : "";
861 ✗ if (abs(remaining) >= 3600) {
862 ✗ expinfo += StringifyInt(remaining/3600) + "h";
863 ✗ } else if (abs(remaining) >= 60) {
864 ✗ expinfo += StringifyInt(remaining/60) + "m";
865 } else {
866 ✗ expinfo += StringifyInt(remaining) + "s";
867 }
868 ✗ if (host.status() == dns::kFailOk) {
869 ✗ result += " (" + host.name() + ", " + expinfo + ")";
870 } else {
871 ✗ result += " (:unresolved:, " + expinfo + ")";
872 }
873 ✗ return result;
874 }
875
876
877 /**
878 * Gets an idle CURL handle from the pool. Creates a new one and adds it to
879 * the pool if necessary.
880 */
881 203 CURL *DownloadManager::AcquireCurlHandle() {
882 CURL *handle;
883
884
2/2
✓ Branch 1 taken 77 times.
✓ Branch 2 taken 126 times.
 203 if (pool_handles_idle_->empty()) {
885 // Create a new handle
886
1/2
✓ Branch 1 taken 77 times.
✗ Branch 2 not taken.
 77 handle = curl_easy_init();
887
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 77 times.
 77 assert(handle != NULL);
888
889
1/2
✓ Branch 1 taken 77 times.
✗ Branch 2 not taken.
 77 curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1);
890 // curl_easy_setopt(curl_default, CURLOPT_FAILONERROR, 1);
891
1/2
✓ Branch 1 taken 77 times.
✗ Branch 2 not taken.
 77 curl_easy_setopt(handle, CURLOPT_HEADERFUNCTION, CallbackCurlHeader);
892
1/2
✓ Branch 1 taken 77 times.
✗ Branch 2 not taken.
 77 curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, CallbackCurlData);
893 } else {
894 126 handle = *(pool_handles_idle_->begin());
895
1/2
✓ Branch 2 taken 126 times.
✗ Branch 3 not taken.
 126 pool_handles_idle_->erase(pool_handles_idle_->begin());
896 }
897
898
1/2
✓ Branch 1 taken 203 times.
✗ Branch 2 not taken.
 203 pool_handles_inuse_->insert(handle);
899
900 203 return handle;
901 }
902
903
904 203 void DownloadManager::ReleaseCurlHandle(CURL *handle) {
905
1/2
✓ Branch 1 taken 203 times.
✗ Branch 2 not taken.
 203 set<CURL *>::iterator elem = pool_handles_inuse_->find(handle);
906
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 203 times.
 203 assert(elem != pool_handles_inuse_->end());
907
908
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 203 times.
 203 if (pool_handles_idle_->size() > pool_max_handles_) {
909 ✗ curl_easy_cleanup(*elem);
910 } else {
911
1/2
✓ Branch 2 taken 203 times.
✗ Branch 3 not taken.
 203 pool_handles_idle_->insert(*elem);
912 }
913
914
1/2
✓ Branch 1 taken 203 times.
✗ Branch 2 not taken.
 203 pool_handles_inuse_->erase(elem);
915 203 }
916
917
918 /**
919 * HTTP request options: set the URL and other options such as timeout and
920 * proxy.
921 */
922 203 void DownloadManager::InitializeRequest(JobInfo *info, CURL *handle) {
923 // Initialize internal download state
924 203 info->SetCurlHandle(handle);
925 203 info->SetErrorCode(kFailOk);
926 203 info->SetHttpCode(-1);
927 203 info->SetFollowRedirects(follow_redirects_);
928 203 info->SetNumUsedProxies(1);
929 203 info->SetNumUsedHosts(1);
930 203 info->SetNumRetries(0);
931 203 info->SetBackoffMs(0);
932 203 info->SetHeaders(header_lists_->DuplicateList(default_headers_));
933
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 203 times.
 203 if (info->info_header()) {
934 ✗ header_lists_->AppendHeader(info->headers(), info->info_header());
935 }
936
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 203 times.
 203 if (enable_http_tracing_) {
937 ✗ for (unsigned int i = 0; i < http_tracing_headers_.size(); i++) {
938 ✗ header_lists_->AppendHeader(info->headers(),
939 ✗ (http_tracing_headers_)[i].c_str());
940 }
941
942 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_pid());
943 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_gid());
944 ✗ header_lists_->AppendHeader(info->headers(), info->tracing_header_uid());
945
946 ✗ LogCvmfs(kLogDownload, kLogDebug, "(manager '%s' - id %" PRId64 ") "
947 "CURL Header for URL: %s is:\n %s",
948 name_.c_str(), info->id(), info->url()->c_str(),
949 ✗ header_lists_->Print(info->headers()).c_str());
950 }
951
952
2/2
✓ Branch 1 taken 3 times.
✓ Branch 2 taken 200 times.
 203 if (info->force_nocache()) {
953 3 SetNocache(info);
954 } else {
955 200 info->SetNocache(false);
956 }
957
2/2
✓ Branch 1 taken 129 times.
✓ Branch 2 taken 74 times.
 203 if (info->compressed()) {
958 129 zlib::DecompressInit(info->GetZstreamPtr());
959 }
960
2/2
✓ Branch 1 taken 130 times.
✓ Branch 2 taken 73 times.
 203 if (info->expected_hash()) {
961
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 130 times.
 130 assert(info->hash_context().buffer != NULL);
962 130 shash::Init(info->hash_context());
963 }
964
965
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 203 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 203 times.
 203 if ((info->range_offset() != -1) && (info->range_size())) {
966 char byte_range_array[100];
967 ✗ const int64_t range_lower = static_cast<int64_t>(info->range_offset());
968 const int64_t range_upper = static_cast<int64_t>(
969 ✗ info->range_offset() + info->range_size() - 1);
970 ✗ if (snprintf(byte_range_array, sizeof(byte_range_array),
971 "%" PRId64 "-%" PRId64,
972 ✗ range_lower, range_upper) == 100)
973 {
974 ✗ PANIC(NULL); // Should be impossible given limits on offset size.
975 }
976 ✗ curl_easy_setopt(handle, CURLOPT_RANGE, byte_range_array);
977 } else {
978 203 curl_easy_setopt(handle, CURLOPT_RANGE, NULL);
979 }
980
981 // Set curl parameters
982 203 curl_easy_setopt(handle, CURLOPT_PRIVATE, static_cast<void *>(info));
983 203 curl_easy_setopt(handle, CURLOPT_WRITEHEADER,
984 static_cast<void *>(info));
985 203 curl_easy_setopt(handle, CURLOPT_WRITEDATA, static_cast<void *>(info));
986 203 curl_easy_setopt(handle, CURLOPT_HTTPHEADER, info->headers());
987
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 203 times.
 203 if (info->head_request()) {
988 ✗ curl_easy_setopt(handle, CURLOPT_NOBODY, 1);
989 } else {
990 203 curl_easy_setopt(handle, CURLOPT_HTTPGET, 1);
991 }
992
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 203 times.
 203 if (opt_ipv4_only_) {
993 ✗ curl_easy_setopt(handle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
994 }
995
2/2
✓ Branch 0 taken 53 times.
✓ Branch 1 taken 150 times.
 203 if (follow_redirects_) {
996 53 curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1);
997 53 curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 4);
998 }
999 #ifdef DEBUGMSG
1000 203 curl_easy_setopt(handle, CURLOPT_VERBOSE, 1);
1001 203 curl_easy_setopt(handle, CURLOPT_DEBUGFUNCTION, CallbackCurlDebug);
1002 #endif
1003 203 }
1004
1005
1006 /**
1007 * Sets the URL specific options such as host to use and timeout. It might also
1008 * set an error code, in which case the further processing should react on.
1009 */
1010 206 void DownloadManager::SetUrlOptions(JobInfo *info) {
1011 206 CURL *curl_handle = info->curl_handle();
1012 206 string url_prefix;
1013
1014 206 MutexLockGuard m(lock_options_);
1015
1016 // sharding policy
1017
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 206 times.
 206 if (sharding_policy_.UseCount() > 0) {
1018 ✗ if (info->proxy() != "") {
1019 // proxy already set, so this is a failover event
1020 ✗ perf::Inc(counters_->n_proxy_failover);
1021 }
1022 ✗ info->SetProxy(sharding_policy_->GetNextProxy(info->url(), info->proxy(),
1023 ✗ info->range_offset() == -1 ? 0 : info->range_offset()));
1024
1025 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, info->proxy().c_str());
1026 } else { // no sharding policy
1027 // Check if proxy group needs to be reset from backup to primary
1028
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 206 times.
 206 if (opt_timestamp_backup_proxies_ > 0) {
1029 ✗ const time_t now = time(NULL);
1030 ✗ if (static_cast<int64_t>(now) >
1031 ✗ static_cast<int64_t>(opt_timestamp_backup_proxies_ +
1032 ✗ opt_proxy_groups_reset_after_))
1033 {
1034 ✗ opt_proxy_groups_current_ = 0;
1035 ✗ opt_timestamp_backup_proxies_ = 0;
1036 ✗ RebalanceProxiesUnlocked("Reset proxy group from backup to primary");
1037 }
1038 }
1039 // Check if load-balanced proxies within the group need to be reset
1040
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 206 times.
 206 if (opt_timestamp_failover_proxies_ > 0) {
1041 ✗ const time_t now = time(NULL);
1042 ✗ if (static_cast<int64_t>(now) >
1043 ✗ static_cast<int64_t>(opt_timestamp_failover_proxies_ +
1044 ✗ opt_proxy_groups_reset_after_))
1045 {
1046 ✗ RebalanceProxiesUnlocked(
1047 "Reset load-balanced proxies within the active group");
1048 }
1049 }
1050 // Check if host needs to be reset
1051
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 206 times.
 206 if (opt_timestamp_backup_host_ > 0) {
1052 ✗ const time_t now = time(NULL);
1053 ✗ if (static_cast<int64_t>(now) >
1054 ✗ static_cast<int64_t>(opt_timestamp_backup_host_ +
1055 ✗ opt_host_reset_after_))
1056 {
1057 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1058 "(manager %s - id %" PRId64 ") "
1059 "switching host from %s to %s (reset host)", name_.c_str(),
1060 ✗ info->id(), (*opt_host_chain_)[opt_host_chain_current_].c_str(),
1061 ✗ (*opt_host_chain_)[0].c_str());
1062 ✗ opt_host_chain_current_ = 0;
1063 ✗ opt_timestamp_backup_host_ = 0;
1064 }
1065 }
1066
1067
1/2
✓ Branch 2 taken 206 times.
✗ Branch 3 not taken.
 206 ProxyInfo *proxy = ChooseProxyUnlocked(info->expected_hash());
1068
6/6
✓ Branch 0 taken 87 times.
✓ Branch 1 taken 119 times.
✓ Branch 3 taken 83 times.
✓ Branch 4 taken 4 times.
✓ Branch 5 taken 202 times.
✓ Branch 6 taken 4 times.
 206 if (!proxy || (proxy->url == "DIRECT")) {
1069
2/4
✓ Branch 2 taken 202 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 202 times.
✗ Branch 6 not taken.
 202 info->SetProxy("DIRECT");
1070
1/2
✓ Branch 2 taken 202 times.
✗ Branch 3 not taken.
 202 curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "");
1071 } else {
1072 // Note: inside ValidateProxyIpsUnlocked() we may change the proxy data
1073 // structure, so we must not pass proxy->... (== current_proxy())
1074 // parameters directly
1075
1/2
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
 4 std::string purl = proxy->url;
1076
1/2
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
 4 dns::Host phost = proxy->host;
1077
1/2
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
 4 const bool changed = ValidateProxyIpsUnlocked(purl, phost);
1078 // Current proxy may have changed
1079
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
 4 if (changed) {
1080 ✗ proxy = ChooseProxyUnlocked(info->expected_hash());
1081 }
1082
1/2
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
 4 info->SetProxy(proxy->url);
1083
1/2
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
 4 if (proxy->host.status() == dns::kFailOk) {
1084
2/4
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
✓ Branch 6 taken 4 times.
✗ Branch 7 not taken.
 4 curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY,
1085 info->proxy().c_str());
1086 } else {
1087 // We know it can't work, don't even try to download
1088 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "0.0.0.0");
1089 }
1090 4 }
1091 } // end !sharding
1092
1093
1/2
✓ Branch 1 taken 206 times.
✗ Branch 2 not taken.
 206 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_LIMIT, opt_low_speed_limit_);
1094
3/6
✓ Branch 1 taken 206 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 4 times.
✓ Branch 6 taken 202 times.
 206 if (info->proxy() != "DIRECT") {
1095
1/2
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
 4 curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT, opt_timeout_proxy_);
1096
1/2
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
 4 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_TIME, opt_timeout_proxy_);
1097 } else {
1098
1/2
✓ Branch 1 taken 202 times.
✗ Branch 2 not taken.
 202 curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT, opt_timeout_direct_);
1099
1/2
✓ Branch 1 taken 202 times.
✗ Branch 2 not taken.
 202 curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_TIME, opt_timeout_direct_);
1100 }
1101
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 206 times.
 206 if (!opt_dns_server_.empty())
1102 ✗ curl_easy_setopt(curl_handle, CURLOPT_DNS_SERVERS, opt_dns_server_.c_str());
1103
1104
5/6
✓ Branch 1 taken 104 times.
✓ Branch 2 taken 102 times.
✓ Branch 3 taken 104 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 104 times.
✓ Branch 6 taken 102 times.
 206 if (info->probe_hosts() && opt_host_chain_) {
1105
1/2
✓ Branch 2 taken 104 times.
✗ Branch 3 not taken.
 104 url_prefix = (*opt_host_chain_)[opt_host_chain_current_];
1106 104 info->SetCurrentHostChainIndex(opt_host_chain_current_);
1107 }
1108
1109
1/2
✓ Branch 2 taken 206 times.
✗ Branch 3 not taken.
 206 string url = url_prefix + *(info->url());
1110
1111
1/2
✓ Branch 1 taken 206 times.
✗ Branch 2 not taken.
 206 curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, 1L);
1112
2/6
✓ Branch 1 taken 206 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 206 times.
 206 if (url.substr(0, 5) == "https") {
1113 ✗ bool rvb = ssl_certificate_store_.ApplySslCertificatePath(curl_handle);
1114 ✗ if (!rvb) {
1115 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1116 "(manager %s - id %" PRId64 ") "
1117 "Failed to set SSL certificate path %s", name_.c_str(),
1118 ✗ info->id(), ssl_certificate_store_.GetCaPath().c_str());
1119 }
1120 ✗ if (info->pid() != -1) {
1121 ✗ if (credentials_attachment_ == NULL) {
1122 ✗ LogCvmfs(kLogDownload, kLogDebug, "(manager %s - id %" PRId64 ") "
1123 "uses secure downloads but no credentials attachment set",
1124 name_.c_str(), info->id());
1125 } else {
1126 ✗ bool retval = credentials_attachment_->ConfigureCurlHandle(
1127 curl_handle, info->pid(), info->GetCredDataPtr());
1128 ✗ if (!retval) {
1129 ✗ LogCvmfs(kLogDownload, kLogDebug, "(manager %s - id %" PRId64 ") "
1130 "failed attaching credentials",
1131 name_.c_str(), info->id());
1132 }
1133 }
1134 }
1135 // The download manager disables signal handling in the curl library;
1136 // as OpenSSL's implementation of TLS will generate a sigpipe in some
1137 // error paths, we must explicitly disable SIGPIPE here.
1138 // TODO(jblomer): it should be enough to do this once
1139 ✗ signal(SIGPIPE, SIG_IGN);
1140 }
1141
1142
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 206 times.
 206 if (url.find("@proxy@") != string::npos) {
1143 // This is used in Geo-API requests (only), to replace a portion of the
1144 // URL with the current proxy name for the sake of caching the result.
1145 // Replace the @proxy@ either with a passed in "forced" template (which
1146 // is set from $CVMFS_PROXY_TEMPLATE) if there is one, or a "direct"
1147 // template (which is the uuid) if there's no proxy, or the name of the
1148 // proxy.
1149 ✗ string replacement;
1150 ✗ if (proxy_template_forced_ != "") {
1151 ✗ replacement = proxy_template_forced_;
1152 ✗ } else if (info->proxy() == "DIRECT") {
1153 ✗ replacement = proxy_template_direct_;
1154 } else {
1155 ✗ if (opt_proxy_groups_current_ >= opt_proxy_groups_fallback_) {
1156 // It doesn't make sense to use the fallback proxies in Geo-API requests
1157 // since the fallback proxies are supposed to get sorted, too.
1158 ✗ info->SetProxy("DIRECT");
1159 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_PROXY, "");
1160 ✗ replacement = proxy_template_direct_;
1161 } else {
1162 ✗ replacement = ChooseProxyUnlocked(info->expected_hash())->host.name();
1163 }
1164 }
1165 ✗ replacement = (replacement == "") ? proxy_template_direct_ : replacement;
1166 ✗ LogCvmfs(kLogDownload, kLogDebug, "(manager %s - id %" PRId64 ") "
1167 "replacing @proxy@ by %s",
1168 name_.c_str(), info->id(), replacement.c_str());
1169 ✗ url = ReplaceAll(url, "@proxy@", replacement);
1170 }
1171
1172 // TODO(heretherebedragons) before removing
1173 // static_cast<cvmfs::MemSink*>(info->sink)->size() == 0
1174 // and just always call info->sink->Reserve()
1175 // we should do a speed check
1176
4/6
✓ Branch 2 taken 206 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 91 times.
✓ Branch 5 taken 115 times.
✓ Branch 6 taken 91 times.
✗ Branch 7 not taken.
 297 if ((info->sink() != NULL) && info->sink()->RequiresReserve() &&
1177
1/4
✓ Branch 1 taken 206 times.
✗ Branch 2 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
 503 (static_cast<cvmfs::MemSink*>(info->sink())->size() == 0) &&
1178
10/17
✓ Branch 2 taken 91 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 91 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 78 times.
✓ Branch 8 taken 13 times.
✓ Branch 9 taken 91 times.
✓ Branch 10 taken 115 times.
✓ Branch 12 taken 91 times.
✓ Branch 13 taken 115 times.
✓ Branch 15 taken 78 times.
✓ Branch 16 taken 128 times.
✗ Branch 17 not taken.
✗ Branch 18 not taken.
✗ Branch 20 not taken.
✗ Branch 21 not taken.
 297 HasPrefix(url, "file://", false)) {
1179 platform_stat64 stat_buf;
1180 78 int retval = platform_stat(url.c_str(), &stat_buf);
1181
1/2
✓ Branch 0 taken 78 times.
✗ Branch 1 not taken.
 78 if (retval != 0) {
1182 // this is an error: file does not exist or out of memory
1183 // error is caught in other code section.
1184
1/2
✓ Branch 2 taken 78 times.
✗ Branch 3 not taken.
 78 info->sink()->Reserve(64ul * 1024ul);
1185 } else {
1186 ✗ info->sink()->Reserve(stat_buf.st_size);
1187 }
1188 }
1189
1190
2/4
✓ Branch 2 taken 206 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 206 times.
✗ Branch 7 not taken.
 206 curl_easy_setopt(curl_handle, CURLOPT_URL,
1191 EscapeUrl(info->id(), url).c_str());
1192 206 }
1193
1194
1195 /**
1196 * Checks if the name resolving information is still up to date. The host
1197 * object should be one from the current load-balance group. If the information
1198 * changed, gather new set of resolved IPs and, if different, exchange them in
1199 * the load-balance group on the fly. In the latter case, also rebalance the
1200 * proxies. The options mutex needs to be open.
1201 *
1202 * Returns true if proxies may have changed.
1203 */
1204 4 bool DownloadManager::ValidateProxyIpsUnlocked(
1205 const string &url,
1206 const dns::Host &host)
1207 {
1208
2/4
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 4 times.
✗ Branch 4 not taken.
 4 if (!host.IsExpired())
1209 4 return false;
1210 ✗ LogCvmfs(kLogDownload, kLogDebug, "(manager '%s') validate DNS entry for %s",
1211 ✗ name_.c_str(), host.name().c_str());
1212
1213 ✗ unsigned group_idx = opt_proxy_groups_current_;
1214 ✗ dns::Host new_host = resolver_->Resolve(host.name());
1215
1216 ✗ bool update_only = true; // No changes to the list of IP addresses.
1217 ✗ if (new_host.status() != dns::kFailOk) {
1218 // Try again later in case resolving fails.
1219 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1220 "(manager '%s') failed to resolve IP addresses for %s (%d - %s)",
1221 ✗ name_.c_str(), host.name().c_str(), new_host.status(),
1222 dns::Code2Ascii(new_host.status()));
1223 ✗ new_host = dns::Host::ExtendDeadline(host, resolver_->min_ttl());
1224 ✗ } else if (!host.IsEquivalent(new_host)) {
1225 ✗ update_only = false;
1226 }
1227
1228 ✗ if (update_only) {
1229 ✗ for (unsigned i = 0; i < (*opt_proxy_groups_)[group_idx].size(); ++i) {
1230 ✗ if ((*opt_proxy_groups_)[group_idx][i].host.id() == host.id())
1231 ✗ (*opt_proxy_groups_)[group_idx][i].host = new_host;
1232 }
1233 ✗ return false;
1234 }
1235
1236 ✗ assert(new_host.status() == dns::kFailOk);
1237
1238 // Remove old host objects, insert new objects, and rebalance.
1239 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslog,
1240 "(manager '%s') DNS entries for proxy %s changed, adjusting",
1241 ✗ name_.c_str(), host.name().c_str());
1242 ✗ vector<ProxyInfo> *group = current_proxy_group();
1243 ✗ opt_num_proxies_ -= group->size();
1244 ✗ for (unsigned i = 0; i < group->size(); ) {
1245 ✗ if ((*group)[i].host.id() == host.id()) {
1246 ✗ group->erase(group->begin() + i);
1247 } else {
1248 ✗ i++;
1249 }
1250 }
1251 ✗ vector<ProxyInfo> new_infos;
1252 ✗ set<string> best_addresses = new_host.ViewBestAddresses(opt_ip_preference_);
1253 ✗ set<string>::const_iterator iter_ips = best_addresses.begin();
1254 ✗ for (; iter_ips != best_addresses.end(); ++iter_ips) {
1255 ✗ string url_ip = dns::RewriteUrl(url, *iter_ips);
1256 ✗ new_infos.push_back(ProxyInfo(new_host, url_ip));
1257 }
1258 ✗ group->insert(group->end(), new_infos.begin(), new_infos.end());
1259 ✗ opt_num_proxies_ += new_infos.size();
1260
1261 ✗ std::string msg = "DNS entries for proxy " + host.name() + " changed";
1262
1263 ✗ RebalanceProxiesUnlocked(msg);
1264 ✗ return true;
1265 }
1266
1267
1268 /**
1269 * Adds transfer time and downloaded bytes to the global counters.
1270 */
1271 214 void DownloadManager::UpdateStatistics(CURL *handle) {
1272 double val;
1273 int retval;
1274 214 int64_t sum = 0;
1275
1276
1/2
✓ Branch 1 taken 214 times.
✗ Branch 2 not taken.
 214 retval = curl_easy_getinfo(handle, CURLINFO_SIZE_DOWNLOAD, &val);
1277
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 214 times.
 214 assert(retval == CURLE_OK);
1278 214 sum += static_cast<int64_t>(val);
1279 /*retval = curl_easy_getinfo(handle, CURLINFO_HEADER_SIZE, &val);
1280 assert(retval == CURLE_OK);
1281 sum += static_cast<int64_t>(val);*/
1282 214 perf::Xadd(counters_->sz_transferred_bytes, sum);
1283 214 }
1284
1285
1286 /**
1287 * Retry if possible if not on no-cache and if not already done too often.
1288 */
1289 214 bool DownloadManager::CanRetry(const JobInfo *info) {
1290 214 MutexLockGuard m(lock_options_);
1291 214 unsigned max_retries = opt_max_retries_;
1292
1293
5/6
✓ Branch 1 taken 208 times.
✓ Branch 2 taken 6 times.
✓ Branch 4 taken 77 times.
✓ Branch 5 taken 131 times.
✓ Branch 6 taken 77 times.
✗ Branch 7 not taken.
 368 return !(info->nocache()) && (info->num_retries() < max_retries) &&
1294
2/2
✓ Branch 2 taken 5 times.
✓ Branch 3 taken 72 times.
 154 (IsProxyTransferError(info->error_code()) ||
1295 77 IsHostTransferError(info->error_code()));
1296 214 }
1297
1298 /**
1299 * Backoff for retry to introduce a jitter into a cluster of requesting
1300 * cvmfs nodes.
1301 * Retry only when HTTP caching is on.
1302 *
1303 * \return true if backoff has been performed, false otherwise
1304 */
1305 5 void DownloadManager::Backoff(JobInfo *info) {
1306 5 unsigned backoff_init_ms = 0;
1307 5 unsigned backoff_max_ms = 0;
1308 {
1309 5 MutexLockGuard m(lock_options_);
1310 5 backoff_init_ms = opt_backoff_init_ms_;
1311 5 backoff_max_ms = opt_backoff_max_ms_;
1312 5 }
1313
1314 5 info->SetNumRetries(info->num_retries() + 1);
1315 5 perf::Inc(counters_->n_retries);
1316
2/2
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 3 times.
 5 if (info->backoff_ms() == 0) {
1317 2 info->SetBackoffMs(prng_.Next(backoff_init_ms + 1)); // Must be != 0
1318 } else {
1319 3 info->SetBackoffMs(info->backoff_ms() * 2);
1320 }
1321
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5 times.
 5 if (info->backoff_ms() > backoff_max_ms) {
1322 ✗ info->SetBackoffMs(backoff_max_ms);
1323 }
1324
1325 5 LogCvmfs(kLogDownload, kLogDebug,
1326 "(manager '%s' - id %" PRId64 ") backing off for %d ms",
1327 name_.c_str(), info->id(), info->backoff_ms());
1328 5 SafeSleepMs(info->backoff_ms());
1329 5 }
1330
1331 6 void DownloadManager::SetNocache(JobInfo *info) {
1332
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6 times.
 6 if (info->nocache())
1333 ✗ return;
1334 6 header_lists_->AppendHeader(info->headers(), "Pragma: no-cache");
1335 6 header_lists_->AppendHeader(info->headers(), "Cache-Control: no-cache");
1336 6 curl_easy_setopt(info->curl_handle(), CURLOPT_HTTPHEADER, info->headers());
1337 6 info->SetNocache(true);
1338 }
1339
1340
1341 /**
1342 * Reverse operation of SetNocache. Makes sure that "no-cache" header
1343 * disappears from the list of headers to let proxies work normally.
1344 */
1345 11 void DownloadManager::SetRegularCache(JobInfo *info) {
1346
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 if (info->nocache() == false)
1347 11 return;
1348 ✗ header_lists_->CutHeader("Pragma: no-cache", info->GetHeadersPtr());
1349 ✗ header_lists_->CutHeader("Cache-Control: no-cache", info->GetHeadersPtr());
1350 ✗ curl_easy_setopt(info->curl_handle(), CURLOPT_HTTPHEADER, info->headers());
1351 ✗ info->SetNocache(false);
1352 }
1353
1354
1355 /**
1356 * Frees the storage associated with the authz attachment from the job
1357 */
1358 206 void DownloadManager::ReleaseCredential(JobInfo *info) {
1359
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 206 times.
 206 if (info->cred_data()) {
1360 ✗ assert(credentials_attachment_ != NULL); // Someone must have set it
1361 ✗ credentials_attachment_->ReleaseCurlHandle(info->curl_handle(),
1362 info->cred_data());
1363 ✗ info->SetCredData(NULL);
1364 }
1365 206 }
1366
1367
1368 /**
1369 * Checks the result of a curl download and implements the failure logic, such
1370 * as changing the proxy server. Takes care of cleanup.
1371 *
1372 * \return true if another download should be performed, false otherwise
1373 */
1374 214 bool DownloadManager::VerifyAndFinalize(const int curl_error, JobInfo *info) {
1375
1/2
✓ Branch 6 taken 214 times.
✗ Branch 7 not taken.
 214 LogCvmfs(kLogDownload, kLogDebug, "(manager '%s' - id %" PRId64 ") "
1376 "Verify downloaded url %s, proxy %s (curl error %d)",
1377 name_.c_str(), info->id(), info->url()->c_str(),
1378 428 info->proxy().c_str(), curl_error);
1379 214 UpdateStatistics(info->curl_handle());
1380
1381 // Verification and error classification
1382
4/14
✓ Branch 0 taken 194 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 3 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 15 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
 214 switch (curl_error) {
1383 194 case CURLE_OK:
1384 // Verify content hash
1385
2/2
✓ Branch 1 taken 125 times.
✓ Branch 2 taken 69 times.
 194 if (info->expected_hash()) {
1386
1/2
✓ Branch 1 taken 125 times.
✗ Branch 2 not taken.
 125 shash::Any match_hash;
1387
1/2
✓ Branch 2 taken 125 times.
✗ Branch 3 not taken.
 125 shash::Final(info->hash_context(), &match_hash);
1388
3/4
✓ Branch 2 taken 125 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 6 times.
✓ Branch 5 taken 119 times.
 125 if (match_hash != *(info->expected_hash())) {
1389
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6 times.
 6 if (ignore_signature_failures_) {
1390 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
1391 "(manager '%s' - id %" PRId64 ") "
1392 "ignoring failed hash verification of %s (expected %s, got %s)",
1393 name_.c_str(), info->id(), info->url()->c_str(),
1394 ✗ info->expected_hash()->ToString().c_str(),
1395 ✗ match_hash.ToString().c_str());
1396 } else {
1397
1/2
✓ Branch 7 taken 6 times.
✗ Branch 8 not taken.
 12 LogCvmfs(kLogDownload, kLogDebug, "(manager '%s' - id %" PRId64 ") "
1398 "hash verification of %s failed (expected %s, got %s)",
1399 name_.c_str(), info->id(), info->url()->c_str(),
1400
1/2
✓ Branch 2 taken 6 times.
✗ Branch 3 not taken.
 12 info->expected_hash()->ToString().c_str(),
1401
1/2
✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
 12 match_hash.ToString().c_str());
1402 6 info->SetErrorCode(kFailBadData);
1403 6 break;
1404 }
1405 }
1406 }
1407
1408 188 info->SetErrorCode(kFailOk);
1409 188 break;
1410 ✗ case CURLE_UNSUPPORTED_PROTOCOL:
1411 ✗ info->SetErrorCode(kFailUnsupportedProtocol);
1412 ✗ break;
1413 2 case CURLE_URL_MALFORMAT:
1414 2 info->SetErrorCode(kFailBadUrl);
1415 2 break;
1416 ✗ case CURLE_COULDNT_RESOLVE_PROXY:
1417 ✗ info->SetErrorCode(kFailProxyResolve);
1418 ✗ break;
1419 ✗ case CURLE_COULDNT_RESOLVE_HOST:
1420 ✗ info->SetErrorCode(kFailHostResolve);
1421 ✗ break;
1422 3 case CURLE_OPERATION_TIMEDOUT:
1423
1/2
✓ Branch 2 taken 3 times.
✗ Branch 3 not taken.
 3 info->SetErrorCode((info->proxy() == "DIRECT") ?
1424 kFailHostTooSlow : kFailProxyTooSlow);
1425 3 break;
1426 ✗ case CURLE_PARTIAL_FILE:
1427 case CURLE_GOT_NOTHING:
1428 case CURLE_RECV_ERROR:
1429 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ?
1430 kFailHostShortTransfer : kFailProxyShortTransfer);
1431 ✗ break;
1432 15 case CURLE_FILE_COULDNT_READ_FILE:
1433 case CURLE_COULDNT_CONNECT:
1434
2/2
✓ Branch 3 taken 1 times.
✓ Branch 4 taken 14 times.
 15 if (info->proxy() != "DIRECT") {
1435 // This is a guess. Fail-over can still change to switching host
1436 1 info->SetErrorCode(kFailProxyConnection);
1437 } else {
1438 14 info->SetErrorCode(kFailHostConnection);
1439 }
1440 15 break;
1441 ✗ case CURLE_TOO_MANY_REDIRECTS:
1442 ✗ info->SetErrorCode(kFailHostConnection);
1443 ✗ break;
1444 ✗ case CURLE_SSL_CACERT_BADFILE:
1445 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
1446 "(manager '%s' -id %" PRId64 ") "
1447 "Failed to load certificate bundle. "
1448 "X509_CERT_BUNDLE might point to the wrong location.",
1449 name_.c_str(), info->id());
1450 ✗ info->SetErrorCode(kFailHostConnection);
1451 ✗ break;
1452 // As of curl 7.62.0, CURLE_SSL_CACERT is the same as
1453 // CURLE_PEER_FAILED_VERIFICATION
1454 ✗ case CURLE_PEER_FAILED_VERIFICATION:
1455 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr,
1456 "(manager '%s' - id %" PRId64 ") "
1457 "invalid SSL certificate of remote host. "
1458 "X509_CERT_DIR and/or X509_CERT_BUNDLE might point to the wrong "
1459 "location.", name_.c_str(), info->id());
1460 ✗ info->SetErrorCode(kFailHostConnection);
1461 ✗ break;
1462 ✗ case CURLE_ABORTED_BY_CALLBACK:
1463 case CURLE_WRITE_ERROR:
1464 // Error set by callback
1465 ✗ break;
1466 ✗ case CURLE_SEND_ERROR:
1467 // The curl error CURLE_SEND_ERROR can be seen when a cache is misbehaving
1468 // and closing connections before the http request send is completed.
1469 // Handle this error, treating it as a short transfer error.
1470 ✗ info->SetErrorCode((info->proxy() == "DIRECT") ?
1471 kFailHostShortTransfer : kFailProxyShortTransfer);
1472 ✗ break;
1473 ✗ default:
1474 ✗ LogCvmfs(kLogDownload, kLogSyslogErr, "(manager '%s' - id %" PRId64 ") "
1475 "unexpected curl error (%d) while trying to fetch %s",
1476 name_.c_str(), info->id(), curl_error, info->url()->c_str());
1477 ✗ info->SetErrorCode(kFailOther);
1478 ✗ break;
1479 }
1480
1481 214 std::vector<std::string> *host_chain = opt_host_chain_;
1482
1483 // Determination if download should be repeated
1484 214 bool try_again = false;
1485 214 bool same_url_retry = CanRetry(info);
1486
2/2
✓ Branch 1 taken 26 times.
✓ Branch 2 taken 188 times.
 214 if (info->error_code() != kFailOk) {
1487 26 MutexLockGuard m(lock_options_);
1488
2/2
✓ Branch 1 taken 6 times.
✓ Branch 2 taken 20 times.
 26 if (info->error_code() == kFailBadData) {
1489
2/2
✓ Branch 1 taken 3 times.
✓ Branch 2 taken 3 times.
 6 if (!info->nocache()) {
1490 3 try_again = true;
1491 } else {
1492 // Make it a host failure
1493
1/2
✓ Branch 3 taken 3 times.
✗ Branch 4 not taken.
 3 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1494 "(manager '%s' - id %" PRId64 ") "
1495 "data corruption with no-cache header, try another host",
1496 name_.c_str(), info->id());
1497
1498 3 info->SetErrorCode(kFailHostHttp);
1499 }
1500 }
1501
5/6
✓ Branch 0 taken 21 times.
✓ Branch 1 taken 5 times.
✓ Branch 2 taken 21 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 8 times.
✓ Branch 5 taken 18 times.
 68 if ( same_url_retry || (
1502
2/2
✓ Branch 1 taken 9 times.
✓ Branch 2 taken 12 times.
 42 ( (info->error_code() == kFailHostResolve) ||
1503
2/2
✓ Branch 2 taken 3 times.
✓ Branch 3 taken 6 times.
 30 IsHostTransferError(info->error_code()) ||
1504
2/2
✓ Branch 1 taken 9 times.
✓ Branch 2 taken 6 times.
 24 (info->error_code() == kFailHostHttp)) &&
1505
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 24 info->probe_hosts() &&
1506
2/2
✓ Branch 2 taken 3 times.
✓ Branch 3 taken 6 times.
 9 host_chain && (info->num_used_hosts() < host_chain->size()))
1507 )
1508 {
1509 8 try_again = true;
1510 }
1511
5/6
✓ Branch 0 taken 21 times.
✓ Branch 1 taken 5 times.
✓ Branch 2 taken 21 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 6 times.
✓ Branch 5 taken 20 times.
 68 if ( same_url_retry || (
1512
2/2
✓ Branch 1 taken 20 times.
✓ Branch 2 taken 1 times.
 42 ( (info->error_code() == kFailProxyResolve) ||
1513
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 20 times.
 41 IsProxyTransferError(info->error_code()) ||
1514 20 (info->error_code() == kFailProxyHttp)) )
1515 )
1516 {
1517
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6 times.
 6 if (sharding_policy_.UseCount() > 0) { // sharding policy
1518 ✗ try_again = true;
1519 ✗ same_url_retry = false;
1520 } else { // no sharding
1521 6 try_again = true;
1522 // If all proxies failed, do a next round with the next host
1523
4/6
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 5 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 6 times.
 6 if (!same_url_retry && (info->num_used_proxies() >= opt_num_proxies_)) {
1524 // Check if this can be made a host fail-over
1525 ✗ if (info->probe_hosts() &&
1526 ✗ host_chain &&
1527 ✗ (info->num_used_hosts() < host_chain->size()))
1528 {
1529 // reset proxy group if not already performed by other handle
1530 ✗ if (opt_proxy_groups_) {
1531 ✗ if ((opt_proxy_groups_current_ > 0) ||
1532 ✗ (opt_proxy_groups_current_burned_ > 0))
1533 {
1534 ✗ opt_proxy_groups_current_ = 0;
1535 ✗ opt_timestamp_backup_proxies_ = 0;
1536 ✗ RebalanceProxiesUnlocked("reset proxies for host failover");
1537 }
1538 }
1539
1540 // Make it a host failure
1541 ✗ LogCvmfs(kLogDownload, kLogDebug,
1542 "(manager '%s' - id %" PRId64 ") make it a host failure",
1543 name_.c_str(), info->id());
1544 ✗ info->SetNumUsedProxies(1);
1545 ✗ info->SetErrorCode(kFailHostAfterProxy);
1546 } else {
1547 ✗ if (failover_indefinitely_) {
1548 // Instead of giving up, reset the num_used_proxies counter,
1549 // switch proxy and try again
1550 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
1551 "(manager '%s' - id %" PRId64 ") "
1552 "VerifyAndFinalize() would fail the download here. "
1553 "Instead switch proxy and retry download. "
1554 "info->probe_hosts=%d host_chain=%p info->num_used_hosts=%d "
1555 "host_chain->size()=%lu same_url_retry=%d "
1556 "info->num_used_proxies=%d opt_num_proxies_=%d",
1557 name_.c_str(), info->id(),
1558 ✗ static_cast<int>(info->probe_hosts()),
1559 ✗ host_chain, info->num_used_hosts(),
1560 host_chain ?
1561 ✗ host_chain->size() : -1, static_cast<int>(same_url_retry),
1562 ✗ info->num_used_proxies(), opt_num_proxies_);
1563 ✗ info->SetNumUsedProxies(1);
1564 ✗ RebalanceProxiesUnlocked(
1565 "download failed - failover indefinitely");
1566 ✗ try_again = !Interrupted(fqrn_, info);
1567 } else {
1568 ✗ try_again = false;
1569 }
1570 }
1571 } // Make a proxy failure a host failure
1572 } // Proxy failure assumed
1573 } // end !sharding
1574 26 }
1575
1576
2/2
✓ Branch 0 taken 12 times.
✓ Branch 1 taken 202 times.
 214 if (try_again) {
1577 12 LogCvmfs(kLogDownload, kLogDebug, "(manager '%s' - id %" PRId64 ") "
1578 "Trying again on same curl handle, same url: %d, "
1579 "error code %d no-cache %d",
1580 name_.c_str(), info->id(), same_url_retry,
1581 12 info->error_code(), info->nocache());
1582 // Reset internal state and destination
1583
3/6
✓ Branch 1 taken 12 times.
✗ Branch 2 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 12 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 12 times.
 12 if (info->sink() != NULL && info->sink()->Reset() != 0) {
1584 ✗ info->SetErrorCode(kFailLocalIO);
1585 ✗ goto verify_and_finalize_stop;
1586 }
1587
5/6
✓ Branch 1 taken 1 times.
✓ Branch 2 taken 11 times.
✓ Branch 5 taken 1 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 1 times.
✓ Branch 8 taken 11 times.
 12 if (info->interrupt_cue() && info->interrupt_cue()->IsCanceled()) {
1588 1 info->SetErrorCode(kFailCanceled);
1589 1 goto verify_and_finalize_stop;
1590 }
1591
1592
2/2
✓ Branch 1 taken 6 times.
✓ Branch 2 taken 5 times.
 11 if (info->expected_hash()) {
1593 6 shash::Init(info->hash_context());
1594 }
1595
2/2
✓ Branch 1 taken 6 times.
✓ Branch 2 taken 5 times.
 11 if (info->compressed()) {
1596 6 zlib::DecompressInit(info->GetZstreamPtr());
1597 }
1598
1599
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 11 times.
 11 if (sharding_policy_.UseCount() > 0) { // sharding policy
1600 ✗ ReleaseCredential(info);
1601 ✗ SetUrlOptions(info);
1602 } else { // no sharding policy
1603 11 SetRegularCache(info);
1604
1605 // Failure handling
1606 11 bool switch_proxy = false;
1607 11 bool switch_host = false;
1608
2/4
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 8 times.
 11 switch (info->error_code()) {
1609 3 case kFailBadData:
1610 3 SetNocache(info);
1611 3 break;
1612 ✗ case kFailProxyResolve:
1613 case kFailProxyHttp:
1614 ✗ switch_proxy = true;
1615 ✗ break;
1616 ✗ case kFailHostResolve:
1617 case kFailHostHttp:
1618 case kFailHostAfterProxy:
1619 ✗ switch_host = true;
1620 ✗ break;
1621 8 default:
1622
2/2
✓ Branch 2 taken 1 times.
✓ Branch 3 taken 7 times.
 8 if (IsProxyTransferError(info->error_code())) {
1623
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 times.
 1 if (same_url_retry) {
1624 ✗ Backoff(info);
1625 } else {
1626 1 switch_proxy = true;
1627 }
1628
1/2
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
 7 } else if (IsHostTransferError(info->error_code())) {
1629
2/2
✓ Branch 0 taken 5 times.
✓ Branch 1 taken 2 times.
 7 if (same_url_retry) {
1630 5 Backoff(info);
1631 } else {
1632 2 switch_host = true;
1633 }
1634 } else {
1635 // No other errors expected when retrying
1636 ✗ PANIC(NULL);
1637 }
1638 }
1639
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 10 times.
 11 if (switch_proxy) {
1640 1 ReleaseCredential(info);
1641 1 SwitchProxy(info);
1642 1 info->SetNumUsedProxies(info->num_used_proxies() + 1);
1643 1 SetUrlOptions(info);
1644 }
1645
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 9 times.
 11 if (switch_host) {
1646 2 ReleaseCredential(info);
1647 2 SwitchHost(info);
1648 2 info->SetNumUsedHosts(info->num_used_hosts() + 1);
1649 2 SetUrlOptions(info);
1650 }
1651 } // end !sharding
1652
1653
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11 times.
 11 if (failover_indefinitely_) {
1654 // try again, breaking if there's a cvmfs reload happening and we are in a
1655 // proxy failover. This will EIO the call application.
1656 ✗ return !Interrupted(fqrn_, info);
1657 }
1658 11 return true; // try again
1659 }
1660
1661 202 verify_and_finalize_stop:
1662 // Finalize, flush destination file
1663 203 ReleaseCredential(info);
1664
3/6
✓ Branch 1 taken 203 times.
✗ Branch 2 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 203 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 203 times.
 203 if (info->sink() != NULL && info->sink()->Flush() != 0) {
1665 ✗ info->SetErrorCode(kFailLocalIO);
1666 }
1667
1668
2/2
✓ Branch 1 taken 129 times.
✓ Branch 2 taken 74 times.
 203 if (info->compressed())
1669 129 zlib::DecompressFini(info->GetZstreamPtr());
1670
1671
1/2
✓ Branch 1 taken 203 times.
✗ Branch 2 not taken.
 203 if (info->headers()) {
1672 203 header_lists_->PutList(info->headers());
1673 203 info->SetHeaders(NULL);
1674 }
1675
1676 203 return false; // stop transfer and return to Fetch()
1677 }
1678
1679 157 DownloadManager::~DownloadManager() {
1680 // cleaned up fini
1681
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 157 times.
 157 if (sharding_policy_.UseCount() > 0) {
1682 ✗ sharding_policy_.Reset();
1683 }
1684
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 157 times.
 157 if (health_check_.UseCount() > 0) {
1685 ✗ if (health_check_.Unique()) {
1686 ✗ LogCvmfs(kLogDownload, kLogDebug,
1687 "(manager '%s') Stopping healthcheck thread", name_.c_str());
1688 ✗ health_check_->StopHealthcheck();
1689 }
1690 ✗ health_check_.Reset();
1691 }
1692
1693
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 157 times.
 157 if (atomic_xadd32(&multi_threaded_, 0) == 1) {
1694 // Shutdown I/O thread
1695 ✗ pipe_terminate_->Write(kPipeTerminateSignal);
1696 ✗ pthread_join(thread_download_, NULL);
1697 // All handles are removed from the multi stack
1698 ✗ pipe_terminate_.Destroy();
1699 ✗ pipe_jobs_.Destroy();
1700 }
1701
1702 314 for (set<CURL *>::iterator i = pool_handles_idle_->begin(),
1703
2/2
✓ Branch 3 taken 77 times.
✓ Branch 4 taken 157 times.
 391 iEnd = pool_handles_idle_->end(); i != iEnd; ++i)
1704 {
1705 77 curl_easy_cleanup(*i);
1706 }
1707
1708
1/2
✓ Branch 0 taken 157 times.
✗ Branch 1 not taken.
 157 delete pool_handles_idle_;
1709
1/2
✓ Branch 0 taken 157 times.
✗ Branch 1 not taken.
 157 delete pool_handles_inuse_;
1710 157 curl_multi_cleanup(curl_multi_);
1711
1712
1/2
✓ Branch 0 taken 157 times.
✗ Branch 1 not taken.
 157 delete header_lists_;
1713
1/2
✓ Branch 0 taken 157 times.
✗ Branch 1 not taken.
 157 if (user_agent_)
1714 157 free(user_agent_);
1715
1716
1/2
✓ Branch 0 taken 157 times.
✗ Branch 1 not taken.
 157 delete counters_;
1717
2/2
✓ Branch 0 taken 77 times.
✓ Branch 1 taken 80 times.
 157 delete opt_host_chain_;
1718
2/2
✓ Branch 0 taken 77 times.
✓ Branch 1 taken 80 times.
 157 delete opt_host_chain_rtt_;
1719
2/2
✓ Branch 0 taken 67 times.
✓ Branch 1 taken 90 times.
 157 delete opt_proxy_groups_;
1720
1721 157 curl_global_cleanup();
1722
1/2
✓ Branch 0 taken 157 times.
✗ Branch 1 not taken.
 157 delete resolver_;
1723
1724 // old destructor
1725 157 pthread_mutex_destroy(lock_options_);
1726 157 pthread_mutex_destroy(lock_synchronous_mode_);
1727 157 free(lock_options_);
1728 157 free(lock_synchronous_mode_);
1729 157 }
1730
1731 157 void DownloadManager::InitHeaders() {
1732 // User-Agent
1733
1/2
✓ Branch 2 taken 157 times.
✗ Branch 3 not taken.
 157 string cernvm_id = "User-Agent: cvmfs ";
1734 #ifdef CVMFS_LIBCVMFS
1735
1/2
✓ Branch 1 taken 157 times.
✗ Branch 2 not taken.
 157 cernvm_id += "libcvmfs ";
1736 #else
1737 ✗ cernvm_id += "Fuse ";
1738 #endif
1739
2/4
✓ Branch 2 taken 157 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 157 times.
✗ Branch 6 not taken.
 157 cernvm_id += string(VERSION);
1740
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 157 times.
 157 if (getenv("CERNVM_UUID") != NULL) {
1741 ✗ cernvm_id += " " +
1742 ✗ sanitizer::InputSanitizer("az AZ 09 -").Filter(getenv("CERNVM_UUID"));
1743 }
1744 157 user_agent_ = strdup(cernvm_id.c_str());
1745
1746
1/2
✓ Branch 1 taken 157 times.
✗ Branch 2 not taken.
 157 header_lists_ = new HeaderLists();
1747
1748
1/2
✓ Branch 1 taken 157 times.
✗ Branch 2 not taken.
 157 default_headers_ = header_lists_->GetList("Connection: Keep-Alive");
1749
1/2
✓ Branch 1 taken 157 times.
✗ Branch 2 not taken.
 157 header_lists_->AppendHeader(default_headers_, "Pragma:");
1750
1/2
✓ Branch 1 taken 157 times.
✗ Branch 2 not taken.
 157 header_lists_->AppendHeader(default_headers_, user_agent_);
1751 157 }
1752
1753 157 DownloadManager::DownloadManager(const unsigned max_pool_handles,
1754 const perf::StatisticsTemplate &statistics,
1755 157 const std::string &name) :
1756 157 prng_(Prng()),
1757 157 pool_handles_idle_(new set<CURL *>),
1758 157 pool_handles_inuse_(new set<CURL *>),
1759 157 pool_max_handles_(max_pool_handles),
1760 157 pipe_terminate_(NULL),
1761
1/2
✓ Branch 1 taken 157 times.
✗ Branch 2 not taken.
 157 pipe_jobs_(NULL),
1762 157 watch_fds_(NULL),
1763 157 watch_fds_size_(0),
1764 157 watch_fds_inuse_(0),
1765 157 watch_fds_max_(4 * max_pool_handles),
1766 157 opt_timeout_proxy_(5),
1767 157 opt_timeout_direct_(10),
1768 157 opt_low_speed_limit_(1024),
1769 157 opt_max_retries_(0),
1770 157 opt_backoff_init_ms_(0),
1771 157 opt_backoff_max_ms_(0),
1772 157 enable_info_header_(false),
1773 157 opt_ipv4_only_(false),
1774 157 follow_redirects_(false),
1775 157 ignore_signature_failures_(false),
1776 157 enable_http_tracing_(false),
1777 157 opt_host_chain_(NULL),
1778 157 opt_host_chain_rtt_(NULL),
1779 157 opt_host_chain_current_(0),
1780 157 opt_proxy_groups_(NULL),
1781 157 opt_proxy_groups_current_(0),
1782 157 opt_proxy_groups_current_burned_(0),
1783 157 opt_proxy_groups_fallback_(0),
1784 157 opt_num_proxies_(0),
1785 157 opt_proxy_shard_(false),
1786 157 failover_indefinitely_(false),
1787
1/2
✓ Branch 1 taken 157 times.
✗ Branch 2 not taken.
 157 name_(name),
1788 157 opt_ip_preference_(dns::kIpPreferSystem),
1789 157 opt_timestamp_backup_proxies_(0),
1790 157 opt_timestamp_failover_proxies_(0),
1791 157 opt_proxy_groups_reset_after_(0),
1792 157 opt_timestamp_backup_host_(0),
1793 157 opt_host_reset_after_(0),
1794 157 credentials_attachment_(NULL),
1795
4/8
✓ Branch 12 taken 157 times.
✗ Branch 13 not taken.
✓ Branch 15 taken 157 times.
✗ Branch 16 not taken.
✓ Branch 18 taken 157 times.
✗ Branch 19 not taken.
✓ Branch 22 taken 157 times.
✗ Branch 23 not taken.
 314 counters_(new Counters(statistics))
1796 {
1797 157 atomic_init32(&multi_threaded_);
1798
1799 157 lock_options_ =
1800 157 reinterpret_cast<pthread_mutex_t *>(smalloc(sizeof(pthread_mutex_t)));
1801 157 int retval = pthread_mutex_init(lock_options_, NULL);
1802
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 157 times.
 157 assert(retval == 0);
1803 157 lock_synchronous_mode_ =
1804 157 reinterpret_cast<pthread_mutex_t *>(smalloc(sizeof(pthread_mutex_t)));
1805 157 retval = pthread_mutex_init(lock_synchronous_mode_, NULL);
1806
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 157 times.
 157 assert(retval == 0);
1807
1808
1/2
✓ Branch 1 taken 157 times.
✗ Branch 2 not taken.
 157 retval = curl_global_init(CURL_GLOBAL_ALL);
1809
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 157 times.
 157 assert(retval == CURLE_OK);
1810
1811
1/2
✓ Branch 1 taken 157 times.
✗ Branch 2 not taken.
 157 InitHeaders();
1812
1813
1/2
✓ Branch 1 taken 157 times.
✗ Branch 2 not taken.
 157 curl_multi_ = curl_multi_init();
1814
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 157 times.
 157 assert(curl_multi_ != NULL);
1815
1/2
✓ Branch 1 taken 157 times.
✗ Branch 2 not taken.
 157 curl_multi_setopt(curl_multi_, CURLMOPT_SOCKETFUNCTION, CallbackCurlSocket);
1816
1/2
✓ Branch 1 taken 157 times.
✗ Branch 2 not taken.
 157 curl_multi_setopt(curl_multi_, CURLMOPT_SOCKETDATA,
1817 static_cast<void *>(this));
1818
1/2
✓ Branch 1 taken 157 times.
✗ Branch 2 not taken.
 157 curl_multi_setopt(curl_multi_, CURLMOPT_MAXCONNECTS, watch_fds_max_);
1819
1/2
✓ Branch 1 taken 157 times.
✗ Branch 2 not taken.
 157 curl_multi_setopt(curl_multi_, CURLMOPT_MAX_TOTAL_CONNECTIONS,
1820 pool_max_handles_);
1821
1822 157 prng_.InitLocaltime();
1823
1824 // Name resolving
1825
2/4
✗ Branch 1 not taken.
✓ Branch 2 taken 157 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 157 times.
 157 if ((getenv("CVMFS_IPV4_ONLY") != NULL) &&
1826 ✗ (strlen(getenv("CVMFS_IPV4_ONLY")) > 0))
1827 {
1828 ✗ opt_ipv4_only_ = true;
1829 }
1830
1/2
✓ Branch 1 taken 157 times.
✗ Branch 2 not taken.
 157 resolver_ = dns::NormalResolver::Create(opt_ipv4_only_,
1831 kDnsDefaultRetries, kDnsDefaultTimeoutMs);
1832
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 157 times.
 157 assert(resolver_);
1833 157 }
1834
1835 /**
1836 * Spawns the I/O worker thread and switches the module in multi-threaded mode.
1837 * No way back except Fini(); Init();
1838 */
1839 ✗ void DownloadManager::Spawn() {
1840 ✗ pipe_terminate_ = new Pipe<kPipeThreadTerminator>();
1841 ✗ pipe_jobs_ = new Pipe<kPipeDownloadJobs>();
1842
1843 ✗ int retval = pthread_create(&thread_download_, NULL, MainDownload,
1844 static_cast<void *>(this));
1845 ✗ assert(retval == 0);
1846
1847 ✗ atomic_inc32(&multi_threaded_);
1848
1849 ✗ if (health_check_.UseCount() > 0) {
1850 ✗ LogCvmfs(kLogDownload, kLogDebug,
1851 "(manager '%s') Starting healthcheck thread", name_.c_str());
1852 ✗ health_check_->StartHealthcheck();
1853 }
1854 }
1855
1856
1857 /**
1858 * Downloads data from an insecure outside channel (currently HTTP or file).
1859 */
1860 203 Failures DownloadManager::Fetch(JobInfo *info) {
1861
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 203 times.
 203 assert(info != NULL);
1862
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 203 times.
 203 assert(info->url() != NULL);
1863
1864 Failures result;
1865
1/2
✓ Branch 1 taken 203 times.
✗ Branch 2 not taken.
 203 result = PrepareDownloadDestination(info);
1866
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 203 times.
 203 if (result != kFailOk)
1867 ✗ return result;
1868
1869
2/2
✓ Branch 1 taken 130 times.
✓ Branch 2 taken 73 times.
 203 if (info->expected_hash()) {
1870 130 const shash::Algorithms algorithm = info->expected_hash()->algorithm;
1871 130 info->GetHashContextPtr()->algorithm = algorithm;
1872
1/2
✓ Branch 1 taken 130 times.
✗ Branch 2 not taken.
 130 info->GetHashContextPtr()->size = shash::GetContextSize(algorithm);
1873 130 info->GetHashContextPtr()->buffer = alloca(info->hash_context().size);
1874 }
1875
1876 // Prepare cvmfs-info: header, allocate string on the stack
1877 203 info->SetInfoHeader(NULL);
1878
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 203 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 203 times.
 203 if (enable_info_header_ && info->extra_info()) {
1879 ✗ const char *header_name = "cvmfs-info: ";
1880 ✗ const size_t header_name_len = strlen(header_name);
1881 ✗ const unsigned header_size = 1 + header_name_len +
1882 ✗ EscapeHeader(*(info->extra_info()), NULL, 0);
1883 ✗ info->SetInfoHeader(static_cast<char *>(alloca(header_size)));
1884 ✗ memcpy(info->info_header(), header_name, header_name_len);
1885 ✗ EscapeHeader(*(info->extra_info()), info->info_header() + header_name_len,
1886 ✗ header_size - header_name_len);
1887 ✗ info->info_header()[header_size-1] = '\0';
1888 }
1889
1890
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 203 times.
 203 if (enable_http_tracing_) {
1891 ✗ const std::string str_pid = "X-CVMFS-PID: " + StringifyInt(info->pid());
1892 ✗ const std::string str_gid = "X-CVMFS-GID: " + StringifyUint(info->gid());
1893 ✗ const std::string str_uid = "X-CVMFS-UID: " + StringifyUint(info->uid());
1894
1895 // will be auto freed at the end of this function Fetch(JobInfo *info)
1896 ✗ info->SetTracingHeaderPid(static_cast<char *>(alloca(str_pid.size() + 1)));
1897 ✗ info->SetTracingHeaderGid(static_cast<char *>(alloca(str_gid.size() + 1)));
1898 ✗ info->SetTracingHeaderUid(static_cast<char *>(alloca(str_uid.size() + 1)));
1899
1900 ✗ memcpy(info->tracing_header_pid(), str_pid.c_str(), str_pid.size() + 1);
1901 ✗ memcpy(info->tracing_header_gid(), str_gid.c_str(), str_gid.size() + 1);
1902 ✗ memcpy(info->tracing_header_uid(), str_uid.c_str(), str_uid.size() + 1);
1903 }
1904
1905
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 203 times.
 203 if (atomic_xadd32(&multi_threaded_, 0) == 1) {
1906 ✗ if (!info->IsValidPipeJobResults()) {
1907 ✗ info->CreatePipeJobResults();
1908 }
1909 ✗ if (!info->IsValidDataTube()) {
1910 ✗ info->CreateDataTube();
1911 }
1912
1913 // LogCvmfs(kLogDownload, kLogDebug, "send job to thread, pipe %d %d",
1914 // info->wait_at[0], info->wait_at[1]);
1915 ✗ pipe_jobs_->Write<JobInfo*>(info);
1916
1917 do {
1918 ✗ DataTubeElement* ele = info->GetDataTubePtr()->PopFront();
1919
1920 ✗ if (ele->action == kActionStop) {
1921 ✗ delete ele;
1922 ✗ break;
1923 }
1924 // TODO(heretherebedragons) add compression
1925 ✗ } while (true);
1926
1927 ✗ info->GetPipeJobResultPtr()->Read<download::Failures>(&result);
1928 // LogCvmfs(kLogDownload, kLogDebug, "got result %d", result);
1929 } else {
1930 203 MutexLockGuard l(lock_synchronous_mode_);
1931
1/2
✓ Branch 1 taken 203 times.
✗ Branch 2 not taken.
 203 CURL *handle = AcquireCurlHandle();
1932
1/2
✓ Branch 1 taken 203 times.
✗ Branch 2 not taken.
 203 InitializeRequest(info, handle);
1933
1/2
✓ Branch 1 taken 203 times.
✗ Branch 2 not taken.
 203 SetUrlOptions(info);
1934 // curl_easy_setopt(handle, CURLOPT_VERBOSE, 1);
1935 int retval;
1936 do {
1937
1/2
✓ Branch 1 taken 214 times.
✗ Branch 2 not taken.
 214 retval = curl_easy_perform(handle);
1938 214 perf::Inc(counters_->n_requests);
1939 double elapsed;
1940
2/4
✓ Branch 1 taken 214 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 214 times.
✗ Branch 4 not taken.
 214 if (curl_easy_getinfo(handle, CURLINFO_TOTAL_TIME, &elapsed) == CURLE_OK)
1941 {
1942 214 perf::Xadd(counters_->sz_transfer_time,
1943 214 static_cast<int64_t>(elapsed * 1000));
1944 }
1945
3/4
✓ Branch 1 taken 214 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 11 times.
✓ Branch 4 taken 203 times.
 214 } while (VerifyAndFinalize(retval, info));
1946 203 result = info->error_code();
1947
1/2
✓ Branch 2 taken 203 times.
✗ Branch 3 not taken.
 203 ReleaseCurlHandle(info->curl_handle());
1948 203 }
1949
1950
2/2
✓ Branch 0 taken 15 times.
✓ Branch 1 taken 188 times.
 203 if (result != kFailOk) {
1951
1/2
✓ Branch 4 taken 15 times.
✗ Branch 5 not taken.
 15 LogCvmfs(kLogDownload, kLogDebug, "(manager '%s' - id %" PRId64 ") "
1952 "download failed (error %d - %s)",
1953 name_.c_str(),
1954 info->id(), result, Code2Ascii(result));
1955
1956
1/2
✓ Branch 1 taken 15 times.
✗ Branch 2 not taken.
 15 if (info->sink() != NULL) {
1957
1/2
✓ Branch 2 taken 15 times.
✗ Branch 3 not taken.
 15 info->sink()->Purge();
1958 }
1959 }
1960
1961 203 return result;
1962 }
1963
1964
1965 /**
1966 * Used by the client to connect the authz session manager to the download
1967 * manager.
1968 */
1969 39 void DownloadManager::SetCredentialsAttachment(CredentialsAttachment *ca) {
1970 39 MutexLockGuard m(lock_options_);
1971 39 credentials_attachment_ = ca;
1972 39 }
1973
1974 /**
1975 * Gets the DNS sever.
1976 */
1977 ✗ std::string DownloadManager::GetDnsServer() const {
1978 ✗ return opt_dns_server_;
1979 }
1980
1981 /**
1982 * Sets a DNS server. Only for testing as it cannot be reverted to the system
1983 * default.
1984 */
1985 ✗ void DownloadManager::SetDnsServer(const string &address) {
1986 ✗ if (!address.empty()) {
1987 ✗ MutexLockGuard m(lock_options_);
1988 ✗ opt_dns_server_ = address;
1989 ✗ assert(!opt_dns_server_.empty());
1990
1991 ✗ vector<string> servers;
1992 ✗ servers.push_back(address);
1993 ✗ bool retval = resolver_->SetResolvers(servers);
1994 ✗ assert(retval);
1995 }
1996 ✗ LogCvmfs(kLogDownload, kLogSyslog, "(manager '%s') set nameserver to %s",
1997 name_.c_str(), address.c_str());
1998 }
1999
2000
2001 /**
2002 * Sets the DNS query timeout parameters.
2003 */
2004 73 void DownloadManager::SetDnsParameters(
2005 const unsigned retries,
2006 const unsigned timeout_ms)
2007 {
2008 73 MutexLockGuard m(lock_options_);
2009
3/6
✓ Branch 1 taken 73 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 73 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 73 times.
✗ Branch 6 not taken.
 146 if ((resolver_->retries() == retries) &&
2010 73 (resolver_->timeout_ms() == timeout_ms))
2011 {
2012 73 return;
2013 }
2014 ✗ delete resolver_;
2015 ✗ resolver_ = NULL;
2016 ✗ resolver_ =
2017 ✗ dns::NormalResolver::Create(opt_ipv4_only_, retries, timeout_ms);
2018 ✗ assert(resolver_);
2019
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 73 times.
 73 }
2020
2021
2022 73 void DownloadManager::SetDnsTtlLimits(
2023 const unsigned min_seconds,
2024 const unsigned max_seconds)
2025 {
2026 73 MutexLockGuard m(lock_options_);
2027 73 resolver_->set_min_ttl(min_seconds);
2028 73 resolver_->set_max_ttl(max_seconds);
2029 73 }
2030
2031
2032 ✗ void DownloadManager::SetIpPreference(dns::IpPreference preference) {
2033 ✗ MutexLockGuard m(lock_options_);
2034 ✗ opt_ip_preference_ = preference;
2035 }
2036
2037
2038 /**
2039 * Sets two timeout values for proxied and for direct connections, respectively.
2040 * The timeout counts for all sorts of connection phases,
2041 * DNS, HTTP connect, etc.
2042 */
2043 95 void DownloadManager::SetTimeout(const unsigned seconds_proxy,
2044 const unsigned seconds_direct)
2045 {
2046 95 MutexLockGuard m(lock_options_);
2047 95 opt_timeout_proxy_ = seconds_proxy;
2048 95 opt_timeout_direct_ = seconds_direct;
2049 95 }
2050
2051
2052 /**
2053 * Sets contains the average transfer speed in bytes per second that the
2054 * transfer should be below during CURLOPT_LOW_SPEED_TIME seconds for libcurl to
2055 * consider it to be too slow and abort. Only effective for new connections.
2056 */
2057 ✗ void DownloadManager::SetLowSpeedLimit(const unsigned low_speed_limit) {
2058 ✗ MutexLockGuard m(lock_options_);
2059 ✗ opt_low_speed_limit_ = low_speed_limit;
2060 }
2061
2062
2063 /**
2064 * Receives the currently active timeout values.
2065 */
2066 32 void DownloadManager::GetTimeout(unsigned *seconds_proxy,
2067 unsigned *seconds_direct)
2068 {
2069 32 MutexLockGuard m(lock_options_);
2070 32 *seconds_proxy = opt_timeout_proxy_;
2071 32 *seconds_direct = opt_timeout_direct_;
2072 32 }
2073
2074
2075 /**
2076 * Parses a list of ';'-separated hosts for the host chain. The empty string
2077 * removes the host list.
2078 */
2079 78 void DownloadManager::SetHostChain(const string &host_list) {
2080
1/2
✓ Branch 2 taken 78 times.
✗ Branch 3 not taken.
 78 SetHostChain(SplitString(host_list, ';'));
2081 78 }
2082
2083
2084 80 void DownloadManager::SetHostChain(const std::vector<std::string> &host_list) {
2085 80 MutexLockGuard m(lock_options_);
2086 80 opt_timestamp_backup_host_ = 0;
2087
2/2
✓ Branch 0 taken 35 times.
✓ Branch 1 taken 45 times.
 80 delete opt_host_chain_;
2088
2/2
✓ Branch 0 taken 35 times.
✓ Branch 1 taken 45 times.
 80 delete opt_host_chain_rtt_;
2089 80 opt_host_chain_current_ = 0;
2090
2091
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 80 times.
 80 if (host_list.empty()) {
2092 ✗ opt_host_chain_ = NULL;
2093 ✗ opt_host_chain_rtt_ = NULL;
2094 ✗ return;
2095 }
2096
2097
2/4
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 80 times.
✗ Branch 5 not taken.
 80 opt_host_chain_ = new vector<string>(host_list);
2098 160 opt_host_chain_rtt_ =
2099
2/4
✓ Branch 2 taken 80 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 80 times.
✗ Branch 6 not taken.
 160 new vector<int>(opt_host_chain_->size(), kProbeUnprobed);
2100 // LogCvmfs(kLogDownload, kLogSyslog, "using host %s",
2101 // (*opt_host_chain_)[0].c_str());
2102
1/2
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
 80 }
2103
2104
2105
2106 /**
2107 * Retrieves the currently set chain of hosts, their round trip times, and the
2108 * currently used host.
2109 */
2110 14 void DownloadManager::GetHostInfo(vector<string> *host_chain, vector<int> *rtt,
2111 unsigned *current_host)
2112 {
2113 14 MutexLockGuard m(lock_options_);
2114
1/2
✓ Branch 0 taken 14 times.
✗ Branch 1 not taken.
 14 if (opt_host_chain_) {
2115
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 14 times.
 14 if (current_host) {*current_host = opt_host_chain_current_;}
2116
2/4
✓ Branch 0 taken 14 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 14 times.
✗ Branch 4 not taken.
 14 if (host_chain) {*host_chain = *opt_host_chain_;}
2117
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 14 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 14 if (rtt) {*rtt = *opt_host_chain_rtt_;}
2118 }
2119 14 }
2120
2121
2122 /**
2123 * Jumps to the next proxy in the ring of forward proxy servers.
2124 * Selects one randomly from a load-balancing group.
2125 *
2126 * Allow for the fact that the proxy may have already been failed by
2127 * another transfer, or that the proxy may no longer be part of the
2128 * current load-balancing group.
2129 */
2130 1 void DownloadManager::SwitchProxy(JobInfo *info) {
2131 1 MutexLockGuard m(lock_options_);
2132
2133
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 times.
 1 if (!opt_proxy_groups_) {
2134 ✗ return;
2135 }
2136
2137 // Fail any matching proxies within the current load-balancing group
2138 1 vector<ProxyInfo> *group = current_proxy_group();
2139 1 const unsigned group_size = group->size();
2140 1 unsigned failed = 0;
2141
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 1 times.
 2 for (unsigned i = 0; i < group_size - opt_proxy_groups_current_burned_; ++i) {
2142
5/14
✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
✓ Branch 4 taken 1 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 1 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 1 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✓ Branch 12 taken 1 times.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
 1 if (info && (info->proxy() == (*group)[i].url)) {
2143 // Move to list of failed proxies
2144 1 opt_proxy_groups_current_burned_++;
2145
1/2
✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
 1 swap((*group)[i],
2146 1 (*group)[group_size - opt_proxy_groups_current_burned_]);
2147 1 perf::Inc(counters_->n_proxy_failover);
2148 1 failed++;
2149 }
2150 }
2151
2152 // Do nothing more unless at least one proxy was marked as failed
2153
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 times.
 1 if (!failed)
2154 ✗ return;
2155
2156 // If all proxies from the current load-balancing group are burned, switch to
2157 // another group
2158
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 if (opt_proxy_groups_current_burned_ == group->size()) {
2159 1 opt_proxy_groups_current_burned_ = 0;
2160
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 if (opt_proxy_groups_->size() > 1) {
2161 2 opt_proxy_groups_current_ = (opt_proxy_groups_current_ + 1) %
2162 1 opt_proxy_groups_->size();
2163 // Remember the timestamp of switching to backup proxies
2164
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 times.
 1 if (opt_proxy_groups_reset_after_ > 0) {
2165 ✗ if (opt_proxy_groups_current_ > 0) {
2166 ✗ if (opt_timestamp_backup_proxies_ == 0)
2167 ✗ opt_timestamp_backup_proxies_ = time(NULL);
2168 // LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2169 // "switched to (another) backup proxy group");
2170 } else {
2171 ✗ opt_timestamp_backup_proxies_ = 0;
2172 // LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2173 // "switched back to primary proxy group");
2174 }
2175 ✗ opt_timestamp_failover_proxies_ = 0;
2176 }
2177 }
2178 } else {
2179 // Record failover time
2180 ✗ if (opt_proxy_groups_reset_after_ > 0) {
2181 ✗ if (opt_timestamp_failover_proxies_ == 0)
2182 ✗ opt_timestamp_failover_proxies_ = time(NULL);
2183 }
2184 }
2185
2186
2/4
✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1 times.
✗ Branch 6 not taken.
 1 UpdateProxiesUnlocked("failed proxy");
2187
1/2
✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
 1 LogCvmfs(kLogDownload, kLogDebug, "(manager '%s' - id %" PRId64 ") "
2188 "%lu proxies remain in group", name_.c_str(), info->id(),
2189 1 current_proxy_group()->size() - opt_proxy_groups_current_burned_);
2190
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 }
2191
2192
2193 /**
2194 * Switches to the next host in the chain. If info is set, switch only if the
2195 * current host is identical to the one used by info, otherwise another transfer
2196 * has already done the switch.
2197 */
2198 3 void DownloadManager::SwitchHost(JobInfo *info) {
2199 3 MutexLockGuard m(lock_options_);
2200
2201
5/6
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 1 times.
✓ Branch 4 taken 2 times.
✓ Branch 5 taken 1 times.
✓ Branch 6 taken 2 times.
 3 if (!opt_host_chain_ || (opt_host_chain_->size() == 1)) {
2202 1 return;
2203 }
2204
2205
3/6
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 2 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 2 times.
 2 if (info && (info->current_host_chain_index() != opt_host_chain_current_)) {
2206 ✗ LogCvmfs(kLogDownload, kLogDebug,
2207 "(manager '%s' - id %" PRId64 ")"
2208 "don't switch host, "
2209 "last used host: %s, current host: %s", name_.c_str(), info->id(),
2210 ✗ (*opt_host_chain_)[info->current_host_chain_index()].c_str(),
2211 ✗ (*opt_host_chain_)[opt_host_chain_current_].c_str());
2212 ✗ return;
2213 }
2214
2215
1/2
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 string reason = "manually triggered";
2216
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 string info_id = "(manager " + name_;
2217
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 if (info) {
2218
1/2
✓ Branch 3 taken 2 times.
✗ Branch 4 not taken.
 2 reason = download::Code2Ascii(info->error_code());
2219
2/4
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2 times.
✗ Branch 6 not taken.
 2 info_id = " - id " + StringifyInt(info->id());
2220 }
2221
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 info_id += ")";
2222
2223
1/2
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 string old_host = (*opt_host_chain_)[opt_host_chain_current_];
2224 2 opt_host_chain_current_ =
2225 2 (opt_host_chain_current_ + 1) % opt_host_chain_->size();
2226 2 perf::Inc(counters_->n_host_failover);
2227
1/3
✓ Branch 5 taken 2 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
 4 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2228 "%s switching host from %s to %s (%s)", info_id.c_str(),
2229 2 old_host.c_str(), (*opt_host_chain_)[opt_host_chain_current_].c_str(),
2230 reason.c_str());
2231
2232 // Remember the timestamp of switching to backup host
2233
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
 2 if (opt_host_reset_after_ > 0) {
2234 ✗ if (opt_host_chain_current_ != 0) {
2235 ✗ if (opt_timestamp_backup_host_ == 0)
2236 ✗ opt_timestamp_backup_host_ = time(NULL);
2237 } else {
2238 ✗ opt_timestamp_backup_host_ = 0;
2239 }
2240 }
2241
2/2
✓ Branch 4 taken 2 times.
✓ Branch 5 taken 1 times.
 3 }
2242
2243 1 void DownloadManager::SwitchHost() {
2244 1 SwitchHost(NULL);
2245 1 }
2246
2247
2248 /**
2249 * Orders the hostlist according to RTT of downloading .cvmfschecksum.
2250 * Sets the current host to the best-responsive host.
2251 * If you change the host list in between by SetHostChain(), it will be
2252 * overwritten by this function.
2253 */
2254 ✗ void DownloadManager::ProbeHosts() {
2255 ✗ vector<string> host_chain;
2256 ✗ vector<int> host_rtt;
2257 unsigned current_host;
2258
2259 ✗ GetHostInfo(&host_chain, &host_rtt, &current_host);
2260
2261 // Stopwatch, two times to fill caches first
2262 unsigned i, retries;
2263 ✗ string url;
2264
2265 ✗ cvmfs::MemSink memsink;
2266 ✗ JobInfo info(&url, false, false, NULL, &memsink);
2267 ✗ for (retries = 0; retries < 2; ++retries) {
2268 ✗ for (i = 0; i < host_chain.size(); ++i) {
2269 ✗ url = host_chain[i] + "/.cvmfspublished";
2270
2271 struct timeval tv_start, tv_end;
2272 ✗ gettimeofday(&tv_start, NULL);
2273 ✗ Failures result = Fetch(&info);
2274 ✗ gettimeofday(&tv_end, NULL);
2275 ✗ memsink.Reset();
2276 ✗ if (result == kFailOk) {
2277 ✗ host_rtt[i] = static_cast<int>(
2278 ✗ DiffTimeSeconds(tv_start, tv_end) * 1000);
2279 ✗ LogCvmfs(kLogDownload, kLogDebug, "(manager '%s' - id %" PRId64 ") "
2280 "probing host %s had %dms rtt",
2281 name_.c_str(), info.id(),
2282 ✗ url.c_str(), host_rtt[i]);
2283 } else {
2284 ✗ LogCvmfs(kLogDownload, kLogDebug, "(manager '%s' - id %" PRId64 ") "
2285 "error while probing host %s: %d %s",
2286 name_.c_str(), info.id(),
2287 url.c_str(), result, Code2Ascii(result));
2288 ✗ host_rtt[i] = INT_MAX;
2289 }
2290 }
2291 }
2292
2293 ✗ SortTeam(&host_rtt, &host_chain);
2294 ✗ for (i = 0; i < host_chain.size(); ++i) {
2295 ✗ if (host_rtt[i] == INT_MAX) host_rtt[i] = kProbeDown;
2296 }
2297
2298 ✗ MutexLockGuard m(lock_options_);
2299 ✗ delete opt_host_chain_;
2300 ✗ delete opt_host_chain_rtt_;
2301 ✗ opt_host_chain_ = new vector<string>(host_chain);
2302 ✗ opt_host_chain_rtt_ = new vector<int>(host_rtt);
2303 ✗ opt_host_chain_current_ = 0;
2304 }
2305
2306 ✗ bool DownloadManager::GeoSortServers(std::vector<std::string> *servers,
2307 std::vector<uint64_t> *output_order) {
2308 ✗ if (!servers) {return false;}
2309 ✗ if (servers->size() == 1) {
2310 ✗ if (output_order) {
2311 ✗ output_order->clear();
2312 ✗ output_order->push_back(0);
2313 }
2314 ✗ return true;
2315 }
2316
2317 ✗ std::vector<std::string> host_chain;
2318 ✗ GetHostInfo(&host_chain, NULL, NULL);
2319
2320 ✗ std::vector<std::string> server_dns_names;
2321 ✗ server_dns_names.reserve(servers->size());
2322 ✗ for (unsigned i = 0; i < servers->size(); ++i) {
2323 ✗ std::string host = dns::ExtractHost((*servers)[i]);
2324 ✗ server_dns_names.push_back(host.empty() ? (*servers)[i] : host);
2325 }
2326 ✗ std::string host_list = JoinStrings(server_dns_names, ",");
2327
2328 ✗ vector<string> host_chain_shuffled;
2329 {
2330 // Protect against concurrent access to prng_
2331 ✗ MutexLockGuard m(lock_options_);
2332 // Determine random hosts for the Geo-API query
2333 ✗ host_chain_shuffled = Shuffle(host_chain, &prng_);
2334 }
2335 // Request ordered list via Geo-API
2336 ✗ bool success = false;
2337 ✗ unsigned max_attempts = std::min(host_chain_shuffled.size(), size_t(3));
2338 ✗ vector<uint64_t> geo_order(servers->size());
2339 ✗ for (unsigned i = 0; i < max_attempts; ++i) {
2340 ✗ string url = host_chain_shuffled[i] + "/api/v1.0/geo/@proxy@/" + host_list;
2341 ✗ LogCvmfs(kLogDownload, kLogDebug,
2342 "(manager '%s') requesting ordered server list from %s",
2343 name_.c_str(), url.c_str());
2344 ✗ cvmfs::MemSink memsink;
2345 ✗ JobInfo info(&url, false, false, NULL, &memsink);
2346 ✗ Failures result = Fetch(&info);
2347 ✗ if (result == kFailOk) {
2348 ✗ string order(reinterpret_cast<char*>(memsink.data()), memsink.pos());
2349 ✗ memsink.Reset();
2350 ✗ bool retval = ValidateGeoReply(order, servers->size(), &geo_order);
2351 ✗ if (!retval) {
2352 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2353 "(manager '%s') retrieved invalid GeoAPI reply from %s [%s]",
2354 name_.c_str(), url.c_str(), order.c_str());
2355 } else {
2356 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslog, "(manager '%s') "
2357 "geographic order of servers retrieved from %s",
2358 name_.c_str(),
2359 ✗ dns::ExtractHost(host_chain_shuffled[i]).c_str());
2360 // remove new line at end of "order"
2361 ✗ LogCvmfs(kLogDownload, kLogDebug, "order is %s",
2362 ✗ Trim(order, true /* trim_newline */).c_str());
2363 ✗ success = true;
2364 ✗ break;
2365 }
2366 ✗ } else {
2367 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2368 "(manager '%s') GeoAPI request %s failed with error %d [%s]",
2369 name_.c_str(), url.c_str(), result, Code2Ascii(result));
2370 }
2371 }
2372 ✗ if (!success) {
2373 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn, "(manager '%s') "
2374 "failed to retrieve geographic order from stratum 1 servers",
2375 name_.c_str());
2376 ✗ return false;
2377 }
2378
2379 ✗ if (output_order) {
2380 ✗ output_order->swap(geo_order);
2381 } else {
2382 ✗ std::vector<std::string> sorted_servers;
2383 ✗ sorted_servers.reserve(geo_order.size());
2384 ✗ for (unsigned i = 0; i < geo_order.size(); ++i) {
2385 ✗ uint64_t orderval = geo_order[i];
2386 ✗ sorted_servers.push_back((*servers)[orderval]);
2387 }
2388 ✗ servers->swap(sorted_servers);
2389 }
2390 ✗ return true;
2391 }
2392
2393
2394 /**
2395 * Uses the Geo-API of Stratum 1s to let any of them order the list of servers
2396 * and fallback proxies (if any).
2397 * Tries at most three random Stratum 1s before giving up.
2398 * If you change the host list in between by SetHostChain() or the fallback
2399 * proxy list by SetProxyChain(), they will be overwritten by this function.
2400 */
2401 ✗ bool DownloadManager::ProbeGeo() {
2402 ✗ vector<string> host_chain;
2403 ✗ vector<int> host_rtt;
2404 unsigned current_host;
2405 ✗ vector< vector<ProxyInfo> > proxy_chain;
2406 unsigned fallback_group;
2407
2408 ✗ GetHostInfo(&host_chain, &host_rtt, &current_host);
2409 ✗ GetProxyInfo(&proxy_chain, NULL, &fallback_group);
2410 ✗ if ((host_chain.size() < 2) && ((proxy_chain.size() - fallback_group) < 2))
2411 ✗ return true;
2412
2413 ✗ vector<string> host_names;
2414 ✗ for (unsigned i = 0; i < host_chain.size(); ++i)
2415 ✗ host_names.push_back(dns::ExtractHost(host_chain[i]));
2416 ✗ SortTeam(&host_names, &host_chain);
2417 ✗ unsigned last_geo_host = host_names.size();
2418
2419 ✗ if ((fallback_group == 0) && (last_geo_host > 1)) {
2420 // There are no non-fallback proxies, which means that the client
2421 // will always use the fallback proxies. Add a keyword separator
2422 // between the hosts and fallback proxies so the geosorting service
2423 // will know to sort the hosts based on the distance from the
2424 // closest fallback proxy rather than the distance from the client.
2425 ✗ host_names.push_back("+PXYSEP+");
2426 }
2427
2428 // Add fallback proxy names to the end of the host list
2429 ✗ unsigned first_geo_fallback = host_names.size();
2430 ✗ for (unsigned i = fallback_group; i < proxy_chain.size(); ++i) {
2431 // We only take the first fallback proxy name from every group under the
2432 // assumption that load-balanced servers are at the same location
2433 ✗ host_names.push_back(proxy_chain[i][0].host.name());
2434 }
2435
2436 ✗ std::vector<uint64_t> geo_order;
2437 ✗ bool success = GeoSortServers(&host_names, &geo_order);
2438 ✗ if (!success) {
2439 // GeoSortServers already logged a failure message.
2440 ✗ return false;
2441 }
2442
2443 // Re-install host chain and proxy chain
2444 ✗ MutexLockGuard m(lock_options_);
2445 ✗ delete opt_host_chain_;
2446 ✗ opt_num_proxies_ = 0;
2447 ✗ opt_host_chain_ = new vector<string>(host_chain.size());
2448
2449 // It's possible that opt_proxy_groups_fallback_ might have changed while
2450 // the lock wasn't held
2451 vector<vector<ProxyInfo> > *proxy_groups = new vector<vector<ProxyInfo> >(
2452 ✗ opt_proxy_groups_fallback_ + proxy_chain.size() - fallback_group);
2453 // First copy the non-fallback part of the current proxy chain
2454 ✗ for (unsigned i = 0; i < opt_proxy_groups_fallback_; ++i) {
2455 ✗ (*proxy_groups)[i] = (*opt_proxy_groups_)[i];
2456 ✗ opt_num_proxies_ += (*opt_proxy_groups_)[i].size();
2457 }
2458
2459 // Copy the host chain and fallback proxies by geo order. Array indices
2460 // in geo_order that are smaller than last_geo_host refer to a stratum 1,
2461 // and those indices greater than or equal to first_geo_fallback refer to
2462 // a fallback proxy.
2463 ✗ unsigned hosti = 0;
2464 ✗ unsigned proxyi = opt_proxy_groups_fallback_;
2465 ✗ for (unsigned i = 0; i < geo_order.size(); ++i) {
2466 ✗ uint64_t orderval = geo_order[i];
2467 ✗ if (orderval < static_cast<uint64_t>(last_geo_host)) {
2468 // LogCvmfs(kLogCvmfs, kLogSyslog, "this is orderval %u at host index
2469 // %u", orderval, hosti);
2470 ✗ (*opt_host_chain_)[hosti++] = host_chain[orderval];
2471 ✗ } else if (orderval >= static_cast<uint64_t>(first_geo_fallback)) {
2472 // LogCvmfs(kLogCvmfs, kLogSyslog,
2473 // "this is orderval %u at proxy index %u, using proxy_chain index %u",
2474 // orderval, proxyi, fallback_group + orderval - first_geo_fallback);
2475 ✗ (*proxy_groups)[proxyi] =
2476 ✗ proxy_chain[fallback_group + orderval - first_geo_fallback];
2477 ✗ opt_num_proxies_ += (*proxy_groups)[proxyi].size();
2478 ✗ proxyi++;
2479 }
2480 }
2481
2482 ✗ opt_proxy_map_.clear();
2483 ✗ delete opt_proxy_groups_;
2484 ✗ opt_proxy_groups_ = proxy_groups;
2485 // In pathological cases, opt_proxy_groups_current_ can be larger now when
2486 // proxies changed in-between.
2487 ✗ if (opt_proxy_groups_current_ > opt_proxy_groups_->size()) {
2488 ✗ if (opt_proxy_groups_->size() == 0) {
2489 ✗ opt_proxy_groups_current_ = 0;
2490 } else {
2491 ✗ opt_proxy_groups_current_ = opt_proxy_groups_->size() - 1;
2492 }
2493 ✗ opt_proxy_groups_current_burned_ = 0;
2494 }
2495
2496 ✗ UpdateProxiesUnlocked("geosort");
2497
2498 ✗ delete opt_host_chain_rtt_;
2499 ✗ opt_host_chain_rtt_ = new vector<int>(host_chain.size(), kProbeGeo);
2500 ✗ opt_host_chain_current_ = 0;
2501
2502 ✗ return true;
2503 }
2504
2505
2506 /**
2507 * Validates a string of the form "1,4,2,3" representing in which order the
2508 * the expected_size number of hosts should be put for optimal geographic
2509 * proximity. Returns false if the reply_order string is invalid, otherwise
2510 * fills in the reply_vals array with zero-based order indexes (e.g.
2511 * [0,3,1,2]) and returns true.
2512 */
2513 14 bool DownloadManager::ValidateGeoReply(
2514 const string &reply_order,
2515 const unsigned expected_size,
2516 vector<uint64_t> *reply_vals)
2517 {
2518
2/2
✓ Branch 1 taken 1 times.
✓ Branch 2 taken 13 times.
 14 if (reply_order.empty())
2519 1 return false;
2520
2/4
✓ Branch 2 taken 13 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 13 times.
✗ Branch 6 not taken.
 26 sanitizer::InputSanitizer sanitizer("09 , \n");
2521
3/4
✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 times.
✓ Branch 4 taken 12 times.
 13 if (!sanitizer.IsValid(reply_order))
2522 1 return false;
2523
2/4
✓ Branch 2 taken 12 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 12 times.
✗ Branch 6 not taken.
 24 sanitizer::InputSanitizer strip_newline("09 ,");
2524 vector<string> reply_strings =
2525
2/4
✓ Branch 1 taken 12 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 12 times.
✗ Branch 5 not taken.
 12 SplitString(strip_newline.Filter(reply_order), ',');
2526 12 vector<uint64_t> tmp_vals;
2527
2/2
✓ Branch 1 taken 23 times.
✓ Branch 2 taken 10 times.
 33 for (unsigned i = 0; i < reply_strings.size(); ++i) {
2528
2/2
✓ Branch 2 taken 2 times.
✓ Branch 3 taken 21 times.
 23 if (reply_strings[i].empty())
2529 2 return false;
2530
2/4
✓ Branch 2 taken 21 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 21 times.
✗ Branch 6 not taken.
 21 tmp_vals.push_back(String2Uint64(reply_strings[i]));
2531 }
2532
2/2
✓ Branch 1 taken 5 times.
✓ Branch 2 taken 5 times.
 10 if (tmp_vals.size() != expected_size)
2533 5 return false;
2534
2535 // Check if tmp_vals contains the number 1..n
2536
1/2
✓ Branch 3 taken 5 times.
✗ Branch 4 not taken.
 5 set<uint64_t> coverage(tmp_vals.begin(), tmp_vals.end());
2537
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 5 times.
 5 if (coverage.size() != tmp_vals.size())
2538 ✗ return false;
2539
5/6
✓ Branch 2 taken 4 times.
✓ Branch 3 taken 1 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 4 times.
✓ Branch 9 taken 1 times.
✓ Branch 10 taken 4 times.
 5 if ((*coverage.begin() != 1) || (*coverage.rbegin() != coverage.size()))
2540 1 return false;
2541
2542
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 4 times.
 13 for (unsigned i = 0; i < expected_size; ++i) {
2543 9 (*reply_vals)[i] = tmp_vals[i] - 1;
2544 }
2545 4 return true;
2546 13 }
2547
2548
2549 /**
2550 * Removes DIRECT from a list of ';' and '|' separated proxies.
2551 * \return true if DIRECT was present, false otherwise
2552 */
2553 77 bool DownloadManager::StripDirect(
2554 const string &proxy_list,
2555 string *cleaned_list)
2556 {
2557
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 77 times.
 77 assert(cleaned_list);
2558
2/2
✓ Branch 1 taken 68 times.
✓ Branch 2 taken 9 times.
 77 if (proxy_list == "") {
2559
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 *cleaned_list = "";
2560 68 return false;
2561 }
2562 9 bool result = false;
2563
2564
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 vector<string> proxy_groups = SplitString(proxy_list, ';');
2565 9 vector<string> cleaned_groups;
2566
2/2
✓ Branch 1 taken 22 times.
✓ Branch 2 taken 9 times.
 31 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2567
1/2
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
 22 vector<string> group = SplitString(proxy_groups[i], '|');
2568 22 vector<string> cleaned;
2569
2/2
✓ Branch 1 taken 37 times.
✓ Branch 2 taken 22 times.
 59 for (unsigned j = 0; j < group.size(); ++j) {
2570
6/6
✓ Branch 2 taken 27 times.
✓ Branch 3 taken 10 times.
✓ Branch 6 taken 13 times.
✓ Branch 7 taken 14 times.
✓ Branch 8 taken 23 times.
✓ Branch 9 taken 14 times.
 37 if ((group[j] == "DIRECT") || (group[j] == "")) {
2571 23 result = true;
2572 } else {
2573
1/2
✓ Branch 2 taken 14 times.
✗ Branch 3 not taken.
 14 cleaned.push_back(group[j]);
2574 }
2575 }
2576
2/2
✓ Branch 1 taken 7 times.
✓ Branch 2 taken 15 times.
 22 if (!cleaned.empty())
2577
3/6
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 7 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 7 times.
✗ Branch 9 not taken.
 7 cleaned_groups.push_back(JoinStrings(cleaned, "|"));
2578 22 }
2579
2580
2/4
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 9 times.
✗ Branch 6 not taken.
 9 *cleaned_list = JoinStrings(cleaned_groups, ";");
2581 9 return result;
2582 9 }
2583
2584
2585 /**
2586 * Parses a list of ';'- and '|'-separated proxy servers and fallback proxy
2587 * servers for the proxy groups.
2588 * The empty string for both removes the proxy chain.
2589 * The set_mode parameter can be used to set either proxies (leaving fallback
2590 * proxies unchanged) or fallback proxies (leaving regular proxies unchanged)
2591 * or both.
2592 */
2593 67 void DownloadManager::SetProxyChain(
2594 const string &proxy_list,
2595 const string &fallback_proxy_list,
2596 const ProxySetModes set_mode)
2597 {
2598 67 MutexLockGuard m(lock_options_);
2599
2600 67 opt_timestamp_backup_proxies_ = 0;
2601 67 opt_timestamp_failover_proxies_ = 0;
2602
1/2
✓ Branch 1 taken 67 times.
✗ Branch 2 not taken.
 67 string set_proxy_list = opt_proxy_list_;
2603
1/2
✓ Branch 1 taken 67 times.
✗ Branch 2 not taken.
 67 string set_proxy_fallback_list = opt_proxy_fallback_list_;
2604 bool contains_direct;
2605
3/4
✓ Branch 0 taken 67 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 64 times.
✓ Branch 3 taken 3 times.
 67 if ((set_mode == kSetProxyFallback) || (set_mode == kSetProxyBoth)) {
2606
1/2
✓ Branch 1 taken 64 times.
✗ Branch 2 not taken.
 64 opt_proxy_fallback_list_ = fallback_proxy_list;
2607 }
2608
3/4
✓ Branch 0 taken 64 times.
✓ Branch 1 taken 3 times.
✓ Branch 2 taken 64 times.
✗ Branch 3 not taken.
 67 if ((set_mode == kSetProxyRegular) || (set_mode == kSetProxyBoth)) {
2609
1/2
✓ Branch 1 taken 67 times.
✗ Branch 2 not taken.
 67 opt_proxy_list_ = proxy_list;
2610 }
2611 contains_direct =
2612
1/2
✓ Branch 1 taken 67 times.
✗ Branch 2 not taken.
 67 StripDirect(opt_proxy_fallback_list_, &set_proxy_fallback_list);
2613
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 67 times.
 67 if (contains_direct) {
2614 ✗ LogCvmfs(kLogDownload, kLogSyslogWarn | kLogDebug,
2615 "(manager '%s') fallback proxies do not support DIRECT, removing",
2616 name_.c_str());
2617 }
2618
1/2
✓ Branch 1 taken 67 times.
✗ Branch 2 not taken.
 67 if (set_proxy_fallback_list == "") {
2619
1/2
✓ Branch 1 taken 67 times.
✗ Branch 2 not taken.
 67 set_proxy_list = opt_proxy_list_;
2620 } else {
2621 ✗ bool contains_direct = StripDirect(opt_proxy_list_, &set_proxy_list);
2622 ✗ if (contains_direct) {
2623 ✗ LogCvmfs(kLogDownload, kLogSyslog | kLogDebug,
2624 "(manager '%s') skipping DIRECT proxy to use fallback proxy",
2625 name_.c_str());
2626 }
2627 }
2628
2629 // From this point on, use set_proxy_list and set_fallback_proxy_list as
2630 // effective proxy lists!
2631
2632 67 opt_proxy_map_.clear();
2633
2/2
✓ Branch 0 taken 32 times.
✓ Branch 1 taken 35 times.
 67 delete opt_proxy_groups_;
2634
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 67 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 67 times.
 67 if ((set_proxy_list == "") && (set_proxy_fallback_list == "")) {
2635 ✗ opt_proxy_groups_ = NULL;
2636 ✗ opt_proxy_groups_current_ = 0;
2637 ✗ opt_proxy_groups_current_burned_ = 0;
2638 ✗ opt_proxy_groups_fallback_ = 0;
2639 ✗ opt_num_proxies_ = 0;
2640 ✗ return;
2641 }
2642
2643 // Determine number of regular proxy groups (== first fallback proxy group)
2644 67 opt_proxy_groups_fallback_ = 0;
2645
1/2
✓ Branch 1 taken 67 times.
✗ Branch 2 not taken.
 67 if (set_proxy_list != "") {
2646
1/2
✓ Branch 1 taken 67 times.
✗ Branch 2 not taken.
 67 opt_proxy_groups_fallback_ = SplitString(set_proxy_list, ';').size();
2647 }
2648
1/2
✓ Branch 2 taken 67 times.
✗ Branch 3 not taken.
 67 LogCvmfs(kLogDownload, kLogDebug, "(manager '%s') "
2649 "first fallback proxy group %u",
2650 name_.c_str(), opt_proxy_groups_fallback_);
2651
2652 // Concatenate regular proxies and fallback proxies, both of which can be
2653 // empty.
2654
1/2
✓ Branch 1 taken 67 times.
✗ Branch 2 not taken.
 67 string all_proxy_list = set_proxy_list;
2655
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 67 times.
 67 if (set_proxy_fallback_list != "") {
2656 ✗ if (all_proxy_list != "")
2657 ✗ all_proxy_list += ";";
2658 ✗ all_proxy_list += set_proxy_fallback_list;
2659 }
2660
1/2
✓ Branch 3 taken 67 times.
✗ Branch 4 not taken.
 67 LogCvmfs(kLogDownload, kLogDebug, "(manager '%s') full proxy list %s",
2661 name_.c_str(), all_proxy_list.c_str());
2662
2663 // Resolve server names in provided urls
2664 67 vector<string> hostnames; // All encountered hostnames
2665 67 vector<string> proxy_groups;
2666
1/2
✓ Branch 1 taken 67 times.
✗ Branch 2 not taken.
 67 if (all_proxy_list != "")
2667
1/2
✓ Branch 1 taken 67 times.
✗ Branch 2 not taken.
 67 proxy_groups = SplitString(all_proxy_list, ';');
2668
2/2
✓ Branch 1 taken 68 times.
✓ Branch 2 taken 67 times.
 135 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2669
1/2
✓ Branch 2 taken 68 times.
✗ Branch 3 not taken.
 68 vector<string> this_group = SplitString(proxy_groups[i], '|');
2670
2/2
✓ Branch 1 taken 68 times.
✓ Branch 2 taken 68 times.
 136 for (unsigned j = 0; j < this_group.size(); ++j) {
2671
1/2
✓ Branch 2 taken 68 times.
✗ Branch 3 not taken.
 68 this_group[j] = dns::AddDefaultScheme(this_group[j]);
2672 // Note: DIRECT strings will be "extracted" to an empty string.
2673
1/2
✓ Branch 2 taken 68 times.
✗ Branch 3 not taken.
 68 string hostname = dns::ExtractHost(this_group[j]);
2674 // Save the hostname. Leave empty (DIRECT) names so indexes will
2675 // match later.
2676
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 hostnames.push_back(hostname);
2677 68 }
2678 68 }
2679 67 vector<dns::Host> hosts;
2680
1/2
✓ Branch 3 taken 67 times.
✗ Branch 4 not taken.
 67 LogCvmfs(kLogDownload, kLogDebug, "(manager '%s') "
2681 "resolving %lu proxy addresses",
2682 name_.c_str(), hostnames.size());
2683
1/2
✓ Branch 1 taken 67 times.
✗ Branch 2 not taken.
 67 resolver_->ResolveMany(hostnames, &hosts);
2684
2685 // Construct opt_proxy_groups_: traverse proxy list in same order and expand
2686 // names to resolved IP addresses.
2687
1/2
✓ Branch 1 taken 67 times.
✗ Branch 2 not taken.
 67 opt_proxy_groups_ = new vector< vector<ProxyInfo> >();
2688 67 opt_num_proxies_ = 0;
2689 67 unsigned num_proxy = 0; // Combined i, j counter
2690
2/2
✓ Branch 1 taken 68 times.
✓ Branch 2 taken 67 times.
 135 for (unsigned i = 0; i < proxy_groups.size(); ++i) {
2691
1/2
✓ Branch 2 taken 68 times.
✗ Branch 3 not taken.
 68 vector<string> this_group = SplitString(proxy_groups[i], '|');
2692 // Construct ProxyInfo objects from proxy string and DNS resolver result for
2693 // every proxy in this_group. One URL can result in multiple ProxyInfo
2694 // objects, one for each IP address.
2695 68 vector<ProxyInfo> infos;
2696
2/2
✓ Branch 1 taken 68 times.
✓ Branch 2 taken 68 times.
 136 for (unsigned j = 0; j < this_group.size(); ++j, ++num_proxy) {
2697
1/2
✓ Branch 2 taken 68 times.
✗ Branch 3 not taken.
 68 this_group[j] = dns::AddDefaultScheme(this_group[j]);
2698
2/2
✓ Branch 2 taken 64 times.
✓ Branch 3 taken 4 times.
 68 if (this_group[j] == "DIRECT") {
2699
3/6
✓ Branch 2 taken 64 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 64 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 64 times.
✗ Branch 9 not taken.
 64 infos.push_back(ProxyInfo("DIRECT"));
2700 64 continue;
2701 }
2702
2703
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 4 times.
 4 if (hosts[num_proxy].status() != dns::kFailOk) {
2704 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn, "(manager '%s') "
2705 "failed to resolve IP addresses for %s (%d - %s)", name_.c_str(),
2706 ✗ hosts[num_proxy].name().c_str(), hosts[num_proxy].status(),
2707 ✗ dns::Code2Ascii(hosts[num_proxy].status()));
2708 dns::Host failed_host =
2709 ✗ dns::Host::ExtendDeadline(hosts[num_proxy], resolver_->min_ttl());
2710 ✗ infos.push_back(ProxyInfo(failed_host, this_group[j]));
2711 ✗ continue;
2712 }
2713
2714 // IPv4 addresses have precedence
2715 set<string> best_addresses =
2716
2/4
✓ Branch 2 taken 4 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 4 times.
✗ Branch 6 not taken.
 4 hosts[num_proxy].ViewBestAddresses(opt_ip_preference_);
2717 4 set<string>::const_iterator iter_ips = best_addresses.begin();
2718
2/2
✓ Branch 3 taken 4 times.
✓ Branch 4 taken 4 times.
 8 for (; iter_ips != best_addresses.end(); ++iter_ips) {
2719
1/2
✓ Branch 3 taken 4 times.
✗ Branch 4 not taken.
 4 string url_ip = dns::RewriteUrl(this_group[j], *iter_ips);
2720
2/4
✓ Branch 2 taken 4 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 4 times.
✗ Branch 6 not taken.
 4 infos.push_back(ProxyInfo(hosts[num_proxy], url_ip));
2721
2722
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4 times.
 4 if (sharding_policy_.UseCount() > 0) {
2723 ✗ sharding_policy_->AddProxy(url_ip);
2724 }
2725 4 }
2726 4 }
2727
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 opt_proxy_groups_->push_back(infos);
2728 68 opt_num_proxies_ += infos.size();
2729 68 }
2730
1/2
✓ Branch 2 taken 67 times.
✗ Branch 3 not taken.
 67 LogCvmfs(kLogDownload, kLogDebug,
2731 "(manager '%s') installed %u proxies in %lu load-balance groups",
2732 67 name_.c_str(), opt_num_proxies_, opt_proxy_groups_->size());
2733 67 opt_proxy_groups_current_ = 0;
2734 67 opt_proxy_groups_current_burned_ = 0;
2735
2736 // Select random start proxy from the first group.
2737
1/2
✓ Branch 1 taken 67 times.
✗ Branch 2 not taken.
 67 if (opt_proxy_groups_->size() > 0) {
2738 // Select random start proxy from the first group.
2739
2/4
✓ Branch 2 taken 67 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 67 times.
✗ Branch 6 not taken.
 67 UpdateProxiesUnlocked("set random start proxy from the first proxy group");
2740 }
2741
3/6
✓ Branch 5 taken 67 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 67 times.
✗ Branch 9 not taken.
✓ Branch 11 taken 67 times.
✗ Branch 12 not taken.
 67 }
2742
2743
2744 /**
2745 * Retrieves the proxy chain, optionally the currently active load-balancing
2746 * group, and optionally the index of the first fallback proxy group.
2747 * If there are no fallback proxies, the index will equal the size of
2748 * the proxy chain.
2749 */
2750 ✗ void DownloadManager::GetProxyInfo(vector< vector<ProxyInfo> > *proxy_chain,
2751 unsigned *current_group,
2752 unsigned *fallback_group)
2753 {
2754 ✗ assert(proxy_chain != NULL);
2755 ✗ MutexLockGuard m(lock_options_);
2756
2757
2758 ✗ if (!opt_proxy_groups_) {
2759 ✗ vector< vector<ProxyInfo> > empty_chain;
2760 ✗ *proxy_chain = empty_chain;
2761 ✗ if (current_group != NULL)
2762 ✗ *current_group = 0;
2763 ✗ if (fallback_group != NULL)
2764 ✗ *fallback_group = 0;
2765 ✗ return;
2766 }
2767
2768 ✗ *proxy_chain = *opt_proxy_groups_;
2769 ✗ if (current_group != NULL)
2770 ✗ *current_group = opt_proxy_groups_current_;
2771 ✗ if (fallback_group != NULL)
2772 ✗ *fallback_group = opt_proxy_groups_fallback_;
2773 }
2774
2775 ✗ string DownloadManager::GetProxyList() {
2776 ✗ return opt_proxy_list_;
2777 }
2778
2779 ✗ string DownloadManager::GetFallbackProxyList() {
2780 ✗ return opt_proxy_fallback_list_;
2781 }
2782
2783 /**
2784 * Choose proxy
2785 */
2786 DownloadManager::ProxyInfo *
2787 206 DownloadManager::ChooseProxyUnlocked(const shash::Any *hash) {
2788
2/2
✓ Branch 0 taken 119 times.
✓ Branch 1 taken 87 times.
 206 if (!opt_proxy_groups_)
2789 119 return NULL;
2790
2791
2/2
✓ Branch 0 taken 43 times.
✓ Branch 1 taken 44 times.
 87 uint32_t key = (hash ? hash->Partial32() : 0);
2792
1/2
✓ Branch 1 taken 87 times.
✗ Branch 2 not taken.
 87 map<uint32_t, ProxyInfo *>::iterator it = opt_proxy_map_.lower_bound(key);
2793 87 ProxyInfo *proxy = it->second;
2794
2795 87 return proxy;
2796 }
2797
2798 /**
2799 * Update currently selected proxy
2800 */
2801 100 void DownloadManager::UpdateProxiesUnlocked(const string &reason) {
2802
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 100 times.
 100 if (!opt_proxy_groups_)
2803 ✗ return;
2804
2805 // Identify number of non-burned proxies within the current group
2806 100 vector<ProxyInfo> *group = current_proxy_group();
2807 100 unsigned num_alive = (group->size() - opt_proxy_groups_current_burned_);
2808
2/4
✓ Branch 2 taken 100 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 100 times.
✗ Branch 6 not taken.
 200 string old_proxy = JoinStrings(opt_proxy_urls_, "|");
2809
2810 // Rebuild proxy map and URL list
2811 100 opt_proxy_map_.clear();
2812 100 opt_proxy_urls_.clear();
2813 100 const uint32_t max_key = 0xffffffffUL;
2814
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 100 times.
 100 if (opt_proxy_shard_) {
2815 // Build a consistent map with multiple entries for each proxy
2816 ✗ for (unsigned i = 0; i < num_alive; ++i) {
2817 ✗ ProxyInfo *proxy = &(*group)[i];
2818 ✗ shash::Any proxy_hash(shash::kSha1);
2819 ✗ HashString(proxy->url, &proxy_hash);
2820 ✗ Prng prng;
2821 ✗ prng.InitSeed(proxy_hash.Partial32());
2822 ✗ for (unsigned j = 0; j < kProxyMapScale; ++j) {
2823 ✗ const std::pair<uint32_t, ProxyInfo *> entry(prng.Next(max_key), proxy);
2824 ✗ opt_proxy_map_.insert(entry);
2825 }
2826 ✗ opt_proxy_urls_.push_back(proxy->url);
2827 }
2828 // Ensure lower_bound() finds a value for all keys
2829 ✗ ProxyInfo *first_proxy = opt_proxy_map_.begin()->second;
2830 ✗ const std::pair<uint32_t, ProxyInfo *> last_entry(max_key, first_proxy);
2831 ✗ opt_proxy_map_.insert(last_entry);
2832 } else {
2833 // Build a map with a single entry for one randomly selected proxy
2834 100 unsigned select = prng_.Next(num_alive);
2835 100 ProxyInfo *proxy = &(*group)[select];
2836 100 const std::pair<uint32_t, ProxyInfo *> entry(max_key, proxy);
2837
1/2
✓ Branch 1 taken 100 times.
✗ Branch 2 not taken.
 100 opt_proxy_map_.insert(entry);
2838
1/2
✓ Branch 1 taken 100 times.
✗ Branch 2 not taken.
 100 opt_proxy_urls_.push_back(proxy->url);
2839 }
2840
1/2
✓ Branch 3 taken 100 times.
✗ Branch 4 not taken.
 100 sort(opt_proxy_urls_.begin(), opt_proxy_urls_.end());
2841
2842 // Report any change in proxy usage
2843
2/4
✓ Branch 2 taken 100 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 100 times.
✗ Branch 6 not taken.
 200 string new_proxy = JoinStrings(opt_proxy_urls_, "|");
2844
2/2
✓ Branch 1 taken 68 times.
✓ Branch 2 taken 32 times.
 100 if (new_proxy != old_proxy) {
2845
4/9
✗ Branch 1 not taken.
✓ Branch 2 taken 68 times.
✓ Branch 3 taken 67 times.
✓ Branch 4 taken 1 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 68 times.
✗ Branch 8 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
 205 LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
2846 "(manager '%s') switching proxy from %s to %s. Reason: %s",
2847 69 name_.c_str(), (old_proxy.empty() ? "(none)" : old_proxy.c_str()),
2848 136 (new_proxy.empty() ? "(none)" : new_proxy.c_str()),
2849 reason.c_str());
2850 }
2851 100 }
2852
2853 /**
2854 * Enable proxy sharding
2855 */
2856 ✗ void DownloadManager::ShardProxies() {
2857 ✗ opt_proxy_shard_ = true;
2858 ✗ RebalanceProxiesUnlocked("enable sharding");
2859 }
2860
2861 /**
2862 * Selects a new random proxy in the current load-balancing group. Resets the
2863 * "burned" counter.
2864 */
2865 ✗ void DownloadManager::RebalanceProxiesUnlocked(const string &reason) {
2866 ✗ if (!opt_proxy_groups_)
2867 ✗ return;
2868
2869 ✗ opt_timestamp_failover_proxies_ = 0;
2870 ✗ opt_proxy_groups_current_burned_ = 0;
2871 ✗ UpdateProxiesUnlocked(reason);
2872 }
2873
2874
2875 ✗ void DownloadManager::RebalanceProxies() {
2876 ✗ MutexLockGuard m(lock_options_);
2877 ✗ RebalanceProxiesUnlocked("rebalance invoked manually");
2878 }
2879
2880
2881 /**
2882 * Switches to the next load-balancing group of proxy servers.
2883 */
2884 ✗ void DownloadManager::SwitchProxyGroup() {
2885 ✗ MutexLockGuard m(lock_options_);
2886
2887 ✗ if (!opt_proxy_groups_ || (opt_proxy_groups_->size() < 2)) {
2888 ✗ return;
2889 }
2890
2891 ✗ opt_proxy_groups_current_ = (opt_proxy_groups_current_ + 1) %
2892 ✗ opt_proxy_groups_->size();
2893 ✗ opt_timestamp_backup_proxies_ = time(NULL);
2894
2895 std::string msg = "switch to proxy group " +
2896 ✗ StringifyUint(opt_proxy_groups_current_);
2897 ✗ RebalanceProxiesUnlocked(msg);
2898 }
2899
2900
2901 ✗ void DownloadManager::SetProxyGroupResetDelay(const unsigned seconds) {
2902 ✗ MutexLockGuard m(lock_options_);
2903 ✗ opt_proxy_groups_reset_after_ = seconds;
2904 ✗ if (opt_proxy_groups_reset_after_ == 0) {
2905 ✗ opt_timestamp_backup_proxies_ = 0;
2906 ✗ opt_timestamp_failover_proxies_ = 0;
2907 }
2908 }
2909
2910
2911 ✗ void DownloadManager::SetHostResetDelay(const unsigned seconds)
2912 {
2913 ✗ MutexLockGuard m(lock_options_);
2914 ✗ opt_host_reset_after_ = seconds;
2915 ✗ if (opt_host_reset_after_ == 0)
2916 ✗ opt_timestamp_backup_host_ = 0;
2917 }
2918
2919
2920 63 void DownloadManager::SetRetryParameters(const unsigned max_retries,
2921 const unsigned backoff_init_ms,
2922 const unsigned backoff_max_ms)
2923 {
2924 63 MutexLockGuard m(lock_options_);
2925 63 opt_max_retries_ = max_retries;
2926 63 opt_backoff_init_ms_ = backoff_init_ms;
2927 63 opt_backoff_max_ms_ = backoff_max_ms;
2928 63 }
2929
2930
2931 34 void DownloadManager::SetMaxIpaddrPerProxy(unsigned limit) {
2932 34 MutexLockGuard m(lock_options_);
2933 34 resolver_->set_throttle(limit);
2934 34 }
2935
2936
2937 39 void DownloadManager::SetProxyTemplates(
2938 const std::string &direct,
2939 const std::string &forced)
2940 {
2941 39 MutexLockGuard m(lock_options_);
2942
1/2
✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
 39 proxy_template_direct_ = direct;
2943
1/2
✓ Branch 1 taken 39 times.
✗ Branch 2 not taken.
 39 proxy_template_forced_ = forced;
2944 39 }
2945
2946
2947 ✗ void DownloadManager::EnableInfoHeader() {
2948 ✗ enable_info_header_ = true;
2949 }
2950
2951
2952 27 void DownloadManager::EnableRedirects() {
2953 27 follow_redirects_ = true;
2954 27 }
2955
2956 ✗ void DownloadManager::EnableIgnoreSignatureFailures() {
2957 ✗ ignore_signature_failures_ = true;
2958 }
2959
2960 ✗ void DownloadManager::EnableHTTPTracing() {
2961 ✗ enable_http_tracing_ = true;
2962 }
2963
2964 ✗ void DownloadManager::AddHTTPTracingHeader(const std::string &header) {
2965 ✗ http_tracing_headers_.push_back(header);
2966 }
2967
2968 24 void DownloadManager::UseSystemCertificatePath() {
2969 24 ssl_certificate_store_.UseSystemCertificatePath();
2970 24 }
2971
2972 ✗ bool DownloadManager::SetShardingPolicy(const ShardingPolicySelector type) {
2973 ✗ bool success = false;
2974 ✗ switch (type) {
2975 default:
2976 ✗ LogCvmfs(kLogDownload, kLogDebug | kLogSyslogErr, "(manager '%s') "
2977 "Proposed sharding policy does not exist. Falling back to default",
2978 name_.c_str());
2979 }
2980 ✗ return success;
2981 }
2982
2983 ✗ void DownloadManager::SetFailoverIndefinitely() {
2984 ✗ failover_indefinitely_ = true;
2985 }
2986
2987 /**
2988 * Creates a copy of the existing download manager. Must only be called in
2989 * single-threaded stage because it calls curl_global_init().
2990 */
2991 34 DownloadManager *DownloadManager::Clone(
2992 const perf::StatisticsTemplate &statistics, const std::string &cloned_name)
2993 {
2994 DownloadManager *clone = new DownloadManager(pool_max_handles_, statistics,
2995
1/2
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
 34 cloned_name);
2996
2997 34 clone->SetDnsParameters(resolver_->retries(), resolver_->timeout_ms());
2998 34 clone->SetDnsTtlLimits(resolver_->min_ttl(), resolver_->max_ttl());
2999 34 clone->SetMaxIpaddrPerProxy(resolver_->throttle());
3000
3001
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 34 times.
 34 if (!opt_dns_server_.empty())
3002 ✗ clone->SetDnsServer(opt_dns_server_);
3003 34 clone->opt_timeout_proxy_ = opt_timeout_proxy_;
3004 34 clone->opt_timeout_direct_ = opt_timeout_direct_;
3005 34 clone->opt_low_speed_limit_ = opt_low_speed_limit_;
3006 34 clone->opt_max_retries_ = opt_max_retries_;
3007 34 clone->opt_backoff_init_ms_ = opt_backoff_init_ms_;
3008 34 clone->opt_backoff_max_ms_ = opt_backoff_max_ms_;
3009 34 clone->enable_info_header_ = enable_info_header_;
3010 34 clone->enable_http_tracing_ = enable_http_tracing_;
3011 34 clone->http_tracing_headers_ = http_tracing_headers_;
3012 34 clone->follow_redirects_ = follow_redirects_;
3013 34 clone->ignore_signature_failures_ = ignore_signature_failures_;
3014
2/2
✓ Branch 0 taken 32 times.
✓ Branch 1 taken 2 times.
 34 if (opt_host_chain_) {
3015
1/2
✓ Branch 2 taken 32 times.
✗ Branch 3 not taken.
 32 clone->opt_host_chain_ = new vector<string>(*opt_host_chain_);
3016
1/2
✓ Branch 2 taken 32 times.
✗ Branch 3 not taken.
 32 clone->opt_host_chain_rtt_ = new vector<int>(*opt_host_chain_rtt_);
3017 }
3018
3019 34 CloneProxyConfig(clone);
3020 34 clone->opt_ip_preference_ = opt_ip_preference_;
3021 34 clone->proxy_template_direct_ = proxy_template_direct_;
3022 34 clone->proxy_template_forced_ = proxy_template_forced_;
3023 34 clone->opt_proxy_groups_reset_after_ = opt_proxy_groups_reset_after_;
3024 34 clone->opt_host_reset_after_ = opt_host_reset_after_;
3025 34 clone->credentials_attachment_ = credentials_attachment_;
3026 34 clone->ssl_certificate_store_ = ssl_certificate_store_;
3027
3028 34 clone->health_check_ = health_check_;
3029 34 clone->sharding_policy_ = sharding_policy_;
3030 34 clone->failover_indefinitely_ = failover_indefinitely_;
3031 34 clone->fqrn_ = fqrn_;
3032
3033 34 return clone;
3034 }
3035
3036
3037 34 void DownloadManager::CloneProxyConfig(DownloadManager *clone) {
3038 34 clone->opt_proxy_groups_current_ = opt_proxy_groups_current_;
3039 34 clone->opt_proxy_groups_current_burned_ = opt_proxy_groups_current_burned_;
3040 34 clone->opt_proxy_groups_fallback_ = opt_proxy_groups_fallback_;
3041 34 clone->opt_num_proxies_ = opt_num_proxies_;
3042 34 clone->opt_proxy_shard_ = opt_proxy_shard_;
3043 34 clone->opt_proxy_list_ = opt_proxy_list_;
3044 34 clone->opt_proxy_fallback_list_ = opt_proxy_fallback_list_;
3045
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 32 times.
 34 if (opt_proxy_groups_ == NULL)
3046 2 return;
3047
3048 32 clone->opt_proxy_groups_ = new vector< vector<ProxyInfo> >(
3049
1/2
✓ Branch 2 taken 32 times.
✗ Branch 3 not taken.
 32 *opt_proxy_groups_);
3050
2/4
✓ Branch 2 taken 32 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 32 times.
✗ Branch 6 not taken.
 32 clone->UpdateProxiesUnlocked("cloned");
3051 }
3052
3053 } // namespace download
3054