| Line |
Branch |
Exec |
Source |
| 1 |
|
|
/** |
| 2 |
|
|
* This file is part of the CernVM File System. |
| 3 |
|
|
*/ |
| 4 |
|
|
|
| 5 |
|
|
#ifndef CVMFS_NETWORK_JOBINFO_H_ |
| 6 |
|
|
#define CVMFS_NETWORK_JOBINFO_H_ |
| 7 |
|
|
|
| 8 |
|
|
#include <poll.h> |
| 9 |
|
|
#include <pthread.h> |
| 10 |
|
|
#include <stdint.h> |
| 11 |
|
|
#include <unistd.h> |
| 12 |
|
|
|
| 13 |
|
|
#include <cstdio> |
| 14 |
|
|
#include <memory> |
| 15 |
|
|
#include <string> |
| 16 |
|
|
|
| 17 |
|
|
#include "compression/compression.h" |
| 18 |
|
|
#include "crypto/hash.h" |
| 19 |
|
|
#include "duplex_curl.h" // IWYU pragma: keep |
| 20 |
|
|
#include "network/network_errors.h" |
| 21 |
|
|
#include "network/sink.h" |
| 22 |
|
|
#include "util/pipe.h" |
| 23 |
|
|
#include "util/tube.h" |
| 24 |
|
|
|
| 25 |
|
|
class InterruptCue; |
| 26 |
|
|
|
| 27 |
|
|
namespace download { |
| 28 |
|
|
|
| 29 |
|
|
enum DataTubeAction { |
| 30 |
|
|
kActionStop = 0, |
| 31 |
|
|
kActionContinue, |
| 32 |
|
|
kActionDecompress, |
| 33 |
|
|
// Marks the boundary of a superseded download attempt (retry / host |
| 34 |
|
|
// fail-over). When the caller pops this it must discard the bytes it has |
| 35 |
|
|
// decompressed so far, reset its sink and zstream, and clear any |
| 36 |
|
|
// decompression error before processing the bytes of the next attempt. |
| 37 |
|
|
kActionReset |
| 38 |
|
|
}; |
| 39 |
|
|
|
| 40 |
|
|
/** |
| 41 |
|
|
* Wrapper for the data tube to transfer data from CallbackCurlData() that is |
| 42 |
|
|
* executed in MainDownload() Thread to Fetch() called by a fuse thread |
| 43 |
|
|
* |
| 44 |
|
|
* TODO(heretherebedragons): do we want to have a pool of those |
| 45 |
|
|
* datatubeelements? |
| 46 |
|
|
*/ |
| 47 |
|
|
struct DataTubeElement : SingleCopy { |
| 48 |
|
|
char *data; |
| 49 |
|
|
size_t size; |
| 50 |
|
|
DataTubeAction action; |
| 51 |
|
|
|
| 52 |
|
✗ |
explicit DataTubeElement(DataTubeAction xact) |
| 53 |
|
✗ |
: data(NULL), size(0), action(xact) { } |
| 54 |
|
✗ |
DataTubeElement(char *mov_data, size_t xsize, DataTubeAction xact) |
| 55 |
|
✗ |
: data(mov_data), size(xsize), action(xact) { } |
| 56 |
|
|
|
| 57 |
|
✗ |
~DataTubeElement() { delete[] data; } |
| 58 |
|
|
}; |
| 59 |
|
|
|
| 60 |
|
|
/** |
| 61 |
|
|
* Contains all the information to specify a download job. |
| 62 |
|
|
*/ |
| 63 |
|
|
class JobInfo { |
| 64 |
|
|
private: |
| 65 |
|
|
static atomic_int64 next_uuid; |
| 66 |
|
|
int64_t id_; |
| 67 |
|
|
/// Pipe used for the return value |
| 68 |
|
|
std::unique_ptr<Pipe<kPipeDownloadJobsResults> > pipe_job_results; |
| 69 |
|
|
/// Tube (bounded thread-safe queue) to transport data from CURL callback |
| 70 |
|
|
/// to be decompressed in Fetch() instead of MainDownload() |
| 71 |
|
|
std::unique_ptr<Tube<DataTubeElement> > data_tube_; |
| 72 |
|
|
const std::string *url_; |
| 73 |
|
|
bool compressed_; |
| 74 |
|
|
bool probe_hosts_; |
| 75 |
|
|
bool head_request_; |
| 76 |
|
|
bool follow_redirects_; |
| 77 |
|
|
bool force_nocache_; |
| 78 |
|
|
pid_t pid_; |
| 79 |
|
|
uid_t uid_; |
| 80 |
|
|
gid_t gid_; |
| 81 |
|
|
void *cred_data_; // Per-transfer credential data |
| 82 |
|
|
InterruptCue *interrupt_cue_; |
| 83 |
|
|
cvmfs::Sink *sink_; |
| 84 |
|
|
const shash::Any *expected_hash_; |
| 85 |
|
|
const std::string *path_info_; |
| 86 |
|
|
|
| 87 |
|
|
// Allow byte ranges to be specified. |
| 88 |
|
|
off_t range_offset_; |
| 89 |
|
|
off_t range_size_; |
| 90 |
|
|
|
| 91 |
|
|
// Internal state |
| 92 |
|
|
CURL *curl_handle_; |
| 93 |
|
|
curl_slist *headers_; |
| 94 |
|
|
char *info_header_; |
| 95 |
|
|
char *tracing_header_pid_; |
| 96 |
|
|
char *tracing_header_gid_; |
| 97 |
|
|
char *tracing_header_uid_; |
| 98 |
|
|
z_stream zstream_; |
| 99 |
|
|
shash::ContextPtr hash_context_; |
| 100 |
|
|
std::string proxy_; |
| 101 |
|
|
std::string link_; |
| 102 |
|
|
bool nocache_; |
| 103 |
|
|
Failures error_code_; |
| 104 |
|
|
int http_code_; |
| 105 |
|
|
unsigned char num_used_proxies_; |
| 106 |
|
|
unsigned char num_used_metalinks_; |
| 107 |
|
|
unsigned char num_used_hosts_; |
| 108 |
|
|
unsigned char num_retries_; |
| 109 |
|
|
unsigned backoff_ms_; |
| 110 |
|
|
int current_metalink_chain_index_; |
| 111 |
|
|
int current_host_chain_index_; |
| 112 |
|
|
|
| 113 |
|
|
// Don't fail-over proxies on download errors. default = false |
| 114 |
|
|
bool allow_failure_; |
| 115 |
|
|
|
| 116 |
|
|
// TODO(heretherebedragons) c++11 allows to delegate constructors (N1986) |
| 117 |
|
|
// Replace Init() with JobInfo() that is called by the other constructors |
| 118 |
|
|
void Init(); |
| 119 |
|
|
|
| 120 |
|
|
public: |
| 121 |
|
|
/** |
| 122 |
|
|
* Sink version: downloads entire data chunk where URL u points to |
| 123 |
|
|
*/ |
| 124 |
|
|
JobInfo(const std::string *u, const bool c, const bool ph, |
| 125 |
|
|
const shash::Any *h, cvmfs::Sink *s); |
| 126 |
|
|
|
| 127 |
|
|
/** |
| 128 |
|
|
* No sink version: Only downloads header where the URL u points to |
| 129 |
|
|
*/ |
| 130 |
|
|
JobInfo(const std::string *u, const bool ph); |
| 131 |
|
|
|
| 132 |
|
7014 |
~JobInfo() { |
| 133 |
|
7014 |
pipe_job_results.reset(); |
| 134 |
|
7014 |
data_tube_.reset(); |
| 135 |
|
7014 |
} |
| 136 |
|
|
|
| 137 |
|
|
static bool EscapeUrlChar(unsigned char input, char output[3]); |
| 138 |
|
|
|
| 139 |
|
✗ |
void CreatePipeJobResults() { |
| 140 |
|
✗ |
pipe_job_results = std::unique_ptr<Pipe<kPipeDownloadJobsResults> >( |
| 141 |
|
✗ |
new Pipe<kPipeDownloadJobsResults>()); |
| 142 |
|
|
} |
| 143 |
|
|
|
| 144 |
|
✗ |
bool IsValidPipeJobResults() { return pipe_job_results.get() != nullptr; } |
| 145 |
|
|
|
| 146 |
|
✗ |
void CreateDataTube() { |
| 147 |
|
|
// TODO(heretherebedragons) change to weighted queue |
| 148 |
|
✗ |
data_tube_ = std::unique_ptr<Tube<DataTubeElement> >( |
| 149 |
|
✗ |
new Tube<DataTubeElement>(500)); |
| 150 |
|
|
} |
| 151 |
|
|
|
| 152 |
|
14497 |
bool IsValidDataTube() { return data_tube_.get() != nullptr; } |
| 153 |
|
|
|
| 154 |
|
|
/** |
| 155 |
|
|
* Tells whether the error is because of a non-existing file. Should only |
| 156 |
|
|
* be called if error_code is not kFailOk |
| 157 |
|
|
*/ |
| 158 |
|
|
bool IsFileNotFound(); |
| 159 |
|
|
|
| 160 |
|
1307 |
pid_t *GetPidPtr() { return &pid_; } |
| 161 |
|
1307 |
uid_t *GetUidPtr() { return &uid_; } |
| 162 |
|
1307 |
gid_t *GetGidPtr() { return &gid_; } |
| 163 |
|
1307 |
InterruptCue **GetInterruptCuePtr() { return &interrupt_cue_; } |
| 164 |
|
15623 |
z_stream *GetZstreamPtr() { return &zstream_; } |
| 165 |
|
|
Failures *GetErrorCodePtr() { return &error_code_; } |
| 166 |
|
✗ |
void **GetCredDataPtr() { return &cred_data_; } |
| 167 |
|
✗ |
curl_slist **GetHeadersPtr() { return &headers_; } |
| 168 |
|
|
CURL **GetCurlHandle() { return &curl_handle_; } |
| 169 |
|
15774 |
shash::ContextPtr *GetHashContextPtr() { return &hash_context_; } |
| 170 |
|
✗ |
Pipe<kPipeDownloadJobsResults> *GetPipeJobResultPtr() { |
| 171 |
|
✗ |
return pipe_job_results.get(); |
| 172 |
|
|
} |
| 173 |
|
✗ |
Tube<DataTubeElement> *GetDataTubePtr() { return data_tube_.get(); } |
| 174 |
|
|
|
| 175 |
|
21936 |
const std::string *url() const { return url_; } |
| 176 |
|
21651 |
bool compressed() const { return compressed_; } |
| 177 |
|
7557 |
bool probe_hosts() const { return probe_hosts_; } |
| 178 |
|
7161 |
bool head_request() const { return head_request_; } |
| 179 |
|
21 |
bool follow_redirects() const { return follow_redirects_; } |
| 180 |
|
7161 |
bool force_nocache() const { return force_nocache_; } |
| 181 |
|
✗ |
pid_t pid() const { return pid_; } |
| 182 |
|
✗ |
uid_t uid() const { return uid_; } |
| 183 |
|
✗ |
gid_t gid() const { return gid_; } |
| 184 |
|
7182 |
void *cred_data() const { return cred_data_; } |
| 185 |
|
270 |
InterruptCue *interrupt_cue() const { return interrupt_cue_; } |
| 186 |
|
107529 |
cvmfs::Sink *sink() const { return sink_; } |
| 187 |
|
46293 |
const shash::Any *expected_hash() const { return expected_hash_; } |
| 188 |
|
|
const std::string *path_info() const { return path_info_; } |
| 189 |
|
|
|
| 190 |
|
7161 |
off_t range_offset() const { return range_offset_; } |
| 191 |
|
✗ |
off_t range_size() const { return range_size_; } |
| 192 |
|
|
|
| 193 |
|
29117 |
CURL *curl_handle() const { return curl_handle_; } |
| 194 |
|
22008 |
curl_slist *headers() const { return headers_; } |
| 195 |
|
7161 |
char *info_header() const { return info_header_; } |
| 196 |
|
✗ |
char *tracing_header_pid() const { return tracing_header_pid_; } |
| 197 |
|
✗ |
char *tracing_header_gid() const { return tracing_header_gid_; } |
| 198 |
|
✗ |
char *tracing_header_uid() const { return tracing_header_uid_; } |
| 199 |
|
|
z_stream zstream() const { return zstream_; } |
| 200 |
|
26081 |
shash::ContextPtr hash_context() const { return hash_context_; } |
| 201 |
|
15075 |
std::string proxy() const { return proxy_; } |
| 202 |
|
✗ |
std::string link() const { return link_; } |
| 203 |
|
8287 |
bool nocache() const { return nocache_; } |
| 204 |
|
28537 |
Failures error_code() const { return error_code_; } |
| 205 |
|
338 |
int http_code() const { return http_code_; } |
| 206 |
|
21 |
unsigned char num_used_proxies() const { return num_used_proxies_; } |
| 207 |
|
✗ |
unsigned char num_used_metalinks() const { return num_used_metalinks_; } |
| 208 |
|
7445 |
unsigned char num_used_hosts() const { return num_used_hosts_; } |
| 209 |
|
7389 |
unsigned char num_retries() const { return num_retries_; } |
| 210 |
|
676 |
unsigned backoff_ms() const { return backoff_ms_; } |
| 211 |
|
7417 |
int current_metalink_chain_index() const { |
| 212 |
|
7417 |
return current_metalink_chain_index_; |
| 213 |
|
|
} |
| 214 |
|
14 |
int current_host_chain_index() const { return current_host_chain_index_; } |
| 215 |
|
|
|
| 216 |
|
✗ |
bool allow_failure() const { return allow_failure_; } |
| 217 |
|
28082 |
int64_t id() const { return id_; } |
| 218 |
|
|
|
| 219 |
|
|
std::string GetInfoHeaderContents(const std::string &templ); |
| 220 |
|
1307 |
void SetUrl(const std::string *url) { url_ = url; } |
| 221 |
|
2343 |
void SetCompressed(bool compressed) { compressed_ = compressed; } |
| 222 |
|
1036 |
void SetProbeHosts(bool probe_hosts) { probe_hosts_ = probe_hosts; } |
| 223 |
|
|
void SetHeadRequest(bool head_request) { head_request_ = head_request; } |
| 224 |
|
7161 |
void SetFollowRedirects(bool follow_redirects) { |
| 225 |
|
7161 |
follow_redirects_ = follow_redirects; |
| 226 |
|
7161 |
} |
| 227 |
|
464 |
void SetForceNocache(bool force_nocache) { force_nocache_ = force_nocache; } |
| 228 |
|
✗ |
void SetCredData(void *cred_data) { cred_data_ = cred_data; } |
| 229 |
|
7 |
void SetInterruptCue(InterruptCue *interrupt_cue) { |
| 230 |
|
7 |
interrupt_cue_ = interrupt_cue; |
| 231 |
|
7 |
} |
| 232 |
|
1307 |
void SetSink(cvmfs::Sink *sink) { sink_ = sink; } |
| 233 |
|
1307 |
void SetExpectedHash(const shash::Any *expected_hash) { |
| 234 |
|
1307 |
expected_hash_ = expected_hash; |
| 235 |
|
1307 |
} |
| 236 |
|
1310 |
void SetPathInfo(const std::string *path_info) { path_info_ = path_info; } |
| 237 |
|
|
|
| 238 |
|
1310 |
void SetRangeOffset(off_t range_offset) { range_offset_ = range_offset; } |
| 239 |
|
1310 |
void SetRangeSize(off_t range_size) { range_size_ = range_size; } |
| 240 |
|
|
|
| 241 |
|
7161 |
void SetCurlHandle(CURL *curl_handle) { curl_handle_ = curl_handle; } |
| 242 |
|
14322 |
void SetHeaders(curl_slist *headers) { headers_ = headers; } |
| 243 |
|
7161 |
void SetInfoHeader(char *info_header) { info_header_ = info_header; } |
| 244 |
|
✗ |
void SetTracingHeaderPid(char *tracing_header_pid) { |
| 245 |
|
✗ |
tracing_header_pid_ = tracing_header_pid; |
| 246 |
|
✗ |
}; |
| 247 |
|
✗ |
void SetTracingHeaderGid(char *tracing_header_gid) { |
| 248 |
|
✗ |
tracing_header_gid_ = tracing_header_gid; |
| 249 |
|
✗ |
}; |
| 250 |
|
✗ |
void SetTracingHeaderUid(char *tracing_header_uid) { |
| 251 |
|
✗ |
tracing_header_uid_ = tracing_header_uid; |
| 252 |
|
✗ |
}; |
| 253 |
|
|
void SetZstream(z_stream zstream) { zstream_ = zstream; } |
| 254 |
|
|
void SetHashContext(shash::ContextPtr hash_context) { |
| 255 |
|
|
hash_context_ = hash_context; |
| 256 |
|
|
} |
| 257 |
|
7182 |
void SetProxy(const std::string &proxy) { proxy_ = proxy; } |
| 258 |
|
7161 |
void SetLink(const std::string &link) { link_ = link; } |
| 259 |
|
7249 |
void SetNocache(bool nocache) { nocache_ = nocache; } |
| 260 |
|
14673 |
void SetErrorCode(Failures error_code) { error_code_ = error_code; } |
| 261 |
|
7245 |
void SetHttpCode(int http_code) { http_code_ = http_code; } |
| 262 |
|
7168 |
void SetNumUsedProxies(unsigned char num_used_proxies) { |
| 263 |
|
7168 |
num_used_proxies_ = num_used_proxies; |
| 264 |
|
7168 |
} |
| 265 |
|
7161 |
void SetNumUsedMetalinks(unsigned char num_used_metalinks) { |
| 266 |
|
7161 |
num_used_metalinks_ = num_used_metalinks; |
| 267 |
|
7161 |
} |
| 268 |
|
7175 |
void SetNumUsedHosts(unsigned char num_used_hosts) { |
| 269 |
|
7175 |
num_used_hosts_ = num_used_hosts; |
| 270 |
|
7175 |
} |
| 271 |
|
7308 |
void SetNumRetries(unsigned char num_retries) { num_retries_ = num_retries; } |
| 272 |
|
7337 |
void SetBackoffMs(unsigned backoff_ms) { backoff_ms_ = backoff_ms; } |
| 273 |
|
✗ |
void SetCurrentMetalinkChainIndex(int current_metalink_chain_index) { |
| 274 |
|
✗ |
current_metalink_chain_index_ = current_metalink_chain_index; |
| 275 |
|
|
} |
| 276 |
|
3184 |
void SetCurrentHostChainIndex(int current_host_chain_index) { |
| 277 |
|
3184 |
current_host_chain_index_ = current_host_chain_index; |
| 278 |
|
3184 |
} |
| 279 |
|
|
|
| 280 |
|
|
void SetAllowFailure(bool allow_failure) { allow_failure_ = allow_failure; } |
| 281 |
|
|
|
| 282 |
|
|
// needed for fetch.h ThreadLocalStorage |
| 283 |
1/2
✓ Branch 6 taken 1036 times.
✗ Branch 7 not taken.
|
1036 |
JobInfo() { Init(); } |
| 284 |
|
|
}; // JobInfo |
| 285 |
|
|
|
| 286 |
|
|
} // namespace download |
| 287 |
|
|
|
| 288 |
|
|
#endif // CVMFS_NETWORK_JOBINFO_H_ |
| 289 |
|
|
|