GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/upload_local.cc
Date: 2024-04-28 02:33:07
Exec Total Coverage
Lines: 87 147 59.2%
Branches: 82 254 32.3%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 */
4
5 #include "upload_local.h"
6 #include "cvmfs_config.h"
7
8 #include <errno.h>
9
10 #include <string>
11
12 #include "compression.h"
13 #include "util/logging.h"
14 #include "util/posix.h"
15
16 namespace upload {
17
18 36 LocalUploader::LocalUploader(const SpoolerDefinition &spooler_definition)
19 : AbstractUploader(spooler_definition),
20 72 backend_file_mode_(default_backend_file_mode_ ^ GetUmask()),
21
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 backend_dir_mode_(default_backend_dir_mode_ ^ GetUmask()),
22
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 upstream_path_(spooler_definition.spooler_configuration),
23
2/4
✓ Branch 2 taken 36 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 36 times.
✗ Branch 6 not taken.
 72 temporary_path_(spooler_definition.temporary_path)
24 {
25
2/4
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 36 times.
✗ Branch 4 not taken.
 36 assert(spooler_definition.IsValid() &&
26 spooler_definition.driver_type == SpoolerDefinition::Local);
27
28 36 atomic_init32(&copy_errors_);
29 36 }
30
31 53 bool LocalUploader::WillHandle(const SpoolerDefinition &spooler_definition) {
32 53 return spooler_definition.driver_type == SpoolerDefinition::Local;
33 }
34
35 65 unsigned int LocalUploader::GetNumberOfErrors() const {
36 65 return atomic_read32(&copy_errors_);
37 }
38
39 ✗ bool LocalUploader::Create() {
40 ✗ return MakeCacheDirectories(upstream_path_ + "/data", backend_dir_mode_) &&
41 ✗ MkdirDeep(upstream_path_ + "/stats", backend_dir_mode_, false);
42 }
43
44 529 void LocalUploader::DoUpload(const std::string &remote_path,
45 IngestionSource *source,
46 const CallbackTN *callback) {
47
1/2
✓ Branch 1 taken 529 times.
✗ Branch 2 not taken.
 529 LogCvmfs(kLogSpooler, kLogVerboseMsg, "FileUpload call started.");
48
49 // create destination in backend storage temporary directory
50 529 std::string tmp_path;
51
2/4
✓ Branch 1 taken 529 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 529 times.
✗ Branch 5 not taken.
 529 FILE *ftmp = CreateTempFile(temporary_path_ + "/upload", 0666, "w",
52 &tmp_path);
53
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 529 times.
 529 if (ftmp == NULL) {
54 ✗ LogCvmfs(kLogSpooler, kLogVerboseMsg,
55 "failed to create temp path for "
56 "upload of file '%s' (errno: %d)",
57 ✗ source->GetPath().c_str(), errno);
58 ✗ atomic_inc32(&copy_errors_);
59 ✗ Respond(callback, UploaderResults(1, source->GetPath()));
60 ✗ return;
61 }
62
63 // copy file into controlled temporary directory location
64
1/2
✓ Branch 1 taken 529 times.
✗ Branch 2 not taken.
 529 bool rvb = source->Open();
65
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 529 times.
 529 if (!rvb) {
66 ✗ fclose(ftmp);
67 ✗ unlink(tmp_path.c_str());
68 ✗ atomic_inc32(&copy_errors_);
69 ✗ Respond(callback, UploaderResults(100, source->GetPath()));
70 ✗ return;
71 }
72 unsigned char buffer[kPageSize];
73 ssize_t rbytes;
74 do {
75
1/2
✓ Branch 1 taken 200201 times.
✗ Branch 2 not taken.
 200201 rbytes = source->Read(buffer, kPageSize);
76 200201 size_t wbytes = 0;
77
2/2
✓ Branch 0 taken 199864 times.
✓ Branch 1 taken 337 times.
 200201 if (rbytes > 0) {
78
1/2
✓ Branch 1 taken 199864 times.
✗ Branch 2 not taken.
 199864 wbytes = fwrite(buffer, 1, rbytes, ftmp);
79 }
80
2/4
✓ Branch 0 taken 200201 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 200201 times.
 200201 if ((rbytes < 0) || (static_cast<size_t>(rbytes) != wbytes)) {
81 ✗ source->Close();
82 ✗ fclose(ftmp);
83 ✗ unlink(tmp_path.c_str());
84 ✗ atomic_inc32(&copy_errors_);
85 ✗ Respond(callback, UploaderResults(100, source->GetPath()));
86 ✗ return;
87 }
88
2/2
✓ Branch 0 taken 199672 times.
✓ Branch 1 taken 529 times.
 200201 } while (rbytes == kPageSize);
89
1/2
✓ Branch 1 taken 529 times.
✗ Branch 2 not taken.
 529 source->Close();
90
1/2
✓ Branch 1 taken 529 times.
✗ Branch 2 not taken.
 529 fclose(ftmp);
91
92 // move the file in place (atomic operation)
93
1/2
✓ Branch 1 taken 529 times.
✗ Branch 2 not taken.
 529 int rvi = Move(tmp_path, remote_path);
94
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 529 times.
 529 if (rvi != 0) {
95 ✗ LogCvmfs(kLogSpooler, kLogVerboseMsg,
96 "failed to move file '%s' from the "
97 "staging area to the final location: "
98 "'%s'",
99 tmp_path.c_str(), remote_path.c_str());
100 ✗ unlink(tmp_path.c_str());
101 ✗ atomic_inc32(&copy_errors_);
102 ✗ Respond(callback, UploaderResults(rvi, source->GetPath()));
103 ✗ return;
104 }
105
106
3/6
✓ Branch 1 taken 529 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 529 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 529 times.
✗ Branch 8 not taken.
 529 Respond(callback, UploaderResults(rvi, source->GetPath()));
107
1/2
✓ Branch 1 taken 529 times.
✗ Branch 2 not taken.
 529 }
108
109 147 UploadStreamHandle *LocalUploader::InitStreamedUpload(
110 const CallbackTN *callback) {
111 147 std::string tmp_path;
112
1/2
✓ Branch 1 taken 147 times.
✗ Branch 2 not taken.
 147 const int tmp_fd = CreateAndOpenTemporaryChunkFile(&tmp_path);
113
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 147 times.
 147 if (tmp_fd < 0) {
114 ✗ atomic_inc32(&copy_errors_);
115 ✗ return NULL;
116 }
117
118
2/4
✓ Branch 1 taken 147 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 147 times.
✗ Branch 5 not taken.
 147 return new LocalStreamHandle(callback, tmp_fd, tmp_path);
119 147 }
120
121 848 void LocalUploader::StreamedUpload(UploadStreamHandle *handle,
122 UploadBuffer buffer,
123 const CallbackTN *callback) {
124 848 LocalStreamHandle *local_handle = static_cast<LocalStreamHandle *>(handle);
125
126 const size_t bytes_written =
127 848 write(local_handle->file_descriptor, buffer.data, buffer.size);
128
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 848 times.
 848 if (bytes_written != buffer.size) {
129 ✗ const int cpy_errno = errno;
130 ✗ LogCvmfs(kLogSpooler, kLogVerboseMsg,
131 "failed to write %lu bytes to '%s' "
132 "(errno: %d)",
133 buffer.size, local_handle->temporary_path.c_str(),
134 cpy_errno);
135 ✗ atomic_inc32(&copy_errors_);
136 ✗ Respond(callback,
137 ✗ UploaderResults(UploaderResults::kBufferUpload, cpy_errno));
138 ✗ return;
139 }
140
141
1/2
✓ Branch 2 taken 848 times.
✗ Branch 3 not taken.
 848 Respond(callback, UploaderResults(UploaderResults::kBufferUpload, 0));
142 }
143
144 147 void LocalUploader::FinalizeStreamedUpload(UploadStreamHandle *handle,
145 const shash::Any &content_hash) {
146 147 int retval = 0;
147 147 LocalStreamHandle *local_handle = static_cast<LocalStreamHandle *>(handle);
148
149
1/2
✓ Branch 1 taken 147 times.
✗ Branch 2 not taken.
 147 retval = close(local_handle->file_descriptor);
150
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 147 times.
 147 if (retval != 0) {
151 ✗ const int cpy_errno = errno;
152 ✗ LogCvmfs(kLogSpooler, kLogVerboseMsg,
153 "failed to close temp file '%s' "
154 "(errno: %d)",
155 local_handle->temporary_path.c_str(), cpy_errno);
156 ✗ atomic_inc32(&copy_errors_);
157 ✗ Respond(handle->commit_callback,
158 ✗ UploaderResults(UploaderResults::kChunkCommit, cpy_errno));
159 ✗ return;
160 }
161
162 147 std::string final_path;
163
2/4
✓ Branch 1 taken 147 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 147 times.
 147 if (local_handle->remote_path != "") {
164 ✗ final_path = local_handle->remote_path;
165 } else {
166
2/4
✓ Branch 1 taken 147 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 147 times.
✗ Branch 5 not taken.
 147 final_path = "data/" + content_hash.MakePath();
167 }
168
2/4
✓ Branch 1 taken 147 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 147 times.
✗ Branch 4 not taken.
 147 if (!Peek(final_path)) {
169
1/2
✓ Branch 1 taken 147 times.
✗ Branch 2 not taken.
 147 retval = Move(local_handle->temporary_path, final_path);
170
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 147 times.
 147 if (retval != 0) {
171 ✗ const int cpy_errno = errno;
172 ✗ LogCvmfs(kLogSpooler, kLogVerboseMsg,
173 "failed to move temp file '%s' to "
174 "final location '%s' (errno: %d)",
175 local_handle->temporary_path.c_str(), final_path.c_str(),
176 cpy_errno);
177 ✗ atomic_inc32(&copy_errors_);
178 ✗ Respond(handle->commit_callback,
179 ✗ UploaderResults(UploaderResults::kChunkCommit, cpy_errno));
180 ✗ return;
181 }
182 147 if (!content_hash.HasSuffix()
183
5/6
✓ Branch 0 taken 47 times.
✓ Branch 1 taken 100 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 47 times.
✓ Branch 4 taken 100 times.
✓ Branch 5 taken 47 times.
 147 || content_hash.suffix == shash::kSuffixPartial) {
184
1/2
✓ Branch 1 taken 100 times.
✗ Branch 2 not taken.
 100 CountUploadedChunks();
185
4/8
✓ Branch 1 taken 100 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 100 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 100 times.
✗ Branch 8 not taken.
✓ Branch 10 taken 100 times.
✗ Branch 11 not taken.
 100 CountUploadedBytes(GetFileSize(upstream_path_ + "/" + final_path));
186
2/2
✓ Branch 0 taken 46 times.
✓ Branch 1 taken 1 times.
 47 } else if (content_hash.suffix == shash::kSuffixCatalog) {
187
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 CountUploadedCatalogs();
188
4/8
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 46 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 46 times.
✗ Branch 8 not taken.
✓ Branch 10 taken 46 times.
✗ Branch 11 not taken.
 46 CountUploadedCatalogBytes(GetFileSize(upstream_path_ + "/" + final_path));
189 }
190 } else {
191 ✗ const int retval = unlink(local_handle->temporary_path.c_str());
192 ✗ if (retval != 0) {
193 ✗ LogCvmfs(kLogSpooler, kLogVerboseMsg,
194 "failed to remove temporary file '%s' (errno: %d)",
195 ✗ local_handle->temporary_path.c_str(), errno);
196 }
197 ✗ CountDuplicates();
198 }
199
200 147 const CallbackTN *callback = handle->commit_callback;
201
1/2
✓ Branch 0 taken 147 times.
✗ Branch 1 not taken.
 147 delete local_handle;
202
203
2/4
✓ Branch 1 taken 147 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 147 times.
✗ Branch 5 not taken.
 147 Respond(callback, UploaderResults(UploaderResults::kChunkCommit, 0));
204
1/2
✓ Branch 1 taken 147 times.
✗ Branch 2 not taken.
 147 }
205
206 /**
207 * TODO(jblomer): investigate if parallelism increases the GC speed on local
208 * disks.
209 */
210 1 void LocalUploader::DoRemoveAsync(const std::string &file_to_delete) {
211
1/2
✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
 1 const int retval = unlink((upstream_path_ + "/" + file_to_delete).c_str());
212
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 1 if ((retval != 0) && (errno != ENOENT))
213 ✗ atomic_inc32(&copy_errors_);
214
1/2
✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
 1 Respond(NULL, UploaderResults());
215 1 }
216
217 152 bool LocalUploader::Peek(const std::string &path) {
218
2/4
✓ Branch 2 taken 152 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 152 times.
✗ Branch 6 not taken.
 152 bool retval = FileExists(upstream_path_ + "/" + path);
219 152 return retval;
220 }
221
222 ✗ bool LocalUploader::Mkdir(const std::string &path) {
223 ✗ return MkdirDeep(upstream_path_ + "/" + path, backend_dir_mode_, false);
224 }
225
226 1 bool LocalUploader::PlaceBootstrappingShortcut(const shash::Any &object) {
227
2/4
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1 times.
✗ Branch 5 not taken.
 1 const std::string src = "data/" + object.MakePath();
228
3/6
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 1 times.
✗ Branch 8 not taken.
 2 const std::string dest = upstream_path_ + "/" + object.MakeAlternativePath();
229
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 2 return SymlinkForced(src, dest);
230 1 }
231
232 676 int LocalUploader::Move(const std::string &local_path,
233 const std::string &remote_path) const {
234
2/4
✓ Branch 1 taken 676 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 676 times.
✗ Branch 5 not taken.
 676 const std::string destination_path = upstream_path_ + "/" + remote_path;
235
236 // make sure the file has the right permissions
237 676 int retval = chmod(local_path.c_str(), backend_file_mode_);
238
1/2
✓ Branch 0 taken 676 times.
✗ Branch 1 not taken.
 676 int retcode = (retval == 0) ? 0 : 101;
239
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 676 times.
 676 if (retcode != 0) {
240 ✗ LogCvmfs(kLogSpooler, kLogVerboseMsg,
241 "failed to set file permission '%s' "
242 "errno: %d",
243 ✗ local_path.c_str(), errno);
244 ✗ return retcode;
245 }
246
247 // move the file in place
248 676 retval = rename(local_path.c_str(), destination_path.c_str());
249
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 676 times.
 676 retcode = (retval == 0) ? 0 : errno;
250
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 676 times.
 676 if (retcode != 0) {
251 ✗ LogCvmfs(kLogSpooler, kLogVerboseMsg,
252 "failed to move file '%s' to '%s' "
253 "errno: %d",
254 ✗ local_path.c_str(), remote_path.c_str(), errno);
255 }
256
257 676 return retcode;
258 676 }
259
260 ✗ int64_t LocalUploader::DoGetObjectSize(const std::string &file_name) {
261 ✗ return GetFileSize(upstream_path_ + "/" + file_name);
262 }
263
264 } // namespace upload
265