GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/upload_local.cc
Date: 2026-07-12 02:35:48
Exec Total Coverage
Lines: 88 148 59.5%
Branches: 82 255 32.2%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 */
4
5 #include "upload_local.h"
6
7 #include <errno.h>
8
9 #include <string>
10
11 #include "compression/compression.h"
12 #include "util/logging.h"
13 #include "util/posix.h"
14
15 namespace upload {
16
17 1371 LocalUploader::LocalUploader(const SpoolerDefinition &spooler_definition)
18 : AbstractUploader(spooler_definition)
19 2742 , backend_file_mode_(default_backend_file_mode_ ^ GetUmask())
20
1/2
✓ Branch 1 taken 1371 times.
✗ Branch 2 not taken.
 1371 , backend_dir_mode_(default_backend_dir_mode_ ^ GetUmask())
21
1/2
✓ Branch 1 taken 1371 times.
✗ Branch 2 not taken.
 1371 , upstream_path_(spooler_definition.spooler_configuration)
22
2/4
✓ Branch 2 taken 1371 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1371 times.
✗ Branch 6 not taken.
 2742 , temporary_path_(spooler_definition.temporary_path) {
23
2/4
✓ Branch 1 taken 1371 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1371 times.
✗ Branch 4 not taken.
 1371 assert(spooler_definition.IsValid()
24 && spooler_definition.driver_type == SpoolerDefinition::Local);
25
26 1371 atomic_init32(&copy_errors_);
27 1371 }
28
29 1819 bool LocalUploader::WillHandle(const SpoolerDefinition &spooler_definition) {
30 1819 return spooler_definition.driver_type == SpoolerDefinition::Local;
31 }
32
33 2905 unsigned int LocalUploader::GetNumberOfErrors() const {
34 2905 return atomic_read32(&copy_errors_);
35 }
36
37 ✗ bool LocalUploader::Create() {
38 ✗ return MakeCacheDirectories(upstream_path_ + "/data", backend_dir_mode_)
39 ✗ && MkdirDeep(upstream_path_ + "/stats", backend_dir_mode_, false);
40 }
41
42 12143 void LocalUploader::DoUpload(const std::string &remote_path,
43 IngestionSource *source,
44 const CallbackTN *callback) {
45
1/2
✓ Branch 1 taken 12143 times.
✗ Branch 2 not taken.
 12143 LogCvmfs(kLogSpooler, kLogVerboseMsg, "FileUpload call started.");
46
47 // create destination in backend storage temporary directory
48 12143 std::string tmp_path;
49
2/4
✓ Branch 1 taken 12143 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 12143 times.
✗ Branch 5 not taken.
 12143 FILE *ftmp = CreateTempFile(temporary_path_ + "/upload", 0666, "w",
50 &tmp_path);
51
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 12143 times.
 12143 if (ftmp == NULL) {
52 ✗ LogCvmfs(kLogSpooler, kLogVerboseMsg,
53 "failed to create temp path for "
54 "upload of file '%s' (errno: %d)",
55 ✗ source->GetPath().c_str(), errno);
56 ✗ atomic_inc32(&copy_errors_);
57 ✗ Respond(callback, UploaderResults(1, source->GetPath()));
58 ✗ return;
59 }
60
61 // copy file into controlled temporary directory location
62
1/2
✓ Branch 1 taken 12143 times.
✗ Branch 2 not taken.
 12143 const bool rvb = source->Open();
63
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 12143 times.
 12143 if (!rvb) {
64 ✗ fclose(ftmp);
65 ✗ unlink(tmp_path.c_str());
66 ✗ atomic_inc32(&copy_errors_);
67 ✗ Respond(callback, UploaderResults(100, source->GetPath()));
68 ✗ return;
69 }
70 unsigned char buffer[kPageSize];
71 ssize_t rbytes;
72 do {
73
1/2
✓ Branch 1 taken 4404927 times.
✗ Branch 2 not taken.
 4404927 rbytes = source->Read(buffer, kPageSize);
74 4404927 size_t wbytes = 0;
75
2/2
✓ Branch 0 taken 4397513 times.
✓ Branch 1 taken 7414 times.
 4404927 if (rbytes > 0) {
76
1/2
✓ Branch 1 taken 4397513 times.
✗ Branch 2 not taken.
 4397513 wbytes = fwrite(buffer, 1, rbytes, ftmp);
77 }
78
2/4
✓ Branch 0 taken 4404927 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 4404927 times.
 4404927 if ((rbytes < 0) || (static_cast<size_t>(rbytes) != wbytes)) {
79 ✗ source->Close();
80 ✗ fclose(ftmp);
81 ✗ unlink(tmp_path.c_str());
82 ✗ atomic_inc32(&copy_errors_);
83 ✗ Respond(callback, UploaderResults(100, source->GetPath()));
84 ✗ return;
85 }
86
2/2
✓ Branch 0 taken 4392784 times.
✓ Branch 1 taken 12143 times.
 4404927 } while (rbytes == kPageSize);
87
1/2
✓ Branch 1 taken 12143 times.
✗ Branch 2 not taken.
 12143 source->Close();
88
1/2
✓ Branch 1 taken 12143 times.
✗ Branch 2 not taken.
 12143 fclose(ftmp);
89
90 // move the file in place (atomic operation)
91
1/2
✓ Branch 1 taken 12143 times.
✗ Branch 2 not taken.
 12143 const int rvi = Move(tmp_path, remote_path);
92
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 12143 times.
 12143 if (rvi != 0) {
93 ✗ LogCvmfs(kLogSpooler, kLogVerboseMsg,
94 "failed to move file '%s' from the "
95 "staging area to the final location: "
96 "'%s'",
97 tmp_path.c_str(), remote_path.c_str());
98 ✗ unlink(tmp_path.c_str());
99 ✗ atomic_inc32(&copy_errors_);
100 ✗ Respond(callback, UploaderResults(rvi, source->GetPath()));
101 ✗ return;
102 }
103
104
3/6
✓ Branch 1 taken 12143 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 12143 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 12143 times.
✗ Branch 8 not taken.
 12143 Respond(callback, UploaderResults(rvi, source->GetPath()));
105
1/2
✓ Branch 1 taken 12143 times.
✗ Branch 2 not taken.
 12143 }
106
107 4454 UploadStreamHandle *LocalUploader::InitStreamedUpload(
108 const CallbackTN *callback) {
109 4454 std::string tmp_path;
110
1/2
✓ Branch 1 taken 4454 times.
✗ Branch 2 not taken.
 4454 const int tmp_fd = CreateAndOpenTemporaryChunkFile(&tmp_path);
111
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4454 times.
 4454 if (tmp_fd < 0) {
112 ✗ atomic_inc32(&copy_errors_);
113 ✗ return NULL;
114 }
115
116
2/4
✓ Branch 1 taken 4454 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 4454 times.
✗ Branch 5 not taken.
 4454 return new LocalStreamHandle(callback, tmp_fd, tmp_path);
117 4454 }
118
119 19876 void LocalUploader::StreamedUpload(UploadStreamHandle *handle,
120 UploadBuffer buffer,
121 const CallbackTN *callback) {
122 19876 LocalStreamHandle *local_handle = static_cast<LocalStreamHandle *>(handle);
123
124 19876 const size_t bytes_written = write(local_handle->file_descriptor, buffer.data,
125 19876 buffer.size);
126
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19876 times.
 19876 if (bytes_written != buffer.size) {
127 ✗ const int cpy_errno = errno;
128 ✗ LogCvmfs(kLogSpooler, kLogVerboseMsg,
129 "failed to write %lu bytes to '%s' "
130 "(errno: %d)",
131 buffer.size, local_handle->temporary_path.c_str(), cpy_errno);
132 ✗ atomic_inc32(&copy_errors_);
133 ✗ Respond(callback,
134 ✗ UploaderResults(UploaderResults::kBufferUpload, cpy_errno));
135 ✗ return;
136 }
137
138
1/2
✓ Branch 2 taken 19876 times.
✗ Branch 3 not taken.
 19876 Respond(callback, UploaderResults(UploaderResults::kBufferUpload, 0));
139 }
140
141 4454 void LocalUploader::FinalizeStreamedUpload(UploadStreamHandle *handle,
142 const shash::Any &content_hash) {
143 4454 int retval = 0;
144 4454 LocalStreamHandle *local_handle = static_cast<LocalStreamHandle *>(handle);
145
146
1/2
✓ Branch 1 taken 4454 times.
✗ Branch 2 not taken.
 4454 retval = close(local_handle->file_descriptor);
147
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4454 times.
 4454 if (retval != 0) {
148 ✗ const int cpy_errno = errno;
149 ✗ LogCvmfs(kLogSpooler, kLogVerboseMsg,
150 "failed to close temp file '%s' "
151 "(errno: %d)",
152 local_handle->temporary_path.c_str(), cpy_errno);
153 ✗ atomic_inc32(&copy_errors_);
154 ✗ Respond(handle->commit_callback,
155 ✗ UploaderResults(UploaderResults::kChunkCommit, cpy_errno));
156 ✗ return;
157 }
158
159 4454 std::string final_path;
160
2/4
✓ Branch 1 taken 4454 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 4454 times.
 4454 if (local_handle->remote_path != "") {
161 ✗ final_path = local_handle->remote_path;
162 } else {
163
2/4
✓ Branch 1 taken 4454 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 4454 times.
✗ Branch 5 not taken.
 4454 final_path = "data/" + content_hash.MakePath();
164 }
165
2/4
✓ Branch 1 taken 4454 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 4454 times.
✗ Branch 4 not taken.
 4454 if (!Peek(final_path)) {
166
1/2
✓ Branch 1 taken 4454 times.
✗ Branch 2 not taken.
 4454 retval = Move(local_handle->temporary_path, final_path);
167
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4454 times.
 4454 if (retval != 0) {
168 ✗ const int cpy_errno = errno;
169 ✗ LogCvmfs(kLogSpooler, kLogVerboseMsg,
170 "failed to move temp file '%s' to "
171 "final location '%s' (errno: %d)",
172 local_handle->temporary_path.c_str(), final_path.c_str(),
173 cpy_errno);
174 ✗ atomic_inc32(&copy_errors_);
175 ✗ Respond(handle->commit_callback,
176 ✗ UploaderResults(UploaderResults::kChunkCommit, cpy_errno));
177 ✗ return;
178 }
179 4454 if (!content_hash.HasSuffix()
180
5/6
✓ Branch 0 taken 2254 times.
✓ Branch 1 taken 2200 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2254 times.
✓ Branch 4 taken 2200 times.
✓ Branch 5 taken 2254 times.
 4454 || content_hash.suffix == shash::kSuffixPartial) {
181
1/2
✓ Branch 1 taken 2200 times.
✗ Branch 2 not taken.
 2200 CountUploadedChunks();
182
4/8
✓ Branch 1 taken 2200 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2200 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 2200 times.
✗ Branch 8 not taken.
✓ Branch 10 taken 2200 times.
✗ Branch 11 not taken.
 2200 CountUploadedBytes(GetFileSize(upstream_path_ + "/" + final_path));
183
2/2
✓ Branch 0 taken 2232 times.
✓ Branch 1 taken 22 times.
 2254 } else if (content_hash.suffix == shash::kSuffixCatalog) {
184
1/2
✓ Branch 1 taken 2232 times.
✗ Branch 2 not taken.
 2232 CountUploadedCatalogs();
185
4/8
✓ Branch 1 taken 2232 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2232 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 2232 times.
✗ Branch 8 not taken.
✓ Branch 10 taken 2232 times.
✗ Branch 11 not taken.
 2232 CountUploadedCatalogBytes(GetFileSize(upstream_path_ + "/" + final_path));
186 }
187 } else {
188 ✗ const int retval = unlink(local_handle->temporary_path.c_str());
189 ✗ if (retval != 0) {
190 ✗ LogCvmfs(kLogSpooler, kLogVerboseMsg,
191 "failed to remove temporary file '%s' (errno: %d)",
192 ✗ local_handle->temporary_path.c_str(), errno);
193 }
194 ✗ CountDuplicates();
195 }
196
197 4454 const CallbackTN *callback = handle->commit_callback;
198
1/2
✓ Branch 0 taken 4454 times.
✗ Branch 1 not taken.
 4454 delete local_handle;
199
200
2/4
✓ Branch 1 taken 4454 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 4454 times.
✗ Branch 5 not taken.
 4454 Respond(callback, UploaderResults(UploaderResults::kChunkCommit, 0));
201
1/2
✓ Branch 1 taken 4454 times.
✗ Branch 2 not taken.
 4454 }
202
203 /**
204 * TODO(jblomer): investigate if parallelism increases the GC speed on local
205 * disks.
206 */
207 22 void LocalUploader::DoRemoveAsync(const std::string &file_to_delete) {
208
1/2
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
 22 const int retval = unlink((upstream_path_ + "/" + file_to_delete).c_str());
209
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 22 if ((retval != 0) && (errno != ENOENT))
210 ✗ atomic_inc32(&copy_errors_);
211
1/2
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
 22 Respond(NULL, UploaderResults());
212 22 }
213
214 4564 bool LocalUploader::Peek(const std::string &path) {
215
2/4
✓ Branch 2 taken 4564 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 4564 times.
✗ Branch 6 not taken.
 4564 const bool retval = FileExists(upstream_path_ + "/" + path);
216 4564 return retval;
217 }
218
219 ✗ bool LocalUploader::Mkdir(const std::string &path) {
220 ✗ return MkdirDeep(upstream_path_ + "/" + path, backend_dir_mode_, false);
221 }
222
223 22 bool LocalUploader::PlaceBootstrappingShortcut(const shash::Any &object) {
224
2/4
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 22 times.
✗ Branch 5 not taken.
 22 const std::string src = "data/" + object.MakePath();
225
3/6
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 22 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 22 times.
✗ Branch 8 not taken.
 44 const std::string dest = upstream_path_ + "/" + object.MakeAlternativePath();
226
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 44 return SymlinkForced(src, dest);
227 22 }
228
229 16597 int LocalUploader::Move(const std::string &local_path,
230 const std::string &remote_path) const {
231
2/4
✓ Branch 1 taken 16597 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 16597 times.
✗ Branch 5 not taken.
 16597 const std::string destination_path = upstream_path_ + "/" + remote_path;
232
233 // make sure the file has the right permissions
234 16597 int retval = chmod(local_path.c_str(), backend_file_mode_);
235
1/2
✓ Branch 0 taken 16597 times.
✗ Branch 1 not taken.
 16597 int retcode = (retval == 0) ? 0 : 101;
236
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 16597 times.
 16597 if (retcode != 0) {
237 ✗ LogCvmfs(kLogSpooler, kLogVerboseMsg,
238 "failed to set file permission '%s' "
239 "errno: %d",
240 ✗ local_path.c_str(), errno);
241 ✗ return retcode;
242 }
243
244 // move the file in place
245 16597 retval = rename(local_path.c_str(), destination_path.c_str());
246
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 16597 times.
 16597 retcode = (retval == 0) ? 0 : errno;
247
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 16597 times.
 16597 if (retcode != 0) {
248 ✗ LogCvmfs(kLogSpooler, kLogVerboseMsg,
249 "failed to move file '%s' to '%s' "
250 "errno: %d",
251 ✗ local_path.c_str(), remote_path.c_str(), errno);
252 }
253
254 16597 return retcode;
255 16597 }
256
257 ✗ int64_t LocalUploader::DoGetObjectSize(const std::string &file_name) {
258 ✗ return GetFileSize(upstream_path_ + "/" + file_name);
259 }
260
261 } // namespace upload
262