GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/cache_ram.cc
Date: 2024-04-28 02:33:07
Exec Total Coverage
Lines: 193 232 83.2%
Branches: 100 208 48.1%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 */
4 #include "cvmfs_config.h"
5 #include "cache_ram.h"
6
7 #include <errno.h>
8 #include <algorithm>
9 #include <cassert>
10 #include <cstring>
11 #include <new>
12
13 #include "kvstore.h"
14 #include "util/concurrency.h"
15 #include "util/logging.h"
16 #include "util/posix.h"
17 #include "util/string.h"
18
19 using namespace std; // NOLINT
20
21 const shash::Any RamCacheManager::kInvalidHandle;
22
23 ✗ string RamCacheManager::Describe() {
24 ✗ return "Internal in-memory cache manager (size " +
25 ✗ StringifyInt(max_size_ / (1024 * 1024)) + "MB)\n";
26 }
27
28
29 29 RamCacheManager::RamCacheManager(
30 uint64_t max_size,
31 unsigned max_entries,
32 MemoryKvStore::MemoryAllocator alloc,
33 29 perf::StatisticsTemplate statistics)
34 29 : max_size_(max_size)
35
1/2
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
 29 , fd_table_(max_entries, ReadOnlyHandle())
36 // TODO(jblomer): the number of slots in the kv-stores should _not_ be the
37 // number of open files.
38
1/2
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
 29 , regular_entries_(max_entries,
39 alloc,
40 max_size,
41
2/4
✓ Branch 2 taken 29 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 29 times.
✗ Branch 6 not taken.
 58 perf::StatisticsTemplate("kv.regular", statistics))
42
1/2
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
 29 , volatile_entries_(max_entries,
43 alloc,
44 max_size,
45
2/4
✓ Branch 2 taken 29 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 29 times.
✗ Branch 6 not taken.
 58 perf::StatisticsTemplate("kv.volatile", statistics))
46
2/4
✓ Branch 3 taken 29 times.
✗ Branch 4 not taken.
✓ Branch 6 taken 29 times.
✗ Branch 7 not taken.
 58 , counters_(statistics)
47 {
48 29 int retval = pthread_rwlock_init(&rwlock_, NULL);
49
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 29 times.
 29 assert(retval == 0);
50
1/2
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
 29 LogCvmfs(kLogCache, kLogDebug, "max %lu B, %u entries",
51 max_size, max_entries);
52
1/2
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
 29 LogCvmfs(kLogCache, kLogDebug | kLogSyslogWarn,
53 "DEPRECATION WARNING: The RAM cache manager is depcreated and "
54 "will be removed from future releases.");
55 29 }
56
57
58 92 RamCacheManager::~RamCacheManager() {
59 58 pthread_rwlock_destroy(&rwlock_);
60 92 }
61
62
63 11043 int RamCacheManager::AddFd(const ReadOnlyHandle &handle) {
64 11043 int result = fd_table_.OpenFd(handle);
65
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 11042 times.
 11043 if (result == -ENFILE) {
66 1 LogCvmfs(kLogCache, kLogDebug, "too many open files");
67 1 perf::Inc(counters_.n_enfile);
68 }
69 11043 return result;
70 }
71
72
73 9 bool RamCacheManager::AcquireQuotaManager(QuotaManager *quota_mgr) {
74
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
 9 assert(quota_mgr != NULL);
75 9 quota_mgr_ = quota_mgr;
76 9 LogCvmfs(kLogCache, kLogDebug, "set quota manager");
77 9 return true;
78 }
79
80
81 23 int RamCacheManager::Open(const LabeledObject &object) {
82 23 WriteLockGuard guard(rwlock_);
83
1/2
✓ Branch 1 taken 23 times.
✗ Branch 2 not taken.
 46 return DoOpen(object.id);
84 23 }
85
86
87 32 int RamCacheManager::DoOpen(const shash::Any &id) {
88 bool ok;
89 bool is_volatile;
90
1/2
✓ Branch 1 taken 32 times.
✗ Branch 2 not taken.
 32 MemoryBuffer buf;
91
92
3/4
✓ Branch 1 taken 32 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 16 times.
✓ Branch 4 taken 16 times.
 32 if (regular_entries_.Contains(id)) {
93 16 is_volatile = false;
94
3/4
✓ Branch 1 taken 16 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2 times.
✓ Branch 4 taken 14 times.
 16 } else if (volatile_entries_.Contains(id)) {
95 2 is_volatile = true;
96 } else {
97
1/2
✓ Branch 2 taken 14 times.
✗ Branch 3 not taken.
 14 LogCvmfs(kLogCache, kLogDebug, "miss for %s",
98
1/2
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
 28 id.ToString().c_str());
99 14 perf::Inc(counters_.n_openmiss);
100 14 return -ENOENT;
101 }
102 18 ReadOnlyHandle generic_handle(id, is_volatile);
103
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 18 int fd = AddFd(generic_handle);
104
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 18 times.
 18 if (fd < 0) {
105 ✗ LogCvmfs(kLogCache, kLogDebug, "error while opening %s: %s",
106 ✗ id.ToString().c_str(), strerror(-fd));
107 ✗ return fd;
108 }
109
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 16 times.
 18 if (is_volatile) {
110
1/2
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 LogCvmfs(kLogCache, kLogDebug, "hit in volatile entries for %s",
111
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 4 id.ToString().c_str());
112 2 perf::Inc(counters_.n_openvolatile);
113 } else {
114
1/2
✓ Branch 2 taken 16 times.
✗ Branch 3 not taken.
 16 LogCvmfs(kLogCache, kLogDebug, "hit in regular entries for %s",
115
1/2
✓ Branch 1 taken 16 times.
✗ Branch 2 not taken.
 32 id.ToString().c_str());
116 16 perf::Inc(counters_.n_openregular);
117 }
118
1/2
✓ Branch 2 taken 18 times.
✗ Branch 3 not taken.
 18 ok = GetStore(generic_handle)->IncRef(id);
119
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 18 times.
 18 assert(ok);
120 18 return fd;
121 }
122
123
124 6 int64_t RamCacheManager::GetSize(int fd) {
125 6 ReadLockGuard guard(rwlock_);
126
1/2
✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
 6 ReadOnlyHandle generic_handle = fd_table_.GetHandle(fd);
127
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 6 times.
 6 if (generic_handle.handle == kInvalidHandle) {
128 ✗ LogCvmfs(kLogCache, kLogDebug, "bad fd %d on GetSize", fd);
129 ✗ return -EBADF;
130 }
131 6 perf::Inc(counters_.n_getsize);
132
1/2
✓ Branch 2 taken 6 times.
✗ Branch 3 not taken.
 6 return GetStore(generic_handle)->GetSize(generic_handle.handle);
133 6 }
134
135
136 11036 int RamCacheManager::Close(int fd) {
137 bool rc;
138
139 11036 WriteLockGuard guard(rwlock_);
140
1/2
✓ Branch 1 taken 11036 times.
✗ Branch 2 not taken.
 11036 ReadOnlyHandle generic_handle = fd_table_.GetHandle(fd);
141
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 11036 times.
 11036 if (generic_handle.handle == kInvalidHandle) {
142 ✗ LogCvmfs(kLogCache, kLogDebug, "bad fd %d on Close", fd);
143 ✗ return -EBADF;
144 }
145
1/2
✓ Branch 2 taken 11036 times.
✗ Branch 3 not taken.
 11036 rc = GetStore(generic_handle)->Unref(generic_handle.handle);
146
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11036 times.
 11036 assert(rc);
147
148
1/2
✓ Branch 1 taken 11036 times.
✗ Branch 2 not taken.
 11036 int rc_int = fd_table_.CloseFd(fd);
149
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11036 times.
 11036 assert(rc_int == 0);
150
1/2
✓ Branch 1 taken 11036 times.
✗ Branch 2 not taken.
 11036 LogCvmfs(kLogCache, kLogDebug, "closed fd %d", fd);
151 11036 perf::Inc(counters_.n_close);
152 11036 return 0;
153 11036 }
154
155
156 4 int64_t RamCacheManager::Pread(
157 int fd,
158 void *buf,
159 uint64_t size,
160 uint64_t offset)
161 {
162 4 ReadLockGuard guard(rwlock_);
163
1/2
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
 4 ReadOnlyHandle generic_handle = fd_table_.GetHandle(fd);
164
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4 times.
 4 if (generic_handle.handle == kInvalidHandle) {
165 ✗ LogCvmfs(kLogCache, kLogDebug, "bad fd %d on Pread", fd);
166 ✗ return -EBADF;
167 }
168 4 perf::Inc(counters_.n_pread);
169
1/2
✓ Branch 2 taken 4 times.
✗ Branch 3 not taken.
 4 return GetStore(generic_handle)->Read(
170 4 generic_handle.handle, buf, size, offset);
171 4 }
172
173
174 11026 int RamCacheManager::Dup(int fd) {
175 bool ok;
176 int rc;
177 11026 WriteLockGuard guard(rwlock_);
178
1/2
✓ Branch 1 taken 11026 times.
✗ Branch 2 not taken.
 11026 ReadOnlyHandle generic_handle = fd_table_.GetHandle(fd);
179
2/2
✓ Branch 1 taken 1 times.
✓ Branch 2 taken 11025 times.
 11026 if (generic_handle.handle == kInvalidHandle) {
180
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 LogCvmfs(kLogCache, kLogDebug, "bad fd %d on Dup", fd);
181 1 return -EBADF;
182 }
183
1/2
✓ Branch 1 taken 11025 times.
✗ Branch 2 not taken.
 11025 rc = AddFd(generic_handle);
184
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 11024 times.
 11025 if (rc < 0) return rc;
185
1/2
✓ Branch 2 taken 11024 times.
✗ Branch 3 not taken.
 11024 ok = GetStore(generic_handle)->IncRef(generic_handle.handle);
186
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11024 times.
 11024 assert(ok);
187
1/2
✓ Branch 1 taken 11024 times.
✗ Branch 2 not taken.
 11024 LogCvmfs(kLogCache, kLogDebug, "dup fd %d", fd);
188 11024 perf::Inc(counters_.n_dup);
189 11024 return rc;
190 11026 }
191
192
193 /**
194 * For a RAM cache, read-ahead is a no-op.
195 */
196 ✗ int RamCacheManager::Readahead(int fd) {
197 ✗ ReadLockGuard guard(rwlock_);
198 ✗ ReadOnlyHandle generic_handle = fd_table_.GetHandle(fd);
199 ✗ if (generic_handle.handle == kInvalidHandle) {
200 ✗ LogCvmfs(kLogCache, kLogDebug, "bad fd %d on Readahead", fd);
201 ✗ return -EBADF;
202 }
203 ✗ LogCvmfs(kLogCache, kLogDebug, "readahead (no-op) on %d", fd);
204 ✗ perf::Inc(counters_.n_readahead);
205 ✗ return 0;
206 }
207
208
209 37 int RamCacheManager::StartTxn(const shash::Any &id, uint64_t size, void *txn) {
210
1/2
✓ Branch 2 taken 37 times.
✗ Branch 3 not taken.
 37 LogCvmfs(kLogCache, kLogDebug, "new transaction with id %s",
211 74 id.ToString().c_str());
212 37 Transaction *transaction = new (txn) Transaction();
213 37 transaction->buffer.id = id;
214 37 transaction->pos = 0;
215 37 transaction->expected_size = size;
216
1/2
✓ Branch 0 taken 37 times.
✗ Branch 1 not taken.
 37 transaction->buffer.size = (size == kSizeUnknown) ? kPageSize : size;
217 37 transaction->buffer.address = malloc(transaction->buffer.size);
218
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 37 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 37 if (!transaction->buffer.address && size > 0) {
219 ✗ LogCvmfs(kLogCache, kLogDebug,
220 "failed to allocate %lu B for %s",
221 ✗ size, id.ToString().c_str());
222 ✗ return -errno;
223 }
224 37 perf::Inc(counters_.n_starttxn);
225 37 return 0;
226 }
227
228
229 8 void RamCacheManager::CtrlTxn(const Label &label, const int /* flags */,
230 void *txn)
231 {
232 8 Transaction *transaction = reinterpret_cast<Transaction *>(txn);
233 8 transaction->description = label.GetDescription();
234 8 transaction->buffer.object_flags = label.flags;
235
1/2
✓ Branch 2 taken 8 times.
✗ Branch 3 not taken.
 8 LogCvmfs(kLogCache, kLogDebug, "modified transaction %s",
236 16 transaction->buffer.id.ToString().c_str());
237 8 }
238
239
240 38 int64_t RamCacheManager::Write(const void *buf, uint64_t size, void *txn) {
241 38 Transaction *transaction = reinterpret_cast<Transaction *>(txn);
242
243
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 38 times.
 38 assert(transaction->pos <= transaction->buffer.size);
244
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 37 times.
 38 if (transaction->pos + size > transaction->buffer.size) {
245
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 times.
 1 if (transaction->expected_size == kSizeUnknown) {
246 ✗ perf::Inc(counters_.n_realloc);
247 ✗ size_t new_size = max(2*transaction->buffer.size,
248 ✗ static_cast<size_t>(size + transaction->pos));
249 ✗ LogCvmfs(kLogCache, kLogDebug, "reallocate transaction for %s to %lu B",
250 ✗ transaction->buffer.id.ToString().c_str(),
251 transaction->buffer.size);
252 ✗ void *new_ptr = realloc(transaction->buffer.address, new_size);
253 ✗ if (!new_ptr) {
254 ✗ LogCvmfs(kLogCache, kLogDebug,
255 "failed to allocate %lu B for %s",
256 ✗ new_size, transaction->buffer.id.ToString().c_str());
257 ✗ return -EIO;
258 }
259 ✗ transaction->buffer.address = new_ptr;
260 ✗ transaction->buffer.size = new_size;
261 } else {
262 1 LogCvmfs(kLogCache, kLogDebug,
263 "attempted to write more than requested (%lu>%zu)",
264 size, transaction->buffer.size);
265 1 return -EFBIG;
266 }
267 }
268
269
2/4
✓ Branch 0 taken 37 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 37 times.
✗ Branch 3 not taken.
 37 if (transaction->buffer.address && buf) {
270 // LogCvmfs(kLogCache, kLogDebug, "copy %u bytes of transaction %s",
271 // size, transaction->id.ToString().c_str());
272 37 memcpy(static_cast<char *>(transaction->buffer.address) + transaction->pos,
273 buf, size);
274 }
275 37 transaction->pos += size;
276 37 perf::Inc(counters_.n_write);
277 37 return size;
278 }
279
280
281 2 int RamCacheManager::Reset(void *txn) {
282 2 Transaction *transaction = reinterpret_cast<Transaction *>(txn);
283 2 transaction->pos = 0;
284
1/2
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 LogCvmfs(kLogCache, kLogDebug, "reset transaction %s",
285 4 transaction->buffer.id.ToString().c_str());
286 2 perf::Inc(counters_.n_reset);
287 2 return 0;
288 }
289
290
291 10 int RamCacheManager::OpenFromTxn(void *txn) {
292 10 WriteLockGuard guard(rwlock_);
293 10 Transaction *transaction = reinterpret_cast<Transaction *>(txn);
294
1/2
✓ Branch 1 taken 10 times.
✗ Branch 2 not taken.
 10 int64_t retval = CommitToKvStore(transaction);
295
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 9 times.
 10 if (retval < 0) {
296
1/2
✓ Branch 3 taken 1 times.
✗ Branch 4 not taken.
 2 LogCvmfs(kLogCache, kLogDebug,
297 "error while committing transaction on %s: %s",
298
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 2 transaction->buffer.id.ToString().c_str(), strerror(-retval));
299 1 return retval;
300 }
301
1/2
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
 9 LogCvmfs(kLogCache, kLogDebug, "open pending transaction for %s",
302
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 18 transaction->buffer.id.ToString().c_str());
303 9 perf::Inc(counters_.n_committxn);
304
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 return DoOpen(transaction->buffer.id);
305 10 }
306
307
308 2 int RamCacheManager::AbortTxn(void *txn) {
309 2 Transaction *transaction = reinterpret_cast<Transaction *>(txn);
310 2 free(transaction->buffer.address);
311
1/2
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 LogCvmfs(kLogCache, kLogDebug, "abort transaction %s",
312 4 transaction->buffer.id.ToString().c_str());
313 2 perf::Inc(counters_.n_aborttxn);
314 2 return 0;
315 }
316
317
318 29 int RamCacheManager::CommitTxn(void *txn) {
319 29 WriteLockGuard guard(rwlock_);
320 29 Transaction *transaction = reinterpret_cast<Transaction *>(txn);
321 29 perf::Inc(counters_.n_committxn);
322
1/2
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
 29 int64_t rc = CommitToKvStore(transaction);
323
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 28 times.
 29 if (rc < 0) return rc;
324 28 free(transaction->buffer.address);
325 28 return rc;
326 29 }
327
328
329 39 int64_t RamCacheManager::CommitToKvStore(Transaction *transaction) {
330 MemoryKvStore *store;
331
332
2/2
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 36 times.
 39 if (transaction->buffer.object_flags & CacheManager::kLabelVolatile)
333 {
334 3 store = &volatile_entries_;
335 } else {
336 36 store = &regular_entries_;
337 }
338
2/2
✓ Branch 0 taken 38 times.
✓ Branch 1 taken 1 times.
 39 if ((transaction->buffer.object_flags & CacheManager::kLabelPinned) ||
339
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 38 times.
 38 (transaction->buffer.object_flags & CacheManager::kLabelCatalog))
340 {
341 1 transaction->buffer.refcount = 1;
342 } else {
343 38 transaction->buffer.refcount = 0;
344 }
345
346 39 int64_t regular_size = regular_entries_.GetUsed();
347 39 int64_t volatile_size = volatile_entries_.GetUsed();
348 39 int64_t overrun = regular_size + volatile_size +
349 39 transaction->buffer.size - max_size_;
350
351
2/2
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 33 times.
 39 if (overrun > 0) {
352 // if we're going to clean the cache, try to remove at least 25%
353 6 overrun = max(overrun, (int64_t) max_size_>>2);
354 6 perf::Inc(counters_.n_overrun);
355
1/2
✓ Branch 2 taken 6 times.
✗ Branch 3 not taken.
 6 volatile_entries_.ShrinkTo(max((int64_t) 0, volatile_size - overrun));
356 }
357 39 overrun -= volatile_size - volatile_entries_.GetUsed();
358
2/2
✓ Branch 0 taken 5 times.
✓ Branch 1 taken 34 times.
 39 if (overrun > 0) {
359
1/2
✓ Branch 2 taken 5 times.
✗ Branch 3 not taken.
 5 regular_entries_.ShrinkTo(max((int64_t) 0, regular_size - overrun));
360 }
361 39 overrun -= regular_size -regular_entries_.GetUsed();
362
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 37 times.
 39 if (overrun > 0) {
363
1/2
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 LogCvmfs(kLogCache, kLogDebug,
364 "transaction for %s would overrun the cache limit by %ld",
365
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 4 transaction->buffer.id.ToString().c_str(), overrun);
366 2 perf::Inc(counters_.n_full);
367 2 return -ENOSPC;
368 }
369
370
1/2
✓ Branch 1 taken 37 times.
✗ Branch 2 not taken.
 37 int rc = store->Commit(transaction->buffer);
371
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 37 times.
 37 if (rc < 0) {
372 ✗ LogCvmfs(kLogCache, kLogDebug,
373 "commit on %s failed",
374 ✗ transaction->buffer.id.ToString().c_str());
375 ✗ return rc;
376 }
377
1/2
✓ Branch 2 taken 37 times.
✗ Branch 3 not taken.
 37 LogCvmfs(kLogCache, kLogDebug, "committed %s to cache",
378
1/2
✓ Branch 1 taken 37 times.
✗ Branch 2 not taken.
 74 transaction->buffer.id.ToString().c_str());
379 37 return 0;
380 }
381