GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/quota_posix.cc
Date: 2026-05-19 11:45:12
Exec Total Coverage
Lines: 985 1419 69.4%
Branches: 769 1966 39.1%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * This module implements a "managed local cache".
5 * This way, we are able to track access times of files in the cache
6 * and remove files based on least recently used strategy.
7 *
8 * We setup another SQLite catalog, a "cache catalog", that helps us
9 * in the bookkeeping of files, file sizes and access times.
10 *
11 * We might choose to not manage the local cache. This is indicated
12 * by limit == 0 and everything succeeds in that case.
13 */
14
15 #define __STDC_LIMIT_MACROS
16
17
18 #include "quota_posix.h"
19
20 #include <dirent.h>
21 #include <errno.h>
22 #include <fcntl.h>
23 #include <inttypes.h>
24 #include <pthread.h>
25 #include <signal.h>
26 #include <stdint.h>
27 #include <sys/dir.h>
28 #include <sys/stat.h>
29 #include <sys/xattr.h>
30
31 #ifndef __APPLE__
32 #include <sys/statfs.h>
33 #endif
34 #include <sys/statvfs.h>
35 #include <sys/types.h>
36 #include <sys/wait.h>
37 #include <unistd.h>
38
39 #include <algorithm>
40 #include <cassert>
41 #include <cstdio>
42 #include <cstdlib>
43 #include <cstring>
44 #include <limits>
45 #include <map>
46 #include <set>
47 #include <string>
48 #include <vector>
49
50 #include "crypto/hash.h"
51 #include "duplex_sqlite3.h"
52 #include "monitor.h"
53 #include "statistics.h"
54 #include "util/capabilities.h"
55 #include "util/concurrency.h"
56 #include "util/exception.h"
57 #include "util/logging.h"
58 #include "util/pointer.h"
59 #include "util/posix.h"
60 #include "util/smalloc.h"
61 #include "util/string.h"
62
63 using namespace std; // NOLINT
64
65
66 3760 int PosixQuotaManager::BindReturnPipe(int pipe_wronly) {
67
2/2
✓ Branch 0 taken 3680 times.
✓ Branch 1 taken 80 times.
 3760 if (!shared_)
68 3680 return pipe_wronly;
69
70 // Connect writer's end
71
1/2
✓ Branch 2 taken 80 times.
✗ Branch 3 not taken.
 80 const int result = open(
72
2/4
✓ Branch 2 taken 80 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 80 times.
✗ Branch 6 not taken.
 160 (workspace_dir_ + "/pipe" + StringifyInt(pipe_wronly)).c_str(),
73 O_WRONLY | O_NONBLOCK);
74
2/2
✓ Branch 0 taken 40 times.
✓ Branch 1 taken 40 times.
 80 if (result >= 0) {
75 40 Nonblock2Block(result);
76 } else {
77 40 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
78 40 "failed to bind return pipe (%d)", errno);
79 }
80 80 return result;
81 }
82
83
84 776 void PosixQuotaManager::CheckHighPinWatermark() {
85 776 const uint64_t watermark = kHighPinWatermark * cleanup_threshold_ / 100;
86
3/4
✓ Branch 0 taken 776 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 120 times.
✓ Branch 3 taken 656 times.
 776 if ((cleanup_threshold_ > 0) && (pinned_ > watermark)) {
87 120 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
88 "high watermark of pinned files (%" PRIu64 "M > %" PRIu64 "M)",
89 120 pinned_ / (1024 * 1024), watermark / (1024 * 1024));
90
2/4
✓ Branch 2 taken 120 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 120 times.
✗ Branch 6 not taken.
 120 BroadcastBackchannels("R"); // clients: please release pinned catalogs
91 }
92 776 }
93
94
95 ✗ void PosixQuotaManager::CleanupPipes() {
96 ✗ DIR *dirp = opendir(workspace_dir_.c_str());
97 ✗ assert(dirp != NULL);
98
99 platform_dirent64 *dent;
100 ✗ bool found_leftovers = false;
101 ✗ while ((dent = platform_readdir(dirp)) != NULL) {
102 ✗ const string name = dent->d_name;
103 ✗ const string path = workspace_dir_ + "/" + name;
104 platform_stat64 info;
105 ✗ const int retval = platform_stat(path.c_str(), &info);
106 ✗ if (retval != 0)
107 ✗ continue;
108 ✗ if (S_ISFIFO(info.st_mode) && (name.substr(0, 4) == "pipe")) {
109 ✗ if (!found_leftovers) {
110 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogWarn,
111 "removing left-over FIFOs from cache directory");
112 }
113 ✗ found_leftovers = true;
114 ✗ unlink(path.c_str());
115 }
116 }
117 ✗ closedir(dirp);
118 }
119
120
121 /**
122 * Cleans up in data cache, until cache size is below leave_size.
123 * The actual unlinking is done in a separate process (fork).
124 *
125 * \return True on success, false otherwise
126 */
127 360 bool PosixQuotaManager::Cleanup(const uint64_t leave_size) {
128
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 360 times.
 360 if (!spawned_)
129 ✗ return DoCleanup(leave_size);
130
131 bool result;
132 int pipe_cleanup[2];
133
1/2
✓ Branch 1 taken 360 times.
✗ Branch 2 not taken.
 360 MakeReturnPipe(pipe_cleanup);
134
135 360 LruCommand cmd;
136 360 cmd.command_type = kCleanup;
137 360 cmd.size = leave_size;
138 360 cmd.return_pipe = pipe_cleanup[1];
139
140
1/2
✓ Branch 1 taken 360 times.
✗ Branch 2 not taken.
 360 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
141
1/2
✓ Branch 1 taken 360 times.
✗ Branch 2 not taken.
 360 ManagedReadHalfPipe(pipe_cleanup[0], &result, sizeof(result));
142
1/2
✓ Branch 1 taken 360 times.
✗ Branch 2 not taken.
 360 CloseReturnPipe(pipe_cleanup);
143
144 360 return result;
145 }
146
147
148 2757 void PosixQuotaManager::CloseDatabase() {
149
1/2
✓ Branch 0 taken 2757 times.
✗ Branch 1 not taken.
 2757 if (stmt_list_catalogs_)
150 2757 sqlite3_finalize(stmt_list_catalogs_);
151
1/2
✓ Branch 0 taken 2757 times.
✗ Branch 1 not taken.
 2757 if (stmt_list_pinned_)
152 2757 sqlite3_finalize(stmt_list_pinned_);
153
1/2
✓ Branch 0 taken 2757 times.
✗ Branch 1 not taken.
 2757 if (stmt_list_volatile_)
154 2757 sqlite3_finalize(stmt_list_volatile_);
155
1/2
✓ Branch 0 taken 2757 times.
✗ Branch 1 not taken.
 2757 if (stmt_list_)
156 2757 sqlite3_finalize(stmt_list_);
157
1/2
✓ Branch 0 taken 2757 times.
✗ Branch 1 not taken.
 2757 if (stmt_lru_)
158 2757 sqlite3_finalize(stmt_lru_);
159
1/2
✓ Branch 0 taken 2757 times.
✗ Branch 1 not taken.
 2757 if (stmt_rm_)
160 2757 sqlite3_finalize(stmt_rm_);
161
1/2
✓ Branch 0 taken 2757 times.
✗ Branch 1 not taken.
 2757 if (stmt_rm_batch_)
162 2757 sqlite3_finalize(stmt_rm_batch_);
163
1/2
✓ Branch 0 taken 2757 times.
✗ Branch 1 not taken.
 2757 if (stmt_size_)
164 2757 sqlite3_finalize(stmt_size_);
165
1/2
✓ Branch 0 taken 2757 times.
✗ Branch 1 not taken.
 2757 if (stmt_touch_)
166 2757 sqlite3_finalize(stmt_touch_);
167
1/2
✓ Branch 0 taken 2757 times.
✗ Branch 1 not taken.
 2757 if (stmt_unpin_)
168 2757 sqlite3_finalize(stmt_unpin_);
169
1/2
✓ Branch 0 taken 2757 times.
✗ Branch 1 not taken.
 2757 if (stmt_block_)
170 2757 sqlite3_finalize(stmt_block_);
171
1/2
✓ Branch 0 taken 2757 times.
✗ Branch 1 not taken.
 2757 if (stmt_unblock_)
172 2757 sqlite3_finalize(stmt_unblock_);
173
1/2
✓ Branch 0 taken 2757 times.
✗ Branch 1 not taken.
 2757 if (stmt_new_)
174 2757 sqlite3_finalize(stmt_new_);
175
1/2
✓ Branch 0 taken 2757 times.
✗ Branch 1 not taken.
 2757 if (database_)
176 2757 sqlite3_close(database_);
177 2757 UnlockFile(fd_lock_cachedb_);
178
179 2757 stmt_list_catalogs_ = NULL;
180 2757 stmt_list_pinned_ = NULL;
181 2757 stmt_list_volatile_ = NULL;
182 2757 stmt_list_ = NULL;
183 2757 stmt_rm_ = NULL;
184 2757 stmt_rm_batch_ = NULL;
185 2757 stmt_size_ = NULL;
186 2757 stmt_touch_ = NULL;
187 2757 stmt_unpin_ = NULL;
188 2757 stmt_block_ = NULL;
189 2757 stmt_unblock_ = NULL;
190 2757 stmt_new_ = NULL;
191 2757 database_ = NULL;
192
193 2757 pinned_chunks_.clear();
194 2757 }
195
196
197 3560 void PosixQuotaManager::CloseReturnPipe(int pipe[2]) {
198
2/2
✓ Branch 0 taken 80 times.
✓ Branch 1 taken 3480 times.
 3560 if (shared_) {
199 80 close(pipe[0]);
200 80 UnlinkReturnPipe(pipe[1]);
201 } else {
202 3480 ClosePipe(pipe);
203 }
204 3560 }
205
206
207 4002178 bool PosixQuotaManager::Contains(const string &hash_str) {
208 4002178 bool result = false;
209
210 4002178 sqlite3_bind_text(stmt_size_, 1, &hash_str[0], hash_str.length(),
211 SQLITE_STATIC);
212
2/2
✓ Branch 1 taken 562 times.
✓ Branch 2 taken 4001616 times.
 4002178 if (sqlite3_step(stmt_size_) == SQLITE_ROW)
213 562 result = true;
214 4002178 sqlite3_reset(stmt_size_);
215 4002178 LogCvmfs(kLogQuota, kLogDebug, "contains %s returns %d", hash_str.c_str(),
216 result);
217
218 4002178 return result;
219 }
220
221
222 2678 void PosixQuotaManager::CheckFreeSpace() {
223
3/4
✓ Branch 0 taken 2678 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 40 times.
✓ Branch 3 taken 2638 times.
 2678 if ((limit_ == 0) || (gauge_ >= limit_))
224 40 return;
225
226 struct statvfs vfs_info;
227
1/2
✓ Branch 1 taken 2638 times.
✗ Branch 2 not taken.
 2638 const int retval = statvfs((cache_dir_ + "/cachedb").c_str(), &vfs_info);
228
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2638 times.
 2638 if (retval != 0) {
229 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
230 "failed to query %s for free space (%d)", cache_dir_.c_str(),
231 ✗ errno);
232 ✗ return;
233 }
234 2638 const int64_t free_space_byte = vfs_info.f_bavail * vfs_info.f_bsize;
235
1/2
✓ Branch 1 taken 2638 times.
✗ Branch 2 not taken.
 2638 LogCvmfs(kLogQuota, kLogDebug, "free space: %" PRId64 " MB",
236 free_space_byte / (1024 * 1024));
237
238 2638 const int64_t required_byte = limit_ - gauge_;
239
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2638 times.
 2638 if (free_space_byte < required_byte) {
240 ✗ LogCvmfs(kLogQuota, kLogSyslogWarn,
241 "too little free space on the file system hosting the cache,"
242 " %" PRId64 " MB available",
243 free_space_byte / (1024 * 1024));
244 }
245 }
246
247
248 2798 PosixQuotaManager *PosixQuotaManager::Create(const string &cache_workspace,
249 const uint64_t limit,
250 const uint64_t cleanup_threshold,
251 const bool rebuild_database) {
252
2/2
✓ Branch 0 taken 80 times.
✓ Branch 1 taken 2718 times.
 2798 if (cleanup_threshold >= limit) {
253 80 LogCvmfs(kLogQuota, kLogDebug,
254 "invalid parameters: limit %" PRIu64 ", "
255 "cleanup_threshold %" PRIu64,
256 limit, cleanup_threshold);
257 80 return NULL;
258 }
259
260 PosixQuotaManager *quota_manager = new PosixQuotaManager(
261
1/2
✓ Branch 2 taken 2718 times.
✗ Branch 3 not taken.
 2718 limit, cleanup_threshold, cache_workspace);
262
263 // Initialize cache catalog
264
2/2
✓ Branch 1 taken 40 times.
✓ Branch 2 taken 2678 times.
 2718 if (!quota_manager->InitDatabase(rebuild_database)) {
265
1/2
✓ Branch 0 taken 40 times.
✗ Branch 1 not taken.
 40 delete quota_manager;
266 40 return NULL;
267 }
268 2678 quota_manager->CheckFreeSpace();
269 2678 MakePipe(quota_manager->pipe_lru_);
270
271 2678 quota_manager->protocol_revision_ = kProtocolRevision;
272 2678 quota_manager->initialized_ = true;
273 2678 return quota_manager;
274 }
275
276
277 /**
278 * Connects to a running shared local quota manager. Creates one if necessary.
279 */
280 80 PosixQuotaManager *PosixQuotaManager::CreateShared(
281 const std::string &exe_path,
282 const std::string &cache_workspace,
283 const uint64_t limit,
284 const uint64_t cleanup_threshold,
285 bool foreground) {
286 80 string cache_dir;
287 80 string workspace_dir;
288
2/4
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 80 times.
✗ Branch 5 not taken.
 80 ParseDirectories(cache_workspace, &cache_dir, &workspace_dir);
289
290 pid_t new_cachemgr_pid;
291
292 // Create lock file: only one fuse client at a time
293
2/4
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 80 times.
✗ Branch 5 not taken.
 80 const int fd_lockfile = LockFile(workspace_dir + "/lock_cachemgr");
294
2/2
✓ Branch 0 taken 40 times.
✓ Branch 1 taken 40 times.
 80 if (fd_lockfile < 0) {
295
1/2
✓ Branch 2 taken 40 times.
✗ Branch 3 not taken.
 40 LogCvmfs(kLogQuota, kLogDebug, "could not open lock file %s (%d)",
296
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 80 (workspace_dir + "/lock_cachemgr").c_str(), errno);
297 40 return NULL;
298 }
299
300 PosixQuotaManager *quota_mgr = new PosixQuotaManager(limit, cleanup_threshold,
301
2/4
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 40 times.
✗ Branch 5 not taken.
 40 cache_workspace);
302 40 quota_mgr->shared_ = true;
303 40 quota_mgr->spawned_ = true;
304
305 // Try to connect to pipe
306
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 const string fifo_path = workspace_dir + "/cachemgr";
307
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 LogCvmfs(kLogQuota, kLogDebug, "trying to connect to existing pipe");
308
1/2
✓ Branch 2 taken 40 times.
✗ Branch 3 not taken.
 40 quota_mgr->pipe_lru_[1] = open(fifo_path.c_str(), O_WRONLY | O_NONBLOCK);
309
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 40 times.
 40 if (quota_mgr->pipe_lru_[1] >= 0) {
310 ✗ const int fd_lockfile_rw = open((workspace_dir + "/lock_cachemgr").c_str(),
311 O_RDWR, 0600);
312 ✗ unsigned lockfile_magicnumber = 0;
313 ✗ const ssize_t result_mn = SafeRead(fd_lockfile_rw, &lockfile_magicnumber,
314 sizeof(lockfile_magicnumber));
315 ✗ const ssize_t result = SafeRead(fd_lockfile_rw, &new_cachemgr_pid,
316 sizeof(new_cachemgr_pid));
317 ✗ close(fd_lockfile_rw);
318
319 ✗ if ((lockfile_magicnumber != kLockFileMagicNumber) || (result < 0)
320 ✗ || (result_mn < 0)
321 ✗ || (static_cast<size_t>(result) < sizeof(new_cachemgr_pid))) {
322 ✗ if (result != 0) {
323 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
324 "could not read cache manager pid from lockfile");
325 ✗ UnlockFile(fd_lockfile);
326 ✗ delete quota_mgr;
327 ✗ return NULL;
328 } else {
329 // support reload from old versions of the cache manager
330 // lock file is empty in this case, try a plain ReadHalfPipe to get pid
331 ✗ quota_mgr->SetCacheMgrPid(quota_mgr->GetPid());
332 }
333 } else {
334 ✗ quota_mgr->SetCacheMgrPid(new_cachemgr_pid);
335 }
336
337
338 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to existing cache manager pipe");
339 ✗ quota_mgr->initialized_ = true;
340 ✗ Nonblock2Block(quota_mgr->pipe_lru_[1]);
341 ✗ UnlockFile(fd_lockfile);
342 ✗ quota_mgr->GetLimits(&quota_mgr->limit_, &quota_mgr->cleanup_threshold_);
343 ✗ LogCvmfs(kLogQuota, kLogDebug,
344 "received limit %" PRIu64 ", threshold %" PRIu64,
345 quota_mgr->limit_, quota_mgr->cleanup_threshold_);
346 ✗ if (FileExists(workspace_dir + "/cachemgr.protocol")) {
347 ✗ quota_mgr->protocol_revision_ = quota_mgr->GetProtocolRevision();
348 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected protocol revision %u",
349 quota_mgr->protocol_revision_);
350 } else {
351 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to ancient cache manager");
352 }
353 ✗ return quota_mgr;
354 }
355 40 const int connect_error = errno;
356
357 // Lock file: let existing cache manager finish first
358
2/4
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 40 times.
✗ Branch 5 not taken.
 40 const int fd_lockfile_fifo = LockFile(workspace_dir + "/lock_cachemgr.fifo");
359
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 40 times.
 40 if (fd_lockfile_fifo < 0) {
360 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not open lock file %s (%d)",
361 ✗ (workspace_dir + "/lock_cachemgr.fifo").c_str(), errno);
362 ✗ UnlockFile(fd_lockfile);
363 ✗ delete quota_mgr;
364 ✗ return NULL;
365 }
366
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 UnlockFile(fd_lockfile_fifo);
367
368
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 40 times.
 40 if (connect_error == ENXIO) {
369 ✗ LogCvmfs(kLogQuota, kLogDebug, "left-over FIFO found, unlinking");
370 ✗ unlink(fifo_path.c_str());
371 }
372
373 // Creating a new FIFO for the cache manager (to be bound later)
374 40 int retval = mkfifo(fifo_path.c_str(), 0600);
375
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 40 times.
 40 if (retval != 0) {
376 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to create cache manager FIFO (%d)",
377 ✗ errno);
378 ✗ UnlockFile(fd_lockfile);
379 ✗ delete quota_mgr;
380 ✗ return NULL;
381 }
382
383 // Create new cache manager
384 int pipe_boot[2];
385 int pipe_handshake[2];
386
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 MakePipe(pipe_boot);
387
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 MakePipe(pipe_handshake);
388
389 40 vector<string> command_line;
390
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 command_line.push_back(exe_path);
391
2/4
✓ Branch 2 taken 40 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 40 times.
✗ Branch 6 not taken.
 40 command_line.push_back("__cachemgr__");
392
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 command_line.push_back(cache_workspace);
393
2/4
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 40 times.
✗ Branch 5 not taken.
 40 command_line.push_back(StringifyInt(pipe_boot[1]));
394
2/4
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 40 times.
✗ Branch 5 not taken.
 40 command_line.push_back(StringifyInt(pipe_handshake[0]));
395
2/4
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 40 times.
✗ Branch 5 not taken.
 40 command_line.push_back(StringifyInt(limit));
396
2/4
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 40 times.
✗ Branch 5 not taken.
 40 command_line.push_back(StringifyInt(cleanup_threshold));
397 // do not propagate foreground in order to reliably get pid from exec
398 // instead, daemonize right here
399
2/4
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 40 times.
✗ Branch 5 not taken.
 40 command_line.push_back(StringifyInt(true)); // foreground
400
3/6
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 40 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 40 times.
✗ Branch 8 not taken.
 40 command_line.push_back(StringifyInt(GetLogSyslogLevel()));
401
3/6
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 40 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 40 times.
✗ Branch 8 not taken.
 40 command_line.push_back(StringifyInt(GetLogSyslogFacility()));
402
5/14
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 40 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 40 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 40 times.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 40 times.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
 40 command_line.push_back(GetLogDebugFile() + ":" + GetLogMicroSyslog());
403
404 40 set<int> preserve_filedes;
405
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 preserve_filedes.insert(0);
406
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 preserve_filedes.insert(1);
407
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 preserve_filedes.insert(2);
408
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 preserve_filedes.insert(pipe_boot[1]);
409
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 preserve_filedes.insert(pipe_handshake[0]);
410
411
1/2
✓ Branch 0 taken 40 times.
✗ Branch 1 not taken.
 40 if (foreground) {
412
1/2
✓ Branch 2 taken 40 times.
✗ Branch 3 not taken.
 40 retval = ManagedExec(command_line, preserve_filedes, map<int, int>(),
413 /*drop_credentials*/ false,
414 /*clear_env*/ false,
415 /*double_fork*/ true, &new_cachemgr_pid);
416 } else {
417 ✗ retval = ExecAsDaemon(command_line, &new_cachemgr_pid);
418 }
419
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 40 times.
 40 if (!retval) {
420 ✗ UnlockFile(fd_lockfile);
421 ✗ ClosePipe(pipe_boot);
422 ✗ ClosePipe(pipe_handshake);
423 ✗ delete quota_mgr;
424 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to start cache manager");
425 ✗ return NULL;
426 }
427
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 LogCvmfs(kLogQuota, kLogDebug,
428 "new cache manager pid: %d protocol revision %d", new_cachemgr_pid,
429 QuotaManager::kProtocolRevision);
430 40 quota_mgr->SetCacheMgrPid(new_cachemgr_pid);
431
2/4
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 40 times.
✗ Branch 6 not taken.
 40 const int fd_lockfile_rw = open((workspace_dir + "/lock_cachemgr").c_str(),
432 O_RDWR | O_TRUNC, 0600);
433 40 const unsigned magic_number = PosixQuotaManager::kLockFileMagicNumber;
434
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 const bool result_mn = SafeWrite(fd_lockfile_rw, &magic_number,
435 sizeof(magic_number));
436
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 const bool result = SafeWrite(fd_lockfile_rw, &new_cachemgr_pid,
437 sizeof(new_cachemgr_pid));
438
2/4
✓ Branch 0 taken 40 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 40 times.
 40 if (!result || !result_mn) {
439 ✗ PANIC(kLogSyslogErr, "could not write cache manager pid to lockfile");
440 }
441
442
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 close(fd_lockfile_rw);
443 // Wait for cache manager to be ready
444
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 close(pipe_boot[1]);
445
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 close(pipe_handshake[0]);
446 char buf;
447
2/4
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 40 times.
✗ Branch 4 not taken.
 40 if (read(pipe_boot[0], &buf, 1) != 1) {
448
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 UnlockFile(fd_lockfile);
449
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 close(pipe_boot[0]);
450
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 close(pipe_handshake[1]);
451
1/2
✓ Branch 0 taken 40 times.
✗ Branch 1 not taken.
 40 delete quota_mgr;
452
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
453 "cache manager did not start");
454 40 return NULL;
455 }
456 ✗ close(pipe_boot[0]);
457
458 // Connect write end
459 ✗ quota_mgr->pipe_lru_[1] = open(fifo_path.c_str(), O_WRONLY | O_NONBLOCK);
460 ✗ if (quota_mgr->pipe_lru_[1] < 0) {
461 ✗ LogCvmfs(kLogQuota, kLogDebug,
462 ✗ "failed to connect to newly created FIFO (%d)", errno);
463 ✗ close(pipe_handshake[1]);
464 ✗ UnlockFile(fd_lockfile);
465 ✗ delete quota_mgr;
466 ✗ return NULL;
467 }
468
469 // Finalize handshake
470 ✗ buf = 'C';
471 ✗ if (write(pipe_handshake[1], &buf, 1) != 1) {
472 ✗ UnlockFile(fd_lockfile);
473 ✗ close(pipe_handshake[1]);
474 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not finalize handshake");
475 ✗ delete quota_mgr;
476 ✗ return NULL;
477 }
478 ✗ close(pipe_handshake[1]);
479
480 ✗ Nonblock2Block(quota_mgr->pipe_lru_[1]);
481 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to a new cache manager");
482 ✗ quota_mgr->protocol_revision_ = kProtocolRevision;
483
484 ✗ UnlockFile(fd_lockfile);
485
486 ✗ quota_mgr->initialized_ = true;
487 ✗ quota_mgr->GetLimits(&quota_mgr->limit_, &quota_mgr->cleanup_threshold_);
488 ✗ LogCvmfs(kLogQuota, kLogDebug,
489 "received limit %" PRIu64 ", "
490 "threshold %" PRIu64,
491 quota_mgr->limit_, quota_mgr->cleanup_threshold_);
492 ✗ return quota_mgr;
493 80 }
494
495
496 400 bool PosixQuotaManager::DoCleanup(const uint64_t leave_size) {
497
2/2
✓ Branch 0 taken 80 times.
✓ Branch 1 taken 320 times.
 400 if (gauge_ <= leave_size)
498 80 return true;
499
500 // TODO(jblomer) transaction
501
1/2
✓ Branch 1 taken 320 times.
✗ Branch 2 not taken.
 320 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug,
502 "clean up cache until at most %lu KB is used", leave_size / 1024);
503
1/2
✓ Branch 1 taken 320 times.
✗ Branch 2 not taken.
 320 LogCvmfs(kLogQuota, kLogDebug, "gauge %" PRIu64, gauge_);
504
1/2
✓ Branch 1 taken 320 times.
✗ Branch 2 not taken.
 320 cleanup_recorder_.Tick();
505
506 bool result;
507 320 vector<string> trash;
508
509 // Note that volatile files start counting from the smallest int64 number:
510 // the absolute sequence number with the first bit set in two's complement.
511 // So -1 can be a marker that will never appear in the database.
512 320 int64_t max_acseq = -1;
513 320 std::vector<EvictCandidate> lru_ordered_open;
514
515 do {
516
1/2
✓ Branch 1 taken 2320 times.
✗ Branch 2 not taken.
 2320 sqlite3_reset(stmt_lru_);
517
3/4
✓ Branch 0 taken 320 times.
✓ Branch 1 taken 2000 times.
✓ Branch 3 taken 2320 times.
✗ Branch 4 not taken.
 2640 sqlite3_bind_int64(stmt_lru_, 1,
518 320 (max_acseq == -1) ? std::numeric_limits<int64_t>::min()
519 : (max_acseq + 1));
520
521 2320 std::vector<EvictCandidate> candidates;
522
1/2
✓ Branch 1 taken 2320 times.
✗ Branch 2 not taken.
 2320 candidates.reserve(kEvictBatchSize);
523 2320 string hash_str;
524 2320 unsigned i = 0;
525
3/4
✓ Branch 1 taken 2082720 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2080400 times.
✓ Branch 4 taken 2320 times.
 2082720 while (sqlite3_step(stmt_lru_) == SQLITE_ROW) {
526 hash_str = reinterpret_cast<const char *>(
527
2/4
✓ Branch 1 taken 2080400 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2080400 times.
✗ Branch 5 not taken.
 2080400 sqlite3_column_text(stmt_lru_, 0));
528
1/2
✓ Branch 2 taken 2080400 times.
✗ Branch 3 not taken.
 2080400 LogCvmfs(kLogQuota, kLogDebug, "add %s to candidates for eviction",
529 hash_str.c_str());
530
1/2
✓ Branch 1 taken 2080400 times.
✗ Branch 2 not taken.
 2080400 candidates.push_back(
531
1/2
✓ Branch 1 taken 2080400 times.
✗ Branch 2 not taken.
 2080400 EvictCandidate(shash::MkFromHexPtr(shash::HexPtr(hash_str)),
532 2080400 sqlite3_column_int64(stmt_lru_, 1),
533
2/4
✓ Branch 1 taken 2080400 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2080400 times.
✗ Branch 5 not taken.
 2080400 sqlite3_column_int64(stmt_lru_, 2)));
534 2080400 i++;
535 }
536
2/2
✓ Branch 1 taken 40 times.
✓ Branch 2 taken 2280 times.
 2320 if (candidates.empty()) {
537
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 LogCvmfs(kLogQuota, kLogDebug, "no more entries to evict");
538 40 break;
539 }
540
541 2280 const unsigned N = candidates.size();
542
543 2280 open_files_.clear();
544
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2280 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 4560 open_files_ = (cleanup_unused_first_) ? CollectAllOpenHashes()
545 2280 : std::vector<shash::Short>();
546
547
2/2
✓ Branch 0 taken 2000360 times.
✓ Branch 1 taken 2000 times.
 2002360 for (i = 0; i < N; ++i) {
548 // That's a critical condition. We must not delete a not yet inserted
549 // pinned file as it is already reserved (but will be inserted later).
550 // Instead, set the pin bit in the db to not run into an endless loop
551
1/2
✓ Branch 2 taken 2000360 times.
✗ Branch 3 not taken.
 2000360 const bool is_pinned = pinned_chunks_.find(candidates[i].hash)
552 4000720 != pinned_chunks_.end();
553
554 // Avoid evicting open files hopping there are enough more recently used
555 // files to satisfy the cleanup request
556 /*
557 const bool is_open = std::find_if(
558 open_files_.begin(), open_files_.end(),
559 [&candidates, &i](const auto &elem) -> bool
560 { return elem.Collide(candidates[i].hash);
561 })
562 != open_files_.end();
563 */
564 2000360 bool is_open = false;
565
1/2
✗ Branch 4 not taken.
✓ Branch 5 taken 2000360 times.
 2000360 for (auto it = open_files_.begin(); it != open_files_.end(); ++it) {
566 ✗ if (it->Collide(candidates[i].hash)) {
567 ✗ is_open = true;
568 ✗ break;
569 }
570 }
571
572
2/2
✓ Branch 0 taken 40 times.
✓ Branch 1 taken 2000320 times.
 2000360 if (is_pinned) {
573
1/2
✓ Branch 2 taken 40 times.
✗ Branch 3 not taken.
 40 SkipEviction(candidates[i]);
574 40 continue;
575 }
576
577
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2000320 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 2000320 if (cleanup_unused_first_ and is_open) {
578 ✗ SkipEviction(candidates[i]);
579 ✗ lru_ordered_open.push_back(candidates[i]);
580 ✗ continue;
581 }
582
583
2/4
✓ Branch 1 taken 2000320 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2000320 times.
✗ Branch 5 not taken.
 4000640 trash.push_back(cache_dir_ + "/"
584
2/4
✓ Branch 2 taken 2000320 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2000320 times.
✗ Branch 6 not taken.
 6000960 + candidates[i].hash.MakePathWithoutSuffix());
585 2000320 gauge_ -= candidates[i].size;
586 2000320 max_acseq = candidates[i].acseq;
587
1/2
✓ Branch 2 taken 2000320 times.
✗ Branch 3 not taken.
 2000320 LogCvmfs(kLogQuota, kLogDebug, "lru cleanup %s, new gauge %" PRIu64,
588
1/2
✓ Branch 2 taken 2000320 times.
✗ Branch 3 not taken.
 4000640 candidates[i].hash.ToString().c_str(), gauge_);
589
590
2/2
✓ Branch 0 taken 280 times.
✓ Branch 1 taken 2000040 times.
 2000320 if (gauge_ <= leave_size)
591 280 break;
592 }
593
6/6
✓ Branch 1 taken 2280 times.
✓ Branch 2 taken 40 times.
✓ Branch 4 taken 2280 times.
✓ Branch 5 taken 40 times.
✓ Branch 6 taken 2000 times.
✓ Branch 7 taken 280 times.
 4640 } while (gauge_ > leave_size);
594
595
1/2
✓ Branch 0 taken 320 times.
✗ Branch 1 not taken.
 320 if (max_acseq != -1) {
596
1/2
✓ Branch 1 taken 320 times.
✗ Branch 2 not taken.
 320 sqlite3_bind_int64(stmt_rm_batch_, 1, max_acseq);
597
1/2
✓ Branch 1 taken 320 times.
✗ Branch 2 not taken.
 320 result = (sqlite3_step(stmt_rm_batch_) == SQLITE_DONE);
598
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 320 times.
 320 assert(result);
599
1/2
✓ Branch 1 taken 320 times.
✗ Branch 2 not taken.
 320 sqlite3_reset(stmt_rm_batch_);
600
601
1/2
✓ Branch 1 taken 320 times.
✗ Branch 2 not taken.
 320 result = (sqlite3_step(stmt_unblock_) == SQLITE_DONE);
602
1/2
✓ Branch 1 taken 320 times.
✗ Branch 2 not taken.
 320 sqlite3_reset(stmt_unblock_);
603
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 320 times.
 320 assert(result);
604 }
605
606
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 320 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 320 times.
 320 while (!lru_ordered_open.empty() and gauge_ > leave_size) {
607 // cleanup files in use
608 ✗ auto &candidate = lru_ordered_open[0];
609 ✗ trash.push_back(cache_dir_ + "/" + candidate.hash.MakePathWithoutSuffix());
610 ✗ gauge_ -= candidate.size;
611 ✗ max_acseq = candidate.acseq;
612 ✗ LogCvmfs(kLogQuota, kLogDebug, "lru cleanup %s, new gauge %" PRIu64,
613 ✗ candidate.hash.ToString().c_str(), gauge_);
614 ✗ lru_ordered_open.erase(lru_ordered_open.begin());
615 }
616
617
2/4
✓ Branch 1 taken 320 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 320 times.
 320 if (!EmptyTrash(trash))
618 ✗ return false;
619
620
2/2
✓ Branch 0 taken 40 times.
✓ Branch 1 taken 280 times.
 320 if (gauge_ > leave_size) {
621
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
622 "request to clean until %" PRIu64 ", "
623 "but effective gauge is %" PRIu64,
624 leave_size, gauge_);
625 40 return false;
626 }
627 280 return true;
628 320 }
629
630 320 bool PosixQuotaManager::EmptyTrash(const std::vector<std::string> &trash) {
631
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 320 times.
 320 if (trash.empty())
632 ✗ return true;
633
634
2/2
✓ Branch 0 taken 240 times.
✓ Branch 1 taken 80 times.
 320 if (async_delete_) {
635 // Double fork avoids zombie, forked removal process must not flush file
636 // buffers
637 pid_t pid;
638 int statloc;
639
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 240 times.
 240 if ((pid = fork()) == 0) {
640 // TODO(jblomer): eviciting files in the cache should perhaps become a
641 // thread. This would also allow to block the chunks and prevent the
642 // race with re-insertion. Then again, a thread can block umount.
643 #ifndef DEBUGMSG
644 ✗ CloseAllFildes(std::set<int>());
645 #endif
646 ✗ if (fork() == 0) {
647 ✗ for (unsigned i = 0, iEnd = trash.size(); i < iEnd; ++i) {
648 ✗ LogCvmfs(kLogQuota, kLogDebug, "unlink %s", trash[i].c_str());
649 ✗ unlink(trash[i].c_str());
650 }
651 ✗ _exit(0);
652 }
653 ✗ _exit(0);
654 } else {
655
1/2
✓ Branch 0 taken 240 times.
✗ Branch 1 not taken.
 240 if (pid > 0)
656
1/2
✓ Branch 1 taken 240 times.
✗ Branch 2 not taken.
 240 waitpid(pid, &statloc, 0);
657 else
658 ✗ return false;
659 }
660 } else { // !async_delete_
661
2/2
✓ Branch 1 taken 120 times.
✓ Branch 2 taken 80 times.
 200 for (unsigned i = 0, iEnd = trash.size(); i < iEnd; ++i) {
662 120 LogCvmfs(kLogQuota, kLogDebug, "unlink %s", trash[i].c_str());
663 120 unlink(trash[i].c_str());
664 }
665 }
666 320 return true;
667 }
668
669
670 4001595 void PosixQuotaManager::DoInsert(const shash::Any &hash,
671 const uint64_t size,
672 const string &description,
673 const CommandType command_type) {
674
1/2
✓ Branch 1 taken 4001595 times.
✗ Branch 2 not taken.
 4001595 const string hash_str = hash.ToString();
675
1/2
✓ Branch 3 taken 4001595 times.
✗ Branch 4 not taken.
 4001595 LogCvmfs(kLogQuota, kLogDebug, "insert into lru %s, path %s, method %d",
676 hash_str.c_str(), description.c_str(), command_type);
677 4001595 const unsigned desc_length = (description.length() > kMaxDescription)
678 ? kMaxDescription
679
1/2
✓ Branch 0 taken 4001595 times.
✗ Branch 1 not taken.
 4001595 : description.length();
680
681 LruCommand *cmd = reinterpret_cast<LruCommand *>(
682 4001595 alloca(sizeof(LruCommand) + desc_length));
683 4001595 new (cmd) LruCommand;
684 4001595 cmd->command_type = command_type;
685 4001595 cmd->SetSize(size);
686
1/2
✓ Branch 1 taken 4001595 times.
✗ Branch 2 not taken.
 4001595 cmd->StoreHash(hash);
687 4001595 cmd->desc_length = desc_length;
688 4001595 memcpy(reinterpret_cast<char *>(cmd) + sizeof(LruCommand), &description[0],
689 desc_length);
690
1/2
✓ Branch 1 taken 4001595 times.
✗ Branch 2 not taken.
 4001595 WritePipe(pipe_lru_[1], cmd, sizeof(LruCommand) + desc_length);
691 4001595 }
692
693
694 1480 vector<string> PosixQuotaManager::DoList(const CommandType list_command) {
695 1480 vector<string> result;
696
697 int pipe_list[2];
698
1/2
✓ Branch 1 taken 1480 times.
✗ Branch 2 not taken.
 1480 MakeReturnPipe(pipe_list);
699 char description_buffer[kMaxDescription];
700
701 1480 LruCommand cmd;
702 1480 cmd.command_type = list_command;
703 1480 cmd.return_pipe = pipe_list[1];
704
1/2
✓ Branch 1 taken 1480 times.
✗ Branch 2 not taken.
 1480 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
705
706 int length;
707 do {
708
1/2
✓ Branch 1 taken 4003120 times.
✗ Branch 2 not taken.
 4003120 ManagedReadHalfPipe(pipe_list[0], &length, sizeof(length));
709
2/2
✓ Branch 0 taken 4001640 times.
✓ Branch 1 taken 1480 times.
 4003120 if (length > 0) {
710
1/2
✓ Branch 1 taken 4001640 times.
✗ Branch 2 not taken.
 4001640 ReadPipe(pipe_list[0], description_buffer, length);
711
2/4
✓ Branch 2 taken 4001640 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 4001640 times.
✗ Branch 6 not taken.
 4001640 result.push_back(string(description_buffer, length));
712 }
713
2/2
✓ Branch 0 taken 4001640 times.
✓ Branch 1 taken 1480 times.
 4003120 } while (length >= 0);
714
715
1/2
✓ Branch 1 taken 1480 times.
✗ Branch 2 not taken.
 1480 CloseReturnPipe(pipe_list);
716 2960 return result;
717 }
718
719
720 358 uint64_t PosixQuotaManager::GetCapacity() {
721
1/2
✓ Branch 0 taken 358 times.
✗ Branch 1 not taken.
 358 if (limit_ != (uint64_t)(-1))
722 358 return limit_;
723
724 // Unrestricted cache, look at free space on cache dir fs
725 struct statfs info;
726 ✗ if (statfs(".", &info) == 0) {
727 ✗ return info.f_bavail * info.f_bsize;
728 } else {
729 ✗ LogCvmfs(kLogQuota, kLogSyslogErr | kLogDebug,
730 ✗ "failed to query file system info of cache (%d)", errno);
731 ✗ return limit_;
732 }
733 }
734
735
736 ✗ void PosixQuotaManager::GetLimits(uint64_t *limit,
737 uint64_t *cleanup_threshold) {
738 int pipe_limits[2];
739 ✗ MakeReturnPipe(pipe_limits);
740
741 ✗ LruCommand cmd;
742 ✗ cmd.command_type = kLimits;
743 ✗ cmd.return_pipe = pipe_limits[1];
744 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
745 ✗ ManagedReadHalfPipe(pipe_limits[0], limit, sizeof(*limit));
746 ✗ ReadPipe(pipe_limits[0], cleanup_threshold, sizeof(*cleanup_threshold));
747 ✗ CloseReturnPipe(pipe_limits);
748 }
749
750
751 /**
752 * Since we only cleanup until cleanup_threshold, we can only add
753 * files smaller than limit-cleanup_threshold.
754 */
755 148 uint64_t PosixQuotaManager::GetMaxFileSize() {
756 148 return limit_ - cleanup_threshold_;
757 }
758
759
760 40 pid_t PosixQuotaManager::GetPid() {
761
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 40 if (!shared_ || !spawned_) {
762 40 return getpid();
763 }
764 ✗ if (cachemgr_pid_) {
765 ✗ return cachemgr_pid_;
766 }
767
768 pid_t result;
769 int pipe_pid[2];
770 ✗ MakeReturnPipe(pipe_pid);
771
772 ✗ LruCommand cmd;
773 ✗ cmd.command_type = kPid;
774 ✗ cmd.return_pipe = pipe_pid[1];
775 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
776 ✗ ReadHalfPipe(pipe_pid[0], &result, sizeof(result));
777 ✗ CloseReturnPipe(pipe_pid);
778 ✗ return result;
779 }
780
781
782 40 uint32_t PosixQuotaManager::GetProtocolRevision() {
783 int pipe_revision[2];
784
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 MakeReturnPipe(pipe_revision);
785
786 40 LruCommand cmd;
787 40 cmd.command_type = kGetProtocolRevision;
788 40 cmd.return_pipe = pipe_revision[1];
789
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
790
791 uint32_t revision;
792
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 ManagedReadHalfPipe(pipe_revision[0], &revision, sizeof(revision));
793
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 CloseReturnPipe(pipe_revision);
794 40 return revision;
795 }
796
797 358 void PosixQuotaManager::SetCleanupPolicy(bool cleanup_unused_first) {
798
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 358 times.
 358 if (protocol_revision_ < 3)
799 ✗ return;
800
801
3/4
✓ Branch 0 taken 40 times.
✓ Branch 1 taken 318 times.
✓ Branch 3 taken 358 times.
✗ Branch 4 not taken.
 358 LogCvmfs(
802 kLogQuota, kLogDebug, "Set cleanup policy to %s",
803 (cleanup_unused_first) ? "cleanup unused files first." : "vanilla lru.");
804
805
2/2
✓ Branch 0 taken 40 times.
✓ Branch 1 taken 318 times.
 358 char policy = (cleanup_unused_first) ? 'S' : 'R'; // S: smart, R: regular;
806
807 LruCommand *cmd = reinterpret_cast<LruCommand *>(
808 358 alloca(sizeof(LruCommand) + sizeof(policy)));
809 358 new (cmd) LruCommand;
810 358 cmd->command_type = kSetCleanupPolicy;
811 358 cmd->desc_length = sizeof(policy);
812 358 memcpy(reinterpret_cast<char *>(cmd) + sizeof(LruCommand), &policy,
813 sizeof(policy));
814
1/2
✓ Branch 1 taken 358 times.
✗ Branch 2 not taken.
 358 WritePipe(pipe_lru_[1], cmd, sizeof(LruCommand) + sizeof(policy));
815 }
816
817 40 void PosixQuotaManager::RegisterMountpoint(const std::string &mountpoint) {
818
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 40 times.
 40 if (protocol_revision_ < 3)
819 ✗ return;
820
821 40 LogCvmfs(kLogQuota, kLogDebug, "Register Mountpoint %s", mountpoint.c_str());
822
823 40 const unsigned desc_length = (mountpoint.size() > kMaxDescription)
824 ? kMaxDescription
825
1/2
✓ Branch 0 taken 40 times.
✗ Branch 1 not taken.
 40 : mountpoint.size();
826 LruCommand *cmd = reinterpret_cast<LruCommand *>(
827 40 alloca(sizeof(LruCommand) + desc_length));
828 40 new (cmd) LruCommand;
829 40 cmd->command_type = kRegisterMountpoint;
830 40 cmd->desc_length = desc_length;
831 40 memcpy(reinterpret_cast<char *>(cmd) + sizeof(LruCommand), mountpoint.data(),
832 desc_length);
833 40 WritePipe(pipe_lru_[1], cmd, sizeof(LruCommand) + desc_length);
834 }
835
836 40 std::string PosixQuotaManager::ReadPipeString(int fd, size_t size) {
837
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 40 times.
 40 if (size == 0)
838 ✗ return "";
839
840
1/2
✓ Branch 2 taken 40 times.
✗ Branch 3 not taken.
 40 std::vector<char> buf(size);
841
1/2
✓ Branch 2 taken 40 times.
✗ Branch 3 not taken.
 40 ManagedReadHalfPipe(fd, buf.data(), size);
842
1/2
✓ Branch 3 taken 40 times.
✗ Branch 4 not taken.
 40 return std::string(buf.data(), size);
843 40 }
844
845 ✗ std::string PosixQuotaManager::GetMountpoints() {
846 ✗ if (protocol_revision_ < 3)
847 ✗ return "";
848
849 int pipe_mp[2];
850 ✗ MakeReturnPipe(pipe_mp);
851
852 ✗ LruCommand cmd;
853 ✗ cmd.command_type = kGetMountpoints;
854 ✗ cmd.return_pipe = pipe_mp[1];
855 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
856 ✗ size_t mp_str_size = 0;
857 ✗ ManagedReadHalfPipe(pipe_mp[0], &mp_str_size, sizeof(size_t));
858 ✗ const std::string result = ReadPipeString(pipe_mp[0], mp_str_size);
859 ✗ CloseReturnPipe(pipe_mp);
860 ✗ return result;
861 }
862
863 ✗ std::string PosixQuotaManager::GetGroupHashes() {
864 ✗ if (protocol_revision_ < 3)
865 ✗ return "";
866
867 int pipe_gh[2];
868 ✗ MakeReturnPipe(pipe_gh);
869
870 ✗ LruCommand cmd;
871 ✗ cmd.command_type = kGetGroupHashes;
872 ✗ cmd.return_pipe = pipe_gh[1];
873 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
874 ✗ size_t mp_str_size = 0;
875 ✗ ManagedReadHalfPipe(pipe_gh[0], &mp_str_size, sizeof(size_t));
876 ✗ const std::string result = ReadPipeString(pipe_gh[0], mp_str_size);
877 ✗ CloseReturnPipe(pipe_gh);
878 ✗ return result;
879 }
880
881 /**
882 * Queries the shared local hard disk quota manager.
883 */
884 720 void PosixQuotaManager::GetSharedStatus(uint64_t *gauge, uint64_t *pinned) {
885 int pipe_status[2];
886
1/2
✓ Branch 1 taken 720 times.
✗ Branch 2 not taken.
 720 MakeReturnPipe(pipe_status);
887
888 720 LruCommand cmd;
889 720 cmd.command_type = kStatus;
890 720 cmd.return_pipe = pipe_status[1];
891
1/2
✓ Branch 1 taken 720 times.
✗ Branch 2 not taken.
 720 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
892
1/2
✓ Branch 1 taken 720 times.
✗ Branch 2 not taken.
 720 ManagedReadHalfPipe(pipe_status[0], gauge, sizeof(*gauge));
893
1/2
✓ Branch 1 taken 720 times.
✗ Branch 2 not taken.
 720 ReadPipe(pipe_status[0], pinned, sizeof(*pinned));
894
1/2
✓ Branch 1 taken 720 times.
✗ Branch 2 not taken.
 720 CloseReturnPipe(pipe_status);
895 720 }
896
897 40 bool PosixQuotaManager::SetSharedLimit(uint64_t limit) {
898 int pipe_set_limit[2];
899 bool result;
900
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 MakeReturnPipe(pipe_set_limit);
901
902 40 LruCommand cmd;
903 40 cmd.command_type = kSetLimit;
904 40 cmd.size = limit;
905 40 cmd.return_pipe = pipe_set_limit[1];
906
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
907
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 ReadHalfPipe(pipe_set_limit[0], &result, sizeof(result));
908
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 CloseReturnPipe(pipe_set_limit);
909 40 return result;
910 }
911
912
913 40 bool PosixQuotaManager::SetLimit(uint64_t size) {
914
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 40 times.
 40 if (!spawned_) {
915 ✗ limit_ = size;
916 ✗ cleanup_threshold_ = size / 2;
917 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
918 "Quota limit set to %lu / threshold %lu", limit_,
919 cleanup_threshold_);
920 ✗ return true;
921 }
922 40 return SetSharedLimit(size);
923 }
924
925 1276 uint64_t PosixQuotaManager::GetSize() {
926
2/2
✓ Branch 0 taken 636 times.
✓ Branch 1 taken 640 times.
 1276 if (!spawned_)
927 636 return gauge_;
928 uint64_t gauge, size_pinned;
929
1/2
✓ Branch 1 taken 640 times.
✗ Branch 2 not taken.
 640 GetSharedStatus(&gauge, &size_pinned);
930 640 return gauge;
931 }
932
933
934 80 uint64_t PosixQuotaManager::GetSizePinned() {
935
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 80 times.
 80 if (!spawned_)
936 ✗ return pinned_;
937 uint64_t gauge, size_pinned;
938
1/2
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
 80 GetSharedStatus(&gauge, &size_pinned);
939 80 return size_pinned;
940 }
941
942
943 160 uint64_t PosixQuotaManager::GetCleanupRate(uint64_t period_s) {
944
2/4
✓ Branch 0 taken 160 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 160 times.
 160 if (!spawned_ || (protocol_revision_ < 2))
945 ✗ return 0;
946 uint64_t cleanup_rate;
947
948 int pipe_cleanup_rate[2];
949
1/2
✓ Branch 1 taken 160 times.
✗ Branch 2 not taken.
 160 MakeReturnPipe(pipe_cleanup_rate);
950 160 LruCommand cmd;
951 160 cmd.command_type = kCleanupRate;
952 160 cmd.size = period_s;
953 160 cmd.return_pipe = pipe_cleanup_rate[1];
954
1/2
✓ Branch 1 taken 160 times.
✗ Branch 2 not taken.
 160 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
955
1/2
✓ Branch 1 taken 160 times.
✗ Branch 2 not taken.
 160 ManagedReadHalfPipe(pipe_cleanup_rate[0], &cleanup_rate,
956 sizeof(cleanup_rate));
957
1/2
✓ Branch 1 taken 160 times.
✗ Branch 2 not taken.
 160 CloseReturnPipe(pipe_cleanup_rate);
958
959 160 return cleanup_rate;
960 }
961
962
963 2918 bool PosixQuotaManager::InitDatabase(const bool rebuild_database) {
964 2918 string sql;
965 sqlite3_stmt *stmt;
966
967
2/4
✓ Branch 1 taken 2918 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2918 times.
✗ Branch 5 not taken.
 2918 fd_lock_cachedb_ = LockFile(workspace_dir_ + "/lock_cachedb");
968
2/2
✓ Branch 0 taken 40 times.
✓ Branch 1 taken 2878 times.
 2918 if (fd_lock_cachedb_ < 0) {
969
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 LogCvmfs(kLogQuota, kLogDebug, "failed to create cachedb lock");
970 40 return false;
971 }
972
973 2878 bool retry = false;
974
1/2
✓ Branch 1 taken 2878 times.
✗ Branch 2 not taken.
 2878 const string db_file = cache_dir_ + "/cachedb";
975
2/2
✓ Branch 0 taken 2711 times.
✓ Branch 1 taken 167 times.
 2878 if (rebuild_database) {
976
1/2
✓ Branch 2 taken 167 times.
✗ Branch 3 not taken.
 167 LogCvmfs(kLogQuota, kLogDebug, "rebuild database, unlinking existing (%s)",
977 db_file.c_str());
978 167 unlink(db_file.c_str());
979
1/2
✓ Branch 1 taken 167 times.
✗ Branch 2 not taken.
 167 unlink((db_file + "-journal").c_str());
980 }
981
982 2711 init_recover:
983
1/2
✓ Branch 2 taken 2878 times.
✗ Branch 3 not taken.
 2878 int err = sqlite3_open(db_file.c_str(), &database_);
984
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2878 times.
 2878 if (err != SQLITE_OK) {
985 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not open cache database (%d)", err);
986 ✗ goto init_database_fail;
987 }
988 // TODO(reneme): make this a `QuotaDatabase : public sqlite::Database`
989 sql = "PRAGMA synchronous=0; PRAGMA locking_mode=EXCLUSIVE; "
990 "PRAGMA auto_vacuum=1; "
991 "CREATE TABLE IF NOT EXISTS cache_catalog (sha1 TEXT, size INTEGER, "
992 " acseq INTEGER, path TEXT, type INTEGER, pinned INTEGER, "
993 "CONSTRAINT pk_cache_catalog PRIMARY KEY (sha1)); "
994 "CREATE UNIQUE INDEX IF NOT EXISTS idx_cache_catalog_acseq "
995 " ON cache_catalog (acseq); "
996 "CREATE TEMP TABLE fscache (sha1 TEXT, size INTEGER, actime INTEGER, "
997 "CONSTRAINT pk_fscache PRIMARY KEY (sha1)); "
998 "CREATE INDEX idx_fscache_actime ON fscache (actime); "
999 "CREATE TABLE IF NOT EXISTS properties (key TEXT, value TEXT, "
1000
1/2
✓ Branch 1 taken 2878 times.
✗ Branch 2 not taken.
 2878 " CONSTRAINT pk_properties PRIMARY KEY(key));";
1001
1/2
✓ Branch 2 taken 2878 times.
✗ Branch 3 not taken.
 2878 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1002
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2878 times.
 2878 if (err != SQLITE_OK) {
1003 ✗ if (!retry) {
1004 ✗ retry = true;
1005 ✗ sqlite3_close(database_);
1006 ✗ unlink(db_file.c_str());
1007 ✗ unlink((db_file + "-journal").c_str());
1008 ✗ LogCvmfs(kLogQuota, kLogSyslogWarn,
1009 "LRU database corrupted, re-building");
1010 ✗ goto init_recover;
1011 }
1012 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
1013 sql.c_str());
1014 ✗ goto init_database_fail;
1015 }
1016
1017 // If this an old cache catalog,
1018 // add and initialize new columns to cache_catalog
1019 sql = "ALTER TABLE cache_catalog ADD type INTEGER; "
1020
1/2
✓ Branch 1 taken 2878 times.
✗ Branch 2 not taken.
 2878 "ALTER TABLE cache_catalog ADD pinned INTEGER";
1021
1/2
✓ Branch 2 taken 2878 times.
✗ Branch 3 not taken.
 2878 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1022
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2878 times.
 2878 if (err == SQLITE_OK) {
1023 ✗ sql = "UPDATE cache_catalog SET type=" + StringifyInt(kFileRegular) + ";";
1024 ✗ err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1025 ✗ if (err != SQLITE_OK) {
1026 ✗ LogCvmfs(kLogQuota, kLogDebug,
1027 "could not init cache database (failed: %s)", sql.c_str());
1028 ✗ goto init_database_fail;
1029 }
1030 }
1031
1032 // Set pinned back
1033
1/2
✓ Branch 1 taken 2878 times.
✗ Branch 2 not taken.
 2878 sql = "UPDATE cache_catalog SET pinned=0;";
1034
1/2
✓ Branch 2 taken 2878 times.
✗ Branch 3 not taken.
 2878 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1035
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2878 times.
 2878 if (err != SQLITE_OK) {
1036 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
1037 sql.c_str());
1038 ✗ goto init_database_fail;
1039 }
1040
1041 // Set schema version
1042 sql = "INSERT OR REPLACE INTO properties (key, value) "
1043
1/2
✓ Branch 1 taken 2878 times.
✗ Branch 2 not taken.
 2878 "VALUES ('schema', '1.0')";
1044
1/2
✓ Branch 2 taken 2878 times.
✗ Branch 3 not taken.
 2878 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1045
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2878 times.
 2878 if (err != SQLITE_OK) {
1046 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
1047 sql.c_str());
1048 ✗ goto init_database_fail;
1049 }
1050
1051 // If cache catalog is empty, recreate from file system
1052
1/2
✓ Branch 1 taken 2878 times.
✗ Branch 2 not taken.
 2878 sql = "SELECT count(*) FROM cache_catalog;";
1053
1/2
✓ Branch 2 taken 2878 times.
✗ Branch 3 not taken.
 2878 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1054
2/4
✓ Branch 1 taken 2878 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2878 times.
✗ Branch 4 not taken.
 2878 if (sqlite3_step(stmt) == SQLITE_ROW) {
1055
6/8
✓ Branch 1 taken 2878 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 160 times.
✓ Branch 4 taken 2718 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 160 times.
✓ Branch 7 taken 2718 times.
✓ Branch 8 taken 160 times.
 2878 if ((sqlite3_column_int64(stmt, 0)) == 0 || rebuild_database) {
1056
1/2
✓ Branch 1 taken 2718 times.
✗ Branch 2 not taken.
 2718 LogCvmfs(kLogCvmfs, kLogDebug,
1057 "CernVM-FS: building lru cache database...");
1058
3/4
✓ Branch 1 taken 2718 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 120 times.
✓ Branch 4 taken 2598 times.
 2718 if (!RebuildDatabase()) {
1059
1/2
✓ Branch 1 taken 120 times.
✗ Branch 2 not taken.
 120 LogCvmfs(kLogQuota, kLogDebug,
1060 "could not build cache database from file system");
1061
1/2
✓ Branch 1 taken 120 times.
✗ Branch 2 not taken.
 120 sqlite3_finalize(stmt);
1062 120 goto init_database_fail;
1063 }
1064 }
1065
1/2
✓ Branch 1 taken 2758 times.
✗ Branch 2 not taken.
 2758 sqlite3_finalize(stmt);
1066 } else {
1067 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not select on cache catalog");
1068 ✗ sqlite3_finalize(stmt);
1069 ✗ goto init_database_fail;
1070 }
1071
1072 // How many bytes do we already have in cache?
1073
1/2
✓ Branch 1 taken 2758 times.
✗ Branch 2 not taken.
 2758 sql = "SELECT sum(size) FROM cache_catalog;";
1074
1/2
✓ Branch 2 taken 2758 times.
✗ Branch 3 not taken.
 2758 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1075
2/4
✓ Branch 1 taken 2758 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2758 times.
✗ Branch 4 not taken.
 2758 if (sqlite3_step(stmt) == SQLITE_ROW) {
1076
1/2
✓ Branch 1 taken 2758 times.
✗ Branch 2 not taken.
 2758 gauge_ = sqlite3_column_int64(stmt, 0);
1077 } else {
1078 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not determine cache size");
1079 ✗ sqlite3_finalize(stmt);
1080 ✗ goto init_database_fail;
1081 }
1082
1/2
✓ Branch 1 taken 2758 times.
✗ Branch 2 not taken.
 2758 sqlite3_finalize(stmt);
1083
1084 // Highest seq-no?
1085
1/2
✓ Branch 1 taken 2758 times.
✗ Branch 2 not taken.
 2758 sql = "SELECT coalesce(max(acseq & (~(1<<63))), 0) FROM cache_catalog;";
1086
1/2
✓ Branch 2 taken 2758 times.
✗ Branch 3 not taken.
 2758 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1087
2/4
✓ Branch 1 taken 2758 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2758 times.
✗ Branch 4 not taken.
 2758 if (sqlite3_step(stmt) == SQLITE_ROW) {
1088
1/2
✓ Branch 1 taken 2758 times.
✗ Branch 2 not taken.
 2758 seq_ = sqlite3_column_int64(stmt, 0) + 1;
1089 } else {
1090 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not determine highest seq-no");
1091 ✗ sqlite3_finalize(stmt);
1092 ✗ goto init_database_fail;
1093 }
1094
1/2
✓ Branch 1 taken 2758 times.
✗ Branch 2 not taken.
 2758 sqlite3_finalize(stmt);
1095
1096 // Prepare touch, new, remove statements
1097
1/2
✓ Branch 1 taken 2758 times.
✗ Branch 2 not taken.
 2758 sqlite3_prepare_v2(database_,
1098 "UPDATE cache_catalog SET acseq=:seq | (acseq&(1<<63)) "
1099 "WHERE sha1=:sha1;",
1100 -1, &stmt_touch_, NULL);
1101
1/2
✓ Branch 1 taken 2758 times.
✗ Branch 2 not taken.
 2758 sqlite3_prepare_v2(database_,
1102 "UPDATE cache_catalog SET pinned=0 "
1103 "WHERE sha1=:sha1;",
1104 -1, &stmt_unpin_, NULL);
1105
1/2
✓ Branch 1 taken 2758 times.
✗ Branch 2 not taken.
 2758 sqlite3_prepare_v2(database_,
1106 "UPDATE cache_catalog SET pinned=2 "
1107 "WHERE sha1=:sha1;",
1108 -1, &stmt_block_, NULL);
1109
1/2
✓ Branch 1 taken 2758 times.
✗ Branch 2 not taken.
 2758 sqlite3_prepare_v2(database_,
1110 "UPDATE cache_catalog SET pinned=1 "
1111 "WHERE pinned=2;",
1112 -1, &stmt_unblock_, NULL);
1113
1/2
✓ Branch 1 taken 2758 times.
✗ Branch 2 not taken.
 2758 sqlite3_prepare_v2(database_,
1114 "INSERT OR REPLACE INTO cache_catalog "
1115 "(sha1, size, acseq, path, type, pinned) "
1116 "VALUES (:sha1, :s, :seq, :p, :t, :pin);",
1117 -1, &stmt_new_, NULL);
1118
1/2
✓ Branch 1 taken 2758 times.
✗ Branch 2 not taken.
 2758 sqlite3_prepare_v2(database_,
1119 "SELECT size, pinned FROM cache_catalog WHERE sha1=:sha1;",
1120 -1, &stmt_size_, NULL);
1121
1/2
✓ Branch 1 taken 2758 times.
✗ Branch 2 not taken.
 2758 sqlite3_prepare_v2(database_, "DELETE FROM cache_catalog WHERE sha1=:sha1;",
1122 -1, &stmt_rm_, NULL);
1123
1/2
✓ Branch 1 taken 2758 times.
✗ Branch 2 not taken.
 2758 sqlite3_prepare_v2(database_,
1124 "DELETE FROM cache_catalog WHERE acseq<=:a AND pinned<>2;",
1125 -1, &stmt_rm_batch_, NULL);
1126
1/2
✓ Branch 2 taken 2758 times.
✗ Branch 3 not taken.
 2758 sqlite3_prepare_v2(database_,
1127
1/2
✓ Branch 2 taken 2758 times.
✗ Branch 3 not taken.
 5516 (std::string("SELECT sha1, size, acseq FROM cache_catalog "
1128 "WHERE pinned<>2 AND acseq>=:a "
1129 "ORDER BY acseq ASC "
1130 "LIMIT ")
1131
3/6
✓ Branch 1 taken 2758 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2758 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 2758 times.
✗ Branch 8 not taken.
 11032 + StringifyInt(kEvictBatchSize) + ";")
1132 .c_str(),
1133 -1, &stmt_lru_, NULL);
1134
1/2
✓ Branch 2 taken 2758 times.
✗ Branch 3 not taken.
 2758 sqlite3_prepare_v2(database_,
1135 ("SELECT path FROM cache_catalog WHERE type="
1136
3/6
✓ Branch 1 taken 2758 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2758 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 2758 times.
✗ Branch 8 not taken.
 5516 + StringifyInt(kFileRegular) + ";")
1137 .c_str(),
1138 -1, &stmt_list_, NULL);
1139
1/2
✓ Branch 1 taken 2758 times.
✗ Branch 2 not taken.
 2758 sqlite3_prepare_v2(database_,
1140 "SELECT path FROM cache_catalog WHERE pinned<>0;", -1,
1141 &stmt_list_pinned_, NULL);
1142
1/2
✓ Branch 1 taken 2758 times.
✗ Branch 2 not taken.
 2758 sqlite3_prepare_v2(database_,
1143 "SELECT path FROM cache_catalog WHERE acseq < 0;", -1,
1144 &stmt_list_volatile_, NULL);
1145
1/2
✓ Branch 2 taken 2758 times.
✗ Branch 3 not taken.
 2758 sqlite3_prepare_v2(database_,
1146 ("SELECT path FROM cache_catalog WHERE type="
1147
3/6
✓ Branch 1 taken 2758 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2758 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 2758 times.
✗ Branch 8 not taken.
 5516 + StringifyInt(kFileCatalog) + ";")
1148 .c_str(),
1149 -1, &stmt_list_catalogs_, NULL);
1150 2758 return true;
1151
1152 120 init_database_fail:
1153
1/2
✓ Branch 1 taken 120 times.
✗ Branch 2 not taken.
 120 sqlite3_close(database_);
1154 120 database_ = NULL;
1155
1/2
✓ Branch 1 taken 120 times.
✗ Branch 2 not taken.
 120 UnlockFile(fd_lock_cachedb_);
1156 120 return false;
1157 2918 }
1158
1159
1160 /**
1161 * Inserts a new file into cache catalog. This file gets a new,
1162 * highest sequence number. Does cache cleanup if necessary.
1163 */
1164 4000915 void PosixQuotaManager::Insert(const shash::Any &any_hash,
1165 const uint64_t size,
1166 const string &description) {
1167 4000915 DoInsert(any_hash, size, description, kInsert);
1168 4000915 }
1169
1170
1171 /**
1172 * Inserts a new file into cache catalog. This file is marked as volatile
1173 * and gets a new highest sequence number with the first bit set. Cache cleanup
1174 * treats these files with priority.
1175 */
1176 160 void PosixQuotaManager::InsertVolatile(const shash::Any &any_hash,
1177 const uint64_t size,
1178 const string &description) {
1179 160 DoInsert(any_hash, size, description, kInsertVolatile);
1180 160 }
1181
1182
1183 /**
1184 * Lists all path names from the cache db.
1185 */
1186 920 vector<string> PosixQuotaManager::List() { return DoList(kList); }
1187
1188
1189 /**
1190 * Lists all pinned files from the cache db.
1191 */
1192 320 vector<string> PosixQuotaManager::ListPinned() { return DoList(kListPinned); }
1193
1194
1195 /**
1196 * Lists all sqlite catalog files from the cache db.
1197 */
1198 120 vector<string> PosixQuotaManager::ListCatalogs() {
1199 120 return DoList(kListCatalogs);
1200 }
1201
1202
1203 /**
1204 * Lists only files flagged as volatile (priority removal)
1205 */
1206 120 vector<string> PosixQuotaManager::ListVolatile() {
1207 120 return DoList(kListVolatile);
1208 }
1209
1210
1211 /**
1212 * Entry point for the shared cache manager process
1213 */
1214 ✗ int PosixQuotaManager::MainCacheManager(int argc, char **argv) {
1215 ✗ LogCvmfs(kLogQuota, kLogDebug, "starting quota manager");
1216 int retval;
1217
1218 ✗ PosixQuotaManager shared_manager(0, 0, "");
1219 ✗ shared_manager.shared_ = true;
1220 ✗ shared_manager.spawned_ = true;
1221 ✗ shared_manager.pinned_ = 0;
1222
1223 // Process command line arguments
1224 ✗ ParseDirectories(string(argv[2]),
1225 &shared_manager.cache_dir_,
1226 &shared_manager.workspace_dir_);
1227 ✗ const int pipe_boot = String2Int64(argv[3]);
1228 ✗ const int pipe_handshake = String2Int64(argv[4]);
1229 ✗ shared_manager.limit_ = String2Int64(argv[5]);
1230 ✗ shared_manager.cleanup_threshold_ = String2Int64(argv[6]);
1231 ✗ const int foreground = String2Int64(argv[7]);
1232 ✗ const int syslog_level = String2Int64(argv[8]);
1233 ✗ const int syslog_facility = String2Int64(argv[9]);
1234 ✗ vector<string> logfiles = SplitString(argv[10], ':');
1235
1236 ✗ SetLogSyslogLevel(syslog_level);
1237 ✗ SetLogSyslogFacility(syslog_facility);
1238 ✗ if ((logfiles.size() > 0) && (logfiles[0] != ""))
1239 ✗ SetLogDebugFile(logfiles[0] + ".cachemgr");
1240 ✗ if (logfiles.size() > 1)
1241 ✗ SetLogMicroSyslog(logfiles[1]);
1242
1243 ✗ if (!foreground)
1244 ✗ Daemonize();
1245
1246 ✗ if ((geteuid() != 0) && SetuidCapabilityPermitted()) {
1247 // Permanently drop credentials
1248 ✗ const std::vector<cap_value_t> nocaps;
1249 ✗ assert(ClearPermittedCapabilities(nocaps, nocaps));
1250 // Leave this process ptraceable
1251 ✗ assert(platform_set_dumpable());
1252 // but without core dumps
1253 ✗ assert(SetLimitCore(0));
1254 }
1255
1256 const UniquePtr<Watchdog> watchdog(
1257 ✗ Watchdog::Create(NULL, false /* needs_read_environ */));
1258 ✗ assert(watchdog.IsValid());
1259 ✗ watchdog->Spawn("./stacktrace.cachemgr");
1260
1261 // Initialize pipe, open non-blocking as cvmfs is not yet connected
1262 ✗ const int fd_lockfile_fifo = LockFile(shared_manager.workspace_dir_
1263 ✗ + "/lock_cachemgr.fifo");
1264 ✗ if (fd_lockfile_fifo < 0) {
1265 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1266 "could not open lock file "
1267 "%s (%d)",
1268 ✗ (shared_manager.workspace_dir_ + "/lock_cachemgr.fifo").c_str(),
1269 ✗ errno);
1270 ✗ return 1;
1271 }
1272 ✗ const string crash_guard = shared_manager.cache_dir_ + "/cachemgr.running";
1273 ✗ const bool rebuild = FileExists(crash_guard);
1274 ✗ retval = open(crash_guard.c_str(), O_RDONLY | O_CREAT, 0600);
1275 ✗ if (retval < 0) {
1276 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1277 "failed to create shared cache manager crash guard");
1278 ✗ UnlockFile(fd_lockfile_fifo);
1279 ✗ return 1;
1280 }
1281 ✗ close(retval);
1282
1283 // Redirect SQlite temp directory to cache (global variable)
1284 ✗ const string tmp_dir = shared_manager.workspace_dir_;
1285 ✗ sqlite3_temp_directory = static_cast<char *>(
1286 ✗ sqlite3_malloc(tmp_dir.length() + 1));
1287 ✗ snprintf(sqlite3_temp_directory, tmp_dir.length() + 1, "%s", tmp_dir.c_str());
1288
1289 // Cleanup leftover named pipes
1290 ✗ shared_manager.CleanupPipes();
1291
1292 ✗ if (!shared_manager.InitDatabase(rebuild)) {
1293 ✗ UnlockFile(fd_lockfile_fifo);
1294 ✗ return 1;
1295 }
1296 ✗ shared_manager.CheckFreeSpace();
1297
1298 // Save protocol revision to file. If the file is not found, it indicates
1299 // to the client that the cache manager is from times before the protocol
1300 // was versioned.
1301 const string protocol_revision_path = shared_manager.workspace_dir_
1302 ✗ + "/cachemgr.protocol";
1303 ✗ retval = open(protocol_revision_path.c_str(), O_WRONLY | O_CREAT, 0600);
1304 ✗ if (retval < 0) {
1305 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1306 ✗ "failed to open protocol revision file (%d)", errno);
1307 ✗ UnlockFile(fd_lockfile_fifo);
1308 ✗ return 1;
1309 }
1310 ✗ const string revision = StringifyInt(kProtocolRevision);
1311 ✗ const int written = write(retval, revision.data(), revision.length());
1312 ✗ close(retval);
1313 ✗ if ((written < 0) || static_cast<unsigned>(written) != revision.length()) {
1314 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1315 ✗ "failed to write protocol revision (%d)", errno);
1316 ✗ UnlockFile(fd_lockfile_fifo);
1317 ✗ return 1;
1318 }
1319
1320 ✗ const string fifo_path = shared_manager.workspace_dir_ + "/cachemgr";
1321 ✗ shared_manager.pipe_lru_[0] = open(fifo_path.c_str(), O_RDONLY | O_NONBLOCK);
1322 ✗ if (shared_manager.pipe_lru_[0] < 0) {
1323 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to listen on FIFO %s (%d)",
1324 ✗ fifo_path.c_str(), errno);
1325 ✗ UnlockFile(fd_lockfile_fifo);
1326 ✗ return 1;
1327 }
1328 ✗ Nonblock2Block(shared_manager.pipe_lru_[0]);
1329 ✗ LogCvmfs(kLogQuota, kLogDebug, "shared cache manager listening");
1330
1331 ✗ char buf = 'C';
1332 ✗ WritePipe(pipe_boot, &buf, 1);
1333 ✗ close(pipe_boot);
1334
1335 ✗ ReadPipe(pipe_handshake, &buf, 1);
1336 ✗ close(pipe_handshake);
1337 ✗ LogCvmfs(kLogQuota, kLogDebug, "shared cache manager handshake done");
1338
1339 // Ensure that broken pipes from clients do not kill the cache manager
1340 ✗ signal(SIGPIPE, SIG_IGN);
1341 // Don't let Ctrl-C ungracefully kill interactive session
1342 ✗ signal(SIGINT, SIG_IGN);
1343
1344 ✗ shared_manager.MainCommandServer(&shared_manager);
1345 ✗ unlink(fifo_path.c_str());
1346 ✗ unlink(protocol_revision_path.c_str());
1347 ✗ shared_manager.CloseDatabase();
1348 ✗ unlink(crash_guard.c_str());
1349 ✗ UnlockFile(fd_lockfile_fifo);
1350
1351 ✗ if (sqlite3_temp_directory) {
1352 ✗ sqlite3_free(sqlite3_temp_directory);
1353 ✗ sqlite3_temp_directory = NULL;
1354 }
1355
1356 ✗ return 0;
1357 }
1358
1359
1360 1360 void *PosixQuotaManager::MainCommandServer(void *data) {
1361 1360 PosixQuotaManager *quota_mgr = static_cast<PosixQuotaManager *>(data);
1362
1363
1/2
✓ Branch 1 taken 1360 times.
✗ Branch 2 not taken.
 1360 LogCvmfs(kLogQuota, kLogDebug, "starting quota manager");
1364
1/2
✓ Branch 1 taken 1360 times.
✗ Branch 2 not taken.
 1360 sqlite3_soft_heap_limit(quota_mgr->kSqliteMemPerThread);
1365
1366
2/2
✓ Branch 1 taken 43520 times.
✓ Branch 2 taken 1360 times.
 44880 LruCommand command_buffer[kCommandBufferSize];
1367 char description_buffer[kCommandBufferSize * kMaxDescription];
1368 1360 unsigned num_commands = 0;
1369
1370
1/2
✓ Branch 1 taken 6006800 times.
✗ Branch 2 not taken.
 6006800 while (read(quota_mgr->pipe_lru_[0], &command_buffer[num_commands],
1371 sizeof(command_buffer[0]))
1372
2/2
✓ Branch 0 taken 6005440 times.
✓ Branch 1 taken 1360 times.
 6006800 == sizeof(command_buffer[0])) {
1373 6005440 const CommandType command_type = command_buffer[num_commands].command_type;
1374
1/2
✓ Branch 1 taken 6005440 times.
✗ Branch 2 not taken.
 6005440 LogCvmfs(kLogQuota, kLogDebug, "received command %d", command_type);
1375 6005440 const uint64_t size = command_buffer[num_commands].GetSize();
1376
1377 // Inserts and pins come with a description (usually a path)
1378
4/4
✓ Branch 0 taken 2004680 times.
✓ Branch 1 taken 4000760 times.
✓ Branch 2 taken 2004520 times.
✓ Branch 3 taken 160 times.
 6005440 if ((command_type == kInsert) || (command_type == kInsertVolatile)
1379
4/4
✓ Branch 0 taken 2004440 times.
✓ Branch 1 taken 80 times.
✓ Branch 2 taken 2004000 times.
✓ Branch 3 taken 440 times.
 2004520 || (command_type == kPin) || (command_type == kPinRegular)
1380
2/2
✓ Branch 0 taken 2003960 times.
✓ Branch 1 taken 40 times.
 2004000 || (command_type == kRegisterMountpoint)
1381
2/2
✓ Branch 0 taken 40 times.
✓ Branch 1 taken 2003920 times.
 2003960 || (command_type == kSetCleanupPolicy)) {
1382 4001520 const int desc_length = command_buffer[num_commands].desc_length;
1383 4001520 ReadPipe(quota_mgr->pipe_lru_[0],
1384
1/2
✓ Branch 1 taken 4001520 times.
✗ Branch 2 not taken.
 4001520 &description_buffer[kMaxDescription * num_commands],
1385 desc_length);
1386 }
1387
1388 // The protocol revision is returned immediately
1389
2/2
✓ Branch 0 taken 40 times.
✓ Branch 1 taken 6005400 times.
 6005440 if (command_type == kGetProtocolRevision) {
1390
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 const int return_pipe = quota_mgr->BindReturnPipe(
1391 command_buffer[num_commands].return_pipe);
1392
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 40 times.
 40 if (return_pipe < 0)
1393 ✗ continue;
1394
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 WritePipe(return_pipe, &quota_mgr->kProtocolRevision,
1395 sizeof(quota_mgr->kProtocolRevision));
1396
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 quota_mgr->UnbindReturnPipe(return_pipe);
1397 40 continue;
1398 40 }
1399
1400 // Register a new mountpoint
1401
2/2
✓ Branch 0 taken 40 times.
✓ Branch 1 taken 6005360 times.
 6005400 if (command_type == kRegisterMountpoint) {
1402 40 const std::string mountpoint(&description_buffer[num_commands*kMaxDescription],
1403
1/2
✓ Branch 2 taken 40 times.
✗ Branch 3 not taken.
 40 command_buffer[num_commands].desc_length);
1404
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 quota_mgr->mountpoints_.push_back(mountpoint);
1405
1/2
✓ Branch 2 taken 40 times.
✗ Branch 3 not taken.
 40 LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
1406 "Mountpoint %s registered in the group", mountpoint.c_str());
1407 40 continue;
1408 40 }
1409
1410 // Set Cleanup Policy
1411
2/2
✓ Branch 0 taken 40 times.
✓ Branch 1 taken 6005320 times.
 6005360 if (command_type == kSetCleanupPolicy) {
1412 40 quota_mgr->cleanup_unused_first_ = (description_buffer[num_commands * kMaxDescription] == 'S') ? true : false;
1413 40 continue;
1414 }
1415 // Mountpoints are returned immediately
1416
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6005320 times.
 6005320 if (command_type == kGetMountpoints) {
1417 ✗ const int return_pipe = quota_mgr->BindReturnPipe(
1418 command_buffer[num_commands].return_pipe);
1419 ✗ if (return_pipe < 0)
1420 ✗ continue;
1421
1422 ✗ std::string mps;
1423 ✗ for (auto it = quota_mgr->mountpoints_.begin();
1424 ✗ it != quota_mgr->mountpoints_.end();
1425 ✗ ++it) {
1426 ✗ mps += *it + "\n";
1427 }
1428 ✗ size_t mp_size = mps.size();
1429 ✗ WritePipe(return_pipe, &mp_size, sizeof(size_t));
1430 ✗ WritePipe(return_pipe, mps.c_str(), mp_size);
1431 ✗ quota_mgr->UnbindReturnPipe(return_pipe);
1432 ✗ continue;
1433 }
1434
1435 // Group hashes are returned immediately
1436
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6005320 times.
 6005320 if (command_type == kGetGroupHashes) {
1437 ✗ const int return_pipe = quota_mgr->BindReturnPipe(
1438 command_buffer[num_commands].return_pipe);
1439 ✗ if (return_pipe < 0)
1440 ✗ continue;
1441
1442 ✗ std::vector<shash::Short> gh = quota_mgr->CollectAllOpenHashes();
1443 ✗ std::string result;
1444 ✗ for (auto it = gh.begin(); it != gh.end(); ++it) {
1445 ✗ result += (*it).ToString() + "\n";
1446 }
1447 ✗ size_t result_size = result.size();
1448 ✗ WritePipe(return_pipe, &result_size, sizeof(size_t));
1449 ✗ WritePipe(return_pipe, result.c_str(), result_size);
1450 ✗ quota_mgr->UnbindReturnPipe(return_pipe);
1451 ✗ continue;
1452 }
1453
1454 // The cleanup rate is returned immediately
1455
2/2
✓ Branch 0 taken 160 times.
✓ Branch 1 taken 6005160 times.
 6005320 if (command_type == kCleanupRate) {
1456
1/2
✓ Branch 1 taken 160 times.
✗ Branch 2 not taken.
 160 const int return_pipe = quota_mgr->BindReturnPipe(
1457 command_buffer[num_commands].return_pipe);
1458
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 160 times.
 160 if (return_pipe < 0)
1459 ✗ continue;
1460 const uint64_t
1461 160 period_s = size; // use the size field to transmit the period
1462
1/2
✓ Branch 1 taken 160 times.
✗ Branch 2 not taken.
 160 uint64_t rate = quota_mgr->cleanup_recorder_.GetNoTicks(period_s);
1463
1/2
✓ Branch 1 taken 160 times.
✗ Branch 2 not taken.
 160 WritePipe(return_pipe, &rate, sizeof(rate));
1464
1/2
✓ Branch 1 taken 160 times.
✗ Branch 2 not taken.
 160 quota_mgr->UnbindReturnPipe(return_pipe);
1465 160 continue;
1466 160 }
1467
1468
2/2
✓ Branch 0 taken 40 times.
✓ Branch 1 taken 6005120 times.
 6005160 if (command_type == kSetLimit) {
1469
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 const int return_pipe = quota_mgr->BindReturnPipe(
1470 command_buffer[num_commands].return_pipe);
1471
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 40 times.
 40 if (return_pipe < 0)
1472 ✗ continue;
1473 40 quota_mgr->limit_ = size; // use the size field to transmit the size
1474 40 quota_mgr->cleanup_threshold_ = size / 2;
1475
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
1476 "Quota limit set to %lu / threshold %lu", quota_mgr->limit_,
1477 quota_mgr->cleanup_threshold_);
1478 40 bool ret = true;
1479
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 WritePipe(return_pipe, &ret, sizeof(ret));
1480
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 quota_mgr->UnbindReturnPipe(return_pipe);
1481 40 continue;
1482 40 }
1483
1484 // Reservations are handled immediately and "out of band"
1485
2/2
✓ Branch 0 taken 560 times.
✓ Branch 1 taken 6004560 times.
 6005120 if (command_type == kReserve) {
1486 560 bool success = true;
1487
1/2
✓ Branch 1 taken 560 times.
✗ Branch 2 not taken.
 560 const int return_pipe = quota_mgr->BindReturnPipe(
1488 command_buffer[num_commands].return_pipe);
1489
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 560 times.
 560 if (return_pipe < 0)
1490 ✗ continue;
1491
1492
1/2
✓ Branch 1 taken 560 times.
✗ Branch 2 not taken.
 560 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1493
1/2
✓ Branch 1 taken 560 times.
✗ Branch 2 not taken.
 560 const string hash_str(hash.ToString());
1494
1/2
✓ Branch 2 taken 560 times.
✗ Branch 3 not taken.
 560 LogCvmfs(kLogQuota, kLogDebug, "reserve %lu bytes for %s", size,
1495 hash_str.c_str());
1496
1497
1/2
✓ Branch 1 taken 560 times.
✗ Branch 2 not taken.
 560 if (quota_mgr->pinned_chunks_.find(hash)
1498
2/2
✓ Branch 2 taken 480 times.
✓ Branch 3 taken 80 times.
 1120 == quota_mgr->pinned_chunks_.end()) {
1499
2/2
✓ Branch 0 taken 40 times.
✓ Branch 1 taken 440 times.
 480 if ((quota_mgr->pinned_ + size) > quota_mgr->cleanup_threshold_) {
1500
1/2
✓ Branch 2 taken 40 times.
✗ Branch 3 not taken.
 40 LogCvmfs(kLogQuota, kLogDebug,
1501 "failed to insert %s (pinned), no space", hash_str.c_str());
1502 40 success = false;
1503 } else {
1504
1/2
✓ Branch 1 taken 440 times.
✗ Branch 2 not taken.
 440 quota_mgr->pinned_chunks_[hash] = size;
1505 440 quota_mgr->pinned_ += size;
1506
1/2
✓ Branch 1 taken 440 times.
✗ Branch 2 not taken.
 440 quota_mgr->CheckHighPinWatermark();
1507 }
1508 }
1509
1510
1/2
✓ Branch 1 taken 560 times.
✗ Branch 2 not taken.
 560 WritePipe(return_pipe, &success, sizeof(success));
1511
1/2
✓ Branch 1 taken 560 times.
✗ Branch 2 not taken.
 560 quota_mgr->UnbindReturnPipe(return_pipe);
1512 560 continue;
1513 560 }
1514
1515 // Back channels are also handled out of band
1516
2/2
✓ Branch 0 taken 160 times.
✓ Branch 1 taken 6004400 times.
 6004560 if (command_type == kRegisterBackChannel) {
1517
1/2
✓ Branch 1 taken 160 times.
✗ Branch 2 not taken.
 160 const int return_pipe = quota_mgr->BindReturnPipe(
1518 command_buffer[num_commands].return_pipe);
1519
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 160 times.
 160 if (return_pipe < 0)
1520 ✗ continue;
1521
1522
1/2
✓ Branch 1 taken 160 times.
✗ Branch 2 not taken.
 160 quota_mgr->UnlinkReturnPipe(command_buffer[num_commands].return_pipe);
1523
1/2
✓ Branch 1 taken 160 times.
✗ Branch 2 not taken.
 160 Block2Nonblock(return_pipe); // back channels are opportunistic
1524
1/2
✓ Branch 1 taken 160 times.
✗ Branch 2 not taken.
 160 shash::Md5 hash;
1525 160 memcpy(hash.digest, command_buffer[num_commands].digest,
1526 160 shash::kDigestSizes[shash::kMd5]);
1527
1528 160 quota_mgr->LockBackChannels();
1529 const map<shash::Md5, int>::const_iterator
1530
1/2
✓ Branch 1 taken 160 times.
✗ Branch 2 not taken.
 160 iter = quota_mgr->back_channels_.find(hash);
1531
1/2
✗ Branch 3 not taken.
✓ Branch 4 taken 160 times.
 160 if (iter != quota_mgr->back_channels_.end()) {
1532 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
1533 ✗ "closing left-over back channel %s", hash.ToString().c_str());
1534 ✗ close(iter->second);
1535 }
1536
1/2
✓ Branch 1 taken 160 times.
✗ Branch 2 not taken.
 160 quota_mgr->back_channels_[hash] = return_pipe;
1537 160 quota_mgr->UnlockBackChannels();
1538
1539 160 char success = 'S';
1540
1/2
✓ Branch 1 taken 160 times.
✗ Branch 2 not taken.
 160 WritePipe(return_pipe, &success, sizeof(success));
1541
1/2
✓ Branch 2 taken 160 times.
✗ Branch 3 not taken.
 160 LogCvmfs(kLogQuota, kLogDebug, "register back channel %s on fd %d",
1542
1/2
✓ Branch 1 taken 160 times.
✗ Branch 2 not taken.
 320 hash.ToString().c_str(), return_pipe);
1543
1544 160 continue;
1545 160 }
1546
1547
2/2
✓ Branch 0 taken 80 times.
✓ Branch 1 taken 6004320 times.
 6004400 if (command_type == kUnregisterBackChannel) {
1548
1/2
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
 80 shash::Md5 hash;
1549 80 memcpy(hash.digest, command_buffer[num_commands].digest,
1550 80 shash::kDigestSizes[shash::kMd5]);
1551
1552 80 quota_mgr->LockBackChannels();
1553 const map<shash::Md5, int>::iterator iter = quota_mgr->back_channels_
1554
1/2
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
 80 .find(hash);
1555
1/2
✓ Branch 2 taken 80 times.
✗ Branch 3 not taken.
 80 if (iter != quota_mgr->back_channels_.end()) {
1556
1/2
✓ Branch 2 taken 80 times.
✗ Branch 3 not taken.
 80 LogCvmfs(kLogQuota, kLogDebug, "closing back channel %s",
1557
1/2
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
 160 hash.ToString().c_str());
1558
1/2
✓ Branch 2 taken 80 times.
✗ Branch 3 not taken.
 80 close(iter->second);
1559
1/2
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
 80 quota_mgr->back_channels_.erase(iter);
1560 } else {
1561 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
1562 ✗ "did not find back channel %s", hash.ToString().c_str());
1563 }
1564 80 quota_mgr->UnlockBackChannels();
1565
1566 80 continue;
1567 80 }
1568
1569 // Unpinnings are also handled immediately with respect to the pinned gauge
1570
2/2
✓ Branch 0 taken 80 times.
✓ Branch 1 taken 6004240 times.
 6004320 if (command_type == kUnpin) {
1571
1/2
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
 80 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1572
1/2
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
 80 const string hash_str(hash.ToString());
1573
1574 const map<shash::Any, uint64_t>::iterator iter = quota_mgr->pinned_chunks_
1575
1/2
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
 80 .find(hash);
1576
1/2
✓ Branch 2 taken 80 times.
✗ Branch 3 not taken.
 80 if (iter != quota_mgr->pinned_chunks_.end()) {
1577 80 quota_mgr->pinned_ -= iter->second;
1578
1/2
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
 80 quota_mgr->pinned_chunks_.erase(iter);
1579 // It can happen that files get pinned that were removed from the cache
1580 // (see cache.cc). We fix this at this point, where we remove such
1581 // entries from the cache database.
1582
2/4
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 80 times.
✗ Branch 5 not taken.
 160 if (!FileExists(quota_mgr->cache_dir_ + "/"
1583
4/6
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 80 times.
✗ Branch 5 not taken.
✓ Branch 9 taken 40 times.
✓ Branch 10 taken 40 times.
 240 + hash.MakePathWithoutSuffix())) {
1584
1/2
✓ Branch 2 taken 40 times.
✗ Branch 3 not taken.
 40 LogCvmfs(kLogQuota, kLogDebug,
1585 "remove orphaned pinned hash %s from cache database",
1586 hash_str.c_str());
1587
1/2
✓ Branch 2 taken 40 times.
✗ Branch 3 not taken.
 40 sqlite3_bind_text(quota_mgr->stmt_size_, 1, &hash_str[0],
1588 40 hash_str.length(), SQLITE_STATIC);
1589 int retval;
1590
2/4
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 40 times.
✗ Branch 4 not taken.
 40 if ((retval = sqlite3_step(quota_mgr->stmt_size_)) == SQLITE_ROW) {
1591
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 const uint64_t size = sqlite3_column_int64(quota_mgr->stmt_size_,
1592 40 0);
1593
1/2
✓ Branch 2 taken 40 times.
✗ Branch 3 not taken.
 40 sqlite3_bind_text(quota_mgr->stmt_rm_, 1, &(hash_str[0]),
1594 40 hash_str.length(), SQLITE_STATIC);
1595
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 retval = sqlite3_step(quota_mgr->stmt_rm_);
1596
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 40 if ((retval == SQLITE_DONE) || (retval == SQLITE_OK)) {
1597 40 quota_mgr->gauge_ -= size;
1598 } else {
1599 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1600 "failed to delete %s (%d)", hash_str.c_str(), retval);
1601 }
1602
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 sqlite3_reset(quota_mgr->stmt_rm_);
1603 }
1604
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 sqlite3_reset(quota_mgr->stmt_size_);
1605 }
1606 } else {
1607 ✗ LogCvmfs(kLogQuota, kLogDebug, "this chunk was not pinned");
1608 }
1609 80 }
1610
1611 // Immediate commands trigger flushing of the buffer
1612 6004320 const bool immediate_command = (command_type == kCleanup)
1613
2/2
✓ Branch 0 taken 6003040 times.
✓ Branch 1 taken 920 times.
 6003960 || (command_type == kList)
1614
2/2
✓ Branch 0 taken 6002720 times.
✓ Branch 1 taken 320 times.
 6003040 || (command_type == kListPinned)
1615
2/2
✓ Branch 0 taken 6002600 times.
✓ Branch 1 taken 120 times.
 6002720 || (command_type == kListCatalogs)
1616
2/2
✓ Branch 0 taken 6002480 times.
✓ Branch 1 taken 120 times.
 6002600 || (command_type == kListVolatile)
1617
2/2
✓ Branch 0 taken 6002360 times.
✓ Branch 1 taken 120 times.
 6002480 || (command_type == kRemove)
1618
2/2
✓ Branch 0 taken 6001640 times.
✓ Branch 1 taken 720 times.
 6002360 || (command_type == kStatus)
1619
1/2
✓ Branch 0 taken 6001640 times.
✗ Branch 1 not taken.
 6001640 || (command_type == kLimits)
1620
3/4
✓ Branch 0 taken 6003960 times.
✓ Branch 1 taken 360 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 6001640 times.
 12008280 || (command_type == kPid);
1621
2/2
✓ Branch 0 taken 6001640 times.
✓ Branch 1 taken 2680 times.
 6004320 if (!immediate_command)
1622 6001640 num_commands++;
1623
1624
4/4
✓ Branch 0 taken 5816840 times.
✓ Branch 1 taken 187480 times.
✓ Branch 2 taken 2680 times.
✓ Branch 3 taken 5814160 times.
 6004320 if ((num_commands == kCommandBufferSize) || immediate_command) {
1625
1/2
✓ Branch 1 taken 190160 times.
✗ Branch 2 not taken.
 190160 quota_mgr->ProcessCommandBunch(num_commands, command_buffer,
1626 description_buffer);
1627
2/2
✓ Branch 0 taken 187480 times.
✓ Branch 1 taken 2680 times.
 190160 if (!immediate_command)
1628 187480 num_commands = 0;
1629 }
1630
1631
2/2
✓ Branch 0 taken 2680 times.
✓ Branch 1 taken 6001640 times.
 6004320 if (immediate_command) {
1632 // Process cleanup, listings
1633
1/2
✓ Branch 1 taken 2680 times.
✗ Branch 2 not taken.
 2680 const int return_pipe = quota_mgr->BindReturnPipe(
1634 command_buffer[num_commands].return_pipe);
1635
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2680 times.
 2680 if (return_pipe < 0) {
1636 ✗ num_commands = 0;
1637 ✗ continue;
1638 }
1639
1640 int retval;
1641 2680 sqlite3_stmt *this_stmt_list = NULL;
1642
7/10
✓ Branch 0 taken 120 times.
✓ Branch 1 taken 360 times.
✓ Branch 2 taken 920 times.
✓ Branch 3 taken 320 times.
✓ Branch 4 taken 120 times.
✓ Branch 5 taken 120 times.
✓ Branch 6 taken 720 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
 2680 switch (command_type) {
1643 120 case kRemove: {
1644
1/2
✓ Branch 1 taken 120 times.
✗ Branch 2 not taken.
 120 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1645
1/2
✓ Branch 1 taken 120 times.
✗ Branch 2 not taken.
 120 const string hash_str = hash.ToString();
1646
1/2
✓ Branch 2 taken 120 times.
✗ Branch 3 not taken.
 120 LogCvmfs(kLogQuota, kLogDebug, "manually removing %s",
1647 hash_str.c_str());
1648 120 bool success = false;
1649
1650
1/2
✓ Branch 2 taken 120 times.
✗ Branch 3 not taken.
 120 sqlite3_bind_text(quota_mgr->stmt_size_, 1, &hash_str[0],
1651 120 hash_str.length(), SQLITE_STATIC);
1652 int retval;
1653
3/4
✓ Branch 1 taken 120 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 80 times.
✓ Branch 4 taken 40 times.
 120 if ((retval = sqlite3_step(quota_mgr->stmt_size_)) == SQLITE_ROW) {
1654
1/2
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
 80 const uint64_t size = sqlite3_column_int64(quota_mgr->stmt_size_,
1655 80 0);
1656
1/2
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
 80 const uint64_t is_pinned = sqlite3_column_int64(
1657 80 quota_mgr->stmt_size_, 1);
1658
1659
1/2
✓ Branch 2 taken 80 times.
✗ Branch 3 not taken.
 80 sqlite3_bind_text(quota_mgr->stmt_rm_, 1, &(hash_str[0]),
1660 80 hash_str.length(), SQLITE_STATIC);
1661
1/2
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
 80 retval = sqlite3_step(quota_mgr->stmt_rm_);
1662
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 80 if ((retval == SQLITE_DONE) || (retval == SQLITE_OK)) {
1663 80 success = true;
1664 80 quota_mgr->gauge_ -= size;
1665
2/2
✓ Branch 0 taken 40 times.
✓ Branch 1 taken 40 times.
 80 if (is_pinned) {
1666
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 quota_mgr->pinned_chunks_.erase(hash);
1667 40 quota_mgr->pinned_ -= size;
1668 }
1669 } else {
1670 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1671 "failed to delete %s (%d)", hash_str.c_str(), retval);
1672 }
1673
1/2
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
 80 sqlite3_reset(quota_mgr->stmt_rm_);
1674 } else {
1675 // File does not exist
1676 40 success = true;
1677 }
1678
1/2
✓ Branch 1 taken 120 times.
✗ Branch 2 not taken.
 120 sqlite3_reset(quota_mgr->stmt_size_);
1679
1680
1/2
✓ Branch 1 taken 120 times.
✗ Branch 2 not taken.
 120 WritePipe(return_pipe, &success, sizeof(success));
1681 120 break;
1682 120 }
1683 360 case kCleanup:
1684
1/2
✓ Branch 1 taken 360 times.
✗ Branch 2 not taken.
 360 retval = quota_mgr->DoCleanup(size);
1685
1/2
✓ Branch 1 taken 360 times.
✗ Branch 2 not taken.
 360 WritePipe(return_pipe, &retval, sizeof(retval));
1686 360 break;
1687 920 case kList:
1688
1/2
✓ Branch 0 taken 920 times.
✗ Branch 1 not taken.
 920 if (!this_stmt_list)
1689 920 this_stmt_list = quota_mgr->stmt_list_;
1690 case kListPinned:
1691
2/2
✓ Branch 0 taken 320 times.
✓ Branch 1 taken 920 times.
 1240 if (!this_stmt_list)
1692 320 this_stmt_list = quota_mgr->stmt_list_pinned_;
1693 case kListCatalogs:
1694
2/2
✓ Branch 0 taken 120 times.
✓ Branch 1 taken 1240 times.
 1360 if (!this_stmt_list)
1695 120 this_stmt_list = quota_mgr->stmt_list_catalogs_;
1696 case kListVolatile:
1697
2/2
✓ Branch 0 taken 120 times.
✓ Branch 1 taken 1360 times.
 1480 if (!this_stmt_list)
1698 120 this_stmt_list = quota_mgr->stmt_list_volatile_;
1699
1700 // Pipe back the list, one by one
1701 int length;
1702
3/4
✓ Branch 1 taken 4003120 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 4001640 times.
✓ Branch 4 taken 1480 times.
 4003120 while (sqlite3_step(this_stmt_list) == SQLITE_ROW) {
1703
1/2
✓ Branch 2 taken 4001640 times.
✗ Branch 3 not taken.
 4001640 string path = "(NULL)";
1704
2/4
✓ Branch 1 taken 4001640 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 4001640 times.
✗ Branch 4 not taken.
 4001640 if (sqlite3_column_type(this_stmt_list, 0) != SQLITE_NULL) {
1705 8003280 path = string(reinterpret_cast<const char *>(
1706
2/4
✓ Branch 1 taken 4001640 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 4001640 times.
✗ Branch 5 not taken.
 4001640 sqlite3_column_text(this_stmt_list, 0)));
1707 }
1708 4001640 length = path.length();
1709
1/2
✓ Branch 1 taken 4001640 times.
✗ Branch 2 not taken.
 4001640 WritePipe(return_pipe, &length, sizeof(length));
1710
1/2
✓ Branch 0 taken 4001640 times.
✗ Branch 1 not taken.
 4001640 if (length > 0)
1711
2/4
✓ Branch 1 taken 4001640 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 4001640 times.
✗ Branch 5 not taken.
 4001640 WritePipe(return_pipe, &path[0], length);
1712 4001640 }
1713 1480 length = -1;
1714
1/2
✓ Branch 1 taken 1480 times.
✗ Branch 2 not taken.
 1480 WritePipe(return_pipe, &length, sizeof(length));
1715
1/2
✓ Branch 1 taken 1480 times.
✗ Branch 2 not taken.
 1480 sqlite3_reset(this_stmt_list);
1716 1480 break;
1717 720 case kStatus:
1718
1/2
✓ Branch 1 taken 720 times.
✗ Branch 2 not taken.
 720 WritePipe(return_pipe, &quota_mgr->gauge_, sizeof(quota_mgr->gauge_));
1719
1/2
✓ Branch 1 taken 720 times.
✗ Branch 2 not taken.
 720 WritePipe(return_pipe, &quota_mgr->pinned_,
1720 sizeof(quota_mgr->pinned_));
1721 720 break;
1722 ✗ case kLimits:
1723 ✗ WritePipe(return_pipe, &quota_mgr->limit_, sizeof(quota_mgr->limit_));
1724 ✗ WritePipe(return_pipe, &quota_mgr->cleanup_threshold_,
1725 sizeof(quota_mgr->cleanup_threshold_));
1726 ✗ break;
1727 ✗ case kPid: {
1728 ✗ pid_t pid = getpid();
1729 ✗ WritePipe(return_pipe, &pid, sizeof(pid));
1730 ✗ break;
1731 }
1732 ✗ default:
1733 ✗ PANIC(NULL); // other types are handled by the bunch processor
1734 }
1735
1/2
✓ Branch 1 taken 2680 times.
✗ Branch 2 not taken.
 2680 quota_mgr->UnbindReturnPipe(return_pipe);
1736 2680 num_commands = 0;
1737 }
1738 }
1739
1740
1/2
✓ Branch 1 taken 1360 times.
✗ Branch 2 not taken.
 1360 LogCvmfs(kLogQuota, kLogDebug, "stopping cache manager (%d)", errno);
1741
1/2
✓ Branch 1 taken 1360 times.
✗ Branch 2 not taken.
 1360 close(quota_mgr->pipe_lru_[0]);
1742
1/2
✓ Branch 1 taken 1360 times.
✗ Branch 2 not taken.
 1360 quota_mgr->ProcessCommandBunch(num_commands, command_buffer,
1743 description_buffer);
1744
1745 // Unpin
1746 1360 command_buffer[0].command_type = kTouch;
1747 1360 for (map<shash::Any, uint64_t>::const_iterator
1748 1360 i = quota_mgr->pinned_chunks_.begin(),
1749 1360 iEnd = quota_mgr->pinned_chunks_.end();
1750
2/2
✓ Branch 1 taken 400 times.
✓ Branch 2 taken 1360 times.
 1760 i != iEnd;
1751 400 ++i) {
1752
1/2
✓ Branch 2 taken 400 times.
✗ Branch 3 not taken.
 400 command_buffer[0].StoreHash(i->first);
1753
1/2
✓ Branch 1 taken 400 times.
✗ Branch 2 not taken.
 400 quota_mgr->ProcessCommandBunch(1, command_buffer, description_buffer);
1754 }
1755
1756 1360 return NULL;
1757 }
1758
1759
1760 3760 void PosixQuotaManager::MakeReturnPipe(int pipe[2]) {
1761
2/2
✓ Branch 0 taken 3640 times.
✓ Branch 1 taken 120 times.
 3760 if (!shared_) {
1762 3640 MakePipe(pipe);
1763 3640 return;
1764 }
1765
1766 // Create FIFO in cache directory, store path name (number) in pipe write end
1767 120 int i = 0;
1768 int retval;
1769 do {
1770
2/4
✓ Branch 2 taken 160 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 160 times.
✗ Branch 6 not taken.
 160 retval = mkfifo((workspace_dir_ + "/pipe" + StringifyInt(i)).c_str(), 0600);
1771 160 pipe[1] = i;
1772 160 i++;
1773
3/4
✓ Branch 0 taken 40 times.
✓ Branch 1 taken 120 times.
✓ Branch 2 taken 40 times.
✗ Branch 3 not taken.
 160 } while ((retval == -1) && (errno == EEXIST));
1774
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 120 times.
 120 assert(retval == 0);
1775
1776 // Connect reader's end
1777
3/6
✓ Branch 2 taken 120 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 120 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 120 times.
✗ Branch 10 not taken.
 120 pipe[0] = open((workspace_dir_ + "/pipe" + StringifyInt(pipe[1])).c_str(),
1778 O_RDONLY | O_NONBLOCK);
1779
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 120 times.
 120 assert(pipe[0] >= 0);
1780 120 Nonblock2Block(pipe[0]);
1781 }
1782
1783
1784 2878 void PosixQuotaManager::ParseDirectories(const std::string cache_workspace,
1785 std::string *cache_dir,
1786 std::string *workspace_dir) {
1787
1/2
✓ Branch 1 taken 2878 times.
✗ Branch 2 not taken.
 2878 vector<string> dir_tokens(SplitString(cache_workspace, ':'));
1788
2/3
✓ Branch 1 taken 2798 times.
✓ Branch 2 taken 80 times.
✗ Branch 3 not taken.
 2878 switch (dir_tokens.size()) {
1789 2798 case 1:
1790
2/4
✓ Branch 2 taken 2798 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2798 times.
✗ Branch 6 not taken.
 2798 *cache_dir = *workspace_dir = dir_tokens[0];
1791 2798 break;
1792 80 case 2:
1793
1/2
✓ Branch 2 taken 80 times.
✗ Branch 3 not taken.
 80 *cache_dir = dir_tokens[0];
1794
1/2
✓ Branch 2 taken 80 times.
✗ Branch 3 not taken.
 80 *workspace_dir = dir_tokens[1];
1795 80 break;
1796 ✗ default:
1797 ✗ PANIC(NULL);
1798 }
1799 2878 }
1800
1801 40 void PosixQuotaManager::SkipEviction(const EvictCandidate &candidate) {
1802 40 bool res = true;
1803
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 std::string hash_str = candidate.hash.ToString();
1804
1/2
✓ Branch 2 taken 40 times.
✗ Branch 3 not taken.
 40 LogCvmfs(kLogQuota, kLogDebug, "Exclude %s from eviction", hash_str.c_str());
1805
2/4
✓ Branch 2 taken 40 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 40 times.
✗ Branch 6 not taken.
 40 sqlite3_bind_text(stmt_block_, 1, &hash_str[0], hash_str.length(),
1806 SQLITE_STATIC);
1807
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 res = (sqlite3_step(stmt_block_) == SQLITE_DONE);
1808
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 sqlite3_reset(stmt_block_);
1809
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 40 times.
 40 assert(res);
1810 40 }
1811
1812 /**
1813 * Immediately inserts a new pinned catalog. Does cache cleanup if necessary.
1814 *
1815 * \return True on success, false otherwise
1816 */
1817 1258 bool PosixQuotaManager::Pin(const shash::Any &hash,
1818 const uint64_t size,
1819 const string &description,
1820 const bool is_catalog) {
1821
3/4
✓ Branch 0 taken 80 times.
✓ Branch 1 taken 1178 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 80 times.
 1258 assert((size > 0) || !is_catalog);
1822
1823
1/2
✓ Branch 1 taken 1258 times.
✗ Branch 2 not taken.
 1258 const string hash_str = hash.ToString();
1824
1/2
✓ Branch 3 taken 1258 times.
✗ Branch 4 not taken.
 1258 LogCvmfs(kLogQuota, kLogDebug, "pin into lru %s, path %s", hash_str.c_str(),
1825 description.c_str());
1826
1827 // Has to run when not yet spawned (cvmfs initialization)
1828
2/2
✓ Branch 0 taken 698 times.
✓ Branch 1 taken 560 times.
 1258 if (!spawned_) {
1829 // Code duplication here
1830
3/4
✓ Branch 2 taken 698 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 416 times.
✓ Branch 6 taken 282 times.
 698 if (pinned_chunks_.find(hash) == pinned_chunks_.end()) {
1831
2/2
✓ Branch 0 taken 80 times.
✓ Branch 1 taken 336 times.
 416 if (pinned_ + size > cleanup_threshold_) {
1832
1/2
✓ Branch 2 taken 80 times.
✗ Branch 3 not taken.
 80 LogCvmfs(kLogQuota, kLogDebug, "failed to insert %s (pinned), no space",
1833 hash_str.c_str());
1834 80 return false;
1835 } else {
1836
1/2
✓ Branch 1 taken 336 times.
✗ Branch 2 not taken.
 336 pinned_chunks_[hash] = size;
1837 336 pinned_ += size;
1838
1/2
✓ Branch 1 taken 336 times.
✗ Branch 2 not taken.
 336 CheckHighPinWatermark();
1839 }
1840 }
1841
1/2
✓ Branch 1 taken 618 times.
✗ Branch 2 not taken.
 618 const bool exists = Contains(hash_str);
1842
4/4
✓ Branch 0 taken 336 times.
✓ Branch 1 taken 282 times.
✓ Branch 2 taken 40 times.
✓ Branch 3 taken 296 times.
 618 if (!exists && (gauge_ + size > limit_)) {
1843
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 LogCvmfs(kLogQuota, kLogDebug, "over limit, gauge %lu, file size %lu",
1844 gauge_, size);
1845
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 const int retval = DoCleanup(cleanup_threshold_);
1846
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 40 times.
 40 assert(retval != 0);
1847 }
1848
1/2
✓ Branch 3 taken 618 times.
✗ Branch 4 not taken.
 618 sqlite3_bind_text(stmt_new_, 1, &hash_str[0], hash_str.length(),
1849 SQLITE_STATIC);
1850
1/2
✓ Branch 1 taken 618 times.
✗ Branch 2 not taken.
 618 sqlite3_bind_int64(stmt_new_, 2, size);
1851
1/2
✓ Branch 1 taken 618 times.
✗ Branch 2 not taken.
 618 sqlite3_bind_int64(stmt_new_, 3, seq_++);
1852
1/2
✓ Branch 3 taken 618 times.
✗ Branch 4 not taken.
 618 sqlite3_bind_text(stmt_new_, 4, &description[0], description.length(),
1853 SQLITE_STATIC);
1854
3/4
✓ Branch 0 taken 498 times.
✓ Branch 1 taken 120 times.
✓ Branch 3 taken 618 times.
✗ Branch 4 not taken.
 618 sqlite3_bind_int64(stmt_new_, 5, is_catalog ? kFileCatalog : kFileRegular);
1855
1/2
✓ Branch 1 taken 618 times.
✗ Branch 2 not taken.
 618 sqlite3_bind_int64(stmt_new_, 6, 1);
1856
1/2
✓ Branch 1 taken 618 times.
✗ Branch 2 not taken.
 618 const int retval = sqlite3_step(stmt_new_);
1857
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 618 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 618 assert((retval == SQLITE_DONE) || (retval == SQLITE_OK));
1858
1/2
✓ Branch 1 taken 618 times.
✗ Branch 2 not taken.
 618 sqlite3_reset(stmt_new_);
1859
2/2
✓ Branch 0 taken 336 times.
✓ Branch 1 taken 282 times.
 618 if (!exists)
1860 336 gauge_ += size;
1861 618 return true;
1862 }
1863
1864 int pipe_reserve[2];
1865
1/2
✓ Branch 1 taken 560 times.
✗ Branch 2 not taken.
 560 MakeReturnPipe(pipe_reserve);
1866
1867 560 LruCommand cmd;
1868 560 cmd.command_type = kReserve;
1869 560 cmd.SetSize(size);
1870
1/2
✓ Branch 1 taken 560 times.
✗ Branch 2 not taken.
 560 cmd.StoreHash(hash);
1871 560 cmd.return_pipe = pipe_reserve[1];
1872
1/2
✓ Branch 1 taken 560 times.
✗ Branch 2 not taken.
 560 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
1873 bool result;
1874
1/2
✓ Branch 1 taken 560 times.
✗ Branch 2 not taken.
 560 ManagedReadHalfPipe(pipe_reserve[0], &result, sizeof(result));
1875
1/2
✓ Branch 1 taken 560 times.
✗ Branch 2 not taken.
 560 CloseReturnPipe(pipe_reserve);
1876
1877
2/2
✓ Branch 0 taken 40 times.
✓ Branch 1 taken 520 times.
 560 if (!result)
1878 40 return false;
1879
3/4
✓ Branch 0 taken 80 times.
✓ Branch 1 taken 440 times.
✓ Branch 3 taken 520 times.
✗ Branch 4 not taken.
 520 DoInsert(hash, size, description, is_catalog ? kPin : kPinRegular);
1880
1881 520 return true;
1882 1258 }
1883
1884
1885 2798 PosixQuotaManager::PosixQuotaManager(const uint64_t limit,
1886 const uint64_t cleanup_threshold,
1887 2798 const string &cache_workspace)
1888 2798 : shared_(false)
1889 2798 , spawned_(false)
1890 2798 , limit_(limit)
1891 2798 , cleanup_threshold_(cleanup_threshold)
1892 2798 , gauge_(0)
1893 2798 , pinned_(0)
1894 2798 , seq_(0)
1895 2798 , cache_dir_() // initialized in body
1896 2798 , workspace_dir_() // initialized in body
1897 2798 , fd_lock_cachedb_(-1)
1898 2798 , async_delete_(true)
1899 2798 , cachemgr_pid_(0)
1900 2798 , database_(NULL)
1901 2798 , stmt_touch_(NULL)
1902 2798 , stmt_unpin_(NULL)
1903 2798 , stmt_block_(NULL)
1904 2798 , stmt_unblock_(NULL)
1905 2798 , stmt_new_(NULL)
1906 2798 , stmt_lru_(NULL)
1907 2798 , stmt_size_(NULL)
1908 2798 , stmt_rm_(NULL)
1909 2798 , stmt_rm_batch_(NULL)
1910 2798 , stmt_list_(NULL)
1911 2798 , stmt_list_pinned_(NULL)
1912 2798 , stmt_list_catalogs_(NULL)
1913 2798 , stmt_list_volatile_(NULL)
1914 2798 , initialized_(false)
1915 5596 , cleanup_unused_first_(false) {
1916
2/4
✓ Branch 1 taken 2798 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2798 times.
✗ Branch 5 not taken.
 2798 ParseDirectories(cache_workspace, &cache_dir_, &workspace_dir_);
1917 2798 pipe_lru_[0] = pipe_lru_[1] = -1;
1918
1/2
✓ Branch 1 taken 2798 times.
✗ Branch 2 not taken.
 2798 cleanup_recorder_.AddRecorder(1, 90); // last 1.5 min with second resolution
1919 // last 1.5 h with minute resolution
1920
1/2
✓ Branch 1 taken 2798 times.
✗ Branch 2 not taken.
 2798 cleanup_recorder_.AddRecorder(60, 90 * 60);
1921 // last 18 hours with 20 min resolution
1922
1/2
✓ Branch 1 taken 2798 times.
✗ Branch 2 not taken.
 2798 cleanup_recorder_.AddRecorder(20 * 60, 60 * 60 * 18);
1923 // last 4 days with hour resolution
1924
1/2
✓ Branch 1 taken 2798 times.
✗ Branch 2 not taken.
 2798 cleanup_recorder_.AddRecorder(60 * 60, 60 * 60 * 24 * 4);
1925
1926 2798 lock_open_files_ = reinterpret_cast<pthread_mutex_t *>(
1927 2798 smalloc(sizeof(pthread_mutex_t)));
1928 2798 }
1929
1930
1931 11188 PosixQuotaManager::~PosixQuotaManager() {
1932 5594 free(lock_open_files_);
1933
1934
2/2
✓ Branch 0 taken 120 times.
✓ Branch 1 taken 2677 times.
 5594 if (!initialized_)
1935 240 return;
1936
1937
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2677 times.
 5354 if (shared_) {
1938 // Most of cleanup is done elsewhen by shared cache manager
1939 ✗ close(pipe_lru_[1]);
1940 ✗ return;
1941 }
1942
1943
2/2
✓ Branch 0 taken 1360 times.
✓ Branch 1 taken 1317 times.
 5354 if (spawned_) {
1944 2720 char fin = 0;
1945 2720 WritePipe(pipe_lru_[1], &fin, 1);
1946 2720 close(pipe_lru_[1]);
1947 2720 pthread_join(thread_lru_, NULL);
1948 } else {
1949 2634 ClosePipe(pipe_lru_);
1950 }
1951
1952 5354 CloseDatabase();
1953
14/14
✓ Branch 1 taken 2677 times.
✓ Branch 2 taken 120 times.
✓ Branch 4 taken 2677 times.
✓ Branch 5 taken 120 times.
✓ Branch 7 taken 2677 times.
✓ Branch 8 taken 120 times.
✓ Branch 10 taken 2677 times.
✓ Branch 11 taken 120 times.
✓ Branch 13 taken 2677 times.
✓ Branch 14 taken 120 times.
✓ Branch 16 taken 2677 times.
✓ Branch 17 taken 120 times.
✓ Branch 19 taken 2677 times.
✓ Branch 20 taken 120 times.
 12628 }
1954
1955
1956 191920 void PosixQuotaManager::ProcessCommandBunch(const unsigned num,
1957 const LruCommand *commands,
1958 const char *descriptions) {
1959 191920 int retval = sqlite3_exec(database_, "BEGIN", NULL, NULL, NULL);
1960
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 191920 times.
 191920 assert(retval == SQLITE_OK);
1961
1962
2/2
✓ Branch 0 taken 6002040 times.
✓ Branch 1 taken 191920 times.
 6193960 for (unsigned i = 0; i < num; ++i) {
1963
1/2
✓ Branch 1 taken 6002040 times.
✗ Branch 2 not taken.
 6002040 const shash::Any hash = commands[i].RetrieveHash();
1964
1/2
✓ Branch 1 taken 6002040 times.
✗ Branch 2 not taken.
 6002040 const string hash_str = hash.ToString();
1965 6002040 const unsigned size = commands[i].GetSize();
1966
1/2
✓ Branch 1 taken 6002040 times.
✗ Branch 2 not taken.
 6002040 LogCvmfs(kLogQuota, kLogDebug, "processing %s (%d)", hash_str.c_str(),
1967 6002040 commands[i].command_type);
1968
1969 bool exists;
1970
3/4
✓ Branch 0 taken 2000520 times.
✓ Branch 1 taken 80 times.
✓ Branch 2 taken 4001440 times.
✗ Branch 3 not taken.
 6002040 switch (commands[i].command_type) {
1971 2000520 case kTouch:
1972
1/2
✓ Branch 1 taken 2000520 times.
✗ Branch 2 not taken.
 2000520 sqlite3_bind_int64(stmt_touch_, 1, seq_++);
1973
1/2
✓ Branch 3 taken 2000520 times.
✗ Branch 4 not taken.
 2000520 sqlite3_bind_text(stmt_touch_, 2, &hash_str[0], hash_str.length(),
1974 SQLITE_STATIC);
1975
1/2
✓ Branch 1 taken 2000520 times.
✗ Branch 2 not taken.
 2000520 retval = sqlite3_step(stmt_touch_);
1976
1/2
✓ Branch 1 taken 2000520 times.
✗ Branch 2 not taken.
 2000520 LogCvmfs(kLogQuota, kLogDebug, "touching %s (%ld): %d",
1977 2000520 hash_str.c_str(), seq_ - 1, retval);
1978
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2000520 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 2000520 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
1979 ✗ PANIC(kLogSyslogErr, "failed to update %s in cachedb, error %d",
1980 hash_str.c_str(), retval);
1981 }
1982
1/2
✓ Branch 1 taken 2000520 times.
✗ Branch 2 not taken.
 2000520 sqlite3_reset(stmt_touch_);
1983 2000520 break;
1984 80 case kUnpin:
1985
1/2
✓ Branch 3 taken 80 times.
✗ Branch 4 not taken.
 80 sqlite3_bind_text(stmt_unpin_, 1, &hash_str[0], hash_str.length(),
1986 SQLITE_STATIC);
1987
1/2
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
 80 retval = sqlite3_step(stmt_unpin_);
1988
1/2
✓ Branch 2 taken 80 times.
✗ Branch 3 not taken.
 80 LogCvmfs(kLogQuota, kLogDebug, "unpinning %s: %d", hash_str.c_str(),
1989 retval);
1990
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 80 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
1991 ✗ PANIC(kLogSyslogErr, "failed to unpin %s in cachedb, error %d",
1992 hash_str.c_str(), retval);
1993 }
1994
1/2
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
 80 sqlite3_reset(stmt_unpin_);
1995 80 break;
1996 4001440 case kPin:
1997 case kPinRegular:
1998 case kInsert:
1999 case kInsertVolatile:
2000 // It could already be in, check
2001
1/2
✓ Branch 1 taken 4001440 times.
✗ Branch 2 not taken.
 4001440 exists = Contains(hash_str);
2002
2003 // Cleanup, move to trash and unlink
2004
3/4
✓ Branch 0 taken 4001240 times.
✓ Branch 1 taken 200 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 4001240 times.
 4001440 if (!exists && (gauge_ + size > limit_)) {
2005 ✗ LogCvmfs(kLogQuota, kLogDebug, "over limit, gauge %lu, file size %u",
2006 gauge_, size);
2007 ✗ retval = DoCleanup(cleanup_threshold_);
2008 ✗ assert(retval != 0);
2009 }
2010
2011 // Insert or replace
2012
1/2
✓ Branch 3 taken 4001440 times.
✗ Branch 4 not taken.
 4001440 sqlite3_bind_text(stmt_new_, 1, &hash_str[0], hash_str.length(),
2013 SQLITE_STATIC);
2014
1/2
✓ Branch 1 taken 4001440 times.
✗ Branch 2 not taken.
 4001440 sqlite3_bind_int64(stmt_new_, 2, size);
2015
2/2
✓ Branch 0 taken 160 times.
✓ Branch 1 taken 4001280 times.
 4001440 if (commands[i].command_type == kInsertVolatile) {
2016
1/2
✓ Branch 1 taken 160 times.
✗ Branch 2 not taken.
 160 sqlite3_bind_int64(stmt_new_, 3, (seq_++) | kVolatileFlag);
2017 } else {
2018
1/2
✓ Branch 1 taken 4001280 times.
✗ Branch 2 not taken.
 4001280 sqlite3_bind_int64(stmt_new_, 3, seq_++);
2019 }
2020 4001440 sqlite3_bind_text(stmt_new_, 4, &descriptions[i * kMaxDescription],
2021
1/2
✓ Branch 1 taken 4001440 times.
✗ Branch 2 not taken.
 4001440 commands[i].desc_length, SQLITE_STATIC);
2022
1/2
✓ Branch 1 taken 4001440 times.
✗ Branch 2 not taken.
 4001440 sqlite3_bind_int64(
2023 stmt_new_, 5,
2024
2/2
✓ Branch 0 taken 80 times.
✓ Branch 1 taken 4001360 times.
 4001440 (commands[i].command_type == kPin) ? kFileCatalog : kFileRegular);
2025
1/2
✓ Branch 1 taken 4001440 times.
✗ Branch 2 not taken.
 4001440 sqlite3_bind_int64(stmt_new_, 6,
2026
2/2
✓ Branch 0 taken 4001360 times.
✓ Branch 1 taken 80 times.
 4001440 ((commands[i].command_type == kPin)
2027
2/2
✓ Branch 0 taken 440 times.
✓ Branch 1 taken 4000920 times.
 4001360 || (commands[i].command_type == kPinRegular))
2028 ? 1
2029 : 0);
2030
1/2
✓ Branch 1 taken 4001440 times.
✗ Branch 2 not taken.
 4001440 retval = sqlite3_step(stmt_new_);
2031
1/2
✓ Branch 1 taken 4001440 times.
✗ Branch 2 not taken.
 4001440 LogCvmfs(kLogQuota, kLogDebug, "insert or replace %s, method %d: %d",
2032 4001440 hash_str.c_str(), commands[i].command_type, retval);
2033
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 4001440 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 4001440 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
2034 ✗ PANIC(kLogSyslogErr, "failed to insert %s in cachedb, error %d",
2035 hash_str.c_str(), retval);
2036 }
2037
1/2
✓ Branch 1 taken 4001440 times.
✗ Branch 2 not taken.
 4001440 sqlite3_reset(stmt_new_);
2038
2039
2/2
✓ Branch 0 taken 4001240 times.
✓ Branch 1 taken 200 times.
 4001440 if (!exists)
2040 4001240 gauge_ += size;
2041 4001440 break;
2042 ✗ default:
2043 // other types should have been taken care of by event loop
2044 ✗ PANIC(NULL);
2045 }
2046 6002040 }
2047
2048 191920 retval = sqlite3_exec(database_, "COMMIT", NULL, NULL, NULL);
2049
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 191920 times.
 191920 if (retval != SQLITE_OK) {
2050 ✗ PANIC(kLogSyslogErr, "failed to commit to cachedb, error %d", retval);
2051 }
2052 191920 }
2053
2054
2055 2718 bool PosixQuotaManager::RebuildDatabase() {
2056 2718 bool result = false;
2057 2718 string sql;
2058 2718 sqlite3_stmt *stmt_select = NULL;
2059 2718 sqlite3_stmt *stmt_insert = NULL;
2060 int sqlerr;
2061 2718 int seq = 0;
2062 char hex[4];
2063 struct stat info;
2064 platform_dirent64 *d;
2065 2718 DIR *dirp = NULL;
2066 2718 string path;
2067
2068
1/2
✓ Branch 1 taken 2718 times.
✗ Branch 2 not taken.
 2718 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug, "re-building cache database");
2069
2070 // Empty cache catalog and fscache
2071
1/2
✓ Branch 1 taken 2718 times.
✗ Branch 2 not taken.
 2718 sql = "DELETE FROM cache_catalog; DELETE FROM fscache;";
2072
1/2
✓ Branch 2 taken 2718 times.
✗ Branch 3 not taken.
 2718 sqlerr = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
2073
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2718 times.
 2718 if (sqlerr != SQLITE_OK) {
2074 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not clear cache database");
2075 ✗ goto build_return;
2076 }
2077
2078 2718 gauge_ = 0;
2079
2080 // Insert files from cache sub-directories 00 - ff
2081 // TODO(jblomer): fs_traversal
2082
1/2
✓ Branch 1 taken 2718 times.
✗ Branch 2 not taken.
 2718 sqlite3_prepare_v2(database_,
2083 "INSERT INTO fscache (sha1, size, actime) "
2084 "VALUES (:sha1, :s, :t);",
2085 -1, &stmt_insert, NULL);
2086
2087
2/2
✓ Branch 0 taken 665208 times.
✓ Branch 1 taken 2598 times.
 667806 for (int i = 0; i <= 0xff; i++) {
2088 665208 snprintf(hex, sizeof(hex), "%02x", i);
2089
3/6
✓ Branch 2 taken 665208 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 665208 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 665208 times.
✗ Branch 9 not taken.
 665208 path = cache_dir_ + "/" + string(hex);
2090
3/4
✓ Branch 2 taken 665208 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 120 times.
✓ Branch 5 taken 665088 times.
 665208 if ((dirp = opendir(path.c_str())) == NULL) {
2091
1/2
✓ Branch 2 taken 120 times.
✗ Branch 3 not taken.
 120 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
2092 "failed to open directory %s (tmpwatch interfering?)",
2093 path.c_str());
2094 120 goto build_return;
2095 }
2096
3/4
✓ Branch 1 taken 1995344 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1330256 times.
✓ Branch 4 taken 665088 times.
 1995344 while ((d = platform_readdir(dirp)) != NULL) {
2097
3/6
✓ Branch 2 taken 1330256 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1330256 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1330256 times.
✗ Branch 9 not taken.
 2660512 const string file_path = path + "/" + string(d->d_name);
2098
1/2
✓ Branch 2 taken 1330256 times.
✗ Branch 3 not taken.
 1330256 if (stat(file_path.c_str(), &info) == 0) {
2099
2/2
✓ Branch 0 taken 1330176 times.
✓ Branch 1 taken 80 times.
 1330256 if (!S_ISREG(info.st_mode))
2100 1330216 continue;
2101
2/2
✓ Branch 0 taken 40 times.
✓ Branch 1 taken 40 times.
 80 if (info.st_size == 0) {
2102
1/2
✓ Branch 2 taken 40 times.
✗ Branch 3 not taken.
 40 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug,
2103 "removing empty file %s during automatic cache db rebuild",
2104 file_path.c_str());
2105 40 unlink(file_path.c_str());
2106 40 continue;
2107 }
2108
2109
3/6
✓ Branch 2 taken 40 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 40 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 40 times.
✗ Branch 10 not taken.
 80 string hash = string(hex) + string(d->d_name);
2110
1/2
✓ Branch 3 taken 40 times.
✗ Branch 4 not taken.
 40 sqlite3_bind_text(stmt_insert, 1, hash.data(), hash.length(),
2111 SQLITE_STATIC);
2112
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 sqlite3_bind_int64(stmt_insert, 2, info.st_size);
2113
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 sqlite3_bind_int64(stmt_insert, 3, info.st_atime);
2114
2/4
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 40 times.
 40 if (sqlite3_step(stmt_insert) != SQLITE_DONE) {
2115 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not insert into temp table");
2116 ✗ goto build_return;
2117 }
2118
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 sqlite3_reset(stmt_insert);
2119
2120 40 gauge_ += info.st_size;
2121
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 } else {
2122 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not stat %s", file_path.c_str());
2123 }
2124
2/3
✓ Branch 1 taken 40 times.
✓ Branch 2 taken 1330216 times.
✗ Branch 3 not taken.
 1330256 }
2125
1/2
✓ Branch 1 taken 665088 times.
✗ Branch 2 not taken.
 665088 closedir(dirp);
2126 665088 dirp = NULL;
2127 }
2128
1/2
✓ Branch 1 taken 2598 times.
✗ Branch 2 not taken.
 2598 sqlite3_finalize(stmt_insert);
2129 2598 stmt_insert = NULL;
2130
2131 // Transfer from temp table in cache catalog
2132
1/2
✓ Branch 1 taken 2598 times.
✗ Branch 2 not taken.
 2598 sqlite3_prepare_v2(database_,
2133 "SELECT sha1, size FROM fscache ORDER BY actime;", -1,
2134 &stmt_select, NULL);
2135
1/2
✓ Branch 1 taken 2598 times.
✗ Branch 2 not taken.
 2598 sqlite3_prepare_v2(
2136 database_,
2137 "INSERT INTO cache_catalog (sha1, size, acseq, path, type, pinned) "
2138 "VALUES (:sha1, :s, :seq, 'unknown (automatic rebuild)', :t, 0);",
2139 -1, &stmt_insert, NULL);
2140
3/4
✓ Branch 1 taken 2638 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 40 times.
✓ Branch 4 taken 2598 times.
 2638 while (sqlite3_step(stmt_select) == SQLITE_ROW) {
2141 const string hash = string(
2142
2/4
✓ Branch 2 taken 40 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 40 times.
✗ Branch 6 not taken.
 40 reinterpret_cast<const char *>(sqlite3_column_text(stmt_select, 0)));
2143
1/2
✓ Branch 3 taken 40 times.
✗ Branch 4 not taken.
 40 sqlite3_bind_text(stmt_insert, 1, &hash[0], hash.length(), SQLITE_STATIC);
2144
2/4
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 40 times.
✗ Branch 5 not taken.
 40 sqlite3_bind_int64(stmt_insert, 2, sqlite3_column_int64(stmt_select, 1));
2145
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 sqlite3_bind_int64(stmt_insert, 3, seq++);
2146 // Might also be a catalog (information is lost)
2147
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 sqlite3_bind_int64(stmt_insert, 4, kFileRegular);
2148
2149
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 const int retval = sqlite3_step(stmt_insert);
2150
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 40 times.
 40 if (retval != SQLITE_DONE) {
2151 // If the file system hosting the cache is full, we'll likely notice here
2152 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
2153 "could not insert into cache catalog (%d - %s)", retval,
2154 sqlite3_errstr(retval));
2155 ✗ goto build_return;
2156 }
2157
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 sqlite3_reset(stmt_insert);
2158
1/2
✓ Branch 1 taken 40 times.
✗ Branch 2 not taken.
 40 }
2159
2160 // Delete temporary table
2161
1/2
✓ Branch 1 taken 2598 times.
✗ Branch 2 not taken.
 2598 sql = "DELETE FROM fscache;";
2162
1/2
✓ Branch 2 taken 2598 times.
✗ Branch 3 not taken.
 2598 sqlerr = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
2163
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2598 times.
 2598 if (sqlerr != SQLITE_OK) {
2164 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not clear temporary table (%d)",
2165 sqlerr);
2166 ✗ goto build_return;
2167 }
2168
2169 2598 seq_ = seq;
2170 2598 result = true;
2171
1/2
✓ Branch 1 taken 2598 times.
✗ Branch 2 not taken.
 2598 LogCvmfs(kLogQuota, kLogDebug,
2172 "rebuilding finished, sequence %" PRIu64 ", gauge %" PRIu64, seq_,
2173 gauge_);
2174
2175 2718 build_return:
2176
1/2
✓ Branch 0 taken 2718 times.
✗ Branch 1 not taken.
 2718 if (stmt_insert)
2177
1/2
✓ Branch 1 taken 2718 times.
✗ Branch 2 not taken.
 2718 sqlite3_finalize(stmt_insert);
2178
2/2
✓ Branch 0 taken 2598 times.
✓ Branch 1 taken 120 times.
 2718 if (stmt_select)
2179
1/2
✓ Branch 1 taken 2598 times.
✗ Branch 2 not taken.
 2598 sqlite3_finalize(stmt_select);
2180
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2718 times.
 2718 if (dirp)
2181 ✗ closedir(dirp);
2182 2718 return result;
2183 2718 }
2184
2185
2186 /**
2187 * Register a channel that allows the cache manager to trigger action to its
2188 * clients. Currently used for releasing pinned catalogs.
2189 */
2190 160 void PosixQuotaManager::RegisterBackChannel(int back_channel[2],
2191 const string &channel_id) {
2192
1/2
✓ Branch 0 taken 160 times.
✗ Branch 1 not taken.
 160 if (protocol_revision_ >= 1) {
2193
1/2
✓ Branch 2 taken 160 times.
✗ Branch 3 not taken.
 160 shash::Md5 hash = shash::Md5(shash::AsciiPtr(channel_id));
2194
1/2
✓ Branch 1 taken 160 times.
✗ Branch 2 not taken.
 160 MakeReturnPipe(back_channel);
2195
2196 160 LruCommand cmd;
2197 160 cmd.command_type = kRegisterBackChannel;
2198 160 cmd.return_pipe = back_channel[1];
2199 // Not StoreHash(). This is an MD5 hash.
2200 160 memcpy(cmd.digest, hash.digest, hash.GetDigestSize());
2201
1/2
✓ Branch 1 taken 160 times.
✗ Branch 2 not taken.
 160 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2202
2203 char success;
2204
1/2
✓ Branch 1 taken 160 times.
✗ Branch 2 not taken.
 160 ManagedReadHalfPipe(back_channel[0], &success, sizeof(success));
2205 // At this point, the named FIFO is unlinked, so don't use CloseReturnPipe
2206
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 160 times.
 160 if (success != 'S') {
2207 ✗ PANIC(kLogDebug | kLogSyslogErr,
2208 "failed to register quota back channel (%c)", success);
2209 }
2210 } else {
2211 // Dummy pipe to return valid file descriptors
2212 ✗ MakePipe(back_channel);
2213 }
2214 160 }
2215
2216
2217 /**
2218 * Removes a chunk from cache, if it exists.
2219 */
2220 120 void PosixQuotaManager::Remove(const shash::Any &hash) {
2221
1/2
✓ Branch 1 taken 120 times.
✗ Branch 2 not taken.
 120 const string hash_str = hash.ToString();
2222
2223 int pipe_remove[2];
2224
1/2
✓ Branch 1 taken 120 times.
✗ Branch 2 not taken.
 120 MakeReturnPipe(pipe_remove);
2225
2226 120 LruCommand cmd;
2227 120 cmd.command_type = kRemove;
2228 120 cmd.return_pipe = pipe_remove[1];
2229
1/2
✓ Branch 1 taken 120 times.
✗ Branch 2 not taken.
 120 cmd.StoreHash(hash);
2230
1/2
✓ Branch 1 taken 120 times.
✗ Branch 2 not taken.
 120 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2231
2232 bool success;
2233
1/2
✓ Branch 1 taken 120 times.
✗ Branch 2 not taken.
 120 ManagedReadHalfPipe(pipe_remove[0], &success, sizeof(success));
2234
1/2
✓ Branch 1 taken 120 times.
✗ Branch 2 not taken.
 120 CloseReturnPipe(pipe_remove);
2235
2236
3/6
✓ Branch 1 taken 120 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 120 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 120 times.
✗ Branch 8 not taken.
 120 unlink((cache_dir_ + "/" + hash.MakePathWithoutSuffix()).c_str());
2237 120 }
2238
2239
2240 1440 void PosixQuotaManager::Spawn() {
2241
2/2
✓ Branch 0 taken 80 times.
✓ Branch 1 taken 1360 times.
 1440 if (spawned_)
2242 80 return;
2243
2244 1360 if (pthread_create(&thread_lru_, NULL, MainCommandServer,
2245 static_cast<void *>(this))
2246
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1360 times.
 1360 != 0) {
2247 ✗ PANIC(kLogDebug, "could not create lru thread");
2248 }
2249
2250 1360 spawned_ = true;
2251 }
2252
2253
2254 /**
2255 * Updates the sequence number of the file specified by the hash.
2256 */
2257 2000383 void PosixQuotaManager::Touch(const shash::Any &hash) {
2258 2000383 LruCommand cmd;
2259 2000383 cmd.command_type = kTouch;
2260
1/2
✓ Branch 1 taken 2000383 times.
✗ Branch 2 not taken.
 2000383 cmd.StoreHash(hash);
2261
1/2
✓ Branch 1 taken 2000383 times.
✗ Branch 2 not taken.
 2000383 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2262 2000383 }
2263
2264
2265 3520 void PosixQuotaManager::UnbindReturnPipe(int pipe_wronly) {
2266
2/2
✓ Branch 0 taken 40 times.
✓ Branch 1 taken 3480 times.
 3520 if (shared_)
2267 40 close(pipe_wronly);
2268 3520 }
2269
2270
2271 280 void PosixQuotaManager::UnlinkReturnPipe(int pipe_wronly) {
2272
2/2
✓ Branch 0 taken 120 times.
✓ Branch 1 taken 160 times.
 280 if (shared_)
2273
2/4
✓ Branch 2 taken 120 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 120 times.
✗ Branch 6 not taken.
 120 unlink((workspace_dir_ + "/pipe" + StringifyInt(pipe_wronly)).c_str());
2274 280 }
2275
2276
2277 298 void PosixQuotaManager::Unpin(const shash::Any &hash) {
2278
2/4
✓ Branch 1 taken 298 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 298 times.
✗ Branch 6 not taken.
 298 LogCvmfs(kLogQuota, kLogDebug, "Unpin %s", hash.ToString().c_str());
2279
2280 298 LruCommand cmd;
2281 298 cmd.command_type = kUnpin;
2282
1/2
✓ Branch 1 taken 298 times.
✗ Branch 2 not taken.
 298 cmd.StoreHash(hash);
2283
1/2
✓ Branch 1 taken 298 times.
✗ Branch 2 not taken.
 298 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2284 298 }
2285
2286
2287 80 void PosixQuotaManager::UnregisterBackChannel(int back_channel[2],
2288 const string &channel_id) {
2289
1/2
✓ Branch 0 taken 80 times.
✗ Branch 1 not taken.
 80 if (protocol_revision_ >= 1) {
2290
1/2
✓ Branch 2 taken 80 times.
✗ Branch 3 not taken.
 80 shash::Md5 hash = shash::Md5(shash::AsciiPtr(channel_id));
2291
2292 80 LruCommand cmd;
2293 80 cmd.command_type = kUnregisterBackChannel;
2294 // Not StoreHash(). This is an MD5 hash.
2295 80 memcpy(cmd.digest, hash.digest, hash.GetDigestSize());
2296
1/2
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
 80 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2297
2298 // Writer's end will be closed by cache manager, FIFO is already unlinked
2299
1/2
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
 80 close(back_channel[0]);
2300 } else {
2301 ✗ ClosePipe(back_channel);
2302 }
2303 80 }
2304
2305 4005280 void PosixQuotaManager::ManagedReadHalfPipe(int fd, void *buf, size_t nbyte) {
2306
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4005280 times.
 4005280 const unsigned timeout_ms = cachemgr_pid_ ? 1000 : 0;
2307 4005280 bool result = false;
2308 do {
2309 4005280 result = ReadHalfPipe(fd, buf, nbyte, timeout_ms);
2310 // try only as long as the cachemgr is still alive
2311
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 4005280 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 4005280 times.
 4005280 } while (!result && getpgid(cachemgr_pid_) >= 0);
2312
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4005280 times.
 4005280 if (!result) {
2313 ✗ PANIC(kLogStderr,
2314 "Error: quota manager could not read from cachemanager pipe");
2315 }
2316 4005280 }
2317
2318 ✗ void *PosixQuotaManager::CollectMountpointsHashes(void *data) {
2319 #ifndef __APPLE__
2320 ✗ pthread_setname_np(pthread_self(), "hash_collector");
2321 ✗ auto *handler = static_cast<CollectorHandler *>(data);
2322
2323 ✗ const std::string mountpoint = handler->mp[handler->i];
2324 ✗ ssize_t n = getxattr(mountpoint.c_str(), "user.list_open_hashes", nullptr, 0);
2325 ✗ if (n < 0) {
2326 ✗ pthread_exit(nullptr);
2327 }
2328 ✗ std::vector<char> buf((size_t)n);
2329 ✗ n = getxattr(mountpoint.c_str(), "user.list_open_hashes", buf.data(),
2330 buf.size());
2331 ✗ if (n < 0) {
2332 ✗ pthread_exit(nullptr);
2333 }
2334
2335 ✗ std::vector<std::string> hash_strs;
2336 ✗ std::string hash_str;
2337 ✗ for (const char c : buf) {
2338 ✗ if (c == '\n') {
2339 ✗ hash_strs.push_back(hash_str);
2340 ✗ hash_str.clear();
2341 } else {
2342 ✗ hash_str += c;
2343 }
2344 }
2345 ✗ const MutexLockGuard lock_guard(handler->l);
2346 ✗ for (auto hash_str : hash_strs) {
2347 ✗ handler->of.push_back(
2348 ✗ shash::Short(shash::MkFromHexPtr(shash::HexPtr(hash_str))));
2349 }
2350 #endif
2351 ✗ pthread_exit(nullptr);
2352 }
2353
2354 ✗ std::vector<shash::Short> PosixQuotaManager::CollectAllOpenHashes() {
2355 ✗ std::vector<CollectorHandler *> handlers;
2356 ✗ std::vector<pthread_t *> threads;
2357 ✗ open_files_.clear();
2358 #ifndef __APPLE__
2359 ✗ auto &&a_after_b = [](const struct timespec a, const struct timespec b) {
2360 ✗ return (a.tv_sec > b.tv_sec) ? true : false;
2361 };
2362
2363 ✗ for (size_t i = 0; i < mountpoints_.size(); ++i) {
2364 ✗ handlers.push_back(
2365 ✗ new CollectorHandler{open_files_, mountpoints_, lock_open_files_, i});
2366 ✗ threads.push_back(new pthread_t);
2367 }
2368
2369 ✗ const int retval = pthread_mutex_init(lock_open_files_, NULL);
2370 ✗ assert(retval == 0);
2371
2372 ✗ for (size_t i = 0; i < mountpoints_.size(); ++i) {
2373 ✗ pthread_create(threads[i], nullptr, CollectMountpointsHashes, handlers[i]);
2374 }
2375
2376 ✗ std::vector<bool> joined(handlers.size(), false);
2377 struct timespec reference, current;
2378 ✗ clock_gettime(CLOCK_REALTIME, &reference);
2379 ✗ clock_gettime(CLOCK_REALTIME, &current);
2380 ✗ reference.tv_sec += 10; // Give 10sec for hash collection
2381 ✗ size_t i = 0;
2382 ✗ while (
2383 ✗ (not std::all_of(joined.begin(), joined.end(), [](bool b) { return b; }))
2384 ✗ and a_after_b(reference, current)) {
2385 // as long as there are still threads that haven't joined yet
2386 // and for 10 seconds
2387 ✗ if (not joined[i]) {
2388 ✗ const int s = pthread_tryjoin_np(*threads[i], NULL);
2389 ✗ if (s == 0) {
2390 ✗ joined[i] = true;
2391 }
2392 }
2393 ✗ ++i;
2394 ✗ i = i % handlers.size();
2395 ✗ clock_gettime(CLOCK_REALTIME, &current);
2396 }
2397
2398 ✗ for (size_t i = 0; i < handlers.size(); ++i) {
2399 ✗ delete handlers[i];
2400 }
2401
2402 ✗ pthread_mutex_destroy(lock_open_files_);
2403 #endif
2404 ✗ return open_files_;
2405 }
2406
2407