GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/quota_posix.cc
Date: 2026-10-25 02:40:14
Exec Total Coverage
Lines: 987 1424 69.3%
Branches: 769 1966 39.1%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * This module implements a "managed local cache".
5 * This way, we are able to track access times of files in the cache
6 * and remove files based on least recently used strategy.
7 *
8 * We setup another SQLite catalog, a "cache catalog", that helps us
9 * in the bookkeeping of files, file sizes and access times.
10 *
11 * We might choose to not manage the local cache. This is indicated
12 * by limit == 0 and everything succeeds in that case.
13 */
14
15 #define __STDC_LIMIT_MACROS
16
17
18 #include "quota_posix.h"
19
20 #include <dirent.h>
21 #include <errno.h>
22 #include <fcntl.h>
23 #include <inttypes.h>
24 #include <pthread.h>
25 #include <signal.h>
26 #include <stdint.h>
27 #include <sys/dir.h>
28 #include <sys/stat.h>
29 #include <sys/xattr.h>
30
31 #ifndef __APPLE__
32 #include <sys/statfs.h>
33 #endif
34 #include <sys/statvfs.h>
35 #include <sys/types.h>
36 #include <sys/wait.h>
37 #include <unistd.h>
38
39 #include <algorithm>
40 #include <cassert>
41 #include <cstdio>
42 #include <cstdlib>
43 #include <cstring>
44 #include <limits>
45 #include <map>
46 #include <memory>
47 #include <set>
48 #include <string>
49 #include <vector>
50
51 #include "crypto/hash.h"
52 #include "duplex_sqlite3.h"
53 #include "monitor.h"
54 #include "statistics.h"
55 #include "util/capabilities.h"
56 #include "util/concurrency.h"
57 #include "util/exception.h"
58 #include "util/logging.h"
59 #include "util/posix.h"
60 #include "util/smalloc.h"
61 #include "util/string.h"
62
63 using namespace std; // NOLINT
64
65
66 846 int PosixQuotaManager::BindReturnPipe(int pipe_wronly) {
67
2/2
✓ Branch 0 taken 828 times.
✓ Branch 1 taken 18 times.
 846 if (!shared_)
68 828 return pipe_wronly;
69
70 // Connect writer's end
71
1/2
✓ Branch 2 taken 18 times.
✗ Branch 3 not taken.
 18 const int result = open(
72
2/4
✓ Branch 2 taken 18 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 18 times.
✗ Branch 6 not taken.
 36 (workspace_dir_ + "/pipe" + StringifyInt(pipe_wronly)).c_str(),
73 O_WRONLY | O_NONBLOCK);
74
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 9 times.
 18 if (result >= 0) {
75 9 Nonblock2Block(result);
76 } else {
77 9 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
78 9 "failed to bind return pipe (%d)", errno);
79 }
80 18 return result;
81 }
82
83
84 741 void PosixQuotaManager::CheckHighPinWatermark() {
85 741 const uint64_t watermark = kHighPinWatermark * cleanup_threshold_ / 100;
86
3/4
✓ Branch 0 taken 741 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 27 times.
✓ Branch 3 taken 714 times.
 741 if ((cleanup_threshold_ > 0) && (pinned_ > watermark)) {
87 27 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
88 "high watermark of pinned files (%" PRIu64 "M > %" PRIu64 "M)",
89 27 pinned_ / (1024 * 1024), watermark / (1024 * 1024));
90
2/4
✓ Branch 2 taken 27 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 27 times.
✗ Branch 6 not taken.
 27 BroadcastBackchannels("R"); // clients: please release pinned catalogs
91 }
92 741 }
93
94
95 ✗ void PosixQuotaManager::CleanupPipes() {
96 ✗ DIR *dirp = opendir(workspace_dir_.c_str());
97 ✗ assert(dirp != NULL);
98
99 platform_dirent64 *dent;
100 ✗ bool found_leftovers = false;
101 ✗ while ((dent = platform_readdir(dirp)) != NULL) {
102 ✗ const string name = dent->d_name;
103 ✗ const string path = workspace_dir_ + "/" + name;
104 platform_stat64 info;
105 ✗ const int retval = platform_stat(path.c_str(), &info);
106 ✗ if (retval != 0)
107 ✗ continue;
108
109 // CVMFS FIFOs are prefixed with "pipe"
110 ✗ if (S_ISFIFO(info.st_mode) && (name.substr(0, 4) == "pipe")) {
111 ✗ if (!found_leftovers) {
112 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogWarn,
113 "removing left-over FIFOs from cache directory");
114 }
115 ✗ found_leftovers = true;
116 ✗ unlink(path.c_str());
117 }
118 }
119 ✗ closedir(dirp);
120 }
121
122
123 /**
124 * Cleans up in data cache, until cache size is below leave_size.
125 * The actual unlinking is done in a separate process (fork).
126 *
127 * \return True on success, false otherwise
128 */
129 81 bool PosixQuotaManager::Cleanup(const uint64_t leave_size) {
130
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 81 times.
 81 if (!spawned_)
131 ✗ return DoCleanup(leave_size);
132
133 bool result;
134 int pipe_cleanup[2];
135
1/2
✓ Branch 1 taken 81 times.
✗ Branch 2 not taken.
 81 MakeReturnPipe(pipe_cleanup);
136
137 81 LruCommand cmd;
138 81 cmd.command_type = kCleanup;
139 81 cmd.size = leave_size;
140 81 cmd.return_pipe = pipe_cleanup[1];
141
142
1/2
✓ Branch 1 taken 81 times.
✗ Branch 2 not taken.
 81 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
143
1/2
✓ Branch 1 taken 81 times.
✗ Branch 2 not taken.
 81 ManagedReadHalfPipe(pipe_cleanup[0], &result, sizeof(result));
144
1/2
✓ Branch 1 taken 81 times.
✗ Branch 2 not taken.
 81 CloseReturnPipe(pipe_cleanup);
145
146 81 return result;
147 }
148
149
150 2163 void PosixQuotaManager::CloseDatabase() {
151
1/2
✓ Branch 0 taken 2163 times.
✗ Branch 1 not taken.
 2163 if (stmt_list_catalogs_)
152 2163 sqlite3_finalize(stmt_list_catalogs_);
153
1/2
✓ Branch 0 taken 2163 times.
✗ Branch 1 not taken.
 2163 if (stmt_list_pinned_)
154 2163 sqlite3_finalize(stmt_list_pinned_);
155
1/2
✓ Branch 0 taken 2163 times.
✗ Branch 1 not taken.
 2163 if (stmt_list_volatile_)
156 2163 sqlite3_finalize(stmt_list_volatile_);
157
1/2
✓ Branch 0 taken 2163 times.
✗ Branch 1 not taken.
 2163 if (stmt_list_)
158 2163 sqlite3_finalize(stmt_list_);
159
1/2
✓ Branch 0 taken 2163 times.
✗ Branch 1 not taken.
 2163 if (stmt_lru_)
160 2163 sqlite3_finalize(stmt_lru_);
161
1/2
✓ Branch 0 taken 2163 times.
✗ Branch 1 not taken.
 2163 if (stmt_rm_)
162 2163 sqlite3_finalize(stmt_rm_);
163
1/2
✓ Branch 0 taken 2163 times.
✗ Branch 1 not taken.
 2163 if (stmt_rm_batch_)
164 2163 sqlite3_finalize(stmt_rm_batch_);
165
1/2
✓ Branch 0 taken 2163 times.
✗ Branch 1 not taken.
 2163 if (stmt_size_)
166 2163 sqlite3_finalize(stmt_size_);
167
1/2
✓ Branch 0 taken 2163 times.
✗ Branch 1 not taken.
 2163 if (stmt_touch_)
168 2163 sqlite3_finalize(stmt_touch_);
169
1/2
✓ Branch 0 taken 2163 times.
✗ Branch 1 not taken.
 2163 if (stmt_unpin_)
170 2163 sqlite3_finalize(stmt_unpin_);
171
1/2
✓ Branch 0 taken 2163 times.
✗ Branch 1 not taken.
 2163 if (stmt_block_)
172 2163 sqlite3_finalize(stmt_block_);
173
1/2
✓ Branch 0 taken 2163 times.
✗ Branch 1 not taken.
 2163 if (stmt_unblock_)
174 2163 sqlite3_finalize(stmt_unblock_);
175
1/2
✓ Branch 0 taken 2163 times.
✗ Branch 1 not taken.
 2163 if (stmt_new_)
176 2163 sqlite3_finalize(stmt_new_);
177
1/2
✓ Branch 0 taken 2163 times.
✗ Branch 1 not taken.
 2163 if (database_)
178 2163 sqlite3_close(database_);
179 2163 UnlockFile(fd_lock_cachedb_);
180
181 2163 stmt_list_catalogs_ = NULL;
182 2163 stmt_list_pinned_ = NULL;
183 2163 stmt_list_volatile_ = NULL;
184 2163 stmt_list_ = NULL;
185 2163 stmt_rm_ = NULL;
186 2163 stmt_rm_batch_ = NULL;
187 2163 stmt_size_ = NULL;
188 2163 stmt_touch_ = NULL;
189 2163 stmt_unpin_ = NULL;
190 2163 stmt_block_ = NULL;
191 2163 stmt_unblock_ = NULL;
192 2163 stmt_new_ = NULL;
193 2163 database_ = NULL;
194
195 2163 pinned_chunks_.clear();
196 2163 }
197
198
199 801 void PosixQuotaManager::CloseReturnPipe(int pipe[2]) {
200
2/2
✓ Branch 0 taken 18 times.
✓ Branch 1 taken 783 times.
 801 if (shared_) {
201 18 close(pipe[0]);
202 18 UnlinkReturnPipe(pipe[1]);
203 } else {
204 783 ClosePipe(pipe);
205 }
206 801 }
207
208
209 901530 bool PosixQuotaManager::Contains(const string &hash_str) {
210 901530 bool result = false;
211
212 901530 sqlite3_bind_text(stmt_size_, 1, &hash_str[0], hash_str.length(),
213 SQLITE_STATIC);
214
2/2
✓ Branch 1 taken 600 times.
✓ Branch 2 taken 900930 times.
 901530 if (sqlite3_step(stmt_size_) == SQLITE_ROW)
215 600 result = true;
216 901530 sqlite3_reset(stmt_size_);
217 901530 LogCvmfs(kLogQuota, kLogDebug, "contains %s returns %d", hash_str.c_str(),
218 result);
219
220 901530 return result;
221 }
222
223
224 2146 void PosixQuotaManager::CheckFreeSpace() {
225
3/4
✓ Branch 0 taken 2146 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 9 times.
✓ Branch 3 taken 2137 times.
 2146 if ((limit_ == 0) || (gauge_ >= limit_))
226 9 return;
227
228 struct statvfs vfs_info;
229
1/2
✓ Branch 1 taken 2137 times.
✗ Branch 2 not taken.
 2137 const int retval = statvfs((cache_dir_ + "/cachedb").c_str(), &vfs_info);
230
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2137 times.
 2137 if (retval != 0) {
231 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
232 "failed to query %s for free space (%d)", cache_dir_.c_str(),
233 ✗ errno);
234 ✗ return;
235 }
236 2137 const int64_t free_space_byte = vfs_info.f_bavail * vfs_info.f_bsize;
237
1/2
✓ Branch 1 taken 2137 times.
✗ Branch 2 not taken.
 2137 LogCvmfs(kLogQuota, kLogDebug, "free space: %" PRId64 " MB",
238 free_space_byte / (1024 * 1024));
239
240 2137 const int64_t required_byte = limit_ - gauge_;
241
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2137 times.
 2137 if (free_space_byte < required_byte) {
242 ✗ LogCvmfs(kLogQuota, kLogSyslogWarn,
243 "too little free space on the file system hosting the cache,"
244 " %" PRId64 " MB available",
245 free_space_byte / (1024 * 1024));
246 }
247 }
248
249
250 2173 PosixQuotaManager *PosixQuotaManager::Create(const string &cache_workspace,
251 const uint64_t limit,
252 const uint64_t cleanup_threshold,
253 const bool rebuild_database) {
254
2/2
✓ Branch 0 taken 18 times.
✓ Branch 1 taken 2155 times.
 2173 if (cleanup_threshold >= limit) {
255 18 LogCvmfs(kLogQuota, kLogDebug,
256 "invalid parameters: limit %" PRIu64 ", "
257 "cleanup_threshold %" PRIu64,
258 limit, cleanup_threshold);
259 18 return NULL;
260 }
261
262 PosixQuotaManager *quota_manager = new PosixQuotaManager(
263
1/2
✓ Branch 2 taken 2155 times.
✗ Branch 3 not taken.
 2155 limit, cleanup_threshold, cache_workspace);
264
265 // Initialize cache catalog
266
2/2
✓ Branch 1 taken 9 times.
✓ Branch 2 taken 2146 times.
 2155 if (!quota_manager->InitDatabase(rebuild_database)) {
267
1/2
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
 9 delete quota_manager;
268 9 return NULL;
269 }
270 2146 quota_manager->CheckFreeSpace();
271 2146 MakePipe(quota_manager->pipe_lru_);
272
273 2146 quota_manager->protocol_revision_ = kProtocolRevision;
274 2146 quota_manager->initialized_ = true;
275 2146 return quota_manager;
276 }
277
278
279 /**
280 * Connects to a running shared local quota manager. Creates one if necessary.
281 */
282 18 PosixQuotaManager *PosixQuotaManager::CreateShared(
283 const std::string &exe_path,
284 const std::string &cache_workspace,
285 const uint64_t limit,
286 const uint64_t cleanup_threshold,
287 bool foreground) {
288 18 string cache_dir;
289 18 string workspace_dir;
290
2/4
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 18 times.
✗ Branch 5 not taken.
 18 ParseDirectories(cache_workspace, &cache_dir, &workspace_dir);
291
292 pid_t new_cachemgr_pid;
293
294 // Create lock file: only one fuse client at a time
295
2/4
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 18 times.
✗ Branch 5 not taken.
 18 const int fd_lockfile = LockFile(workspace_dir + "/lock_cachemgr");
296
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 9 times.
 18 if (fd_lockfile < 0) {
297
1/2
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
 9 LogCvmfs(kLogQuota, kLogDebug, "could not open lock file %s (%d)",
298
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 18 (workspace_dir + "/lock_cachemgr").c_str(), errno);
299 9 return NULL;
300 }
301
302 PosixQuotaManager *quota_mgr = new PosixQuotaManager(limit, cleanup_threshold,
303
2/4
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 9 times.
✗ Branch 5 not taken.
 9 cache_workspace);
304 9 quota_mgr->shared_ = true;
305 9 quota_mgr->spawned_ = true;
306
307 // Try to connect to pipe
308
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 const string fifo_path = workspace_dir + "/cachemgr";
309
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 LogCvmfs(kLogQuota, kLogDebug, "trying to connect to existing pipe");
310
1/2
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
 9 quota_mgr->pipe_lru_[1] = open(fifo_path.c_str(), O_WRONLY | O_NONBLOCK);
311
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
 9 if (quota_mgr->pipe_lru_[1] >= 0) {
312 ✗ const int fd_lockfile_rw = open((workspace_dir + "/lock_cachemgr").c_str(),
313 O_RDWR, 0600);
314 ✗ unsigned lockfile_magicnumber = 0;
315 ✗ const ssize_t result_mn = SafeRead(fd_lockfile_rw, &lockfile_magicnumber,
316 sizeof(lockfile_magicnumber));
317 ✗ const ssize_t result = SafeRead(fd_lockfile_rw, &new_cachemgr_pid,
318 sizeof(new_cachemgr_pid));
319 ✗ close(fd_lockfile_rw);
320
321 ✗ if ((lockfile_magicnumber != kLockFileMagicNumber) || (result < 0)
322 ✗ || (result_mn < 0)
323 ✗ || (static_cast<size_t>(result) < sizeof(new_cachemgr_pid))) {
324 ✗ if (result != 0) {
325 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
326 "could not read cache manager pid from lockfile");
327 ✗ UnlockFile(fd_lockfile);
328 ✗ delete quota_mgr;
329 ✗ return NULL;
330 } else {
331 // support reload from old versions of the cache manager
332 // lock file is empty in this case, try a plain ReadHalfPipe to get pid
333 ✗ quota_mgr->SetCacheMgrPid(quota_mgr->GetPid());
334 }
335 } else {
336 ✗ quota_mgr->SetCacheMgrPid(new_cachemgr_pid);
337 }
338
339
340 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to existing cache manager pipe");
341 ✗ quota_mgr->initialized_ = true;
342 ✗ Nonblock2Block(quota_mgr->pipe_lru_[1]);
343 ✗ UnlockFile(fd_lockfile);
344 ✗ quota_mgr->GetLimits(&quota_mgr->limit_, &quota_mgr->cleanup_threshold_);
345 ✗ LogCvmfs(kLogQuota, kLogDebug,
346 "received limit %" PRIu64 ", threshold %" PRIu64,
347 quota_mgr->limit_, quota_mgr->cleanup_threshold_);
348 ✗ if (FileExists(workspace_dir + "/cachemgr.protocol")) {
349 ✗ quota_mgr->protocol_revision_ = quota_mgr->GetProtocolRevision();
350 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected protocol revision %u",
351 quota_mgr->protocol_revision_);
352 } else {
353 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to ancient cache manager");
354 }
355 ✗ return quota_mgr;
356 }
357 9 const int connect_error = errno;
358
359 // Lock file: let existing cache manager finish first
360
2/4
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 9 times.
✗ Branch 5 not taken.
 9 const int fd_lockfile_fifo = LockFile(workspace_dir + "/lock_cachemgr.fifo");
361
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
 9 if (fd_lockfile_fifo < 0) {
362 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not open lock file %s (%d)",
363 ✗ (workspace_dir + "/lock_cachemgr.fifo").c_str(), errno);
364 ✗ UnlockFile(fd_lockfile);
365 ✗ delete quota_mgr;
366 ✗ return NULL;
367 }
368
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 UnlockFile(fd_lockfile_fifo);
369
370
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
 9 if (connect_error == ENXIO) {
371 ✗ LogCvmfs(kLogQuota, kLogDebug, "left-over FIFO found, unlinking");
372 ✗ unlink(fifo_path.c_str());
373 }
374
375 // Creating a new FIFO for the cache manager (to be bound later)
376 9 int retval = mkfifo(fifo_path.c_str(), 0600);
377
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
 9 if (retval != 0) {
378 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to create cache manager FIFO (%d)",
379 ✗ errno);
380 ✗ UnlockFile(fd_lockfile);
381 ✗ delete quota_mgr;
382 ✗ return NULL;
383 }
384
385 // Create new cache manager
386 int pipe_boot[2];
387 int pipe_handshake[2];
388
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 MakePipe(pipe_boot);
389
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 MakePipe(pipe_handshake);
390
391 9 vector<string> command_line;
392
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 command_line.push_back(exe_path);
393
2/4
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 9 times.
✗ Branch 6 not taken.
 9 command_line.push_back("__cachemgr__");
394
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 command_line.push_back(cache_workspace);
395
2/4
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 9 times.
✗ Branch 5 not taken.
 9 command_line.push_back(StringifyInt(pipe_boot[1]));
396
2/4
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 9 times.
✗ Branch 5 not taken.
 9 command_line.push_back(StringifyInt(pipe_handshake[0]));
397
2/4
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 9 times.
✗ Branch 5 not taken.
 9 command_line.push_back(StringifyInt(limit));
398
2/4
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 9 times.
✗ Branch 5 not taken.
 9 command_line.push_back(StringifyInt(cleanup_threshold));
399 // do not propagate foreground in order to reliably get pid from exec
400 // instead, daemonize right here
401
2/4
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 9 times.
✗ Branch 5 not taken.
 9 command_line.push_back(StringifyInt(true)); // foreground
402
3/6
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 9 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 9 times.
✗ Branch 8 not taken.
 9 command_line.push_back(StringifyInt(GetLogSyslogLevel()));
403
3/6
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 9 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 9 times.
✗ Branch 8 not taken.
 9 command_line.push_back(StringifyInt(GetLogSyslogFacility()));
404
5/14
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 9 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 9 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 9 times.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 9 times.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
 9 command_line.push_back(GetLogDebugFile() + ":" + GetLogMicroSyslog());
405
406 9 set<int> preserve_filedes;
407
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 preserve_filedes.insert(0);
408
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 preserve_filedes.insert(1);
409
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 preserve_filedes.insert(2);
410
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 preserve_filedes.insert(pipe_boot[1]);
411
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 preserve_filedes.insert(pipe_handshake[0]);
412
413
1/2
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
 9 if (foreground) {
414
1/2
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
 9 retval = ManagedExec(command_line, preserve_filedes, map<int, int>(),
415 /*drop_credentials*/ false,
416 /*clear_env*/ false,
417 /*double_fork*/ true, &new_cachemgr_pid);
418 } else {
419 ✗ retval = ExecAsDaemon(command_line, &new_cachemgr_pid);
420 }
421
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
 9 if (!retval) {
422 ✗ UnlockFile(fd_lockfile);
423 ✗ ClosePipe(pipe_boot);
424 ✗ ClosePipe(pipe_handshake);
425 ✗ delete quota_mgr;
426 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to start cache manager");
427 ✗ return NULL;
428 }
429
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 LogCvmfs(kLogQuota, kLogDebug,
430 "new cache manager pid: %d protocol revision %d", new_cachemgr_pid,
431 QuotaManager::kProtocolRevision);
432 9 quota_mgr->SetCacheMgrPid(new_cachemgr_pid);
433
2/4
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 9 times.
✗ Branch 6 not taken.
 9 const int fd_lockfile_rw = open((workspace_dir + "/lock_cachemgr").c_str(),
434 O_RDWR | O_TRUNC, 0600);
435 9 const unsigned magic_number = PosixQuotaManager::kLockFileMagicNumber;
436
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 const bool result_mn = SafeWrite(fd_lockfile_rw, &magic_number,
437 sizeof(magic_number));
438
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 const bool result = SafeWrite(fd_lockfile_rw, &new_cachemgr_pid,
439 sizeof(new_cachemgr_pid));
440
2/4
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 9 times.
 9 if (!result || !result_mn) {
441 ✗ PANIC(kLogSyslogErr, "could not write cache manager pid to lockfile");
442 }
443
444
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 close(fd_lockfile_rw);
445 // Wait for cache manager to be ready
446
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 close(pipe_boot[1]);
447
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 close(pipe_handshake[0]);
448 char buf;
449
2/4
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 9 times.
✗ Branch 4 not taken.
 9 if (read(pipe_boot[0], &buf, 1) != 1) {
450
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 UnlockFile(fd_lockfile);
451
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 close(pipe_boot[0]);
452
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 close(pipe_handshake[1]);
453
1/2
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
 9 delete quota_mgr;
454
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
455 "cache manager did not start");
456 9 return NULL;
457 }
458 ✗ close(pipe_boot[0]);
459
460 // Connect write end
461 ✗ quota_mgr->pipe_lru_[1] = open(fifo_path.c_str(), O_WRONLY | O_NONBLOCK);
462 ✗ if (quota_mgr->pipe_lru_[1] < 0) {
463 ✗ LogCvmfs(kLogQuota, kLogDebug,
464 ✗ "failed to connect to newly created FIFO (%d)", errno);
465 ✗ close(pipe_handshake[1]);
466 ✗ UnlockFile(fd_lockfile);
467 ✗ delete quota_mgr;
468 ✗ return NULL;
469 }
470
471 // Finalize handshake
472 ✗ buf = 'C';
473 ✗ if (write(pipe_handshake[1], &buf, 1) != 1) {
474 ✗ UnlockFile(fd_lockfile);
475 ✗ close(pipe_handshake[1]);
476 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not finalize handshake");
477 ✗ delete quota_mgr;
478 ✗ return NULL;
479 }
480 ✗ close(pipe_handshake[1]);
481
482 ✗ Nonblock2Block(quota_mgr->pipe_lru_[1]);
483 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to a new cache manager");
484 ✗ quota_mgr->protocol_revision_ = kProtocolRevision;
485
486 ✗ UnlockFile(fd_lockfile);
487
488 ✗ quota_mgr->initialized_ = true;
489 ✗ quota_mgr->GetLimits(&quota_mgr->limit_, &quota_mgr->cleanup_threshold_);
490 ✗ LogCvmfs(kLogQuota, kLogDebug,
491 "received limit %" PRIu64 ", "
492 "threshold %" PRIu64,
493 quota_mgr->limit_, quota_mgr->cleanup_threshold_);
494 ✗ return quota_mgr;
495 18 }
496
497
498 90 bool PosixQuotaManager::DoCleanup(const uint64_t leave_size) {
499
2/2
✓ Branch 0 taken 18 times.
✓ Branch 1 taken 72 times.
 90 if (gauge_ <= leave_size)
500 18 return true;
501
502 // TODO(jblomer) transaction
503
1/2
✓ Branch 1 taken 72 times.
✗ Branch 2 not taken.
 72 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug,
504 "clean up cache until at most %lu KB is used", leave_size / 1024);
505
1/2
✓ Branch 1 taken 72 times.
✗ Branch 2 not taken.
 72 LogCvmfs(kLogQuota, kLogDebug, "gauge %" PRIu64, gauge_);
506
1/2
✓ Branch 1 taken 72 times.
✗ Branch 2 not taken.
 72 cleanup_recorder_.Tick();
507
508 bool result;
509 72 vector<string> trash;
510
511 // Note that volatile files start counting from the smallest int64 number:
512 // the absolute sequence number with the first bit set in two's complement.
513 // So -1 can be a marker that will never appear in the database.
514 72 int64_t max_acseq = -1;
515 72 std::vector<EvictCandidate> lru_ordered_open;
516
517 do {
518
1/2
✓ Branch 1 taken 522 times.
✗ Branch 2 not taken.
 522 sqlite3_reset(stmt_lru_);
519
3/4
✓ Branch 0 taken 72 times.
✓ Branch 1 taken 450 times.
✓ Branch 3 taken 522 times.
✗ Branch 4 not taken.
 594 sqlite3_bind_int64(stmt_lru_, 1,
520 72 (max_acseq == -1) ? std::numeric_limits<int64_t>::min()
521 : (max_acseq + 1));
522
523 522 std::vector<EvictCandidate> candidates;
524
1/2
✓ Branch 1 taken 522 times.
✗ Branch 2 not taken.
 522 candidates.reserve(kEvictBatchSize);
525 522 string hash_str;
526 522 unsigned i = 0;
527
3/4
✓ Branch 1 taken 468612 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 468090 times.
✓ Branch 4 taken 522 times.
 468612 while (sqlite3_step(stmt_lru_) == SQLITE_ROW) {
528 hash_str = reinterpret_cast<const char *>(
529
2/4
✓ Branch 1 taken 468090 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 468090 times.
✗ Branch 5 not taken.
 468090 sqlite3_column_text(stmt_lru_, 0));
530
1/2
✓ Branch 2 taken 468090 times.
✗ Branch 3 not taken.
 468090 LogCvmfs(kLogQuota, kLogDebug, "add %s to candidates for eviction",
531 hash_str.c_str());
532
1/2
✓ Branch 1 taken 468090 times.
✗ Branch 2 not taken.
 468090 candidates.push_back(
533
1/2
✓ Branch 1 taken 468090 times.
✗ Branch 2 not taken.
 468090 EvictCandidate(shash::MkFromHexPtr(shash::HexPtr(hash_str)),
534 468090 sqlite3_column_int64(stmt_lru_, 1),
535
2/4
✓ Branch 1 taken 468090 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 468090 times.
✗ Branch 5 not taken.
 468090 sqlite3_column_int64(stmt_lru_, 2)));
536 468090 i++;
537 }
538
2/2
✓ Branch 1 taken 9 times.
✓ Branch 2 taken 513 times.
 522 if (candidates.empty()) {
539
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 LogCvmfs(kLogQuota, kLogDebug, "no more entries to evict");
540 9 break;
541 }
542
543 513 const unsigned N = candidates.size();
544
545 513 open_files_.clear();
546
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 513 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 1026 open_files_ = (cleanup_unused_first_) ? CollectAllOpenHashes()
547 513 : std::vector<shash::Short>();
548
549
2/2
✓ Branch 0 taken 450081 times.
✓ Branch 1 taken 450 times.
 450531 for (i = 0; i < N; ++i) {
550 // That's a critical condition. We must not delete a not yet inserted
551 // pinned file as it is already reserved (but will be inserted later).
552 // Instead, set the pin bit in the db to not run into an endless loop
553
1/2
✓ Branch 2 taken 450081 times.
✗ Branch 3 not taken.
 450081 const bool is_pinned = pinned_chunks_.find(candidates[i].hash)
554 900162 != pinned_chunks_.end();
555
556 // Avoid evicting open files hopping there are enough more recently used
557 // files to satisfy the cleanup request
558 /*
559 const bool is_open = std::find_if(
560 open_files_.begin(), open_files_.end(),
561 [&candidates, &i](const auto &elem) -> bool
562 { return elem.Collide(candidates[i].hash);
563 })
564 != open_files_.end();
565 */
566 450081 bool is_open = false;
567
1/2
✗ Branch 4 not taken.
✓ Branch 5 taken 450081 times.
 450081 for (auto it = open_files_.begin(); it != open_files_.end(); ++it) {
568 ✗ if (it->Collide(candidates[i].hash)) {
569 ✗ is_open = true;
570 ✗ break;
571 }
572 }
573
574
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 450072 times.
 450081 if (is_pinned) {
575
1/2
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
 9 SkipEviction(candidates[i]);
576 9 continue;
577 }
578
579
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 450072 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 450072 if (cleanup_unused_first_ and is_open) {
580 ✗ SkipEviction(candidates[i]);
581 ✗ lru_ordered_open.push_back(candidates[i]);
582 ✗ continue;
583 }
584
585
2/4
✓ Branch 1 taken 450072 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 450072 times.
✗ Branch 5 not taken.
 900144 trash.push_back(cache_dir_ + "/"
586
2/4
✓ Branch 2 taken 450072 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 450072 times.
✗ Branch 6 not taken.
 1350216 + candidates[i].hash.MakePathWithoutSuffix());
587 450072 gauge_ -= candidates[i].size;
588 450072 max_acseq = candidates[i].acseq;
589
1/2
✓ Branch 2 taken 450072 times.
✗ Branch 3 not taken.
 450072 LogCvmfs(kLogQuota, kLogDebug, "lru cleanup %s, new gauge %" PRIu64,
590
1/2
✓ Branch 2 taken 450072 times.
✗ Branch 3 not taken.
 900144 candidates[i].hash.ToString().c_str(), gauge_);
591
592
2/2
✓ Branch 0 taken 63 times.
✓ Branch 1 taken 450009 times.
 450072 if (gauge_ <= leave_size)
593 63 break;
594 }
595
6/6
✓ Branch 1 taken 513 times.
✓ Branch 2 taken 9 times.
✓ Branch 4 taken 513 times.
✓ Branch 5 taken 9 times.
✓ Branch 6 taken 450 times.
✓ Branch 7 taken 63 times.
 1044 } while (gauge_ > leave_size);
596
597
1/2
✓ Branch 0 taken 72 times.
✗ Branch 1 not taken.
 72 if (max_acseq != -1) {
598
1/2
✓ Branch 1 taken 72 times.
✗ Branch 2 not taken.
 72 sqlite3_bind_int64(stmt_rm_batch_, 1, max_acseq);
599
1/2
✓ Branch 1 taken 72 times.
✗ Branch 2 not taken.
 72 result = (sqlite3_step(stmt_rm_batch_) == SQLITE_DONE);
600
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 72 times.
 72 assert(result);
601
1/2
✓ Branch 1 taken 72 times.
✗ Branch 2 not taken.
 72 sqlite3_reset(stmt_rm_batch_);
602
603
1/2
✓ Branch 1 taken 72 times.
✗ Branch 2 not taken.
 72 result = (sqlite3_step(stmt_unblock_) == SQLITE_DONE);
604
1/2
✓ Branch 1 taken 72 times.
✗ Branch 2 not taken.
 72 sqlite3_reset(stmt_unblock_);
605
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 72 times.
 72 assert(result);
606 }
607
608
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 72 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 72 times.
 72 while (!lru_ordered_open.empty() and gauge_ > leave_size) {
609 // cleanup files in use
610 ✗ auto &candidate = lru_ordered_open[0];
611 ✗ trash.push_back(cache_dir_ + "/" + candidate.hash.MakePathWithoutSuffix());
612 ✗ gauge_ -= candidate.size;
613 ✗ max_acseq = candidate.acseq;
614 ✗ LogCvmfs(kLogQuota, kLogDebug, "lru cleanup %s, new gauge %" PRIu64,
615 ✗ candidate.hash.ToString().c_str(), gauge_);
616 ✗ lru_ordered_open.erase(lru_ordered_open.begin());
617 }
618
619
2/4
✓ Branch 1 taken 72 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 72 times.
 72 if (!EmptyTrash(trash))
620 ✗ return false;
621
622
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 63 times.
 72 if (gauge_ > leave_size) {
623
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
624 "request to clean until %" PRIu64 ", "
625 "but effective gauge is %" PRIu64,
626 leave_size, gauge_);
627 9 return false;
628 }
629 63 return true;
630 72 }
631
632 72 bool PosixQuotaManager::EmptyTrash(const std::vector<std::string> &trash) {
633
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 72 times.
 72 if (trash.empty())
634 ✗ return true;
635
636
2/2
✓ Branch 0 taken 54 times.
✓ Branch 1 taken 18 times.
 72 if (async_delete_) {
637 // Double fork avoids zombie, forked removal process must not flush file
638 // buffers
639 pid_t pid;
640 int statloc;
641
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 54 times.
 54 if ((pid = fork()) == 0) {
642 // TODO(jblomer): eviciting files in the cache should perhaps become a
643 // thread. This would also allow to block the chunks and prevent the
644 // race with re-insertion. Then again, a thread can block umount.
645 #ifndef DEBUGMSG
646 ✗ CloseAllFildes(std::set<int>());
647 #endif
648 ✗ if (fork() == 0) {
649 ✗ for (unsigned i = 0, iEnd = trash.size(); i < iEnd; ++i) {
650 ✗ LogCvmfs(kLogQuota, kLogDebug, "unlink %s", trash[i].c_str());
651 ✗ unlink(trash[i].c_str());
652 }
653 ✗ _exit(0);
654 }
655 ✗ _exit(0);
656 } else {
657
1/2
✓ Branch 0 taken 54 times.
✗ Branch 1 not taken.
 54 if (pid > 0)
658
1/2
✓ Branch 1 taken 54 times.
✗ Branch 2 not taken.
 54 waitpid(pid, &statloc, 0);
659 else
660 ✗ return false;
661 }
662 } else { // !async_delete_
663
2/2
✓ Branch 1 taken 27 times.
✓ Branch 2 taken 18 times.
 45 for (unsigned i = 0, iEnd = trash.size(); i < iEnd; ++i) {
664 27 LogCvmfs(kLogQuota, kLogDebug, "unlink %s", trash[i].c_str());
665 27 unlink(trash[i].c_str());
666 }
667 }
668 72 return true;
669 }
670
671
672 900708 void PosixQuotaManager::DoInsert(const shash::Any &hash,
673 const uint64_t size,
674 const string &description,
675 const CommandType command_type) {
676
1/2
✓ Branch 1 taken 900708 times.
✗ Branch 2 not taken.
 900708 const string hash_str = hash.ToString();
677
1/2
✓ Branch 3 taken 900708 times.
✗ Branch 4 not taken.
 900708 LogCvmfs(kLogQuota, kLogDebug, "insert into lru %s, path %s, method %d",
678 hash_str.c_str(), description.c_str(), command_type);
679 900708 const unsigned desc_length = (description.length() > kMaxDescription)
680 ? kMaxDescription
681
1/2
✓ Branch 0 taken 900708 times.
✗ Branch 1 not taken.
 900708 : description.length();
682
683 LruCommand *cmd = reinterpret_cast<LruCommand *>(
684 900708 alloca(sizeof(LruCommand) + desc_length));
685 900708 new (cmd) LruCommand;
686 900708 cmd->command_type = command_type;
687 900708 cmd->SetSize(size);
688
1/2
✓ Branch 1 taken 900708 times.
✗ Branch 2 not taken.
 900708 cmd->StoreHash(hash);
689 900708 cmd->desc_length = desc_length;
690 900708 memcpy(reinterpret_cast<char *>(cmd) + sizeof(LruCommand), &description[0],
691 desc_length);
692
1/2
✓ Branch 1 taken 900708 times.
✗ Branch 2 not taken.
 900708 WritePipe(pipe_lru_[1], cmd, sizeof(LruCommand) + desc_length);
693 900708 }
694
695
696 333 vector<string> PosixQuotaManager::DoList(const CommandType list_command) {
697 333 vector<string> result;
698
699 int pipe_list[2];
700
1/2
✓ Branch 1 taken 333 times.
✗ Branch 2 not taken.
 333 MakeReturnPipe(pipe_list);
701 char description_buffer[kMaxDescription];
702
703 333 LruCommand cmd;
704 333 cmd.command_type = list_command;
705 333 cmd.return_pipe = pipe_list[1];
706
1/2
✓ Branch 1 taken 333 times.
✗ Branch 2 not taken.
 333 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
707
708 int length;
709 do {
710
1/2
✓ Branch 1 taken 900702 times.
✗ Branch 2 not taken.
 900702 ManagedReadHalfPipe(pipe_list[0], &length, sizeof(length));
711
2/2
✓ Branch 0 taken 900369 times.
✓ Branch 1 taken 333 times.
 900702 if (length > 0) {
712
1/2
✓ Branch 1 taken 900369 times.
✗ Branch 2 not taken.
 900369 ReadPipe(pipe_list[0], description_buffer, length);
713
2/4
✓ Branch 2 taken 900369 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 900369 times.
✗ Branch 6 not taken.
 900369 result.push_back(string(description_buffer, length));
714 }
715
2/2
✓ Branch 0 taken 900369 times.
✓ Branch 1 taken 333 times.
 900702 } while (length >= 0);
716
717
1/2
✓ Branch 1 taken 333 times.
✗ Branch 2 not taken.
 333 CloseReturnPipe(pipe_list);
718 666 return result;
719 }
720
721
722 1624 uint64_t PosixQuotaManager::GetCapacity() {
723
1/2
✓ Branch 0 taken 1624 times.
✗ Branch 1 not taken.
 1624 if (limit_ != (uint64_t)(-1))
724 1624 return limit_;
725
726 // Unrestricted cache, look at free space on cache dir fs
727 struct statfs info;
728 ✗ if (statfs(".", &info) == 0) {
729 ✗ return info.f_bavail * info.f_bsize;
730 } else {
731 ✗ LogCvmfs(kLogQuota, kLogSyslogErr | kLogDebug,
732 ✗ "failed to query file system info of cache (%d)", errno);
733 ✗ return limit_;
734 }
735 }
736
737
738 ✗ void PosixQuotaManager::GetLimits(uint64_t *limit,
739 uint64_t *cleanup_threshold) {
740 int pipe_limits[2];
741 ✗ MakeReturnPipe(pipe_limits);
742
743 ✗ LruCommand cmd;
744 ✗ cmd.command_type = kLimits;
745 ✗ cmd.return_pipe = pipe_limits[1];
746 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
747 ✗ ManagedReadHalfPipe(pipe_limits[0], limit, sizeof(*limit));
748 ✗ ReadPipe(pipe_limits[0], cleanup_threshold, sizeof(*cleanup_threshold));
749 ✗ CloseReturnPipe(pipe_limits);
750 }
751
752
753 /**
754 * Since we only cleanup until cleanup_threshold, we can only add
755 * files smaller than limit-cleanup_threshold.
756 */
757 336 uint64_t PosixQuotaManager::GetMaxFileSize() {
758 336 return limit_ - cleanup_threshold_;
759 }
760
761
762 9 pid_t PosixQuotaManager::GetPid() {
763
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 9 if (!shared_ || !spawned_) {
764 9 return getpid();
765 }
766 ✗ if (cachemgr_pid_) {
767 ✗ return cachemgr_pid_;
768 }
769
770 pid_t result;
771 int pipe_pid[2];
772 ✗ MakeReturnPipe(pipe_pid);
773
774 ✗ LruCommand cmd;
775 ✗ cmd.command_type = kPid;
776 ✗ cmd.return_pipe = pipe_pid[1];
777 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
778 ✗ ReadHalfPipe(pipe_pid[0], &result, sizeof(result));
779 ✗ CloseReturnPipe(pipe_pid);
780 ✗ return result;
781 }
782
783
784 9 uint32_t PosixQuotaManager::GetProtocolRevision() {
785 int pipe_revision[2];
786
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 MakeReturnPipe(pipe_revision);
787
788 9 LruCommand cmd;
789 9 cmd.command_type = kGetProtocolRevision;
790 9 cmd.return_pipe = pipe_revision[1];
791
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
792
793 uint32_t revision;
794
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 ManagedReadHalfPipe(pipe_revision[0], &revision, sizeof(revision));
795
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 CloseReturnPipe(pipe_revision);
796 9 return revision;
797 }
798
799 1624 void PosixQuotaManager::SetCleanupPolicy(bool cleanup_unused_first) {
800
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1624 times.
 1624 if (protocol_revision_ < 3)
801 ✗ return;
802
803
3/4
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 1615 times.
✓ Branch 3 taken 1624 times.
✗ Branch 4 not taken.
 1624 LogCvmfs(
804 kLogQuota, kLogDebug, "Set cleanup policy to %s",
805 (cleanup_unused_first) ? "cleanup unused files first." : "vanilla lru.");
806
807
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 1615 times.
 1624 char policy = (cleanup_unused_first) ? 'S' : 'R'; // S: smart, R: regular;
808
809 LruCommand *cmd = reinterpret_cast<LruCommand *>(
810 1624 alloca(sizeof(LruCommand) + sizeof(policy)));
811 1624 new (cmd) LruCommand;
812 1624 cmd->command_type = kSetCleanupPolicy;
813 1624 cmd->desc_length = sizeof(policy);
814 1624 memcpy(reinterpret_cast<char *>(cmd) + sizeof(LruCommand), &policy,
815 sizeof(policy));
816
1/2
✓ Branch 1 taken 1624 times.
✗ Branch 2 not taken.
 1624 WritePipe(pipe_lru_[1], cmd, sizeof(LruCommand) + sizeof(policy));
817 }
818
819 9 void PosixQuotaManager::RegisterMountpoint(const std::string &mountpoint) {
820
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
 9 if (protocol_revision_ < 3)
821 ✗ return;
822
823 9 LogCvmfs(kLogQuota, kLogDebug, "Register Mountpoint %s", mountpoint.c_str());
824
825 9 const unsigned desc_length = (mountpoint.size() > kMaxDescription)
826 ? kMaxDescription
827
1/2
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
 9 : mountpoint.size();
828 LruCommand *cmd = reinterpret_cast<LruCommand *>(
829 9 alloca(sizeof(LruCommand) + desc_length));
830 9 new (cmd) LruCommand;
831 9 cmd->command_type = kRegisterMountpoint;
832 9 cmd->desc_length = desc_length;
833 9 memcpy(reinterpret_cast<char *>(cmd) + sizeof(LruCommand), mountpoint.data(),
834 desc_length);
835 9 WritePipe(pipe_lru_[1], cmd, sizeof(LruCommand) + desc_length);
836 }
837
838 9 std::string PosixQuotaManager::ReadPipeString(int fd, size_t size) {
839
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
 9 if (size == 0)
840 ✗ return "";
841
842
1/2
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
 9 std::vector<char> buf(size);
843
1/2
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
 9 ManagedReadHalfPipe(fd, buf.data(), size);
844
1/2
✓ Branch 3 taken 9 times.
✗ Branch 4 not taken.
 9 return std::string(buf.data(), size);
845 9 }
846
847 ✗ std::string PosixQuotaManager::GetMountpoints() {
848 ✗ if (protocol_revision_ < 3)
849 ✗ return "";
850
851 int pipe_mp[2];
852 ✗ MakeReturnPipe(pipe_mp);
853
854 ✗ LruCommand cmd;
855 ✗ cmd.command_type = kGetMountpoints;
856 ✗ cmd.return_pipe = pipe_mp[1];
857 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
858 ✗ size_t mp_str_size = 0;
859 ✗ ManagedReadHalfPipe(pipe_mp[0], &mp_str_size, sizeof(size_t));
860 ✗ const std::string result = ReadPipeString(pipe_mp[0], mp_str_size);
861 ✗ CloseReturnPipe(pipe_mp);
862 ✗ return result;
863 }
864
865 ✗ std::string PosixQuotaManager::GetGroupHashes() {
866 ✗ if (protocol_revision_ < 3)
867 ✗ return "";
868
869 int pipe_gh[2];
870 ✗ MakeReturnPipe(pipe_gh);
871
872 ✗ LruCommand cmd;
873 ✗ cmd.command_type = kGetGroupHashes;
874 ✗ cmd.return_pipe = pipe_gh[1];
875 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
876 ✗ size_t mp_str_size = 0;
877 ✗ ManagedReadHalfPipe(pipe_gh[0], &mp_str_size, sizeof(size_t));
878 ✗ const std::string result = ReadPipeString(pipe_gh[0], mp_str_size);
879 ✗ CloseReturnPipe(pipe_gh);
880 ✗ return result;
881 }
882
883 /**
884 * Queries the shared local hard disk quota manager.
885 */
886 162 void PosixQuotaManager::GetSharedStatus(uint64_t *gauge, uint64_t *pinned) {
887 int pipe_status[2];
888
1/2
✓ Branch 1 taken 162 times.
✗ Branch 2 not taken.
 162 MakeReturnPipe(pipe_status);
889
890 162 LruCommand cmd;
891 162 cmd.command_type = kStatus;
892 162 cmd.return_pipe = pipe_status[1];
893
1/2
✓ Branch 1 taken 162 times.
✗ Branch 2 not taken.
 162 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
894
1/2
✓ Branch 1 taken 162 times.
✗ Branch 2 not taken.
 162 ManagedReadHalfPipe(pipe_status[0], gauge, sizeof(*gauge));
895
1/2
✓ Branch 1 taken 162 times.
✗ Branch 2 not taken.
 162 ReadPipe(pipe_status[0], pinned, sizeof(*pinned));
896
1/2
✓ Branch 1 taken 162 times.
✗ Branch 2 not taken.
 162 CloseReturnPipe(pipe_status);
897 162 }
898
899 9 bool PosixQuotaManager::SetSharedLimit(uint64_t limit) {
900 int pipe_set_limit[2];
901 bool result;
902
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 MakeReturnPipe(pipe_set_limit);
903
904 9 LruCommand cmd;
905 9 cmd.command_type = kSetLimit;
906 9 cmd.size = limit;
907 9 cmd.return_pipe = pipe_set_limit[1];
908
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
909
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 ReadHalfPipe(pipe_set_limit[0], &result, sizeof(result));
910
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 CloseReturnPipe(pipe_set_limit);
911 9 return result;
912 }
913
914
915 9 bool PosixQuotaManager::SetLimit(uint64_t size) {
916
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
 9 if (!spawned_) {
917 ✗ limit_ = size;
918 ✗ cleanup_threshold_ = size / 2;
919 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
920 "Quota limit set to %lu / threshold %lu", limit_,
921 cleanup_threshold_);
922 ✗ return true;
923 }
924 9 return SetSharedLimit(size);
925 }
926
927 3374 uint64_t PosixQuotaManager::GetSize() {
928
2/2
✓ Branch 0 taken 3230 times.
✓ Branch 1 taken 144 times.
 3374 if (!spawned_)
929 3230 return gauge_;
930 uint64_t gauge, size_pinned;
931
1/2
✓ Branch 1 taken 144 times.
✗ Branch 2 not taken.
 144 GetSharedStatus(&gauge, &size_pinned);
932 144 return gauge;
933 }
934
935
936 18 uint64_t PosixQuotaManager::GetSizePinned() {
937
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 18 times.
 18 if (!spawned_)
938 ✗ return pinned_;
939 uint64_t gauge, size_pinned;
940
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 18 GetSharedStatus(&gauge, &size_pinned);
941 18 return size_pinned;
942 }
943
944
945 36 uint64_t PosixQuotaManager::GetCleanupRate(uint64_t period_s) {
946
2/4
✓ Branch 0 taken 36 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 36 times.
 36 if (!spawned_ || (protocol_revision_ < 2))
947 ✗ return 0;
948 uint64_t cleanup_rate;
949
950 int pipe_cleanup_rate[2];
951
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 MakeReturnPipe(pipe_cleanup_rate);
952 36 LruCommand cmd;
953 36 cmd.command_type = kCleanupRate;
954 36 cmd.size = period_s;
955 36 cmd.return_pipe = pipe_cleanup_rate[1];
956
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
957
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 ManagedReadHalfPipe(pipe_cleanup_rate[0], &cleanup_rate,
958 sizeof(cleanup_rate));
959
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 CloseReturnPipe(pipe_cleanup_rate);
960
961 36 return cleanup_rate;
962 }
963
964
965 2200 bool PosixQuotaManager::InitDatabase(const bool rebuild_database) {
966 2200 string sql;
967 sqlite3_stmt *stmt;
968
969
2/4
✓ Branch 1 taken 2200 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2200 times.
✗ Branch 5 not taken.
 2200 fd_lock_cachedb_ = LockFile(workspace_dir_ + "/lock_cachedb");
970
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 2191 times.
 2200 if (fd_lock_cachedb_ < 0) {
971
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 LogCvmfs(kLogQuota, kLogDebug, "failed to create cachedb lock");
972 9 return false;
973 }
974
975 2191 bool retry = false;
976
1/2
✓ Branch 1 taken 2191 times.
✗ Branch 2 not taken.
 2191 const string db_file = cache_dir_ + "/cachedb";
977
2/2
✓ Branch 0 taken 2107 times.
✓ Branch 1 taken 84 times.
 2191 if (rebuild_database) {
978
1/2
✓ Branch 2 taken 84 times.
✗ Branch 3 not taken.
 84 LogCvmfs(kLogQuota, kLogDebug, "rebuild database, unlinking existing (%s)",
979 db_file.c_str());
980 84 unlink(db_file.c_str());
981
1/2
✓ Branch 1 taken 84 times.
✗ Branch 2 not taken.
 84 unlink((db_file + "-journal").c_str());
982 }
983
984 2107 init_recover:
985
1/2
✓ Branch 2 taken 2191 times.
✗ Branch 3 not taken.
 2191 int err = sqlite3_open(db_file.c_str(), &database_);
986
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2191 times.
 2191 if (err != SQLITE_OK) {
987 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not open cache database (%d)", err);
988 ✗ goto init_database_fail;
989 }
990 // TODO(reneme): make this a `QuotaDatabase : public sqlite::Database`
991 sql = "PRAGMA synchronous=0; PRAGMA locking_mode=EXCLUSIVE; "
992 "PRAGMA auto_vacuum=1; "
993 "CREATE TABLE IF NOT EXISTS cache_catalog (sha1 TEXT, size INTEGER, "
994 " acseq INTEGER, path TEXT, type INTEGER, pinned INTEGER, "
995 "CONSTRAINT pk_cache_catalog PRIMARY KEY (sha1)); "
996 "CREATE UNIQUE INDEX IF NOT EXISTS idx_cache_catalog_acseq "
997 " ON cache_catalog (acseq); "
998 "CREATE TEMP TABLE fscache (sha1 TEXT, size INTEGER, actime INTEGER, "
999 "CONSTRAINT pk_fscache PRIMARY KEY (sha1)); "
1000 "CREATE INDEX idx_fscache_actime ON fscache (actime); "
1001 "CREATE TABLE IF NOT EXISTS properties (key TEXT, value TEXT, "
1002
1/2
✓ Branch 1 taken 2191 times.
✗ Branch 2 not taken.
 2191 " CONSTRAINT pk_properties PRIMARY KEY(key));";
1003
1/2
✓ Branch 2 taken 2191 times.
✗ Branch 3 not taken.
 2191 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1004
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2191 times.
 2191 if (err != SQLITE_OK) {
1005 ✗ if (!retry) {
1006 ✗ retry = true;
1007 ✗ sqlite3_close(database_);
1008 ✗ unlink(db_file.c_str());
1009 ✗ unlink((db_file + "-journal").c_str());
1010 ✗ LogCvmfs(kLogQuota, kLogSyslogWarn,
1011 "LRU database corrupted, re-building");
1012 ✗ goto init_recover;
1013 }
1014 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
1015 sql.c_str());
1016 ✗ goto init_database_fail;
1017 }
1018
1019 // If this an old cache catalog,
1020 // add and initialize new columns to cache_catalog
1021 sql = "ALTER TABLE cache_catalog ADD type INTEGER; "
1022
1/2
✓ Branch 1 taken 2191 times.
✗ Branch 2 not taken.
 2191 "ALTER TABLE cache_catalog ADD pinned INTEGER";
1023
1/2
✓ Branch 2 taken 2191 times.
✗ Branch 3 not taken.
 2191 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1024
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2191 times.
 2191 if (err == SQLITE_OK) {
1025 ✗ sql = "UPDATE cache_catalog SET type=" + StringifyInt(kFileRegular) + ";";
1026 ✗ err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1027 ✗ if (err != SQLITE_OK) {
1028 ✗ LogCvmfs(kLogQuota, kLogDebug,
1029 "could not init cache database (failed: %s)", sql.c_str());
1030 ✗ goto init_database_fail;
1031 }
1032 }
1033
1034 // Set pinned back
1035
1/2
✓ Branch 1 taken 2191 times.
✗ Branch 2 not taken.
 2191 sql = "UPDATE cache_catalog SET pinned=0;";
1036
1/2
✓ Branch 2 taken 2191 times.
✗ Branch 3 not taken.
 2191 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1037
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2191 times.
 2191 if (err != SQLITE_OK) {
1038 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
1039 sql.c_str());
1040 ✗ goto init_database_fail;
1041 }
1042
1043 // Set schema version
1044 sql = "INSERT OR REPLACE INTO properties (key, value) "
1045
1/2
✓ Branch 1 taken 2191 times.
✗ Branch 2 not taken.
 2191 "VALUES ('schema', '1.0')";
1046
1/2
✓ Branch 2 taken 2191 times.
✗ Branch 3 not taken.
 2191 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1047
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2191 times.
 2191 if (err != SQLITE_OK) {
1048 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
1049 sql.c_str());
1050 ✗ goto init_database_fail;
1051 }
1052
1053 // If cache catalog is empty, recreate from file system
1054
1/2
✓ Branch 1 taken 2191 times.
✗ Branch 2 not taken.
 2191 sql = "SELECT count(*) FROM cache_catalog;";
1055
1/2
✓ Branch 2 taken 2191 times.
✗ Branch 3 not taken.
 2191 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1056
2/4
✓ Branch 1 taken 2191 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2191 times.
✗ Branch 4 not taken.
 2191 if (sqlite3_step(stmt) == SQLITE_ROW) {
1057
6/8
✓ Branch 1 taken 2191 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 36 times.
✓ Branch 4 taken 2155 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 36 times.
✓ Branch 7 taken 2155 times.
✓ Branch 8 taken 36 times.
 2191 if ((sqlite3_column_int64(stmt, 0)) == 0 || rebuild_database) {
1058
1/2
✓ Branch 1 taken 2155 times.
✗ Branch 2 not taken.
 2155 LogCvmfs(kLogCvmfs, kLogDebug,
1059 "CernVM-FS: building lru cache database...");
1060
3/4
✓ Branch 1 taken 2155 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 27 times.
✓ Branch 4 taken 2128 times.
 2155 if (!RebuildDatabase()) {
1061
1/2
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
 27 LogCvmfs(kLogQuota, kLogDebug,
1062 "could not build cache database from file system");
1063
1/2
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
 27 sqlite3_finalize(stmt);
1064 27 goto init_database_fail;
1065 }
1066 }
1067
1/2
✓ Branch 1 taken 2164 times.
✗ Branch 2 not taken.
 2164 sqlite3_finalize(stmt);
1068 } else {
1069 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not select on cache catalog");
1070 ✗ sqlite3_finalize(stmt);
1071 ✗ goto init_database_fail;
1072 }
1073
1074 // How many bytes do we already have in cache?
1075
1/2
✓ Branch 1 taken 2164 times.
✗ Branch 2 not taken.
 2164 sql = "SELECT sum(size) FROM cache_catalog;";
1076
1/2
✓ Branch 2 taken 2164 times.
✗ Branch 3 not taken.
 2164 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1077
2/4
✓ Branch 1 taken 2164 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2164 times.
✗ Branch 4 not taken.
 2164 if (sqlite3_step(stmt) == SQLITE_ROW) {
1078
1/2
✓ Branch 1 taken 2164 times.
✗ Branch 2 not taken.
 2164 gauge_ = sqlite3_column_int64(stmt, 0);
1079 } else {
1080 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not determine cache size");
1081 ✗ sqlite3_finalize(stmt);
1082 ✗ goto init_database_fail;
1083 }
1084
1/2
✓ Branch 1 taken 2164 times.
✗ Branch 2 not taken.
 2164 sqlite3_finalize(stmt);
1085
1086 // Highest seq-no?
1087
1/2
✓ Branch 1 taken 2164 times.
✗ Branch 2 not taken.
 2164 sql = "SELECT coalesce(max(acseq & (~(1<<63))), 0) FROM cache_catalog;";
1088
1/2
✓ Branch 2 taken 2164 times.
✗ Branch 3 not taken.
 2164 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1089
2/4
✓ Branch 1 taken 2164 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2164 times.
✗ Branch 4 not taken.
 2164 if (sqlite3_step(stmt) == SQLITE_ROW) {
1090
1/2
✓ Branch 1 taken 2164 times.
✗ Branch 2 not taken.
 2164 seq_ = sqlite3_column_int64(stmt, 0) + 1;
1091 } else {
1092 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not determine highest seq-no");
1093 ✗ sqlite3_finalize(stmt);
1094 ✗ goto init_database_fail;
1095 }
1096
1/2
✓ Branch 1 taken 2164 times.
✗ Branch 2 not taken.
 2164 sqlite3_finalize(stmt);
1097
1098 // Prepare touch, new, remove statements
1099
1/2
✓ Branch 1 taken 2164 times.
✗ Branch 2 not taken.
 2164 sqlite3_prepare_v2(database_,
1100 "UPDATE cache_catalog SET acseq=:seq | (acseq&(1<<63)) "
1101 "WHERE sha1=:sha1;",
1102 -1, &stmt_touch_, NULL);
1103
1/2
✓ Branch 1 taken 2164 times.
✗ Branch 2 not taken.
 2164 sqlite3_prepare_v2(database_,
1104 "UPDATE cache_catalog SET pinned=0 "
1105 "WHERE sha1=:sha1;",
1106 -1, &stmt_unpin_, NULL);
1107
1/2
✓ Branch 1 taken 2164 times.
✗ Branch 2 not taken.
 2164 sqlite3_prepare_v2(database_,
1108 "UPDATE cache_catalog SET pinned=2 "
1109 "WHERE sha1=:sha1;",
1110 -1, &stmt_block_, NULL);
1111
1/2
✓ Branch 1 taken 2164 times.
✗ Branch 2 not taken.
 2164 sqlite3_prepare_v2(database_,
1112 "UPDATE cache_catalog SET pinned=1 "
1113 "WHERE pinned=2;",
1114 -1, &stmt_unblock_, NULL);
1115
1/2
✓ Branch 1 taken 2164 times.
✗ Branch 2 not taken.
 2164 sqlite3_prepare_v2(database_,
1116 "INSERT OR REPLACE INTO cache_catalog "
1117 "(sha1, size, acseq, path, type, pinned) "
1118 "VALUES (:sha1, :s, :seq, :p, :t, :pin);",
1119 -1, &stmt_new_, NULL);
1120
1/2
✓ Branch 1 taken 2164 times.
✗ Branch 2 not taken.
 2164 sqlite3_prepare_v2(database_,
1121 "SELECT size, pinned FROM cache_catalog WHERE sha1=:sha1;",
1122 -1, &stmt_size_, NULL);
1123
1/2
✓ Branch 1 taken 2164 times.
✗ Branch 2 not taken.
 2164 sqlite3_prepare_v2(database_, "DELETE FROM cache_catalog WHERE sha1=:sha1;",
1124 -1, &stmt_rm_, NULL);
1125
1/2
✓ Branch 1 taken 2164 times.
✗ Branch 2 not taken.
 2164 sqlite3_prepare_v2(database_,
1126 "DELETE FROM cache_catalog WHERE acseq<=:a AND pinned<>2;",
1127 -1, &stmt_rm_batch_, NULL);
1128
1/2
✓ Branch 2 taken 2164 times.
✗ Branch 3 not taken.
 2164 sqlite3_prepare_v2(database_,
1129
1/2
✓ Branch 2 taken 2164 times.
✗ Branch 3 not taken.
 4328 (std::string("SELECT sha1, size, acseq FROM cache_catalog "
1130 "WHERE pinned<>2 AND acseq>=:a "
1131 "ORDER BY acseq ASC "
1132 "LIMIT ")
1133
3/6
✓ Branch 1 taken 2164 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2164 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 2164 times.
✗ Branch 8 not taken.
 8656 + StringifyInt(kEvictBatchSize) + ";")
1134 .c_str(),
1135 -1, &stmt_lru_, NULL);
1136
1/2
✓ Branch 2 taken 2164 times.
✗ Branch 3 not taken.
 2164 sqlite3_prepare_v2(database_,
1137 ("SELECT path FROM cache_catalog WHERE type="
1138
3/6
✓ Branch 1 taken 2164 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2164 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 2164 times.
✗ Branch 8 not taken.
 4328 + StringifyInt(kFileRegular) + ";")
1139 .c_str(),
1140 -1, &stmt_list_, NULL);
1141
1/2
✓ Branch 1 taken 2164 times.
✗ Branch 2 not taken.
 2164 sqlite3_prepare_v2(database_,
1142 "SELECT path FROM cache_catalog WHERE pinned<>0;", -1,
1143 &stmt_list_pinned_, NULL);
1144
1/2
✓ Branch 1 taken 2164 times.
✗ Branch 2 not taken.
 2164 sqlite3_prepare_v2(database_,
1145 "SELECT path FROM cache_catalog WHERE acseq < 0;", -1,
1146 &stmt_list_volatile_, NULL);
1147
1/2
✓ Branch 2 taken 2164 times.
✗ Branch 3 not taken.
 2164 sqlite3_prepare_v2(database_,
1148 ("SELECT path FROM cache_catalog WHERE type="
1149
3/6
✓ Branch 1 taken 2164 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2164 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 2164 times.
✗ Branch 8 not taken.
 4328 + StringifyInt(kFileCatalog) + ";")
1150 .c_str(),
1151 -1, &stmt_list_catalogs_, NULL);
1152 2164 return true;
1153
1154 27 init_database_fail:
1155
1/2
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
 27 sqlite3_close(database_);
1156 27 database_ = NULL;
1157
1/2
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
 27 UnlockFile(fd_lock_cachedb_);
1158 27 return false;
1159 2200 }
1160
1161
1162 /**
1163 * Inserts a new file into cache catalog. This file gets a new,
1164 * highest sequence number. Does cache cleanup if necessary.
1165 */
1166 900555 void PosixQuotaManager::Insert(const shash::Any &any_hash,
1167 const uint64_t size,
1168 const string &description) {
1169 900555 DoInsert(any_hash, size, description, kInsert);
1170 900555 }
1171
1172
1173 /**
1174 * Inserts a new file into cache catalog. This file is marked as volatile
1175 * and gets a new highest sequence number with the first bit set. Cache cleanup
1176 * treats these files with priority.
1177 */
1178 36 void PosixQuotaManager::InsertVolatile(const shash::Any &any_hash,
1179 const uint64_t size,
1180 const string &description) {
1181 36 DoInsert(any_hash, size, description, kInsertVolatile);
1182 36 }
1183
1184
1185 /**
1186 * Lists all path names from the cache db.
1187 */
1188 207 vector<string> PosixQuotaManager::List() { return DoList(kList); }
1189
1190
1191 /**
1192 * Lists all pinned files from the cache db.
1193 */
1194 72 vector<string> PosixQuotaManager::ListPinned() { return DoList(kListPinned); }
1195
1196
1197 /**
1198 * Lists all sqlite catalog files from the cache db.
1199 */
1200 27 vector<string> PosixQuotaManager::ListCatalogs() {
1201 27 return DoList(kListCatalogs);
1202 }
1203
1204
1205 /**
1206 * Lists only files flagged as volatile (priority removal)
1207 */
1208 27 vector<string> PosixQuotaManager::ListVolatile() {
1209 27 return DoList(kListVolatile);
1210 }
1211
1212
1213 /**
1214 * Entry point for the shared cache manager process
1215 */
1216 ✗ int PosixQuotaManager::MainCacheManager(int argc, char **argv) {
1217 ✗ LogCvmfs(kLogQuota, kLogDebug, "starting quota manager");
1218 int retval;
1219
1220 ✗ PosixQuotaManager shared_manager(0, 0, "");
1221 ✗ shared_manager.shared_ = true;
1222 ✗ shared_manager.spawned_ = true;
1223 ✗ shared_manager.pinned_ = 0;
1224
1225 // Process command line arguments
1226 ✗ ParseDirectories(string(argv[2]),
1227 &shared_manager.cache_dir_,
1228 &shared_manager.workspace_dir_);
1229 ✗ const int pipe_boot = String2Int64(argv[3]);
1230 ✗ const int pipe_handshake = String2Int64(argv[4]);
1231 ✗ shared_manager.limit_ = String2Int64(argv[5]);
1232 ✗ shared_manager.cleanup_threshold_ = String2Int64(argv[6]);
1233 ✗ const int foreground = String2Int64(argv[7]);
1234 ✗ const int syslog_level = String2Int64(argv[8]);
1235 ✗ const int syslog_facility = String2Int64(argv[9]);
1236 ✗ vector<string> logfiles = SplitString(argv[10], ':');
1237
1238 ✗ SetLogSyslogLevel(syslog_level);
1239 ✗ SetLogSyslogFacility(syslog_facility);
1240 ✗ if ((logfiles.size() > 0) && (logfiles[0] != ""))
1241 ✗ SetLogDebugFile(logfiles[0] + ".cachemgr");
1242 ✗ if (logfiles.size() > 1)
1243 ✗ SetLogMicroSyslog(logfiles[1]);
1244
1245 ✗ if (!foreground)
1246 ✗ Daemonize();
1247
1248 ✗ if ((geteuid() != 0) && SetuidCapabilityPermitted()) {
1249 // Permanently drop credentials
1250 ✗ const std::vector<cap_value_t> nocaps;
1251 ✗ if (!ClearPermittedCapabilities(nocaps, nocaps))
1252 ✗ PANIC(kLogStderr | kLogSyslogErr,
1253 "Failed to clear quota manager capabilities");
1254 // Leave this process ptraceable
1255 ✗ if (!platform_set_dumpable())
1256 ✗ PANIC(kLogStderr | kLogSyslogErr,
1257 "Failed to make quota manager process ptraceable");
1258 // but without core dumps
1259 ✗ if (!SetLimitCore(0))
1260 ✗ PANIC(kLogStderr | kLogSyslogErr,
1261 "Failed to disable quota manager core dumps");
1262 }
1263
1264 const std::unique_ptr<Watchdog> watchdog(
1265 ✗ Watchdog::Create(NULL, false /* needs_read_environ */));
1266 ✗ assert(watchdog.get() != nullptr);
1267 ✗ watchdog->Spawn("./stacktrace.cachemgr");
1268
1269 // Initialize pipe, open non-blocking as cvmfs is not yet connected
1270 ✗ const int fd_lockfile_fifo = LockFile(shared_manager.workspace_dir_
1271 ✗ + "/lock_cachemgr.fifo");
1272 ✗ if (fd_lockfile_fifo < 0) {
1273 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1274 "could not open lock file "
1275 "%s (%d)",
1276 ✗ (shared_manager.workspace_dir_ + "/lock_cachemgr.fifo").c_str(),
1277 ✗ errno);
1278 ✗ return 1;
1279 }
1280 ✗ const string crash_guard = shared_manager.cache_dir_ + "/cachemgr.running";
1281 ✗ const bool rebuild = FileExists(crash_guard);
1282 ✗ retval = open(crash_guard.c_str(), O_RDONLY | O_CREAT, 0600);
1283 ✗ if (retval < 0) {
1284 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1285 "failed to create shared cache manager crash guard");
1286 ✗ UnlockFile(fd_lockfile_fifo);
1287 ✗ return 1;
1288 }
1289 ✗ close(retval);
1290
1291 // Redirect SQlite temp directory to cache (global variable)
1292 ✗ const string tmp_dir = shared_manager.workspace_dir_;
1293 ✗ sqlite3_temp_directory = static_cast<char *>(
1294 ✗ sqlite3_malloc(tmp_dir.length() + 1));
1295 ✗ snprintf(sqlite3_temp_directory, tmp_dir.length() + 1, "%s", tmp_dir.c_str());
1296
1297 // Cleanup leftover named pipes
1298 ✗ shared_manager.CleanupPipes();
1299
1300 ✗ if (!shared_manager.InitDatabase(rebuild)) {
1301 ✗ UnlockFile(fd_lockfile_fifo);
1302 ✗ return 1;
1303 }
1304 ✗ shared_manager.CheckFreeSpace();
1305
1306 // Save protocol revision to file. If the file is not found, it indicates
1307 // to the client that the cache manager is from times before the protocol
1308 // was versioned.
1309 const string protocol_revision_path = shared_manager.workspace_dir_
1310 ✗ + "/cachemgr.protocol";
1311 ✗ retval = open(protocol_revision_path.c_str(), O_WRONLY | O_CREAT, 0600);
1312 ✗ if (retval < 0) {
1313 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1314 ✗ "failed to open protocol revision file (%d)", errno);
1315 ✗ UnlockFile(fd_lockfile_fifo);
1316 ✗ return 1;
1317 }
1318 ✗ const string revision = StringifyInt(kProtocolRevision);
1319 ✗ const int written = write(retval, revision.data(), revision.length());
1320 ✗ close(retval);
1321 ✗ if ((written < 0) || static_cast<unsigned>(written) != revision.length()) {
1322 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1323 ✗ "failed to write protocol revision (%d)", errno);
1324 ✗ UnlockFile(fd_lockfile_fifo);
1325 ✗ return 1;
1326 }
1327
1328 ✗ const string fifo_path = shared_manager.workspace_dir_ + "/cachemgr";
1329 ✗ shared_manager.pipe_lru_[0] = open(fifo_path.c_str(), O_RDONLY | O_NONBLOCK);
1330 ✗ if (shared_manager.pipe_lru_[0] < 0) {
1331 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to listen on FIFO %s (%d)",
1332 ✗ fifo_path.c_str(), errno);
1333 ✗ UnlockFile(fd_lockfile_fifo);
1334 ✗ return 1;
1335 }
1336 ✗ Nonblock2Block(shared_manager.pipe_lru_[0]);
1337 ✗ LogCvmfs(kLogQuota, kLogDebug, "shared cache manager listening");
1338
1339 ✗ char buf = 'C';
1340 ✗ WritePipe(pipe_boot, &buf, 1);
1341 ✗ close(pipe_boot);
1342
1343 ✗ ReadPipe(pipe_handshake, &buf, 1);
1344 ✗ close(pipe_handshake);
1345 ✗ LogCvmfs(kLogQuota, kLogDebug, "shared cache manager handshake done");
1346
1347 // Ensure that broken pipes from clients do not kill the cache manager
1348 ✗ signal(SIGPIPE, SIG_IGN);
1349 // Don't let Ctrl-C ungracefully kill interactive session
1350 ✗ signal(SIGINT, SIG_IGN);
1351
1352 ✗ shared_manager.MainCommandServer(&shared_manager);
1353 ✗ unlink(fifo_path.c_str());
1354 ✗ unlink(protocol_revision_path.c_str());
1355 ✗ shared_manager.CloseDatabase();
1356 ✗ unlink(crash_guard.c_str());
1357 ✗ UnlockFile(fd_lockfile_fifo);
1358
1359 ✗ if (sqlite3_temp_directory) {
1360 ✗ sqlite3_free(sqlite3_temp_directory);
1361 ✗ sqlite3_temp_directory = NULL;
1362 }
1363
1364 ✗ return 0;
1365 }
1366
1367
1368 306 void *PosixQuotaManager::MainCommandServer(void *data) {
1369 306 PosixQuotaManager *quota_mgr = static_cast<PosixQuotaManager *>(data);
1370
1371
1/2
✓ Branch 1 taken 306 times.
✗ Branch 2 not taken.
 306 LogCvmfs(kLogQuota, kLogDebug, "starting quota manager");
1372
1/2
✓ Branch 1 taken 306 times.
✗ Branch 2 not taken.
 306 sqlite3_soft_heap_limit(quota_mgr->kSqliteMemPerThread);
1373
1374
2/2
✓ Branch 1 taken 9792 times.
✓ Branch 2 taken 306 times.
 10098 LruCommand command_buffer[kCommandBufferSize];
1375 char description_buffer[kCommandBufferSize * kMaxDescription];
1376 306 unsigned num_commands = 0;
1377
1378
1/2
✓ Branch 1 taken 1351530 times.
✗ Branch 2 not taken.
 1351530 while (read(quota_mgr->pipe_lru_[0], &command_buffer[num_commands],
1379 sizeof(command_buffer[0]))
1380
2/2
✓ Branch 0 taken 1351224 times.
✓ Branch 1 taken 306 times.
 1351530 == sizeof(command_buffer[0])) {
1381 1351224 const CommandType command_type = command_buffer[num_commands].command_type;
1382
1/2
✓ Branch 1 taken 1351224 times.
✗ Branch 2 not taken.
 1351224 LogCvmfs(kLogQuota, kLogDebug, "received command %d", command_type);
1383 1351224 const uint64_t size = command_buffer[num_commands].GetSize();
1384
1385 // Inserts and pins come with a description (usually a path)
1386
4/4
✓ Branch 0 taken 451053 times.
✓ Branch 1 taken 900171 times.
✓ Branch 2 taken 451017 times.
✓ Branch 3 taken 36 times.
 1351224 if ((command_type == kInsert) || (command_type == kInsertVolatile)
1387
4/4
✓ Branch 0 taken 450999 times.
✓ Branch 1 taken 18 times.
✓ Branch 2 taken 450900 times.
✓ Branch 3 taken 99 times.
 451017 || (command_type == kPin) || (command_type == kPinRegular)
1388
2/2
✓ Branch 0 taken 450891 times.
✓ Branch 1 taken 9 times.
 450900 || (command_type == kRegisterMountpoint)
1389
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 450882 times.
 450891 || (command_type == kSetCleanupPolicy)) {
1390 900342 const int desc_length = command_buffer[num_commands].desc_length;
1391 900342 ReadPipe(quota_mgr->pipe_lru_[0],
1392
1/2
✓ Branch 1 taken 900342 times.
✗ Branch 2 not taken.
 900342 &description_buffer[kMaxDescription * num_commands],
1393 desc_length);
1394 }
1395
1396 // The protocol revision is returned immediately
1397
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 1351215 times.
 1351224 if (command_type == kGetProtocolRevision) {
1398
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 const int return_pipe = quota_mgr->BindReturnPipe(
1399 command_buffer[num_commands].return_pipe);
1400
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
 9 if (return_pipe < 0)
1401 ✗ continue;
1402
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 WritePipe(return_pipe, &quota_mgr->kProtocolRevision,
1403 sizeof(quota_mgr->kProtocolRevision));
1404
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 quota_mgr->UnbindReturnPipe(return_pipe);
1405 9 continue;
1406 9 }
1407
1408 // Register a new mountpoint
1409
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 1351206 times.
 1351215 if (command_type == kRegisterMountpoint) {
1410 const std::string mountpoint(
1411 9 &description_buffer[num_commands * kMaxDescription],
1412
1/2
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
 9 command_buffer[num_commands].desc_length);
1413
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 quota_mgr->mountpoints_.push_back(mountpoint);
1414
1/2
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
 9 LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
1415 "Mountpoint %s registered in the group", mountpoint.c_str());
1416 9 continue;
1417 9 }
1418
1419 // Set Cleanup Policy
1420
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 1351197 times.
 1351206 if (command_type == kSetCleanupPolicy) {
1421 9 quota_mgr->cleanup_unused_first_ = (description_buffer[num_commands
1422 9 * kMaxDescription]
1423 == 'S')
1424 9 ? true
1425 : false;
1426 9 continue;
1427 }
1428 // Mountpoints are returned immediately
1429
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1351197 times.
 1351197 if (command_type == kGetMountpoints) {
1430 ✗ const int return_pipe = quota_mgr->BindReturnPipe(
1431 command_buffer[num_commands].return_pipe);
1432 ✗ if (return_pipe < 0)
1433 ✗ continue;
1434
1435 ✗ std::string mps;
1436 ✗ for (auto it = quota_mgr->mountpoints_.begin();
1437 ✗ it != quota_mgr->mountpoints_.end();
1438 ✗ ++it) {
1439 ✗ mps += *it + "\n";
1440 }
1441 ✗ size_t mp_size = mps.size();
1442 ✗ WritePipe(return_pipe, &mp_size, sizeof(size_t));
1443 ✗ WritePipe(return_pipe, mps.c_str(), mp_size);
1444 ✗ quota_mgr->UnbindReturnPipe(return_pipe);
1445 ✗ continue;
1446 }
1447
1448 // Group hashes are returned immediately
1449
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1351197 times.
 1351197 if (command_type == kGetGroupHashes) {
1450 ✗ const int return_pipe = quota_mgr->BindReturnPipe(
1451 command_buffer[num_commands].return_pipe);
1452 ✗ if (return_pipe < 0)
1453 ✗ continue;
1454
1455 ✗ std::vector<shash::Short> gh = quota_mgr->CollectAllOpenHashes();
1456 ✗ std::string result;
1457 ✗ for (auto it = gh.begin(); it != gh.end(); ++it) {
1458 ✗ result += (*it).ToString() + "\n";
1459 }
1460 ✗ size_t result_size = result.size();
1461 ✗ WritePipe(return_pipe, &result_size, sizeof(size_t));
1462 ✗ WritePipe(return_pipe, result.c_str(), result_size);
1463 ✗ quota_mgr->UnbindReturnPipe(return_pipe);
1464 ✗ continue;
1465 }
1466
1467 // The cleanup rate is returned immediately
1468
2/2
✓ Branch 0 taken 36 times.
✓ Branch 1 taken 1351161 times.
 1351197 if (command_type == kCleanupRate) {
1469
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 const int return_pipe = quota_mgr->BindReturnPipe(
1470 command_buffer[num_commands].return_pipe);
1471
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 36 times.
 36 if (return_pipe < 0)
1472 ✗ continue;
1473 const uint64_t
1474 36 period_s = size; // use the size field to transmit the period
1475
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 uint64_t rate = quota_mgr->cleanup_recorder_.GetNoTicks(period_s);
1476
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 WritePipe(return_pipe, &rate, sizeof(rate));
1477
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 quota_mgr->UnbindReturnPipe(return_pipe);
1478 36 continue;
1479 36 }
1480
1481
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 1351152 times.
 1351161 if (command_type == kSetLimit) {
1482
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 const int return_pipe = quota_mgr->BindReturnPipe(
1483 command_buffer[num_commands].return_pipe);
1484
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
 9 if (return_pipe < 0)
1485 ✗ continue;
1486 9 quota_mgr->limit_ = size; // use the size field to transmit the size
1487 9 quota_mgr->cleanup_threshold_ = size / 2;
1488
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
1489 "Quota limit set to %lu / threshold %lu", quota_mgr->limit_,
1490 quota_mgr->cleanup_threshold_);
1491 9 bool ret = true;
1492
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 WritePipe(return_pipe, &ret, sizeof(ret));
1493
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 quota_mgr->UnbindReturnPipe(return_pipe);
1494 9 continue;
1495 9 }
1496
1497 // Reservations are handled immediately and "out of band"
1498
2/2
✓ Branch 0 taken 126 times.
✓ Branch 1 taken 1351026 times.
 1351152 if (command_type == kReserve) {
1499 126 bool success = true;
1500
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 const int return_pipe = quota_mgr->BindReturnPipe(
1501 command_buffer[num_commands].return_pipe);
1502
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 126 times.
 126 if (return_pipe < 0)
1503 ✗ continue;
1504
1505
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1506
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 const string hash_str(hash.ToString());
1507
1/2
✓ Branch 2 taken 126 times.
✗ Branch 3 not taken.
 126 LogCvmfs(kLogQuota, kLogDebug, "reserve %lu bytes for %s", size,
1508 hash_str.c_str());
1509
1510
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 if (quota_mgr->pinned_chunks_.find(hash)
1511
2/2
✓ Branch 2 taken 108 times.
✓ Branch 3 taken 18 times.
 252 == quota_mgr->pinned_chunks_.end()) {
1512
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 99 times.
 108 if ((quota_mgr->pinned_ + size) > quota_mgr->cleanup_threshold_) {
1513
1/2
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
 9 LogCvmfs(kLogQuota, kLogDebug,
1514 "failed to insert %s (pinned), no space", hash_str.c_str());
1515 9 success = false;
1516 } else {
1517
1/2
✓ Branch 1 taken 99 times.
✗ Branch 2 not taken.
 99 quota_mgr->pinned_chunks_[hash] = size;
1518 99 quota_mgr->pinned_ += size;
1519
1/2
✓ Branch 1 taken 99 times.
✗ Branch 2 not taken.
 99 quota_mgr->CheckHighPinWatermark();
1520 }
1521 }
1522
1523
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 WritePipe(return_pipe, &success, sizeof(success));
1524
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 quota_mgr->UnbindReturnPipe(return_pipe);
1525 126 continue;
1526 126 }
1527
1528 // Back channels are also handled out of band
1529
2/2
✓ Branch 0 taken 36 times.
✓ Branch 1 taken 1350990 times.
 1351026 if (command_type == kRegisterBackChannel) {
1530
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 const int return_pipe = quota_mgr->BindReturnPipe(
1531 command_buffer[num_commands].return_pipe);
1532
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 36 times.
 36 if (return_pipe < 0)
1533 ✗ continue;
1534
1535
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 quota_mgr->UnlinkReturnPipe(command_buffer[num_commands].return_pipe);
1536
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 Block2Nonblock(return_pipe); // back channels are opportunistic
1537
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 shash::Md5 hash;
1538 36 memcpy(hash.digest, command_buffer[num_commands].digest,
1539 36 shash::kDigestSizes[shash::kMd5]);
1540
1541 36 quota_mgr->LockBackChannels();
1542 const map<shash::Md5, int>::const_iterator
1543
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 iter = quota_mgr->back_channels_.find(hash);
1544
1/2
✗ Branch 3 not taken.
✓ Branch 4 taken 36 times.
 36 if (iter != quota_mgr->back_channels_.end()) {
1545 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
1546 ✗ "closing left-over back channel %s", hash.ToString().c_str());
1547 ✗ close(iter->second);
1548 }
1549
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 quota_mgr->back_channels_[hash] = return_pipe;
1550 36 quota_mgr->UnlockBackChannels();
1551
1552 36 char success = 'S';
1553
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 WritePipe(return_pipe, &success, sizeof(success));
1554
1/2
✓ Branch 2 taken 36 times.
✗ Branch 3 not taken.
 36 LogCvmfs(kLogQuota, kLogDebug, "register back channel %s on fd %d",
1555
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 72 hash.ToString().c_str(), return_pipe);
1556
1557 36 continue;
1558 36 }
1559
1560
2/2
✓ Branch 0 taken 18 times.
✓ Branch 1 taken 1350972 times.
 1350990 if (command_type == kUnregisterBackChannel) {
1561
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 18 shash::Md5 hash;
1562 18 memcpy(hash.digest, command_buffer[num_commands].digest,
1563 18 shash::kDigestSizes[shash::kMd5]);
1564
1565 18 quota_mgr->LockBackChannels();
1566 const map<shash::Md5, int>::iterator iter = quota_mgr->back_channels_
1567
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 18 .find(hash);
1568
1/2
✓ Branch 2 taken 18 times.
✗ Branch 3 not taken.
 18 if (iter != quota_mgr->back_channels_.end()) {
1569
1/2
✓ Branch 2 taken 18 times.
✗ Branch 3 not taken.
 18 LogCvmfs(kLogQuota, kLogDebug, "closing back channel %s",
1570
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 36 hash.ToString().c_str());
1571
1/2
✓ Branch 2 taken 18 times.
✗ Branch 3 not taken.
 18 close(iter->second);
1572
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 18 quota_mgr->back_channels_.erase(iter);
1573 } else {
1574 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
1575 ✗ "did not find back channel %s", hash.ToString().c_str());
1576 }
1577 18 quota_mgr->UnlockBackChannels();
1578
1579 18 continue;
1580 18 }
1581
1582 // Unpinnings are also handled immediately with respect to the pinned gauge
1583
2/2
✓ Branch 0 taken 18 times.
✓ Branch 1 taken 1350954 times.
 1350972 if (command_type == kUnpin) {
1584
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 18 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1585
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 18 const string hash_str(hash.ToString());
1586
1587 const map<shash::Any, uint64_t>::iterator iter = quota_mgr->pinned_chunks_
1588
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 18 .find(hash);
1589
1/2
✓ Branch 2 taken 18 times.
✗ Branch 3 not taken.
 18 if (iter != quota_mgr->pinned_chunks_.end()) {
1590 18 quota_mgr->pinned_ -= iter->second;
1591
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 18 quota_mgr->pinned_chunks_.erase(iter);
1592 // It can happen that files get pinned that were removed from the cache
1593 // (see cache.cc). We fix this at this point, where we remove such
1594 // entries from the cache database.
1595
2/4
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 18 times.
✗ Branch 5 not taken.
 36 if (!FileExists(quota_mgr->cache_dir_ + "/"
1596
4/6
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 18 times.
✗ Branch 5 not taken.
✓ Branch 9 taken 9 times.
✓ Branch 10 taken 9 times.
 54 + hash.MakePathWithoutSuffix())) {
1597
1/2
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
 9 LogCvmfs(kLogQuota, kLogDebug,
1598 "remove orphaned pinned hash %s from cache database",
1599 hash_str.c_str());
1600
1/2
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
 9 sqlite3_bind_text(quota_mgr->stmt_size_, 1, &hash_str[0],
1601 9 hash_str.length(), SQLITE_STATIC);
1602 int retval;
1603
2/4
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 9 times.
✗ Branch 4 not taken.
 9 if ((retval = sqlite3_step(quota_mgr->stmt_size_)) == SQLITE_ROW) {
1604
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 const uint64_t size = sqlite3_column_int64(quota_mgr->stmt_size_,
1605 9 0);
1606
1/2
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
 9 sqlite3_bind_text(quota_mgr->stmt_rm_, 1, &(hash_str[0]),
1607 9 hash_str.length(), SQLITE_STATIC);
1608
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 retval = sqlite3_step(quota_mgr->stmt_rm_);
1609
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 9 if ((retval == SQLITE_DONE) || (retval == SQLITE_OK)) {
1610 9 quota_mgr->gauge_ -= size;
1611 } else {
1612 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1613 "failed to delete %s (%d)", hash_str.c_str(), retval);
1614 }
1615
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 sqlite3_reset(quota_mgr->stmt_rm_);
1616 }
1617
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 sqlite3_reset(quota_mgr->stmt_size_);
1618 }
1619 } else {
1620 ✗ LogCvmfs(kLogQuota, kLogDebug, "this chunk was not pinned");
1621 }
1622 18 }
1623
1624 // Immediate commands trigger flushing of the buffer
1625 1350972 const bool immediate_command = (command_type == kCleanup)
1626
2/2
✓ Branch 0 taken 1350684 times.
✓ Branch 1 taken 207 times.
 1350891 || (command_type == kList)
1627
2/2
✓ Branch 0 taken 1350612 times.
✓ Branch 1 taken 72 times.
 1350684 || (command_type == kListPinned)
1628
2/2
✓ Branch 0 taken 1350585 times.
✓ Branch 1 taken 27 times.
 1350612 || (command_type == kListCatalogs)
1629
2/2
✓ Branch 0 taken 1350558 times.
✓ Branch 1 taken 27 times.
 1350585 || (command_type == kListVolatile)
1630
2/2
✓ Branch 0 taken 1350531 times.
✓ Branch 1 taken 27 times.
 1350558 || (command_type == kRemove)
1631
2/2
✓ Branch 0 taken 1350369 times.
✓ Branch 1 taken 162 times.
 1350531 || (command_type == kStatus)
1632
1/2
✓ Branch 0 taken 1350369 times.
✗ Branch 1 not taken.
 1350369 || (command_type == kLimits)
1633
3/4
✓ Branch 0 taken 1350891 times.
✓ Branch 1 taken 81 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1350369 times.
 2701863 || (command_type == kPid);
1634
2/2
✓ Branch 0 taken 1350369 times.
✓ Branch 1 taken 603 times.
 1350972 if (!immediate_command)
1635 1350369 num_commands++;
1636
1637
4/4
✓ Branch 0 taken 1308789 times.
✓ Branch 1 taken 42183 times.
✓ Branch 2 taken 603 times.
✓ Branch 3 taken 1308186 times.
 1350972 if ((num_commands == kCommandBufferSize) || immediate_command) {
1638
1/2
✓ Branch 1 taken 42786 times.
✗ Branch 2 not taken.
 42786 quota_mgr->ProcessCommandBunch(num_commands, command_buffer,
1639 description_buffer);
1640
2/2
✓ Branch 0 taken 42183 times.
✓ Branch 1 taken 603 times.
 42786 if (!immediate_command)
1641 42183 num_commands = 0;
1642 }
1643
1644
2/2
✓ Branch 0 taken 603 times.
✓ Branch 1 taken 1350369 times.
 1350972 if (immediate_command) {
1645 // Process cleanup, listings
1646
1/2
✓ Branch 1 taken 603 times.
✗ Branch 2 not taken.
 603 const int return_pipe = quota_mgr->BindReturnPipe(
1647 command_buffer[num_commands].return_pipe);
1648
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 603 times.
 603 if (return_pipe < 0) {
1649 ✗ num_commands = 0;
1650 ✗ continue;
1651 }
1652
1653 int retval;
1654 603 sqlite3_stmt *this_stmt_list = NULL;
1655
7/10
✓ Branch 0 taken 27 times.
✓ Branch 1 taken 81 times.
✓ Branch 2 taken 207 times.
✓ Branch 3 taken 72 times.
✓ Branch 4 taken 27 times.
✓ Branch 5 taken 27 times.
✓ Branch 6 taken 162 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
 603 switch (command_type) {
1656 27 case kRemove: {
1657
1/2
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
 27 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1658
1/2
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
 27 const string hash_str = hash.ToString();
1659
1/2
✓ Branch 2 taken 27 times.
✗ Branch 3 not taken.
 27 LogCvmfs(kLogQuota, kLogDebug, "manually removing %s",
1660 hash_str.c_str());
1661 27 bool success = false;
1662
1663
1/2
✓ Branch 2 taken 27 times.
✗ Branch 3 not taken.
 27 sqlite3_bind_text(quota_mgr->stmt_size_, 1, &hash_str[0],
1664 27 hash_str.length(), SQLITE_STATIC);
1665 int retval;
1666
3/4
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 18 times.
✓ Branch 4 taken 9 times.
 27 if ((retval = sqlite3_step(quota_mgr->stmt_size_)) == SQLITE_ROW) {
1667
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 18 const uint64_t size = sqlite3_column_int64(quota_mgr->stmt_size_,
1668 18 0);
1669
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 18 const uint64_t is_pinned = sqlite3_column_int64(
1670 18 quota_mgr->stmt_size_, 1);
1671
1672
1/2
✓ Branch 2 taken 18 times.
✗ Branch 3 not taken.
 18 sqlite3_bind_text(quota_mgr->stmt_rm_, 1, &(hash_str[0]),
1673 18 hash_str.length(), SQLITE_STATIC);
1674
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 18 retval = sqlite3_step(quota_mgr->stmt_rm_);
1675
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 18 if ((retval == SQLITE_DONE) || (retval == SQLITE_OK)) {
1676 18 success = true;
1677 18 quota_mgr->gauge_ -= size;
1678
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 9 times.
 18 if (is_pinned) {
1679
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 quota_mgr->pinned_chunks_.erase(hash);
1680 9 quota_mgr->pinned_ -= size;
1681 }
1682 } else {
1683 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1684 "failed to delete %s (%d)", hash_str.c_str(), retval);
1685 }
1686
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 18 sqlite3_reset(quota_mgr->stmt_rm_);
1687 } else {
1688 // File does not exist
1689 9 success = true;
1690 }
1691
1/2
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
 27 sqlite3_reset(quota_mgr->stmt_size_);
1692
1693
1/2
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
 27 WritePipe(return_pipe, &success, sizeof(success));
1694 27 break;
1695 27 }
1696 81 case kCleanup:
1697
1/2
✓ Branch 1 taken 81 times.
✗ Branch 2 not taken.
 81 retval = quota_mgr->DoCleanup(size);
1698
1/2
✓ Branch 1 taken 81 times.
✗ Branch 2 not taken.
 81 WritePipe(return_pipe, &retval, sizeof(retval));
1699 81 break;
1700 207 case kList:
1701
1/2
✓ Branch 0 taken 207 times.
✗ Branch 1 not taken.
 207 if (!this_stmt_list)
1702 207 this_stmt_list = quota_mgr->stmt_list_;
1703 case kListPinned:
1704
2/2
✓ Branch 0 taken 72 times.
✓ Branch 1 taken 207 times.
 279 if (!this_stmt_list)
1705 72 this_stmt_list = quota_mgr->stmt_list_pinned_;
1706 case kListCatalogs:
1707
2/2
✓ Branch 0 taken 27 times.
✓ Branch 1 taken 279 times.
 306 if (!this_stmt_list)
1708 27 this_stmt_list = quota_mgr->stmt_list_catalogs_;
1709 case kListVolatile:
1710
2/2
✓ Branch 0 taken 27 times.
✓ Branch 1 taken 306 times.
 333 if (!this_stmt_list)
1711 27 this_stmt_list = quota_mgr->stmt_list_volatile_;
1712
1713 // Pipe back the list, one by one
1714 int length;
1715
3/4
✓ Branch 1 taken 900702 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 900369 times.
✓ Branch 4 taken 333 times.
 900702 while (sqlite3_step(this_stmt_list) == SQLITE_ROW) {
1716
1/2
✓ Branch 2 taken 900369 times.
✗ Branch 3 not taken.
 900369 string path = "(NULL)";
1717
2/4
✓ Branch 1 taken 900369 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 900369 times.
✗ Branch 4 not taken.
 900369 if (sqlite3_column_type(this_stmt_list, 0) != SQLITE_NULL) {
1718 1800738 path = string(reinterpret_cast<const char *>(
1719
2/4
✓ Branch 1 taken 900369 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 900369 times.
✗ Branch 5 not taken.
 900369 sqlite3_column_text(this_stmt_list, 0)));
1720 }
1721 900369 length = path.length();
1722
1/2
✓ Branch 1 taken 900369 times.
✗ Branch 2 not taken.
 900369 WritePipe(return_pipe, &length, sizeof(length));
1723
1/2
✓ Branch 0 taken 900369 times.
✗ Branch 1 not taken.
 900369 if (length > 0)
1724
2/4
✓ Branch 1 taken 900369 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 900369 times.
✗ Branch 5 not taken.
 900369 WritePipe(return_pipe, &path[0], length);
1725 900369 }
1726 333 length = -1;
1727
1/2
✓ Branch 1 taken 333 times.
✗ Branch 2 not taken.
 333 WritePipe(return_pipe, &length, sizeof(length));
1728
1/2
✓ Branch 1 taken 333 times.
✗ Branch 2 not taken.
 333 sqlite3_reset(this_stmt_list);
1729 333 break;
1730 162 case kStatus:
1731
1/2
✓ Branch 1 taken 162 times.
✗ Branch 2 not taken.
 162 WritePipe(return_pipe, &quota_mgr->gauge_, sizeof(quota_mgr->gauge_));
1732
1/2
✓ Branch 1 taken 162 times.
✗ Branch 2 not taken.
 162 WritePipe(return_pipe, &quota_mgr->pinned_,
1733 sizeof(quota_mgr->pinned_));
1734 162 break;
1735 ✗ case kLimits:
1736 ✗ WritePipe(return_pipe, &quota_mgr->limit_, sizeof(quota_mgr->limit_));
1737 ✗ WritePipe(return_pipe, &quota_mgr->cleanup_threshold_,
1738 sizeof(quota_mgr->cleanup_threshold_));
1739 ✗ break;
1740 ✗ case kPid: {
1741 ✗ pid_t pid = getpid();
1742 ✗ WritePipe(return_pipe, &pid, sizeof(pid));
1743 ✗ break;
1744 }
1745 ✗ default:
1746 ✗ PANIC(NULL); // other types are handled by the bunch processor
1747 }
1748
1/2
✓ Branch 1 taken 603 times.
✗ Branch 2 not taken.
 603 quota_mgr->UnbindReturnPipe(return_pipe);
1749 603 num_commands = 0;
1750 }
1751 }
1752
1753
1/2
✓ Branch 1 taken 306 times.
✗ Branch 2 not taken.
 306 LogCvmfs(kLogQuota, kLogDebug, "stopping cache manager (%d)", errno);
1754
1/2
✓ Branch 1 taken 306 times.
✗ Branch 2 not taken.
 306 close(quota_mgr->pipe_lru_[0]);
1755
1/2
✓ Branch 1 taken 306 times.
✗ Branch 2 not taken.
 306 quota_mgr->ProcessCommandBunch(num_commands, command_buffer,
1756 description_buffer);
1757
1758 // Unpin
1759 306 command_buffer[0].command_type = kTouch;
1760 306 for (map<shash::Any, uint64_t>::const_iterator
1761 306 i = quota_mgr->pinned_chunks_.begin(),
1762 306 iEnd = quota_mgr->pinned_chunks_.end();
1763
2/2
✓ Branch 1 taken 90 times.
✓ Branch 2 taken 306 times.
 396 i != iEnd;
1764 90 ++i) {
1765
1/2
✓ Branch 2 taken 90 times.
✗ Branch 3 not taken.
 90 command_buffer[0].StoreHash(i->first);
1766
1/2
✓ Branch 1 taken 90 times.
✗ Branch 2 not taken.
 90 quota_mgr->ProcessCommandBunch(1, command_buffer, description_buffer);
1767 }
1768
1769 306 return NULL;
1770 }
1771
1772
1773 846 void PosixQuotaManager::MakeReturnPipe(int pipe[2]) {
1774
2/2
✓ Branch 0 taken 819 times.
✓ Branch 1 taken 27 times.
 846 if (!shared_) {
1775 819 MakePipe(pipe);
1776 819 return;
1777 }
1778
1779 // Create FIFO in cache directory, store path name (number) in pipe write end
1780 27 int i = 0;
1781 int retval;
1782 do {
1783
2/4
✓ Branch 2 taken 36 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 36 times.
✗ Branch 6 not taken.
 36 retval = mkfifo((workspace_dir_ + "/pipe" + StringifyInt(i)).c_str(), 0600);
1784 36 pipe[1] = i;
1785 36 i++;
1786
3/4
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 27 times.
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
 36 } while ((retval == -1) && (errno == EEXIST));
1787
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 27 times.
 27 assert(retval == 0);
1788
1789 // Connect reader's end
1790
3/6
✓ Branch 2 taken 27 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 27 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 27 times.
✗ Branch 10 not taken.
 27 pipe[0] = open((workspace_dir_ + "/pipe" + StringifyInt(pipe[1])).c_str(),
1791 O_RDONLY | O_NONBLOCK);
1792
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 27 times.
 27 assert(pipe[0] >= 0);
1793 27 Nonblock2Block(pipe[0]);
1794 }
1795
1796
1797 2191 void PosixQuotaManager::ParseDirectories(const std::string cache_workspace,
1798 std::string *cache_dir,
1799 std::string *workspace_dir) {
1800
1/2
✓ Branch 1 taken 2191 times.
✗ Branch 2 not taken.
 2191 vector<string> dir_tokens(SplitString(cache_workspace, ':'));
1801
2/3
✓ Branch 1 taken 2173 times.
✓ Branch 2 taken 18 times.
✗ Branch 3 not taken.
 2191 switch (dir_tokens.size()) {
1802 2173 case 1:
1803
2/4
✓ Branch 2 taken 2173 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2173 times.
✗ Branch 6 not taken.
 2173 *cache_dir = *workspace_dir = dir_tokens[0];
1804 2173 break;
1805 18 case 2:
1806
1/2
✓ Branch 2 taken 18 times.
✗ Branch 3 not taken.
 18 *cache_dir = dir_tokens[0];
1807
1/2
✓ Branch 2 taken 18 times.
✗ Branch 3 not taken.
 18 *workspace_dir = dir_tokens[1];
1808 18 break;
1809 ✗ default:
1810 ✗ PANIC(NULL);
1811 }
1812 2191 }
1813
1814 9 void PosixQuotaManager::SkipEviction(const EvictCandidate &candidate) {
1815 9 bool res = true;
1816
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 std::string hash_str = candidate.hash.ToString();
1817
1/2
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
 9 LogCvmfs(kLogQuota, kLogDebug, "Exclude %s from eviction", hash_str.c_str());
1818
2/4
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 9 times.
✗ Branch 6 not taken.
 9 sqlite3_bind_text(stmt_block_, 1, &hash_str[0], hash_str.length(),
1819 SQLITE_STATIC);
1820
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 res = (sqlite3_step(stmt_block_) == SQLITE_DONE);
1821
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 sqlite3_reset(stmt_block_);
1822
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
 9 assert(res);
1823 9 }
1824
1825 /**
1826 * Immediately inserts a new pinned catalog. Does cache cleanup if necessary.
1827 *
1828 * \return True on success, false otherwise
1829 */
1830 1323 bool PosixQuotaManager::Pin(const shash::Any &hash,
1831 const uint64_t size,
1832 const string &description,
1833 const bool is_catalog) {
1834
3/4
✓ Branch 0 taken 18 times.
✓ Branch 1 taken 1305 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 18 times.
 1323 assert((size > 0) || !is_catalog);
1835
1836
1/2
✓ Branch 1 taken 1323 times.
✗ Branch 2 not taken.
 1323 const string hash_str = hash.ToString();
1837
1/2
✓ Branch 3 taken 1323 times.
✗ Branch 4 not taken.
 1323 LogCvmfs(kLogQuota, kLogDebug, "pin into lru %s, path %s", hash_str.c_str(),
1838 description.c_str());
1839
1840 // Has to run when not yet spawned (cvmfs initialization)
1841
2/2
✓ Branch 0 taken 1197 times.
✓ Branch 1 taken 126 times.
 1323 if (!spawned_) {
1842 // Code duplication here
1843
3/4
✓ Branch 2 taken 1197 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 660 times.
✓ Branch 6 taken 537 times.
 1197 if (pinned_chunks_.find(hash) == pinned_chunks_.end()) {
1844
2/2
✓ Branch 0 taken 18 times.
✓ Branch 1 taken 642 times.
 660 if (pinned_ + size > cleanup_threshold_) {
1845
1/2
✓ Branch 2 taken 18 times.
✗ Branch 3 not taken.
 18 LogCvmfs(kLogQuota, kLogDebug, "failed to insert %s (pinned), no space",
1846 hash_str.c_str());
1847 18 return false;
1848 } else {
1849
1/2
✓ Branch 1 taken 642 times.
✗ Branch 2 not taken.
 642 pinned_chunks_[hash] = size;
1850 642 pinned_ += size;
1851
1/2
✓ Branch 1 taken 642 times.
✗ Branch 2 not taken.
 642 CheckHighPinWatermark();
1852 }
1853 }
1854
1/2
✓ Branch 1 taken 1179 times.
✗ Branch 2 not taken.
 1179 const bool exists = Contains(hash_str);
1855
4/4
✓ Branch 0 taken 642 times.
✓ Branch 1 taken 537 times.
✓ Branch 2 taken 9 times.
✓ Branch 3 taken 633 times.
 1179 if (!exists && (gauge_ + size > limit_)) {
1856
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 LogCvmfs(kLogQuota, kLogDebug, "over limit, gauge %lu, file size %lu",
1857 gauge_, size);
1858
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 const int retval = DoCleanup(cleanup_threshold_);
1859
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
 9 assert(retval != 0);
1860 }
1861
1/2
✓ Branch 3 taken 1179 times.
✗ Branch 4 not taken.
 1179 sqlite3_bind_text(stmt_new_, 1, &hash_str[0], hash_str.length(),
1862 SQLITE_STATIC);
1863
1/2
✓ Branch 1 taken 1179 times.
✗ Branch 2 not taken.
 1179 sqlite3_bind_int64(stmt_new_, 2, size);
1864
1/2
✓ Branch 1 taken 1179 times.
✗ Branch 2 not taken.
 1179 sqlite3_bind_int64(stmt_new_, 3, seq_++);
1865
1/2
✓ Branch 3 taken 1179 times.
✗ Branch 4 not taken.
 1179 sqlite3_bind_text(stmt_new_, 4, &description[0], description.length(),
1866 SQLITE_STATIC);
1867
3/4
✓ Branch 0 taken 1152 times.
✓ Branch 1 taken 27 times.
✓ Branch 3 taken 1179 times.
✗ Branch 4 not taken.
 1179 sqlite3_bind_int64(stmt_new_, 5, is_catalog ? kFileCatalog : kFileRegular);
1868
1/2
✓ Branch 1 taken 1179 times.
✗ Branch 2 not taken.
 1179 sqlite3_bind_int64(stmt_new_, 6, 1);
1869
1/2
✓ Branch 1 taken 1179 times.
✗ Branch 2 not taken.
 1179 const int retval = sqlite3_step(stmt_new_);
1870
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1179 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 1179 assert((retval == SQLITE_DONE) || (retval == SQLITE_OK));
1871
1/2
✓ Branch 1 taken 1179 times.
✗ Branch 2 not taken.
 1179 sqlite3_reset(stmt_new_);
1872
2/2
✓ Branch 0 taken 642 times.
✓ Branch 1 taken 537 times.
 1179 if (!exists)
1873 642 gauge_ += size;
1874 1179 return true;
1875 }
1876
1877 int pipe_reserve[2];
1878
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 MakeReturnPipe(pipe_reserve);
1879
1880 126 LruCommand cmd;
1881 126 cmd.command_type = kReserve;
1882 126 cmd.SetSize(size);
1883
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 cmd.StoreHash(hash);
1884 126 cmd.return_pipe = pipe_reserve[1];
1885
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
1886 bool result;
1887
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 ManagedReadHalfPipe(pipe_reserve[0], &result, sizeof(result));
1888
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 CloseReturnPipe(pipe_reserve);
1889
1890
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 117 times.
 126 if (!result)
1891 9 return false;
1892
3/4
✓ Branch 0 taken 18 times.
✓ Branch 1 taken 99 times.
✓ Branch 3 taken 117 times.
✗ Branch 4 not taken.
 117 DoInsert(hash, size, description, is_catalog ? kPin : kPinRegular);
1893
1894 117 return true;
1895 1323 }
1896
1897
1898 2173 PosixQuotaManager::PosixQuotaManager(const uint64_t limit,
1899 const uint64_t cleanup_threshold,
1900 2173 const string &cache_workspace)
1901 2173 : shared_(false)
1902 2173 , spawned_(false)
1903 2173 , limit_(limit)
1904 2173 , cleanup_threshold_(cleanup_threshold)
1905 2173 , gauge_(0)
1906 2173 , pinned_(0)
1907 2173 , seq_(0)
1908 2173 , cache_dir_() // initialized in body
1909 2173 , workspace_dir_() // initialized in body
1910 2173 , fd_lock_cachedb_(-1)
1911 2173 , async_delete_(true)
1912 2173 , cachemgr_pid_(0)
1913 2173 , database_(NULL)
1914 2173 , stmt_touch_(NULL)
1915 2173 , stmt_unpin_(NULL)
1916 2173 , stmt_block_(NULL)
1917 2173 , stmt_unblock_(NULL)
1918 2173 , stmt_new_(NULL)
1919 2173 , stmt_lru_(NULL)
1920 2173 , stmt_size_(NULL)
1921 2173 , stmt_rm_(NULL)
1922 2173 , stmt_rm_batch_(NULL)
1923 2173 , stmt_list_(NULL)
1924 2173 , stmt_list_pinned_(NULL)
1925 2173 , stmt_list_catalogs_(NULL)
1926 2173 , stmt_list_volatile_(NULL)
1927 2173 , initialized_(false)
1928 4346 , cleanup_unused_first_(false) {
1929
2/4
✓ Branch 1 taken 2173 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2173 times.
✗ Branch 5 not taken.
 2173 ParseDirectories(cache_workspace, &cache_dir_, &workspace_dir_);
1930 2173 pipe_lru_[0] = pipe_lru_[1] = -1;
1931
1/2
✓ Branch 1 taken 2173 times.
✗ Branch 2 not taken.
 2173 cleanup_recorder_.AddRecorder(1, 90); // last 1.5 min with second resolution
1932 // last 1.5 h with minute resolution
1933
1/2
✓ Branch 1 taken 2173 times.
✗ Branch 2 not taken.
 2173 cleanup_recorder_.AddRecorder(60, 90 * 60);
1934 // last 18 hours with 20 min resolution
1935
1/2
✓ Branch 1 taken 2173 times.
✗ Branch 2 not taken.
 2173 cleanup_recorder_.AddRecorder(20 * 60, 60 * 60 * 18);
1936 // last 4 days with hour resolution
1937
1/2
✓ Branch 1 taken 2173 times.
✗ Branch 2 not taken.
 2173 cleanup_recorder_.AddRecorder(60 * 60, 60 * 60 * 24 * 4);
1938
1939 2173 lock_open_files_ = reinterpret_cast<pthread_mutex_t *>(
1940 2173 smalloc(sizeof(pthread_mutex_t)));
1941 2173 }
1942
1943
1944 8688 PosixQuotaManager::~PosixQuotaManager() {
1945 4344 free(lock_open_files_);
1946
1947
2/2
✓ Branch 0 taken 27 times.
✓ Branch 1 taken 2145 times.
 4344 if (!initialized_)
1948 54 return;
1949
1950
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2145 times.
 4290 if (shared_) {
1951 // Most of cleanup is done elsewhen by shared cache manager
1952 ✗ close(pipe_lru_[1]);
1953 ✗ return;
1954 }
1955
1956
2/2
✓ Branch 0 taken 306 times.
✓ Branch 1 taken 1839 times.
 4290 if (spawned_) {
1957 612 char fin = 0;
1958 612 WritePipe(pipe_lru_[1], &fin, 1);
1959 612 close(pipe_lru_[1]);
1960 612 pthread_join(thread_lru_, NULL);
1961 } else {
1962 3678 ClosePipe(pipe_lru_);
1963 }
1964
1965 4290 CloseDatabase();
1966
14/14
✓ Branch 1 taken 2145 times.
✓ Branch 2 taken 27 times.
✓ Branch 4 taken 2145 times.
✓ Branch 5 taken 27 times.
✓ Branch 7 taken 2145 times.
✓ Branch 8 taken 27 times.
✓ Branch 10 taken 2145 times.
✓ Branch 11 taken 27 times.
✓ Branch 13 taken 2145 times.
✓ Branch 14 taken 27 times.
✓ Branch 16 taken 2145 times.
✓ Branch 17 taken 27 times.
✓ Branch 19 taken 2145 times.
✓ Branch 20 taken 27 times.
 9012 }
1967
1968
1969 43182 void PosixQuotaManager::ProcessCommandBunch(const unsigned num,
1970 const LruCommand *commands,
1971 const char *descriptions) {
1972 43182 int retval = sqlite3_exec(database_, "BEGIN", NULL, NULL, NULL);
1973
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 43182 times.
 43182 assert(retval == SQLITE_OK);
1974
1975
2/2
✓ Branch 0 taken 1350459 times.
✓ Branch 1 taken 43182 times.
 1393641 for (unsigned i = 0; i < num; ++i) {
1976
1/2
✓ Branch 1 taken 1350459 times.
✗ Branch 2 not taken.
 1350459 const shash::Any hash = commands[i].RetrieveHash();
1977
1/2
✓ Branch 1 taken 1350459 times.
✗ Branch 2 not taken.
 1350459 const string hash_str = hash.ToString();
1978 1350459 const unsigned size = commands[i].GetSize();
1979
1/2
✓ Branch 1 taken 1350459 times.
✗ Branch 2 not taken.
 1350459 LogCvmfs(kLogQuota, kLogDebug, "processing %s (%d)", hash_str.c_str(),
1980 1350459 commands[i].command_type);
1981
1982 bool exists;
1983
3/4
✓ Branch 0 taken 450117 times.
✓ Branch 1 taken 18 times.
✓ Branch 2 taken 900324 times.
✗ Branch 3 not taken.
 1350459 switch (commands[i].command_type) {
1984 450117 case kTouch:
1985
1/2
✓ Branch 1 taken 450117 times.
✗ Branch 2 not taken.
 450117 sqlite3_bind_int64(stmt_touch_, 1, seq_++);
1986
1/2
✓ Branch 3 taken 450117 times.
✗ Branch 4 not taken.
 450117 sqlite3_bind_text(stmt_touch_, 2, &hash_str[0], hash_str.length(),
1987 SQLITE_STATIC);
1988
1/2
✓ Branch 1 taken 450117 times.
✗ Branch 2 not taken.
 450117 retval = sqlite3_step(stmt_touch_);
1989
1/2
✓ Branch 1 taken 450117 times.
✗ Branch 2 not taken.
 450117 LogCvmfs(kLogQuota, kLogDebug, "touching %s (%ld): %d",
1990 450117 hash_str.c_str(), seq_ - 1, retval);
1991
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 450117 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 450117 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
1992 ✗ PANIC(kLogSyslogErr, "failed to update %s in cachedb, error %d",
1993 hash_str.c_str(), retval);
1994 }
1995
1/2
✓ Branch 1 taken 450117 times.
✗ Branch 2 not taken.
 450117 sqlite3_reset(stmt_touch_);
1996 450117 break;
1997 18 case kUnpin:
1998
1/2
✓ Branch 3 taken 18 times.
✗ Branch 4 not taken.
 18 sqlite3_bind_text(stmt_unpin_, 1, &hash_str[0], hash_str.length(),
1999 SQLITE_STATIC);
2000
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 18 retval = sqlite3_step(stmt_unpin_);
2001
1/2
✓ Branch 2 taken 18 times.
✗ Branch 3 not taken.
 18 LogCvmfs(kLogQuota, kLogDebug, "unpinning %s: %d", hash_str.c_str(),
2002 retval);
2003
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 18 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
2004 ✗ PANIC(kLogSyslogErr, "failed to unpin %s in cachedb, error %d",
2005 hash_str.c_str(), retval);
2006 }
2007
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 18 sqlite3_reset(stmt_unpin_);
2008 18 break;
2009 900324 case kPin:
2010 case kPinRegular:
2011 case kInsert:
2012 case kInsertVolatile:
2013 // It could already be in, check
2014
1/2
✓ Branch 1 taken 900324 times.
✗ Branch 2 not taken.
 900324 exists = Contains(hash_str);
2015
2016 // Cleanup, move to trash and unlink
2017
3/4
✓ Branch 0 taken 900279 times.
✓ Branch 1 taken 45 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 900279 times.
 900324 if (!exists && (gauge_ + size > limit_)) {
2018 ✗ LogCvmfs(kLogQuota, kLogDebug, "over limit, gauge %lu, file size %u",
2019 gauge_, size);
2020 ✗ retval = DoCleanup(cleanup_threshold_);
2021 ✗ assert(retval != 0);
2022 }
2023
2024 // Insert or replace
2025
1/2
✓ Branch 3 taken 900324 times.
✗ Branch 4 not taken.
 900324 sqlite3_bind_text(stmt_new_, 1, &hash_str[0], hash_str.length(),
2026 SQLITE_STATIC);
2027
1/2
✓ Branch 1 taken 900324 times.
✗ Branch 2 not taken.
 900324 sqlite3_bind_int64(stmt_new_, 2, size);
2028
2/2
✓ Branch 0 taken 36 times.
✓ Branch 1 taken 900288 times.
 900324 if (commands[i].command_type == kInsertVolatile) {
2029
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 sqlite3_bind_int64(stmt_new_, 3, (seq_++) | kVolatileFlag);
2030 } else {
2031
1/2
✓ Branch 1 taken 900288 times.
✗ Branch 2 not taken.
 900288 sqlite3_bind_int64(stmt_new_, 3, seq_++);
2032 }
2033 900324 sqlite3_bind_text(stmt_new_, 4, &descriptions[i * kMaxDescription],
2034
1/2
✓ Branch 1 taken 900324 times.
✗ Branch 2 not taken.
 900324 commands[i].desc_length, SQLITE_STATIC);
2035
1/2
✓ Branch 1 taken 900324 times.
✗ Branch 2 not taken.
 900324 sqlite3_bind_int64(
2036 stmt_new_, 5,
2037
2/2
✓ Branch 0 taken 18 times.
✓ Branch 1 taken 900306 times.
 900324 (commands[i].command_type == kPin) ? kFileCatalog : kFileRegular);
2038
1/2
✓ Branch 1 taken 900324 times.
✗ Branch 2 not taken.
 900324 sqlite3_bind_int64(stmt_new_, 6,
2039
2/2
✓ Branch 0 taken 900306 times.
✓ Branch 1 taken 18 times.
 900324 ((commands[i].command_type == kPin)
2040
2/2
✓ Branch 0 taken 99 times.
✓ Branch 1 taken 900207 times.
 900306 || (commands[i].command_type == kPinRegular))
2041 ? 1
2042 : 0);
2043
1/2
✓ Branch 1 taken 900324 times.
✗ Branch 2 not taken.
 900324 retval = sqlite3_step(stmt_new_);
2044
1/2
✓ Branch 1 taken 900324 times.
✗ Branch 2 not taken.
 900324 LogCvmfs(kLogQuota, kLogDebug, "insert or replace %s, method %d: %d",
2045 900324 hash_str.c_str(), commands[i].command_type, retval);
2046
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 900324 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 900324 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
2047 ✗ PANIC(kLogSyslogErr, "failed to insert %s in cachedb, error %d",
2048 hash_str.c_str(), retval);
2049 }
2050
1/2
✓ Branch 1 taken 900324 times.
✗ Branch 2 not taken.
 900324 sqlite3_reset(stmt_new_);
2051
2052
2/2
✓ Branch 0 taken 900279 times.
✓ Branch 1 taken 45 times.
 900324 if (!exists)
2053 900279 gauge_ += size;
2054 900324 break;
2055 ✗ default:
2056 // other types should have been taken care of by event loop
2057 ✗ PANIC(NULL);
2058 }
2059 1350459 }
2060
2061 43182 retval = sqlite3_exec(database_, "COMMIT", NULL, NULL, NULL);
2062
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 43182 times.
 43182 if (retval != SQLITE_OK) {
2063 ✗ PANIC(kLogSyslogErr, "failed to commit to cachedb, error %d", retval);
2064 }
2065 43182 }
2066
2067
2068 2155 bool PosixQuotaManager::RebuildDatabase() {
2069 2155 bool result = false;
2070 2155 string sql;
2071 2155 sqlite3_stmt *stmt_select = NULL;
2072 2155 sqlite3_stmt *stmt_insert = NULL;
2073 int sqlerr;
2074 2155 int seq = 0;
2075 char hex[4];
2076 struct stat info;
2077 platform_dirent64 *d;
2078 2155 DIR *dirp = NULL;
2079 2155 string path;
2080
2081
1/2
✓ Branch 1 taken 2155 times.
✗ Branch 2 not taken.
 2155 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug, "re-building cache database");
2082
2083 // Empty cache catalog and fscache
2084
1/2
✓ Branch 1 taken 2155 times.
✗ Branch 2 not taken.
 2155 sql = "DELETE FROM cache_catalog; DELETE FROM fscache;";
2085
1/2
✓ Branch 2 taken 2155 times.
✗ Branch 3 not taken.
 2155 sqlerr = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
2086
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2155 times.
 2155 if (sqlerr != SQLITE_OK) {
2087 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not clear cache database");
2088 ✗ goto build_return;
2089 }
2090
2091 2155 gauge_ = 0;
2092
2093 // Insert files from cache sub-directories 00 - ff
2094 // TODO(jblomer): fs_traversal
2095
1/2
✓ Branch 1 taken 2155 times.
✗ Branch 2 not taken.
 2155 sqlite3_prepare_v2(database_,
2096 "INSERT INTO fscache (sha1, size, actime) "
2097 "VALUES (:sha1, :s, :t);",
2098 -1, &stmt_insert, NULL);
2099
2100
2/2
✓ Branch 0 taken 544795 times.
✓ Branch 1 taken 2128 times.
 546923 for (int i = 0; i <= 0xff; i++) {
2101 544795 snprintf(hex, sizeof(hex), "%02x", i);
2102
3/6
✓ Branch 2 taken 544795 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 544795 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 544795 times.
✗ Branch 9 not taken.
 544795 path = cache_dir_ + "/" + string(hex);
2103
3/4
✓ Branch 2 taken 544795 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 27 times.
✓ Branch 5 taken 544768 times.
 544795 if ((dirp = opendir(path.c_str())) == NULL) {
2104
1/2
✓ Branch 2 taken 27 times.
✗ Branch 3 not taken.
 27 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
2105 "failed to open directory %s (tmpwatch interfering?)",
2106 path.c_str());
2107 27 goto build_return;
2108 }
2109
3/4
✓ Branch 1 taken 1634322 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1089554 times.
✓ Branch 4 taken 544768 times.
 1634322 while ((d = platform_readdir(dirp)) != NULL) {
2110
3/6
✓ Branch 2 taken 1089554 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1089554 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1089554 times.
✗ Branch 9 not taken.
 2179108 const string file_path = path + "/" + string(d->d_name);
2111
1/2
✓ Branch 2 taken 1089554 times.
✗ Branch 3 not taken.
 1089554 if (stat(file_path.c_str(), &info) == 0) {
2112
2/2
✓ Branch 0 taken 1089536 times.
✓ Branch 1 taken 18 times.
 1089554 if (!S_ISREG(info.st_mode))
2113 1089545 continue;
2114
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 9 times.
 18 if (info.st_size == 0) {
2115
1/2
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
 9 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug,
2116 "removing empty file %s during automatic cache db rebuild",
2117 file_path.c_str());
2118 9 unlink(file_path.c_str());
2119 9 continue;
2120 }
2121
2122
3/6
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 9 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 9 times.
✗ Branch 10 not taken.
 18 string hash = string(hex) + string(d->d_name);
2123
1/2
✓ Branch 3 taken 9 times.
✗ Branch 4 not taken.
 9 sqlite3_bind_text(stmt_insert, 1, hash.data(), hash.length(),
2124 SQLITE_STATIC);
2125
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 sqlite3_bind_int64(stmt_insert, 2, info.st_size);
2126
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 sqlite3_bind_int64(stmt_insert, 3, info.st_atime);
2127
2/4
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 9 times.
 9 if (sqlite3_step(stmt_insert) != SQLITE_DONE) {
2128 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not insert into temp table");
2129 ✗ goto build_return;
2130 }
2131
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 sqlite3_reset(stmt_insert);
2132
2133 9 gauge_ += info.st_size;
2134
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 } else {
2135 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not stat %s", file_path.c_str());
2136 }
2137
2/3
✓ Branch 1 taken 9 times.
✓ Branch 2 taken 1089545 times.
✗ Branch 3 not taken.
 1089554 }
2138
1/2
✓ Branch 1 taken 544768 times.
✗ Branch 2 not taken.
 544768 closedir(dirp);
2139 544768 dirp = NULL;
2140 }
2141
1/2
✓ Branch 1 taken 2128 times.
✗ Branch 2 not taken.
 2128 sqlite3_finalize(stmt_insert);
2142 2128 stmt_insert = NULL;
2143
2144 // Transfer from temp table in cache catalog
2145
1/2
✓ Branch 1 taken 2128 times.
✗ Branch 2 not taken.
 2128 sqlite3_prepare_v2(database_,
2146 "SELECT sha1, size FROM fscache ORDER BY actime;", -1,
2147 &stmt_select, NULL);
2148
1/2
✓ Branch 1 taken 2128 times.
✗ Branch 2 not taken.
 2128 sqlite3_prepare_v2(
2149 database_,
2150 "INSERT INTO cache_catalog (sha1, size, acseq, path, type, pinned) "
2151 "VALUES (:sha1, :s, :seq, 'unknown (automatic rebuild)', :t, 0);",
2152 -1, &stmt_insert, NULL);
2153
3/4
✓ Branch 1 taken 2137 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 9 times.
✓ Branch 4 taken 2128 times.
 2137 while (sqlite3_step(stmt_select) == SQLITE_ROW) {
2154 const string hash = string(
2155
2/4
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 9 times.
✗ Branch 6 not taken.
 9 reinterpret_cast<const char *>(sqlite3_column_text(stmt_select, 0)));
2156
1/2
✓ Branch 3 taken 9 times.
✗ Branch 4 not taken.
 9 sqlite3_bind_text(stmt_insert, 1, &hash[0], hash.length(), SQLITE_STATIC);
2157
2/4
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 9 times.
✗ Branch 5 not taken.
 9 sqlite3_bind_int64(stmt_insert, 2, sqlite3_column_int64(stmt_select, 1));
2158
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 sqlite3_bind_int64(stmt_insert, 3, seq++);
2159 // Might also be a catalog (information is lost)
2160
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 sqlite3_bind_int64(stmt_insert, 4, kFileRegular);
2161
2162
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 const int retval = sqlite3_step(stmt_insert);
2163
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
 9 if (retval != SQLITE_DONE) {
2164 // If the file system hosting the cache is full, we'll likely notice here
2165 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
2166 "could not insert into cache catalog (%d - %s)", retval,
2167 sqlite3_errstr(retval));
2168 ✗ goto build_return;
2169 }
2170
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 sqlite3_reset(stmt_insert);
2171
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 }
2172
2173 // Delete temporary table
2174
1/2
✓ Branch 1 taken 2128 times.
✗ Branch 2 not taken.
 2128 sql = "DELETE FROM fscache;";
2175
1/2
✓ Branch 2 taken 2128 times.
✗ Branch 3 not taken.
 2128 sqlerr = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
2176
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2128 times.
 2128 if (sqlerr != SQLITE_OK) {
2177 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not clear temporary table (%d)",
2178 sqlerr);
2179 ✗ goto build_return;
2180 }
2181
2182 2128 seq_ = seq;
2183 2128 result = true;
2184
1/2
✓ Branch 1 taken 2128 times.
✗ Branch 2 not taken.
 2128 LogCvmfs(kLogQuota, kLogDebug,
2185 "rebuilding finished, sequence %" PRIu64 ", gauge %" PRIu64, seq_,
2186 gauge_);
2187
2188 2155 build_return:
2189
1/2
✓ Branch 0 taken 2155 times.
✗ Branch 1 not taken.
 2155 if (stmt_insert)
2190
1/2
✓ Branch 1 taken 2155 times.
✗ Branch 2 not taken.
 2155 sqlite3_finalize(stmt_insert);
2191
2/2
✓ Branch 0 taken 2128 times.
✓ Branch 1 taken 27 times.
 2155 if (stmt_select)
2192
1/2
✓ Branch 1 taken 2128 times.
✗ Branch 2 not taken.
 2128 sqlite3_finalize(stmt_select);
2193
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2155 times.
 2155 if (dirp)
2194 ✗ closedir(dirp);
2195 2155 return result;
2196 2155 }
2197
2198
2199 /**
2200 * Register a channel that allows the cache manager to trigger action to its
2201 * clients. Currently used for releasing pinned catalogs.
2202 */
2203 36 void PosixQuotaManager::RegisterBackChannel(int back_channel[2],
2204 const string &channel_id) {
2205
1/2
✓ Branch 0 taken 36 times.
✗ Branch 1 not taken.
 36 if (protocol_revision_ >= 1) {
2206
1/2
✓ Branch 2 taken 36 times.
✗ Branch 3 not taken.
 36 shash::Md5 hash = shash::Md5(shash::AsciiPtr(channel_id));
2207
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 MakeReturnPipe(back_channel);
2208
2209 36 LruCommand cmd;
2210 36 cmd.command_type = kRegisterBackChannel;
2211 36 cmd.return_pipe = back_channel[1];
2212 // Not StoreHash(). This is an MD5 hash.
2213 36 memcpy(cmd.digest, hash.digest, hash.GetDigestSize());
2214
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2215
2216 char success;
2217
1/2
✓ Branch 1 taken 36 times.
✗ Branch 2 not taken.
 36 ManagedReadHalfPipe(back_channel[0], &success, sizeof(success));
2218 // At this point, the named FIFO is unlinked, so don't use CloseReturnPipe
2219
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 36 times.
 36 if (success != 'S') {
2220 ✗ PANIC(kLogDebug | kLogSyslogErr,
2221 "failed to register quota back channel (%c)", success);
2222 }
2223 } else {
2224 // Dummy pipe to return valid file descriptors
2225 ✗ MakePipe(back_channel);
2226 }
2227 36 }
2228
2229
2230 /**
2231 * Removes a chunk from cache, if it exists.
2232 */
2233 27 void PosixQuotaManager::Remove(const shash::Any &hash) {
2234
1/2
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
 27 const string hash_str = hash.ToString();
2235
2236 int pipe_remove[2];
2237
1/2
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
 27 MakeReturnPipe(pipe_remove);
2238
2239 27 LruCommand cmd;
2240 27 cmd.command_type = kRemove;
2241 27 cmd.return_pipe = pipe_remove[1];
2242
1/2
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
 27 cmd.StoreHash(hash);
2243
1/2
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
 27 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2244
2245 bool success;
2246
1/2
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
 27 ManagedReadHalfPipe(pipe_remove[0], &success, sizeof(success));
2247
1/2
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
 27 CloseReturnPipe(pipe_remove);
2248
2249
3/6
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 27 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 27 times.
✗ Branch 8 not taken.
 27 unlink((cache_dir_ + "/" + hash.MakePathWithoutSuffix()).c_str());
2250 27 }
2251
2252
2253 324 void PosixQuotaManager::Spawn() {
2254
2/2
✓ Branch 0 taken 18 times.
✓ Branch 1 taken 306 times.
 324 if (spawned_)
2255 18 return;
2256
2257 306 if (pthread_create(&thread_lru_, NULL, MainCommandServer,
2258 static_cast<void *>(this))
2259
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 306 times.
 306 != 0) {
2260 ✗ PANIC(kLogDebug, "could not create lru thread");
2261 }
2262
2263 306 spawned_ = true;
2264 }
2265
2266
2267 /**
2268 * Updates the sequence number of the file specified by the hash.
2269 */
2270 450699 void PosixQuotaManager::Touch(const shash::Any &hash) {
2271 450699 LruCommand cmd;
2272 450699 cmd.command_type = kTouch;
2273
1/2
✓ Branch 1 taken 450699 times.
✗ Branch 2 not taken.
 450699 cmd.StoreHash(hash);
2274
1/2
✓ Branch 1 taken 450699 times.
✗ Branch 2 not taken.
 450699 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2275 450699 }
2276
2277
2278 792 void PosixQuotaManager::UnbindReturnPipe(int pipe_wronly) {
2279
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 783 times.
 792 if (shared_)
2280 9 close(pipe_wronly);
2281 792 }
2282
2283
2284 63 void PosixQuotaManager::UnlinkReturnPipe(int pipe_wronly) {
2285
2/2
✓ Branch 0 taken 27 times.
✓ Branch 1 taken 36 times.
 63 if (shared_)
2286
2/4
✓ Branch 2 taken 27 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 27 times.
✗ Branch 6 not taken.
 27 unlink((workspace_dir_ + "/pipe" + StringifyInt(pipe_wronly)).c_str());
2287 63 }
2288
2289
2290 834 void PosixQuotaManager::Unpin(const shash::Any &hash) {
2291
2/4
✓ Branch 1 taken 834 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 834 times.
✗ Branch 6 not taken.
 834 LogCvmfs(kLogQuota, kLogDebug, "Unpin %s", hash.ToString().c_str());
2292
2293 834 LruCommand cmd;
2294 834 cmd.command_type = kUnpin;
2295
1/2
✓ Branch 1 taken 834 times.
✗ Branch 2 not taken.
 834 cmd.StoreHash(hash);
2296
1/2
✓ Branch 1 taken 834 times.
✗ Branch 2 not taken.
 834 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2297 834 }
2298
2299
2300 18 void PosixQuotaManager::UnregisterBackChannel(int back_channel[2],
2301 const string &channel_id) {
2302
1/2
✓ Branch 0 taken 18 times.
✗ Branch 1 not taken.
 18 if (protocol_revision_ >= 1) {
2303
1/2
✓ Branch 2 taken 18 times.
✗ Branch 3 not taken.
 18 shash::Md5 hash = shash::Md5(shash::AsciiPtr(channel_id));
2304
2305 18 LruCommand cmd;
2306 18 cmd.command_type = kUnregisterBackChannel;
2307 // Not StoreHash(). This is an MD5 hash.
2308 18 memcpy(cmd.digest, hash.digest, hash.GetDigestSize());
2309
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 18 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2310
2311 // Writer's end will be closed by cache manager, FIFO is already unlinked
2312
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 18 close(back_channel[0]);
2313 } else {
2314 ✗ ClosePipe(back_channel);
2315 }
2316 18 }
2317
2318 901188 void PosixQuotaManager::ManagedReadHalfPipe(int fd, void *buf, size_t nbyte) {
2319
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 901188 times.
 901188 const unsigned timeout_ms = cachemgr_pid_ ? 1000 : 0;
2320 901188 bool result = false;
2321 do {
2322 901188 result = ReadHalfPipe(fd, buf, nbyte, timeout_ms);
2323 // try only as long as the cachemgr is still alive
2324
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 901188 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 901188 times.
 901188 } while (!result && getpgid(cachemgr_pid_) >= 0);
2325
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 901188 times.
 901188 if (!result) {
2326 ✗ PANIC(kLogStderr,
2327 "Error: quota manager could not read from cachemanager pipe");
2328 }
2329 901188 }
2330
2331 ✗ void *PosixQuotaManager::CollectMountpointsHashes(void *data) {
2332 #ifndef __APPLE__
2333 ✗ pthread_setname_np(pthread_self(), "hash_collector");
2334 ✗ auto *handler = static_cast<CollectorHandler *>(data);
2335
2336 ✗ const std::string mountpoint = handler->mp[handler->i];
2337 ✗ ssize_t n = getxattr(mountpoint.c_str(), "user.list_open_hashes", nullptr, 0);
2338 ✗ if (n < 0) {
2339 ✗ pthread_exit(nullptr);
2340 }
2341 ✗ std::vector<char> buf((size_t)n);
2342 ✗ n = getxattr(mountpoint.c_str(), "user.list_open_hashes", buf.data(),
2343 buf.size());
2344 ✗ if (n < 0) {
2345 ✗ pthread_exit(nullptr);
2346 }
2347
2348 ✗ std::vector<std::string> hash_strs;
2349 ✗ std::string hash_str;
2350 ✗ for (const char c : buf) {
2351 ✗ if (c == '\n') {
2352 ✗ hash_strs.push_back(hash_str);
2353 ✗ hash_str.clear();
2354 } else {
2355 ✗ hash_str += c;
2356 }
2357 }
2358 ✗ const MutexLockGuard lock_guard(handler->l);
2359 ✗ for (auto hash_str : hash_strs) {
2360 ✗ handler->of.push_back(
2361 ✗ shash::Short(shash::MkFromHexPtr(shash::HexPtr(hash_str))));
2362 }
2363 #endif
2364 ✗ pthread_exit(nullptr);
2365 }
2366
2367 ✗ std::vector<shash::Short> PosixQuotaManager::CollectAllOpenHashes() {
2368 ✗ std::vector<CollectorHandler *> handlers;
2369 ✗ std::vector<pthread_t *> threads;
2370 ✗ open_files_.clear();
2371 #ifndef __APPLE__
2372 ✗ auto &&a_after_b = [](const struct timespec a, const struct timespec b) {
2373 ✗ return (a.tv_sec > b.tv_sec) ? true : false;
2374 };
2375
2376 ✗ for (size_t i = 0; i < mountpoints_.size(); ++i) {
2377 ✗ handlers.push_back(
2378 ✗ new CollectorHandler{open_files_, mountpoints_, lock_open_files_, i});
2379 ✗ threads.push_back(new pthread_t);
2380 }
2381
2382 ✗ const int retval = pthread_mutex_init(lock_open_files_, NULL);
2383 ✗ assert(retval == 0);
2384
2385 ✗ for (size_t i = 0; i < mountpoints_.size(); ++i) {
2386 ✗ pthread_create(threads[i], nullptr, CollectMountpointsHashes, handlers[i]);
2387 }
2388
2389 ✗ std::vector<bool> joined(handlers.size(), false);
2390 struct timespec reference, current;
2391 ✗ clock_gettime(CLOCK_REALTIME, &reference);
2392 ✗ clock_gettime(CLOCK_REALTIME, &current);
2393 ✗ reference.tv_sec += 10; // Give 10sec for hash collection
2394 ✗ size_t i = 0;
2395 ✗ while (
2396 ✗ (not std::all_of(joined.begin(), joined.end(), [](bool b) { return b; }))
2397 ✗ and a_after_b(reference, current)) {
2398 // as long as there are still threads that haven't joined yet
2399 // and for 10 seconds
2400 ✗ if (not joined[i]) {
2401 ✗ const int s = pthread_tryjoin_np(*threads[i], NULL);
2402 ✗ if (s == 0) {
2403 ✗ joined[i] = true;
2404 }
2405 }
2406 ✗ ++i;
2407 ✗ i = i % handlers.size();
2408 ✗ clock_gettime(CLOCK_REALTIME, &current);
2409 }
2410
2411 ✗ for (size_t i = 0; i < handlers.size(); ++i) {
2412 ✗ delete handlers[i];
2413 }
2414
2415 ✗ pthread_mutex_destroy(lock_open_files_);
2416 #endif
2417 ✗ return open_files_;
2418 }
2419