GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/quota_posix.cc
Date: 2026-05-10 02:36:07
Exec Total Coverage
Lines: 985 1419 69.4%
Branches: 769 1966 39.1%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * This module implements a "managed local cache".
5 * This way, we are able to track access times of files in the cache
6 * and remove files based on least recently used strategy.
7 *
8 * We setup another SQLite catalog, a "cache catalog", that helps us
9 * in the bookkeeping of files, file sizes and access times.
10 *
11 * We might choose to not manage the local cache. This is indicated
12 * by limit == 0 and everything succeeds in that case.
13 */
14
15 #define __STDC_LIMIT_MACROS
16
17
18 #include "quota_posix.h"
19
20 #include <dirent.h>
21 #include <errno.h>
22 #include <fcntl.h>
23 #include <inttypes.h>
24 #include <pthread.h>
25 #include <signal.h>
26 #include <stdint.h>
27 #include <sys/dir.h>
28 #include <sys/stat.h>
29 #include <sys/xattr.h>
30
31 #ifndef __APPLE__
32 #include <sys/statfs.h>
33 #endif
34 #include <sys/statvfs.h>
35 #include <sys/types.h>
36 #include <sys/wait.h>
37 #include <unistd.h>
38
39 #include <algorithm>
40 #include <cassert>
41 #include <cstdio>
42 #include <cstdlib>
43 #include <cstring>
44 #include <limits>
45 #include <map>
46 #include <set>
47 #include <string>
48 #include <vector>
49
50 #include "crypto/hash.h"
51 #include "duplex_sqlite3.h"
52 #include "monitor.h"
53 #include "statistics.h"
54 #include "util/capabilities.h"
55 #include "util/concurrency.h"
56 #include "util/exception.h"
57 #include "util/logging.h"
58 #include "util/pointer.h"
59 #include "util/posix.h"
60 #include "util/smalloc.h"
61 #include "util/string.h"
62
63 using namespace std; // NOLINT
64
65
66 4042 int PosixQuotaManager::BindReturnPipe(int pipe_wronly) {
67
2/2
✓ Branch 0 taken 3956 times.
✓ Branch 1 taken 86 times.
 4042 if (!shared_)
68 3956 return pipe_wronly;
69
70 // Connect writer's end
71
1/2
✓ Branch 2 taken 86 times.
✗ Branch 3 not taken.
 86 const int result = open(
72
2/4
✓ Branch 2 taken 86 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 86 times.
✗ Branch 6 not taken.
 172 (workspace_dir_ + "/pipe" + StringifyInt(pipe_wronly)).c_str(),
73 O_WRONLY | O_NONBLOCK);
74
2/2
✓ Branch 0 taken 43 times.
✓ Branch 1 taken 43 times.
 86 if (result >= 0) {
75 43 Nonblock2Block(result);
76 } else {
77 43 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
78 43 "failed to bind return pipe (%d)", errno);
79 }
80 86 return result;
81 }
82
83
84 836 void PosixQuotaManager::CheckHighPinWatermark() {
85 836 const uint64_t watermark = kHighPinWatermark * cleanup_threshold_ / 100;
86
3/4
✓ Branch 0 taken 836 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 129 times.
✓ Branch 3 taken 707 times.
 836 if ((cleanup_threshold_ > 0) && (pinned_ > watermark)) {
87 129 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
88 "high watermark of pinned files (%" PRIu64 "M > %" PRIu64 "M)",
89 129 pinned_ / (1024 * 1024), watermark / (1024 * 1024));
90
2/4
✓ Branch 2 taken 129 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 129 times.
✗ Branch 6 not taken.
 129 BroadcastBackchannels("R"); // clients: please release pinned catalogs
91 }
92 836 }
93
94
95 ✗ void PosixQuotaManager::CleanupPipes() {
96 ✗ DIR *dirp = opendir(workspace_dir_.c_str());
97 ✗ assert(dirp != NULL);
98
99 platform_dirent64 *dent;
100 ✗ bool found_leftovers = false;
101 ✗ while ((dent = platform_readdir(dirp)) != NULL) {
102 ✗ const string name = dent->d_name;
103 ✗ const string path = workspace_dir_ + "/" + name;
104 platform_stat64 info;
105 ✗ const int retval = platform_stat(path.c_str(), &info);
106 ✗ if (retval != 0)
107 ✗ continue;
108 ✗ if (S_ISFIFO(info.st_mode) && (name.substr(0, 4) == "pipe")) {
109 ✗ if (!found_leftovers) {
110 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogWarn,
111 "removing left-over FIFOs from cache directory");
112 }
113 ✗ found_leftovers = true;
114 ✗ unlink(path.c_str());
115 }
116 }
117 ✗ closedir(dirp);
118 }
119
120
121 /**
122 * Cleans up in data cache, until cache size is below leave_size.
123 * The actual unlinking is done in a separate process (fork).
124 *
125 * \return True on success, false otherwise
126 */
127 387 bool PosixQuotaManager::Cleanup(const uint64_t leave_size) {
128
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 387 times.
 387 if (!spawned_)
129 ✗ return DoCleanup(leave_size);
130
131 bool result;
132 int pipe_cleanup[2];
133
1/2
✓ Branch 1 taken 387 times.
✗ Branch 2 not taken.
 387 MakeReturnPipe(pipe_cleanup);
134
135 387 LruCommand cmd;
136 387 cmd.command_type = kCleanup;
137 387 cmd.size = leave_size;
138 387 cmd.return_pipe = pipe_cleanup[1];
139
140
1/2
✓ Branch 1 taken 387 times.
✗ Branch 2 not taken.
 387 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
141
1/2
✓ Branch 1 taken 387 times.
✗ Branch 2 not taken.
 387 ManagedReadHalfPipe(pipe_cleanup[0], &result, sizeof(result));
142
1/2
✓ Branch 1 taken 387 times.
✗ Branch 2 not taken.
 387 CloseReturnPipe(pipe_cleanup);
143
144 387 return result;
145 }
146
147
148 3144 void PosixQuotaManager::CloseDatabase() {
149
1/2
✓ Branch 0 taken 3144 times.
✗ Branch 1 not taken.
 3144 if (stmt_list_catalogs_)
150 3144 sqlite3_finalize(stmt_list_catalogs_);
151
1/2
✓ Branch 0 taken 3144 times.
✗ Branch 1 not taken.
 3144 if (stmt_list_pinned_)
152 3144 sqlite3_finalize(stmt_list_pinned_);
153
1/2
✓ Branch 0 taken 3144 times.
✗ Branch 1 not taken.
 3144 if (stmt_list_volatile_)
154 3144 sqlite3_finalize(stmt_list_volatile_);
155
1/2
✓ Branch 0 taken 3144 times.
✗ Branch 1 not taken.
 3144 if (stmt_list_)
156 3144 sqlite3_finalize(stmt_list_);
157
1/2
✓ Branch 0 taken 3144 times.
✗ Branch 1 not taken.
 3144 if (stmt_lru_)
158 3144 sqlite3_finalize(stmt_lru_);
159
1/2
✓ Branch 0 taken 3144 times.
✗ Branch 1 not taken.
 3144 if (stmt_rm_)
160 3144 sqlite3_finalize(stmt_rm_);
161
1/2
✓ Branch 0 taken 3144 times.
✗ Branch 1 not taken.
 3144 if (stmt_rm_batch_)
162 3144 sqlite3_finalize(stmt_rm_batch_);
163
1/2
✓ Branch 0 taken 3144 times.
✗ Branch 1 not taken.
 3144 if (stmt_size_)
164 3144 sqlite3_finalize(stmt_size_);
165
1/2
✓ Branch 0 taken 3144 times.
✗ Branch 1 not taken.
 3144 if (stmt_touch_)
166 3144 sqlite3_finalize(stmt_touch_);
167
1/2
✓ Branch 0 taken 3144 times.
✗ Branch 1 not taken.
 3144 if (stmt_unpin_)
168 3144 sqlite3_finalize(stmt_unpin_);
169
1/2
✓ Branch 0 taken 3144 times.
✗ Branch 1 not taken.
 3144 if (stmt_block_)
170 3144 sqlite3_finalize(stmt_block_);
171
1/2
✓ Branch 0 taken 3144 times.
✗ Branch 1 not taken.
 3144 if (stmt_unblock_)
172 3144 sqlite3_finalize(stmt_unblock_);
173
1/2
✓ Branch 0 taken 3144 times.
✗ Branch 1 not taken.
 3144 if (stmt_new_)
174 3144 sqlite3_finalize(stmt_new_);
175
1/2
✓ Branch 0 taken 3144 times.
✗ Branch 1 not taken.
 3144 if (database_)
176 3144 sqlite3_close(database_);
177 3144 UnlockFile(fd_lock_cachedb_);
178
179 3144 stmt_list_catalogs_ = NULL;
180 3144 stmt_list_pinned_ = NULL;
181 3144 stmt_list_volatile_ = NULL;
182 3144 stmt_list_ = NULL;
183 3144 stmt_rm_ = NULL;
184 3144 stmt_rm_batch_ = NULL;
185 3144 stmt_size_ = NULL;
186 3144 stmt_touch_ = NULL;
187 3144 stmt_unpin_ = NULL;
188 3144 stmt_block_ = NULL;
189 3144 stmt_unblock_ = NULL;
190 3144 stmt_new_ = NULL;
191 3144 database_ = NULL;
192
193 3144 pinned_chunks_.clear();
194 3144 }
195
196
197 3827 void PosixQuotaManager::CloseReturnPipe(int pipe[2]) {
198
2/2
✓ Branch 0 taken 86 times.
✓ Branch 1 taken 3741 times.
 3827 if (shared_) {
199 86 close(pipe[0]);
200 86 UnlinkReturnPipe(pipe[1]);
201 } else {
202 3741 ClosePipe(pipe);
203 }
204 3827 }
205
206
207 4302352 bool PosixQuotaManager::Contains(const string &hash_str) {
208 4302352 bool result = false;
209
210 4302352 sqlite3_bind_text(stmt_size_, 1, &hash_str[0], hash_str.length(),
211 SQLITE_STATIC);
212
2/2
✓ Branch 1 taken 613 times.
✓ Branch 2 taken 4301739 times.
 4302352 if (sqlite3_step(stmt_size_) == SQLITE_ROW)
213 613 result = true;
214 4302352 sqlite3_reset(stmt_size_);
215 4302352 LogCvmfs(kLogQuota, kLogDebug, "contains %s returns %d", hash_str.c_str(),
216 result);
217
218 4302352 return result;
219 }
220
221
222 3059 void PosixQuotaManager::CheckFreeSpace() {
223
3/4
✓ Branch 0 taken 3059 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 43 times.
✓ Branch 3 taken 3016 times.
 3059 if ((limit_ == 0) || (gauge_ >= limit_))
224 43 return;
225
226 struct statvfs vfs_info;
227
1/2
✓ Branch 1 taken 3016 times.
✗ Branch 2 not taken.
 3016 const int retval = statvfs((cache_dir_ + "/cachedb").c_str(), &vfs_info);
228
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3016 times.
 3016 if (retval != 0) {
229 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
230 "failed to query %s for free space (%d)", cache_dir_.c_str(),
231 ✗ errno);
232 ✗ return;
233 }
234 3016 const int64_t free_space_byte = vfs_info.f_bavail * vfs_info.f_bsize;
235
1/2
✓ Branch 1 taken 3016 times.
✗ Branch 2 not taken.
 3016 LogCvmfs(kLogQuota, kLogDebug, "free space: %" PRId64 " MB",
236 free_space_byte / (1024 * 1024));
237
238 3016 const int64_t required_byte = limit_ - gauge_;
239
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3016 times.
 3016 if (free_space_byte < required_byte) {
240 ✗ LogCvmfs(kLogQuota, kLogSyslogWarn,
241 "too little free space on the file system hosting the cache,"
242 " %" PRId64 " MB available",
243 free_space_byte / (1024 * 1024));
244 }
245 }
246
247
248 3188 PosixQuotaManager *PosixQuotaManager::Create(const string &cache_workspace,
249 const uint64_t limit,
250 const uint64_t cleanup_threshold,
251 const bool rebuild_database) {
252
2/2
✓ Branch 0 taken 86 times.
✓ Branch 1 taken 3102 times.
 3188 if (cleanup_threshold >= limit) {
253 86 LogCvmfs(kLogQuota, kLogDebug,
254 "invalid parameters: limit %" PRIu64 ", "
255 "cleanup_threshold %" PRIu64,
256 limit, cleanup_threshold);
257 86 return NULL;
258 }
259
260 PosixQuotaManager *quota_manager = new PosixQuotaManager(
261
1/2
✓ Branch 2 taken 3102 times.
✗ Branch 3 not taken.
 3102 limit, cleanup_threshold, cache_workspace);
262
263 // Initialize cache catalog
264
2/2
✓ Branch 1 taken 43 times.
✓ Branch 2 taken 3059 times.
 3102 if (!quota_manager->InitDatabase(rebuild_database)) {
265
1/2
✓ Branch 0 taken 43 times.
✗ Branch 1 not taken.
 43 delete quota_manager;
266 43 return NULL;
267 }
268 3059 quota_manager->CheckFreeSpace();
269 3059 MakePipe(quota_manager->pipe_lru_);
270
271 3059 quota_manager->protocol_revision_ = kProtocolRevision;
272 3059 quota_manager->initialized_ = true;
273 3059 return quota_manager;
274 }
275
276
277 /**
278 * Connects to a running shared local quota manager. Creates one if necessary.
279 */
280 86 PosixQuotaManager *PosixQuotaManager::CreateShared(
281 const std::string &exe_path,
282 const std::string &cache_workspace,
283 const uint64_t limit,
284 const uint64_t cleanup_threshold,
285 bool foreground) {
286 86 string cache_dir;
287 86 string workspace_dir;
288
2/4
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 86 times.
✗ Branch 5 not taken.
 86 ParseDirectories(cache_workspace, &cache_dir, &workspace_dir);
289
290 pid_t new_cachemgr_pid;
291
292 // Create lock file: only one fuse client at a time
293
2/4
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 86 times.
✗ Branch 5 not taken.
 86 const int fd_lockfile = LockFile(workspace_dir + "/lock_cachemgr");
294
2/2
✓ Branch 0 taken 43 times.
✓ Branch 1 taken 43 times.
 86 if (fd_lockfile < 0) {
295
1/2
✓ Branch 2 taken 43 times.
✗ Branch 3 not taken.
 43 LogCvmfs(kLogQuota, kLogDebug, "could not open lock file %s (%d)",
296
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 86 (workspace_dir + "/lock_cachemgr").c_str(), errno);
297 43 return NULL;
298 }
299
300 PosixQuotaManager *quota_mgr = new PosixQuotaManager(limit, cleanup_threshold,
301
2/4
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 43 times.
✗ Branch 5 not taken.
 43 cache_workspace);
302 43 quota_mgr->shared_ = true;
303 43 quota_mgr->spawned_ = true;
304
305 // Try to connect to pipe
306
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 const string fifo_path = workspace_dir + "/cachemgr";
307
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 LogCvmfs(kLogQuota, kLogDebug, "trying to connect to existing pipe");
308
1/2
✓ Branch 2 taken 43 times.
✗ Branch 3 not taken.
 43 quota_mgr->pipe_lru_[1] = open(fifo_path.c_str(), O_WRONLY | O_NONBLOCK);
309
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 43 times.
 43 if (quota_mgr->pipe_lru_[1] >= 0) {
310 ✗ const int fd_lockfile_rw = open((workspace_dir + "/lock_cachemgr").c_str(),
311 O_RDWR, 0600);
312 ✗ unsigned lockfile_magicnumber = 0;
313 ✗ const ssize_t result_mn = SafeRead(fd_lockfile_rw, &lockfile_magicnumber,
314 sizeof(lockfile_magicnumber));
315 ✗ const ssize_t result = SafeRead(fd_lockfile_rw, &new_cachemgr_pid,
316 sizeof(new_cachemgr_pid));
317 ✗ close(fd_lockfile_rw);
318
319 ✗ if ((lockfile_magicnumber != kLockFileMagicNumber) || (result < 0)
320 ✗ || (result_mn < 0)
321 ✗ || (static_cast<size_t>(result) < sizeof(new_cachemgr_pid))) {
322 ✗ if (result != 0) {
323 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
324 "could not read cache manager pid from lockfile");
325 ✗ UnlockFile(fd_lockfile);
326 ✗ delete quota_mgr;
327 ✗ return NULL;
328 } else {
329 // support reload from old versions of the cache manager
330 // lock file is empty in this case, try a plain ReadHalfPipe to get pid
331 ✗ quota_mgr->SetCacheMgrPid(quota_mgr->GetPid());
332 }
333 } else {
334 ✗ quota_mgr->SetCacheMgrPid(new_cachemgr_pid);
335 }
336
337
338 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to existing cache manager pipe");
339 ✗ quota_mgr->initialized_ = true;
340 ✗ Nonblock2Block(quota_mgr->pipe_lru_[1]);
341 ✗ UnlockFile(fd_lockfile);
342 ✗ quota_mgr->GetLimits(&quota_mgr->limit_, &quota_mgr->cleanup_threshold_);
343 ✗ LogCvmfs(kLogQuota, kLogDebug,
344 "received limit %" PRIu64 ", threshold %" PRIu64,
345 quota_mgr->limit_, quota_mgr->cleanup_threshold_);
346 ✗ if (FileExists(workspace_dir + "/cachemgr.protocol")) {
347 ✗ quota_mgr->protocol_revision_ = quota_mgr->GetProtocolRevision();
348 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected protocol revision %u",
349 quota_mgr->protocol_revision_);
350 } else {
351 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to ancient cache manager");
352 }
353 ✗ return quota_mgr;
354 }
355 43 const int connect_error = errno;
356
357 // Lock file: let existing cache manager finish first
358
2/4
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 43 times.
✗ Branch 5 not taken.
 43 const int fd_lockfile_fifo = LockFile(workspace_dir + "/lock_cachemgr.fifo");
359
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 43 times.
 43 if (fd_lockfile_fifo < 0) {
360 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not open lock file %s (%d)",
361 ✗ (workspace_dir + "/lock_cachemgr.fifo").c_str(), errno);
362 ✗ UnlockFile(fd_lockfile);
363 ✗ delete quota_mgr;
364 ✗ return NULL;
365 }
366
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 UnlockFile(fd_lockfile_fifo);
367
368
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 43 times.
 43 if (connect_error == ENXIO) {
369 ✗ LogCvmfs(kLogQuota, kLogDebug, "left-over FIFO found, unlinking");
370 ✗ unlink(fifo_path.c_str());
371 }
372
373 // Creating a new FIFO for the cache manager (to be bound later)
374 43 int retval = mkfifo(fifo_path.c_str(), 0600);
375
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 43 times.
 43 if (retval != 0) {
376 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to create cache manager FIFO (%d)",
377 ✗ errno);
378 ✗ UnlockFile(fd_lockfile);
379 ✗ delete quota_mgr;
380 ✗ return NULL;
381 }
382
383 // Create new cache manager
384 int pipe_boot[2];
385 int pipe_handshake[2];
386
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 MakePipe(pipe_boot);
387
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 MakePipe(pipe_handshake);
388
389 43 vector<string> command_line;
390
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 command_line.push_back(exe_path);
391
2/4
✓ Branch 2 taken 43 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 43 times.
✗ Branch 6 not taken.
 43 command_line.push_back("__cachemgr__");
392
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 command_line.push_back(cache_workspace);
393
2/4
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 43 times.
✗ Branch 5 not taken.
 43 command_line.push_back(StringifyInt(pipe_boot[1]));
394
2/4
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 43 times.
✗ Branch 5 not taken.
 43 command_line.push_back(StringifyInt(pipe_handshake[0]));
395
2/4
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 43 times.
✗ Branch 5 not taken.
 43 command_line.push_back(StringifyInt(limit));
396
2/4
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 43 times.
✗ Branch 5 not taken.
 43 command_line.push_back(StringifyInt(cleanup_threshold));
397 // do not propagate foreground in order to reliably get pid from exec
398 // instead, daemonize right here
399
2/4
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 43 times.
✗ Branch 5 not taken.
 43 command_line.push_back(StringifyInt(true)); // foreground
400
3/6
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 43 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 43 times.
✗ Branch 8 not taken.
 43 command_line.push_back(StringifyInt(GetLogSyslogLevel()));
401
3/6
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 43 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 43 times.
✗ Branch 8 not taken.
 43 command_line.push_back(StringifyInt(GetLogSyslogFacility()));
402
5/14
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 43 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 43 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 43 times.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 43 times.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
 43 command_line.push_back(GetLogDebugFile() + ":" + GetLogMicroSyslog());
403
404 43 set<int> preserve_filedes;
405
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 preserve_filedes.insert(0);
406
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 preserve_filedes.insert(1);
407
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 preserve_filedes.insert(2);
408
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 preserve_filedes.insert(pipe_boot[1]);
409
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 preserve_filedes.insert(pipe_handshake[0]);
410
411
1/2
✓ Branch 0 taken 43 times.
✗ Branch 1 not taken.
 43 if (foreground) {
412
1/2
✓ Branch 2 taken 43 times.
✗ Branch 3 not taken.
 43 retval = ManagedExec(command_line, preserve_filedes, map<int, int>(),
413 /*drop_credentials*/ false,
414 /*clear_env*/ false,
415 /*double_fork*/ true, &new_cachemgr_pid);
416 } else {
417 ✗ retval = ExecAsDaemon(command_line, &new_cachemgr_pid);
418 }
419
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 43 times.
 43 if (!retval) {
420 ✗ UnlockFile(fd_lockfile);
421 ✗ ClosePipe(pipe_boot);
422 ✗ ClosePipe(pipe_handshake);
423 ✗ delete quota_mgr;
424 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to start cache manager");
425 ✗ return NULL;
426 }
427
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 LogCvmfs(kLogQuota, kLogDebug,
428 "new cache manager pid: %d protocol revision %d", new_cachemgr_pid,
429 QuotaManager::kProtocolRevision);
430 43 quota_mgr->SetCacheMgrPid(new_cachemgr_pid);
431
2/4
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 43 times.
✗ Branch 6 not taken.
 43 const int fd_lockfile_rw = open((workspace_dir + "/lock_cachemgr").c_str(),
432 O_RDWR | O_TRUNC, 0600);
433 43 const unsigned magic_number = PosixQuotaManager::kLockFileMagicNumber;
434
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 const bool result_mn = SafeWrite(fd_lockfile_rw, &magic_number,
435 sizeof(magic_number));
436
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 const bool result = SafeWrite(fd_lockfile_rw, &new_cachemgr_pid,
437 sizeof(new_cachemgr_pid));
438
2/4
✓ Branch 0 taken 43 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 43 times.
 43 if (!result || !result_mn) {
439 ✗ PANIC(kLogSyslogErr, "could not write cache manager pid to lockfile");
440 }
441
442
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 close(fd_lockfile_rw);
443 // Wait for cache manager to be ready
444
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 close(pipe_boot[1]);
445
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 close(pipe_handshake[0]);
446 char buf;
447
2/4
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 43 times.
✗ Branch 4 not taken.
 43 if (read(pipe_boot[0], &buf, 1) != 1) {
448
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 UnlockFile(fd_lockfile);
449
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 close(pipe_boot[0]);
450
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 close(pipe_handshake[1]);
451
1/2
✓ Branch 0 taken 43 times.
✗ Branch 1 not taken.
 43 delete quota_mgr;
452
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
453 "cache manager did not start");
454 43 return NULL;
455 }
456 ✗ close(pipe_boot[0]);
457
458 // Connect write end
459 ✗ quota_mgr->pipe_lru_[1] = open(fifo_path.c_str(), O_WRONLY | O_NONBLOCK);
460 ✗ if (quota_mgr->pipe_lru_[1] < 0) {
461 ✗ LogCvmfs(kLogQuota, kLogDebug,
462 ✗ "failed to connect to newly created FIFO (%d)", errno);
463 ✗ close(pipe_handshake[1]);
464 ✗ UnlockFile(fd_lockfile);
465 ✗ delete quota_mgr;
466 ✗ return NULL;
467 }
468
469 // Finalize handshake
470 ✗ buf = 'C';
471 ✗ if (write(pipe_handshake[1], &buf, 1) != 1) {
472 ✗ UnlockFile(fd_lockfile);
473 ✗ close(pipe_handshake[1]);
474 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not finalize handshake");
475 ✗ delete quota_mgr;
476 ✗ return NULL;
477 }
478 ✗ close(pipe_handshake[1]);
479
480 ✗ Nonblock2Block(quota_mgr->pipe_lru_[1]);
481 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to a new cache manager");
482 ✗ quota_mgr->protocol_revision_ = kProtocolRevision;
483
484 ✗ UnlockFile(fd_lockfile);
485
486 ✗ quota_mgr->initialized_ = true;
487 ✗ quota_mgr->GetLimits(&quota_mgr->limit_, &quota_mgr->cleanup_threshold_);
488 ✗ LogCvmfs(kLogQuota, kLogDebug,
489 "received limit %" PRIu64 ", "
490 "threshold %" PRIu64,
491 quota_mgr->limit_, quota_mgr->cleanup_threshold_);
492 ✗ return quota_mgr;
493 86 }
494
495
496 430 bool PosixQuotaManager::DoCleanup(const uint64_t leave_size) {
497
2/2
✓ Branch 0 taken 86 times.
✓ Branch 1 taken 344 times.
 430 if (gauge_ <= leave_size)
498 86 return true;
499
500 // TODO(jblomer) transaction
501
1/2
✓ Branch 1 taken 344 times.
✗ Branch 2 not taken.
 344 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug,
502 "clean up cache until at most %lu KB is used", leave_size / 1024);
503
1/2
✓ Branch 1 taken 344 times.
✗ Branch 2 not taken.
 344 LogCvmfs(kLogQuota, kLogDebug, "gauge %" PRIu64, gauge_);
504
1/2
✓ Branch 1 taken 344 times.
✗ Branch 2 not taken.
 344 cleanup_recorder_.Tick();
505
506 bool result;
507 344 vector<string> trash;
508
509 // Note that volatile files start counting from the smallest int64 number:
510 // the absolute sequence number with the first bit set in two's complement.
511 // So -1 can be a marker that will never appear in the database.
512 344 int64_t max_acseq = -1;
513 344 std::vector<EvictCandidate> lru_ordered_open;
514
515 do {
516
1/2
✓ Branch 1 taken 2494 times.
✗ Branch 2 not taken.
 2494 sqlite3_reset(stmt_lru_);
517
3/4
✓ Branch 0 taken 344 times.
✓ Branch 1 taken 2150 times.
✓ Branch 3 taken 2494 times.
✗ Branch 4 not taken.
 2838 sqlite3_bind_int64(stmt_lru_, 1,
518 344 (max_acseq == -1) ? std::numeric_limits<int64_t>::min()
519 : (max_acseq + 1));
520
521 2494 std::vector<EvictCandidate> candidates;
522
1/2
✓ Branch 1 taken 2494 times.
✗ Branch 2 not taken.
 2494 candidates.reserve(kEvictBatchSize);
523 2494 string hash_str;
524 2494 unsigned i = 0;
525
3/4
✓ Branch 1 taken 2238924 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2236430 times.
✓ Branch 4 taken 2494 times.
 2238924 while (sqlite3_step(stmt_lru_) == SQLITE_ROW) {
526 hash_str = reinterpret_cast<const char *>(
527
2/4
✓ Branch 1 taken 2236430 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2236430 times.
✗ Branch 5 not taken.
 2236430 sqlite3_column_text(stmt_lru_, 0));
528
1/2
✓ Branch 2 taken 2236430 times.
✗ Branch 3 not taken.
 2236430 LogCvmfs(kLogQuota, kLogDebug, "add %s to candidates for eviction",
529 hash_str.c_str());
530
1/2
✓ Branch 1 taken 2236430 times.
✗ Branch 2 not taken.
 2236430 candidates.push_back(
531
1/2
✓ Branch 1 taken 2236430 times.
✗ Branch 2 not taken.
 2236430 EvictCandidate(shash::MkFromHexPtr(shash::HexPtr(hash_str)),
532 2236430 sqlite3_column_int64(stmt_lru_, 1),
533
2/4
✓ Branch 1 taken 2236430 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2236430 times.
✗ Branch 5 not taken.
 2236430 sqlite3_column_int64(stmt_lru_, 2)));
534 2236430 i++;
535 }
536
2/2
✓ Branch 1 taken 43 times.
✓ Branch 2 taken 2451 times.
 2494 if (candidates.empty()) {
537
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 LogCvmfs(kLogQuota, kLogDebug, "no more entries to evict");
538 43 break;
539 }
540
541 2451 const unsigned N = candidates.size();
542
543 2451 open_files_.clear();
544
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2451 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 4902 open_files_ = (cleanup_unused_first_) ? CollectAllOpenHashes()
545 2451 : std::vector<shash::Short>();
546
547
2/2
✓ Branch 0 taken 2150387 times.
✓ Branch 1 taken 2150 times.
 2152537 for (i = 0; i < N; ++i) {
548 // That's a critical condition. We must not delete a not yet inserted
549 // pinned file as it is already reserved (but will be inserted later).
550 // Instead, set the pin bit in the db to not run into an endless loop
551
1/2
✓ Branch 2 taken 2150387 times.
✗ Branch 3 not taken.
 2150387 const bool is_pinned = pinned_chunks_.find(candidates[i].hash)
552 4300774 != pinned_chunks_.end();
553
554 // Avoid evicting open files hopping there are enough more recently used
555 // files to satisfy the cleanup request
556 /*
557 const bool is_open = std::find_if(
558 open_files_.begin(), open_files_.end(),
559 [&candidates, &i](const auto &elem) -> bool
560 { return elem.Collide(candidates[i].hash);
561 })
562 != open_files_.end();
563 */
564 2150387 bool is_open = false;
565
1/2
✗ Branch 4 not taken.
✓ Branch 5 taken 2150387 times.
 2150387 for (auto it = open_files_.begin(); it != open_files_.end(); ++it) {
566 ✗ if (it->Collide(candidates[i].hash)) {
567 ✗ is_open = true;
568 ✗ break;
569 }
570 }
571
572
2/2
✓ Branch 0 taken 43 times.
✓ Branch 1 taken 2150344 times.
 2150387 if (is_pinned) {
573
1/2
✓ Branch 2 taken 43 times.
✗ Branch 3 not taken.
 43 SkipEviction(candidates[i]);
574 43 continue;
575 }
576
577
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2150344 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 2150344 if (cleanup_unused_first_ and is_open) {
578 ✗ SkipEviction(candidates[i]);
579 ✗ lru_ordered_open.push_back(candidates[i]);
580 ✗ continue;
581 }
582
583
2/4
✓ Branch 1 taken 2150344 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2150344 times.
✗ Branch 5 not taken.
 4300688 trash.push_back(cache_dir_ + "/"
584
2/4
✓ Branch 2 taken 2150344 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2150344 times.
✗ Branch 6 not taken.
 6451032 + candidates[i].hash.MakePathWithoutSuffix());
585 2150344 gauge_ -= candidates[i].size;
586 2150344 max_acseq = candidates[i].acseq;
587
1/2
✓ Branch 2 taken 2150344 times.
✗ Branch 3 not taken.
 2150344 LogCvmfs(kLogQuota, kLogDebug, "lru cleanup %s, new gauge %" PRIu64,
588
1/2
✓ Branch 2 taken 2150344 times.
✗ Branch 3 not taken.
 4300688 candidates[i].hash.ToString().c_str(), gauge_);
589
590
2/2
✓ Branch 0 taken 301 times.
✓ Branch 1 taken 2150043 times.
 2150344 if (gauge_ <= leave_size)
591 301 break;
592 }
593
6/6
✓ Branch 1 taken 2451 times.
✓ Branch 2 taken 43 times.
✓ Branch 4 taken 2451 times.
✓ Branch 5 taken 43 times.
✓ Branch 6 taken 2150 times.
✓ Branch 7 taken 301 times.
 4988 } while (gauge_ > leave_size);
594
595
1/2
✓ Branch 0 taken 344 times.
✗ Branch 1 not taken.
 344 if (max_acseq != -1) {
596
1/2
✓ Branch 1 taken 344 times.
✗ Branch 2 not taken.
 344 sqlite3_bind_int64(stmt_rm_batch_, 1, max_acseq);
597
1/2
✓ Branch 1 taken 344 times.
✗ Branch 2 not taken.
 344 result = (sqlite3_step(stmt_rm_batch_) == SQLITE_DONE);
598
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 344 times.
 344 assert(result);
599
1/2
✓ Branch 1 taken 344 times.
✗ Branch 2 not taken.
 344 sqlite3_reset(stmt_rm_batch_);
600
601
1/2
✓ Branch 1 taken 344 times.
✗ Branch 2 not taken.
 344 result = (sqlite3_step(stmt_unblock_) == SQLITE_DONE);
602
1/2
✓ Branch 1 taken 344 times.
✗ Branch 2 not taken.
 344 sqlite3_reset(stmt_unblock_);
603
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 344 times.
 344 assert(result);
604 }
605
606
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 344 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 344 times.
 344 while (!lru_ordered_open.empty() and gauge_ > leave_size) {
607 // cleanup files in use
608 ✗ auto &candidate = lru_ordered_open[0];
609 ✗ trash.push_back(cache_dir_ + "/" + candidate.hash.MakePathWithoutSuffix());
610 ✗ gauge_ -= candidate.size;
611 ✗ max_acseq = candidate.acseq;
612 ✗ LogCvmfs(kLogQuota, kLogDebug, "lru cleanup %s, new gauge %" PRIu64,
613 ✗ candidate.hash.ToString().c_str(), gauge_);
614 ✗ lru_ordered_open.erase(lru_ordered_open.begin());
615 }
616
617
2/4
✓ Branch 1 taken 344 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 344 times.
 344 if (!EmptyTrash(trash))
618 ✗ return false;
619
620
2/2
✓ Branch 0 taken 43 times.
✓ Branch 1 taken 301 times.
 344 if (gauge_ > leave_size) {
621
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
622 "request to clean until %" PRIu64 ", "
623 "but effective gauge is %" PRIu64,
624 leave_size, gauge_);
625 43 return false;
626 }
627 301 return true;
628 344 }
629
630 344 bool PosixQuotaManager::EmptyTrash(const std::vector<std::string> &trash) {
631
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 344 times.
 344 if (trash.empty())
632 ✗ return true;
633
634
2/2
✓ Branch 0 taken 258 times.
✓ Branch 1 taken 86 times.
 344 if (async_delete_) {
635 // Double fork avoids zombie, forked removal process must not flush file
636 // buffers
637 pid_t pid;
638 int statloc;
639
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 258 times.
 258 if ((pid = fork()) == 0) {
640 // TODO(jblomer): eviciting files in the cache should perhaps become a
641 // thread. This would also allow to block the chunks and prevent the
642 // race with re-insertion. Then again, a thread can block umount.
643 #ifndef DEBUGMSG
644 ✗ CloseAllFildes(std::set<int>());
645 #endif
646 ✗ if (fork() == 0) {
647 ✗ for (unsigned i = 0, iEnd = trash.size(); i < iEnd; ++i) {
648 ✗ LogCvmfs(kLogQuota, kLogDebug, "unlink %s", trash[i].c_str());
649 ✗ unlink(trash[i].c_str());
650 }
651 ✗ _exit(0);
652 }
653 ✗ _exit(0);
654 } else {
655
1/2
✓ Branch 0 taken 258 times.
✗ Branch 1 not taken.
 258 if (pid > 0)
656
1/2
✓ Branch 1 taken 258 times.
✗ Branch 2 not taken.
 258 waitpid(pid, &statloc, 0);
657 else
658 ✗ return false;
659 }
660 } else { // !async_delete_
661
2/2
✓ Branch 1 taken 129 times.
✓ Branch 2 taken 86 times.
 215 for (unsigned i = 0, iEnd = trash.size(); i < iEnd; ++i) {
662 129 LogCvmfs(kLogQuota, kLogDebug, "unlink %s", trash[i].c_str());
663 129 unlink(trash[i].c_str());
664 }
665 }
666 344 return true;
667 }
668
669
670 4301715 void PosixQuotaManager::DoInsert(const shash::Any &hash,
671 const uint64_t size,
672 const string &description,
673 const CommandType command_type) {
674
1/2
✓ Branch 1 taken 4301715 times.
✗ Branch 2 not taken.
 4301715 const string hash_str = hash.ToString();
675
1/2
✓ Branch 3 taken 4301715 times.
✗ Branch 4 not taken.
 4301715 LogCvmfs(kLogQuota, kLogDebug, "insert into lru %s, path %s, method %d",
676 hash_str.c_str(), description.c_str(), command_type);
677 4301715 const unsigned desc_length = (description.length() > kMaxDescription)
678 ? kMaxDescription
679
1/2
✓ Branch 0 taken 4301715 times.
✗ Branch 1 not taken.
 4301715 : description.length();
680
681 LruCommand *cmd = reinterpret_cast<LruCommand *>(
682 4301715 alloca(sizeof(LruCommand) + desc_length));
683 4301715 new (cmd) LruCommand;
684 4301715 cmd->command_type = command_type;
685 4301715 cmd->SetSize(size);
686
1/2
✓ Branch 1 taken 4301715 times.
✗ Branch 2 not taken.
 4301715 cmd->StoreHash(hash);
687 4301715 cmd->desc_length = desc_length;
688 4301715 memcpy(reinterpret_cast<char *>(cmd) + sizeof(LruCommand), &description[0],
689 desc_length);
690
1/2
✓ Branch 1 taken 4301715 times.
✗ Branch 2 not taken.
 4301715 WritePipe(pipe_lru_[1], cmd, sizeof(LruCommand) + desc_length);
691 4301715 }
692
693
694 1591 vector<string> PosixQuotaManager::DoList(const CommandType list_command) {
695 1591 vector<string> result;
696
697 int pipe_list[2];
698
1/2
✓ Branch 1 taken 1591 times.
✗ Branch 2 not taken.
 1591 MakeReturnPipe(pipe_list);
699 char description_buffer[kMaxDescription];
700
701 1591 LruCommand cmd;
702 1591 cmd.command_type = list_command;
703 1591 cmd.return_pipe = pipe_list[1];
704
1/2
✓ Branch 1 taken 1591 times.
✗ Branch 2 not taken.
 1591 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
705
706 int length;
707 do {
708
1/2
✓ Branch 1 taken 4303354 times.
✗ Branch 2 not taken.
 4303354 ManagedReadHalfPipe(pipe_list[0], &length, sizeof(length));
709
2/2
✓ Branch 0 taken 4301763 times.
✓ Branch 1 taken 1591 times.
 4303354 if (length > 0) {
710
1/2
✓ Branch 1 taken 4301763 times.
✗ Branch 2 not taken.
 4301763 ReadPipe(pipe_list[0], description_buffer, length);
711
2/4
✓ Branch 2 taken 4301763 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 4301763 times.
✗ Branch 6 not taken.
 4301763 result.push_back(string(description_buffer, length));
712 }
713
2/2
✓ Branch 0 taken 4301763 times.
✓ Branch 1 taken 1591 times.
 4303354 } while (length >= 0);
714
715
1/2
✓ Branch 1 taken 1591 times.
✗ Branch 2 not taken.
 1591 CloseReturnPipe(pipe_list);
716 3182 return result;
717 }
718
719
720 565 uint64_t PosixQuotaManager::GetCapacity() {
721
1/2
✓ Branch 0 taken 565 times.
✗ Branch 1 not taken.
 565 if (limit_ != (uint64_t)(-1))
722 565 return limit_;
723
724 // Unrestricted cache, look at free space on cache dir fs
725 struct statfs info;
726 ✗ if (statfs(".", &info) == 0) {
727 ✗ return info.f_bavail * info.f_bsize;
728 } else {
729 ✗ LogCvmfs(kLogQuota, kLogSyslogErr | kLogDebug,
730 ✗ "failed to query file system info of cache (%d)", errno);
731 ✗ return limit_;
732 }
733 }
734
735
736 ✗ void PosixQuotaManager::GetLimits(uint64_t *limit,
737 uint64_t *cleanup_threshold) {
738 int pipe_limits[2];
739 ✗ MakeReturnPipe(pipe_limits);
740
741 ✗ LruCommand cmd;
742 ✗ cmd.command_type = kLimits;
743 ✗ cmd.return_pipe = pipe_limits[1];
744 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
745 ✗ ManagedReadHalfPipe(pipe_limits[0], limit, sizeof(*limit));
746 ✗ ReadPipe(pipe_limits[0], cleanup_threshold, sizeof(*cleanup_threshold));
747 ✗ CloseReturnPipe(pipe_limits);
748 }
749
750
751 /**
752 * Since we only cleanup until cleanup_threshold, we can only add
753 * files smaller than limit-cleanup_threshold.
754 */
755 163 uint64_t PosixQuotaManager::GetMaxFileSize() {
756 163 return limit_ - cleanup_threshold_;
757 }
758
759
760 43 pid_t PosixQuotaManager::GetPid() {
761
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 43 if (!shared_ || !spawned_) {
762 43 return getpid();
763 }
764 ✗ if (cachemgr_pid_) {
765 ✗ return cachemgr_pid_;
766 }
767
768 pid_t result;
769 int pipe_pid[2];
770 ✗ MakeReturnPipe(pipe_pid);
771
772 ✗ LruCommand cmd;
773 ✗ cmd.command_type = kPid;
774 ✗ cmd.return_pipe = pipe_pid[1];
775 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
776 ✗ ReadHalfPipe(pipe_pid[0], &result, sizeof(result));
777 ✗ CloseReturnPipe(pipe_pid);
778 ✗ return result;
779 }
780
781
782 43 uint32_t PosixQuotaManager::GetProtocolRevision() {
783 int pipe_revision[2];
784
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 MakeReturnPipe(pipe_revision);
785
786 43 LruCommand cmd;
787 43 cmd.command_type = kGetProtocolRevision;
788 43 cmd.return_pipe = pipe_revision[1];
789
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
790
791 uint32_t revision;
792
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 ManagedReadHalfPipe(pipe_revision[0], &revision, sizeof(revision));
793
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 CloseReturnPipe(pipe_revision);
794 43 return revision;
795 }
796
797 565 void PosixQuotaManager::SetCleanupPolicy(bool cleanup_unused_first) {
798
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 565 times.
 565 if (protocol_revision_ < 3)
799 ✗ return;
800
801
3/4
✓ Branch 0 taken 43 times.
✓ Branch 1 taken 522 times.
✓ Branch 3 taken 565 times.
✗ Branch 4 not taken.
 565 LogCvmfs(
802 kLogQuota, kLogDebug, "Set cleanup policy to %s",
803 (cleanup_unused_first) ? "cleanup unused files first." : "vanilla lru.");
804
805
2/2
✓ Branch 0 taken 43 times.
✓ Branch 1 taken 522 times.
 565 char policy = (cleanup_unused_first) ? 'S' : 'R'; // S: smart, R: regular;
806
807 LruCommand *cmd = reinterpret_cast<LruCommand *>(
808 565 alloca(sizeof(LruCommand) + sizeof(policy)));
809 565 new (cmd) LruCommand;
810 565 cmd->command_type = kSetCleanupPolicy;
811 565 cmd->desc_length = sizeof(policy);
812 565 memcpy(reinterpret_cast<char *>(cmd) + sizeof(LruCommand), &policy,
813 sizeof(policy));
814
1/2
✓ Branch 1 taken 565 times.
✗ Branch 2 not taken.
 565 WritePipe(pipe_lru_[1], cmd, sizeof(LruCommand) + sizeof(policy));
815 }
816
817 43 void PosixQuotaManager::RegisterMountpoint(const std::string &mountpoint) {
818
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 43 times.
 43 if (protocol_revision_ < 3)
819 ✗ return;
820
821 43 LogCvmfs(kLogQuota, kLogDebug, "Register Mountpoint %s", mountpoint.c_str());
822
823 43 const unsigned desc_length = (mountpoint.size() > kMaxDescription)
824 ? kMaxDescription
825
1/2
✓ Branch 0 taken 43 times.
✗ Branch 1 not taken.
 43 : mountpoint.size();
826 LruCommand *cmd = reinterpret_cast<LruCommand *>(
827 43 alloca(sizeof(LruCommand) + desc_length));
828 43 new (cmd) LruCommand;
829 43 cmd->command_type = kRegisterMountpoint;
830 43 cmd->desc_length = desc_length;
831 43 memcpy(reinterpret_cast<char *>(cmd) + sizeof(LruCommand), mountpoint.data(),
832 desc_length);
833 43 WritePipe(pipe_lru_[1], cmd, sizeof(LruCommand) + desc_length);
834 }
835
836 43 std::string PosixQuotaManager::ReadPipeString(int fd, size_t size) {
837
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 43 times.
 43 if (size == 0)
838 ✗ return "";
839
840
1/2
✓ Branch 2 taken 43 times.
✗ Branch 3 not taken.
 43 std::vector<char> buf(size);
841
1/2
✓ Branch 2 taken 43 times.
✗ Branch 3 not taken.
 43 ManagedReadHalfPipe(fd, buf.data(), size);
842
1/2
✓ Branch 3 taken 43 times.
✗ Branch 4 not taken.
 43 return std::string(buf.data(), size);
843 43 }
844
845 ✗ std::string PosixQuotaManager::GetMountpoints() {
846 ✗ if (protocol_revision_ < 3)
847 ✗ return "";
848
849 int pipe_mp[2];
850 ✗ MakeReturnPipe(pipe_mp);
851
852 ✗ LruCommand cmd;
853 ✗ cmd.command_type = kGetMountpoints;
854 ✗ cmd.return_pipe = pipe_mp[1];
855 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
856 ✗ size_t mp_str_size = 0;
857 ✗ ManagedReadHalfPipe(pipe_mp[0], &mp_str_size, sizeof(size_t));
858 ✗ const std::string result = ReadPipeString(pipe_mp[0], mp_str_size);
859 ✗ CloseReturnPipe(pipe_mp);
860 ✗ return result;
861 }
862
863 ✗ std::string PosixQuotaManager::GetGroupHashes() {
864 ✗ if (protocol_revision_ < 3)
865 ✗ return "";
866
867 int pipe_gh[2];
868 ✗ MakeReturnPipe(pipe_gh);
869
870 ✗ LruCommand cmd;
871 ✗ cmd.command_type = kGetGroupHashes;
872 ✗ cmd.return_pipe = pipe_gh[1];
873 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
874 ✗ size_t mp_str_size = 0;
875 ✗ ManagedReadHalfPipe(pipe_gh[0], &mp_str_size, sizeof(size_t));
876 ✗ const std::string result = ReadPipeString(pipe_gh[0], mp_str_size);
877 ✗ CloseReturnPipe(pipe_gh);
878 ✗ return result;
879 }
880
881 /**
882 * Queries the shared local hard disk quota manager.
883 */
884 774 void PosixQuotaManager::GetSharedStatus(uint64_t *gauge, uint64_t *pinned) {
885 int pipe_status[2];
886
1/2
✓ Branch 1 taken 774 times.
✗ Branch 2 not taken.
 774 MakeReturnPipe(pipe_status);
887
888 774 LruCommand cmd;
889 774 cmd.command_type = kStatus;
890 774 cmd.return_pipe = pipe_status[1];
891
1/2
✓ Branch 1 taken 774 times.
✗ Branch 2 not taken.
 774 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
892
1/2
✓ Branch 1 taken 774 times.
✗ Branch 2 not taken.
 774 ManagedReadHalfPipe(pipe_status[0], gauge, sizeof(*gauge));
893
1/2
✓ Branch 1 taken 774 times.
✗ Branch 2 not taken.
 774 ReadPipe(pipe_status[0], pinned, sizeof(*pinned));
894
1/2
✓ Branch 1 taken 774 times.
✗ Branch 2 not taken.
 774 CloseReturnPipe(pipe_status);
895 774 }
896
897 43 bool PosixQuotaManager::SetSharedLimit(uint64_t limit) {
898 int pipe_set_limit[2];
899 bool result;
900
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 MakeReturnPipe(pipe_set_limit);
901
902 43 LruCommand cmd;
903 43 cmd.command_type = kSetLimit;
904 43 cmd.size = limit;
905 43 cmd.return_pipe = pipe_set_limit[1];
906
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
907
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 ReadHalfPipe(pipe_set_limit[0], &result, sizeof(result));
908
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 CloseReturnPipe(pipe_set_limit);
909 43 return result;
910 }
911
912
913 43 bool PosixQuotaManager::SetLimit(uint64_t size) {
914
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 43 times.
 43 if (!spawned_) {
915 ✗ limit_ = size;
916 ✗ cleanup_threshold_ = size / 2;
917 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
918 "Quota limit set to %lu / threshold %lu", limit_,
919 cleanup_threshold_);
920 ✗ return true;
921 }
922 43 return SetSharedLimit(size);
923 }
924
925 1732 uint64_t PosixQuotaManager::GetSize() {
926
2/2
✓ Branch 0 taken 1044 times.
✓ Branch 1 taken 688 times.
 1732 if (!spawned_)
927 1044 return gauge_;
928 uint64_t gauge, size_pinned;
929
1/2
✓ Branch 1 taken 688 times.
✗ Branch 2 not taken.
 688 GetSharedStatus(&gauge, &size_pinned);
930 688 return gauge;
931 }
932
933
934 86 uint64_t PosixQuotaManager::GetSizePinned() {
935
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 86 times.
 86 if (!spawned_)
936 ✗ return pinned_;
937 uint64_t gauge, size_pinned;
938
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 86 GetSharedStatus(&gauge, &size_pinned);
939 86 return size_pinned;
940 }
941
942
943 172 uint64_t PosixQuotaManager::GetCleanupRate(uint64_t period_s) {
944
2/4
✓ Branch 0 taken 172 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 172 times.
 172 if (!spawned_ || (protocol_revision_ < 2))
945 ✗ return 0;
946 uint64_t cleanup_rate;
947
948 int pipe_cleanup_rate[2];
949
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 MakeReturnPipe(pipe_cleanup_rate);
950 172 LruCommand cmd;
951 172 cmd.command_type = kCleanupRate;
952 172 cmd.size = period_s;
953 172 cmd.return_pipe = pipe_cleanup_rate[1];
954
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
955
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 ManagedReadHalfPipe(pipe_cleanup_rate[0], &cleanup_rate,
956 sizeof(cleanup_rate));
957
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 CloseReturnPipe(pipe_cleanup_rate);
958
959 172 return cleanup_rate;
960 }
961
962
963 3317 bool PosixQuotaManager::InitDatabase(const bool rebuild_database) {
964 3317 string sql;
965 sqlite3_stmt *stmt;
966
967
2/4
✓ Branch 1 taken 3317 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3317 times.
✗ Branch 5 not taken.
 3317 fd_lock_cachedb_ = LockFile(workspace_dir_ + "/lock_cachedb");
968
2/2
✓ Branch 0 taken 43 times.
✓ Branch 1 taken 3274 times.
 3317 if (fd_lock_cachedb_ < 0) {
969
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 LogCvmfs(kLogQuota, kLogDebug, "failed to create cachedb lock");
970 43 return false;
971 }
972
973 3274 bool retry = false;
974
1/2
✓ Branch 1 taken 3274 times.
✗ Branch 2 not taken.
 3274 const string db_file = cache_dir_ + "/cachedb";
975
2/2
✓ Branch 0 taken 3098 times.
✓ Branch 1 taken 176 times.
 3274 if (rebuild_database) {
976
1/2
✓ Branch 2 taken 176 times.
✗ Branch 3 not taken.
 176 LogCvmfs(kLogQuota, kLogDebug, "rebuild database, unlinking existing (%s)",
977 db_file.c_str());
978 176 unlink(db_file.c_str());
979
1/2
✓ Branch 1 taken 176 times.
✗ Branch 2 not taken.
 176 unlink((db_file + "-journal").c_str());
980 }
981
982 3098 init_recover:
983
1/2
✓ Branch 2 taken 3274 times.
✗ Branch 3 not taken.
 3274 int err = sqlite3_open(db_file.c_str(), &database_);
984
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3274 times.
 3274 if (err != SQLITE_OK) {
985 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not open cache database (%d)", err);
986 ✗ goto init_database_fail;
987 }
988 // TODO(reneme): make this a `QuotaDatabase : public sqlite::Database`
989 sql = "PRAGMA synchronous=0; PRAGMA locking_mode=EXCLUSIVE; "
990 "PRAGMA auto_vacuum=1; "
991 "CREATE TABLE IF NOT EXISTS cache_catalog (sha1 TEXT, size INTEGER, "
992 " acseq INTEGER, path TEXT, type INTEGER, pinned INTEGER, "
993 "CONSTRAINT pk_cache_catalog PRIMARY KEY (sha1)); "
994 "CREATE UNIQUE INDEX IF NOT EXISTS idx_cache_catalog_acseq "
995 " ON cache_catalog (acseq); "
996 "CREATE TEMP TABLE fscache (sha1 TEXT, size INTEGER, actime INTEGER, "
997 "CONSTRAINT pk_fscache PRIMARY KEY (sha1)); "
998 "CREATE INDEX idx_fscache_actime ON fscache (actime); "
999 "CREATE TABLE IF NOT EXISTS properties (key TEXT, value TEXT, "
1000
1/2
✓ Branch 1 taken 3274 times.
✗ Branch 2 not taken.
 3274 " CONSTRAINT pk_properties PRIMARY KEY(key));";
1001
1/2
✓ Branch 2 taken 3274 times.
✗ Branch 3 not taken.
 3274 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1002
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3274 times.
 3274 if (err != SQLITE_OK) {
1003 ✗ if (!retry) {
1004 ✗ retry = true;
1005 ✗ sqlite3_close(database_);
1006 ✗ unlink(db_file.c_str());
1007 ✗ unlink((db_file + "-journal").c_str());
1008 ✗ LogCvmfs(kLogQuota, kLogSyslogWarn,
1009 "LRU database corrupted, re-building");
1010 ✗ goto init_recover;
1011 }
1012 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
1013 sql.c_str());
1014 ✗ goto init_database_fail;
1015 }
1016
1017 // If this an old cache catalog,
1018 // add and initialize new columns to cache_catalog
1019 sql = "ALTER TABLE cache_catalog ADD type INTEGER; "
1020
1/2
✓ Branch 1 taken 3274 times.
✗ Branch 2 not taken.
 3274 "ALTER TABLE cache_catalog ADD pinned INTEGER";
1021
1/2
✓ Branch 2 taken 3274 times.
✗ Branch 3 not taken.
 3274 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1022
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3274 times.
 3274 if (err == SQLITE_OK) {
1023 ✗ sql = "UPDATE cache_catalog SET type=" + StringifyInt(kFileRegular) + ";";
1024 ✗ err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1025 ✗ if (err != SQLITE_OK) {
1026 ✗ LogCvmfs(kLogQuota, kLogDebug,
1027 "could not init cache database (failed: %s)", sql.c_str());
1028 ✗ goto init_database_fail;
1029 }
1030 }
1031
1032 // Set pinned back
1033
1/2
✓ Branch 1 taken 3274 times.
✗ Branch 2 not taken.
 3274 sql = "UPDATE cache_catalog SET pinned=0;";
1034
1/2
✓ Branch 2 taken 3274 times.
✗ Branch 3 not taken.
 3274 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1035
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3274 times.
 3274 if (err != SQLITE_OK) {
1036 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
1037 sql.c_str());
1038 ✗ goto init_database_fail;
1039 }
1040
1041 // Set schema version
1042 sql = "INSERT OR REPLACE INTO properties (key, value) "
1043
1/2
✓ Branch 1 taken 3274 times.
✗ Branch 2 not taken.
 3274 "VALUES ('schema', '1.0')";
1044
1/2
✓ Branch 2 taken 3274 times.
✗ Branch 3 not taken.
 3274 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1045
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3274 times.
 3274 if (err != SQLITE_OK) {
1046 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
1047 sql.c_str());
1048 ✗ goto init_database_fail;
1049 }
1050
1051 // If cache catalog is empty, recreate from file system
1052
1/2
✓ Branch 1 taken 3274 times.
✗ Branch 2 not taken.
 3274 sql = "SELECT count(*) FROM cache_catalog;";
1053
1/2
✓ Branch 2 taken 3274 times.
✗ Branch 3 not taken.
 3274 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1054
2/4
✓ Branch 1 taken 3274 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 3274 times.
✗ Branch 4 not taken.
 3274 if (sqlite3_step(stmt) == SQLITE_ROW) {
1055
6/8
✓ Branch 1 taken 3274 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 172 times.
✓ Branch 4 taken 3102 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 172 times.
✓ Branch 7 taken 3102 times.
✓ Branch 8 taken 172 times.
 3274 if ((sqlite3_column_int64(stmt, 0)) == 0 || rebuild_database) {
1056
1/2
✓ Branch 1 taken 3102 times.
✗ Branch 2 not taken.
 3102 LogCvmfs(kLogCvmfs, kLogDebug,
1057 "CernVM-FS: building lru cache database...");
1058
3/4
✓ Branch 1 taken 3102 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 129 times.
✓ Branch 4 taken 2973 times.
 3102 if (!RebuildDatabase()) {
1059
1/2
✓ Branch 1 taken 129 times.
✗ Branch 2 not taken.
 129 LogCvmfs(kLogQuota, kLogDebug,
1060 "could not build cache database from file system");
1061
1/2
✓ Branch 1 taken 129 times.
✗ Branch 2 not taken.
 129 sqlite3_finalize(stmt);
1062 129 goto init_database_fail;
1063 }
1064 }
1065
1/2
✓ Branch 1 taken 3145 times.
✗ Branch 2 not taken.
 3145 sqlite3_finalize(stmt);
1066 } else {
1067 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not select on cache catalog");
1068 ✗ sqlite3_finalize(stmt);
1069 ✗ goto init_database_fail;
1070 }
1071
1072 // How many bytes do we already have in cache?
1073
1/2
✓ Branch 1 taken 3145 times.
✗ Branch 2 not taken.
 3145 sql = "SELECT sum(size) FROM cache_catalog;";
1074
1/2
✓ Branch 2 taken 3145 times.
✗ Branch 3 not taken.
 3145 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1075
2/4
✓ Branch 1 taken 3145 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 3145 times.
✗ Branch 4 not taken.
 3145 if (sqlite3_step(stmt) == SQLITE_ROW) {
1076
1/2
✓ Branch 1 taken 3145 times.
✗ Branch 2 not taken.
 3145 gauge_ = sqlite3_column_int64(stmt, 0);
1077 } else {
1078 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not determine cache size");
1079 ✗ sqlite3_finalize(stmt);
1080 ✗ goto init_database_fail;
1081 }
1082
1/2
✓ Branch 1 taken 3145 times.
✗ Branch 2 not taken.
 3145 sqlite3_finalize(stmt);
1083
1084 // Highest seq-no?
1085
1/2
✓ Branch 1 taken 3145 times.
✗ Branch 2 not taken.
 3145 sql = "SELECT coalesce(max(acseq & (~(1<<63))), 0) FROM cache_catalog;";
1086
1/2
✓ Branch 2 taken 3145 times.
✗ Branch 3 not taken.
 3145 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1087
2/4
✓ Branch 1 taken 3145 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 3145 times.
✗ Branch 4 not taken.
 3145 if (sqlite3_step(stmt) == SQLITE_ROW) {
1088
1/2
✓ Branch 1 taken 3145 times.
✗ Branch 2 not taken.
 3145 seq_ = sqlite3_column_int64(stmt, 0) + 1;
1089 } else {
1090 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not determine highest seq-no");
1091 ✗ sqlite3_finalize(stmt);
1092 ✗ goto init_database_fail;
1093 }
1094
1/2
✓ Branch 1 taken 3145 times.
✗ Branch 2 not taken.
 3145 sqlite3_finalize(stmt);
1095
1096 // Prepare touch, new, remove statements
1097
1/2
✓ Branch 1 taken 3145 times.
✗ Branch 2 not taken.
 3145 sqlite3_prepare_v2(database_,
1098 "UPDATE cache_catalog SET acseq=:seq | (acseq&(1<<63)) "
1099 "WHERE sha1=:sha1;",
1100 -1, &stmt_touch_, NULL);
1101
1/2
✓ Branch 1 taken 3145 times.
✗ Branch 2 not taken.
 3145 sqlite3_prepare_v2(database_,
1102 "UPDATE cache_catalog SET pinned=0 "
1103 "WHERE sha1=:sha1;",
1104 -1, &stmt_unpin_, NULL);
1105
1/2
✓ Branch 1 taken 3145 times.
✗ Branch 2 not taken.
 3145 sqlite3_prepare_v2(database_,
1106 "UPDATE cache_catalog SET pinned=2 "
1107 "WHERE sha1=:sha1;",
1108 -1, &stmt_block_, NULL);
1109
1/2
✓ Branch 1 taken 3145 times.
✗ Branch 2 not taken.
 3145 sqlite3_prepare_v2(database_,
1110 "UPDATE cache_catalog SET pinned=1 "
1111 "WHERE pinned=2;",
1112 -1, &stmt_unblock_, NULL);
1113
1/2
✓ Branch 1 taken 3145 times.
✗ Branch 2 not taken.
 3145 sqlite3_prepare_v2(database_,
1114 "INSERT OR REPLACE INTO cache_catalog "
1115 "(sha1, size, acseq, path, type, pinned) "
1116 "VALUES (:sha1, :s, :seq, :p, :t, :pin);",
1117 -1, &stmt_new_, NULL);
1118
1/2
✓ Branch 1 taken 3145 times.
✗ Branch 2 not taken.
 3145 sqlite3_prepare_v2(database_,
1119 "SELECT size, pinned FROM cache_catalog WHERE sha1=:sha1;",
1120 -1, &stmt_size_, NULL);
1121
1/2
✓ Branch 1 taken 3145 times.
✗ Branch 2 not taken.
 3145 sqlite3_prepare_v2(database_, "DELETE FROM cache_catalog WHERE sha1=:sha1;",
1122 -1, &stmt_rm_, NULL);
1123
1/2
✓ Branch 1 taken 3145 times.
✗ Branch 2 not taken.
 3145 sqlite3_prepare_v2(database_,
1124 "DELETE FROM cache_catalog WHERE acseq<=:a AND pinned<>2;",
1125 -1, &stmt_rm_batch_, NULL);
1126
1/2
✓ Branch 2 taken 3145 times.
✗ Branch 3 not taken.
 3145 sqlite3_prepare_v2(database_,
1127
1/2
✓ Branch 2 taken 3145 times.
✗ Branch 3 not taken.
 6290 (std::string("SELECT sha1, size, acseq FROM cache_catalog "
1128 "WHERE pinned<>2 AND acseq>=:a "
1129 "ORDER BY acseq ASC "
1130 "LIMIT ")
1131
3/6
✓ Branch 1 taken 3145 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3145 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 3145 times.
✗ Branch 8 not taken.
 12580 + StringifyInt(kEvictBatchSize) + ";")
1132 .c_str(),
1133 -1, &stmt_lru_, NULL);
1134
1/2
✓ Branch 2 taken 3145 times.
✗ Branch 3 not taken.
 3145 sqlite3_prepare_v2(database_,
1135 ("SELECT path FROM cache_catalog WHERE type="
1136
3/6
✓ Branch 1 taken 3145 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3145 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 3145 times.
✗ Branch 8 not taken.
 6290 + StringifyInt(kFileRegular) + ";")
1137 .c_str(),
1138 -1, &stmt_list_, NULL);
1139
1/2
✓ Branch 1 taken 3145 times.
✗ Branch 2 not taken.
 3145 sqlite3_prepare_v2(database_,
1140 "SELECT path FROM cache_catalog WHERE pinned<>0;", -1,
1141 &stmt_list_pinned_, NULL);
1142
1/2
✓ Branch 1 taken 3145 times.
✗ Branch 2 not taken.
 3145 sqlite3_prepare_v2(database_,
1143 "SELECT path FROM cache_catalog WHERE acseq < 0;", -1,
1144 &stmt_list_volatile_, NULL);
1145
1/2
✓ Branch 2 taken 3145 times.
✗ Branch 3 not taken.
 3145 sqlite3_prepare_v2(database_,
1146 ("SELECT path FROM cache_catalog WHERE type="
1147
3/6
✓ Branch 1 taken 3145 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3145 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 3145 times.
✗ Branch 8 not taken.
 6290 + StringifyInt(kFileCatalog) + ";")
1148 .c_str(),
1149 -1, &stmt_list_catalogs_, NULL);
1150 3145 return true;
1151
1152 129 init_database_fail:
1153
1/2
✓ Branch 1 taken 129 times.
✗ Branch 2 not taken.
 129 sqlite3_close(database_);
1154 129 database_ = NULL;
1155
1/2
✓ Branch 1 taken 129 times.
✗ Branch 2 not taken.
 129 UnlockFile(fd_lock_cachedb_);
1156 129 return false;
1157 3317 }
1158
1159
1160 /**
1161 * Inserts a new file into cache catalog. This file gets a new,
1162 * highest sequence number. Does cache cleanup if necessary.
1163 */
1164 4300984 void PosixQuotaManager::Insert(const shash::Any &any_hash,
1165 const uint64_t size,
1166 const string &description) {
1167 4300984 DoInsert(any_hash, size, description, kInsert);
1168 4300984 }
1169
1170
1171 /**
1172 * Inserts a new file into cache catalog. This file is marked as volatile
1173 * and gets a new highest sequence number with the first bit set. Cache cleanup
1174 * treats these files with priority.
1175 */
1176 172 void PosixQuotaManager::InsertVolatile(const shash::Any &any_hash,
1177 const uint64_t size,
1178 const string &description) {
1179 172 DoInsert(any_hash, size, description, kInsertVolatile);
1180 172 }
1181
1182
1183 /**
1184 * Lists all path names from the cache db.
1185 */
1186 989 vector<string> PosixQuotaManager::List() { return DoList(kList); }
1187
1188
1189 /**
1190 * Lists all pinned files from the cache db.
1191 */
1192 344 vector<string> PosixQuotaManager::ListPinned() { return DoList(kListPinned); }
1193
1194
1195 /**
1196 * Lists all sqlite catalog files from the cache db.
1197 */
1198 129 vector<string> PosixQuotaManager::ListCatalogs() {
1199 129 return DoList(kListCatalogs);
1200 }
1201
1202
1203 /**
1204 * Lists only files flagged as volatile (priority removal)
1205 */
1206 129 vector<string> PosixQuotaManager::ListVolatile() {
1207 129 return DoList(kListVolatile);
1208 }
1209
1210
1211 /**
1212 * Entry point for the shared cache manager process
1213 */
1214 ✗ int PosixQuotaManager::MainCacheManager(int argc, char **argv) {
1215 ✗ LogCvmfs(kLogQuota, kLogDebug, "starting quota manager");
1216 int retval;
1217
1218 ✗ PosixQuotaManager shared_manager(0, 0, "");
1219 ✗ shared_manager.shared_ = true;
1220 ✗ shared_manager.spawned_ = true;
1221 ✗ shared_manager.pinned_ = 0;
1222
1223 // Process command line arguments
1224 ✗ ParseDirectories(string(argv[2]),
1225 &shared_manager.cache_dir_,
1226 &shared_manager.workspace_dir_);
1227 ✗ const int pipe_boot = String2Int64(argv[3]);
1228 ✗ const int pipe_handshake = String2Int64(argv[4]);
1229 ✗ shared_manager.limit_ = String2Int64(argv[5]);
1230 ✗ shared_manager.cleanup_threshold_ = String2Int64(argv[6]);
1231 ✗ const int foreground = String2Int64(argv[7]);
1232 ✗ const int syslog_level = String2Int64(argv[8]);
1233 ✗ const int syslog_facility = String2Int64(argv[9]);
1234 ✗ vector<string> logfiles = SplitString(argv[10], ':');
1235
1236 ✗ SetLogSyslogLevel(syslog_level);
1237 ✗ SetLogSyslogFacility(syslog_facility);
1238 ✗ if ((logfiles.size() > 0) && (logfiles[0] != ""))
1239 ✗ SetLogDebugFile(logfiles[0] + ".cachemgr");
1240 ✗ if (logfiles.size() > 1)
1241 ✗ SetLogMicroSyslog(logfiles[1]);
1242
1243 ✗ if (!foreground)
1244 ✗ Daemonize();
1245
1246 ✗ if ((geteuid() != 0) && SetuidCapabilityPermitted()) {
1247 // Permanently drop credentials
1248 ✗ const std::vector<cap_value_t> nocaps;
1249 ✗ assert(ClearPermittedCapabilities(nocaps, nocaps));
1250 // Leave this process ptraceable
1251 ✗ assert(platform_set_dumpable());
1252 // but without core dumps
1253 ✗ assert(SetLimitCore(0));
1254 }
1255
1256 const UniquePtr<Watchdog> watchdog(
1257 ✗ Watchdog::Create(NULL, false /* needs_read_environ */));
1258 ✗ assert(watchdog.IsValid());
1259 ✗ watchdog->Spawn("./stacktrace.cachemgr");
1260
1261 // Initialize pipe, open non-blocking as cvmfs is not yet connected
1262 ✗ const int fd_lockfile_fifo = LockFile(shared_manager.workspace_dir_
1263 ✗ + "/lock_cachemgr.fifo");
1264 ✗ if (fd_lockfile_fifo < 0) {
1265 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1266 "could not open lock file "
1267 "%s (%d)",
1268 ✗ (shared_manager.workspace_dir_ + "/lock_cachemgr.fifo").c_str(),
1269 ✗ errno);
1270 ✗ return 1;
1271 }
1272 ✗ const string crash_guard = shared_manager.cache_dir_ + "/cachemgr.running";
1273 ✗ const bool rebuild = FileExists(crash_guard);
1274 ✗ retval = open(crash_guard.c_str(), O_RDONLY | O_CREAT, 0600);
1275 ✗ if (retval < 0) {
1276 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1277 "failed to create shared cache manager crash guard");
1278 ✗ UnlockFile(fd_lockfile_fifo);
1279 ✗ return 1;
1280 }
1281 ✗ close(retval);
1282
1283 // Redirect SQlite temp directory to cache (global variable)
1284 ✗ const string tmp_dir = shared_manager.workspace_dir_;
1285 ✗ sqlite3_temp_directory = static_cast<char *>(
1286 ✗ sqlite3_malloc(tmp_dir.length() + 1));
1287 ✗ snprintf(sqlite3_temp_directory, tmp_dir.length() + 1, "%s", tmp_dir.c_str());
1288
1289 // Cleanup leftover named pipes
1290 ✗ shared_manager.CleanupPipes();
1291
1292 ✗ if (!shared_manager.InitDatabase(rebuild)) {
1293 ✗ UnlockFile(fd_lockfile_fifo);
1294 ✗ return 1;
1295 }
1296 ✗ shared_manager.CheckFreeSpace();
1297
1298 // Save protocol revision to file. If the file is not found, it indicates
1299 // to the client that the cache manager is from times before the protocol
1300 // was versioned.
1301 const string protocol_revision_path = shared_manager.workspace_dir_
1302 ✗ + "/cachemgr.protocol";
1303 ✗ retval = open(protocol_revision_path.c_str(), O_WRONLY | O_CREAT, 0600);
1304 ✗ if (retval < 0) {
1305 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1306 ✗ "failed to open protocol revision file (%d)", errno);
1307 ✗ UnlockFile(fd_lockfile_fifo);
1308 ✗ return 1;
1309 }
1310 ✗ const string revision = StringifyInt(kProtocolRevision);
1311 ✗ const int written = write(retval, revision.data(), revision.length());
1312 ✗ close(retval);
1313 ✗ if ((written < 0) || static_cast<unsigned>(written) != revision.length()) {
1314 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1315 ✗ "failed to write protocol revision (%d)", errno);
1316 ✗ UnlockFile(fd_lockfile_fifo);
1317 ✗ return 1;
1318 }
1319
1320 ✗ const string fifo_path = shared_manager.workspace_dir_ + "/cachemgr";
1321 ✗ shared_manager.pipe_lru_[0] = open(fifo_path.c_str(), O_RDONLY | O_NONBLOCK);
1322 ✗ if (shared_manager.pipe_lru_[0] < 0) {
1323 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to listen on FIFO %s (%d)",
1324 ✗ fifo_path.c_str(), errno);
1325 ✗ UnlockFile(fd_lockfile_fifo);
1326 ✗ return 1;
1327 }
1328 ✗ Nonblock2Block(shared_manager.pipe_lru_[0]);
1329 ✗ LogCvmfs(kLogQuota, kLogDebug, "shared cache manager listening");
1330
1331 ✗ char buf = 'C';
1332 ✗ WritePipe(pipe_boot, &buf, 1);
1333 ✗ close(pipe_boot);
1334
1335 ✗ ReadPipe(pipe_handshake, &buf, 1);
1336 ✗ close(pipe_handshake);
1337 ✗ LogCvmfs(kLogQuota, kLogDebug, "shared cache manager handshake done");
1338
1339 // Ensure that broken pipes from clients do not kill the cache manager
1340 ✗ signal(SIGPIPE, SIG_IGN);
1341 // Don't let Ctrl-C ungracefully kill interactive session
1342 ✗ signal(SIGINT, SIG_IGN);
1343
1344 ✗ shared_manager.MainCommandServer(&shared_manager);
1345 ✗ unlink(fifo_path.c_str());
1346 ✗ unlink(protocol_revision_path.c_str());
1347 ✗ shared_manager.CloseDatabase();
1348 ✗ unlink(crash_guard.c_str());
1349 ✗ UnlockFile(fd_lockfile_fifo);
1350
1351 ✗ if (sqlite3_temp_directory) {
1352 ✗ sqlite3_free(sqlite3_temp_directory);
1353 ✗ sqlite3_temp_directory = NULL;
1354 }
1355
1356 ✗ return 0;
1357 }
1358
1359
1360 1462 void *PosixQuotaManager::MainCommandServer(void *data) {
1361 1462 PosixQuotaManager *quota_mgr = static_cast<PosixQuotaManager *>(data);
1362
1363
1/2
✓ Branch 1 taken 1462 times.
✗ Branch 2 not taken.
 1462 LogCvmfs(kLogQuota, kLogDebug, "starting quota manager");
1364
1/2
✓ Branch 1 taken 1462 times.
✗ Branch 2 not taken.
 1462 sqlite3_soft_heap_limit(quota_mgr->kSqliteMemPerThread);
1365
1366
2/2
✓ Branch 1 taken 46784 times.
✓ Branch 2 taken 1462 times.
 48246 LruCommand command_buffer[kCommandBufferSize];
1367 char description_buffer[kCommandBufferSize * kMaxDescription];
1368 1462 unsigned num_commands = 0;
1369
1370
1/2
✓ Branch 1 taken 6457310 times.
✗ Branch 2 not taken.
 6457310 while (read(quota_mgr->pipe_lru_[0], &command_buffer[num_commands],
1371 sizeof(command_buffer[0]))
1372
2/2
✓ Branch 0 taken 6455848 times.
✓ Branch 1 taken 1462 times.
 6457310 == sizeof(command_buffer[0])) {
1373 6455848 const CommandType command_type = command_buffer[num_commands].command_type;
1374
1/2
✓ Branch 1 taken 6455848 times.
✗ Branch 2 not taken.
 6455848 LogCvmfs(kLogQuota, kLogDebug, "received command %d", command_type);
1375 6455848 const uint64_t size = command_buffer[num_commands].GetSize();
1376
1377 // Inserts and pins come with a description (usually a path)
1378
4/4
✓ Branch 0 taken 2155031 times.
✓ Branch 1 taken 4300817 times.
✓ Branch 2 taken 2154859 times.
✓ Branch 3 taken 172 times.
 6455848 if ((command_type == kInsert) || (command_type == kInsertVolatile)
1379
4/4
✓ Branch 0 taken 2154773 times.
✓ Branch 1 taken 86 times.
✓ Branch 2 taken 2154300 times.
✓ Branch 3 taken 473 times.
 2154859 || (command_type == kPin) || (command_type == kPinRegular)
1380
2/2
✓ Branch 0 taken 2154257 times.
✓ Branch 1 taken 43 times.
 2154300 || (command_type == kRegisterMountpoint)
1381
2/2
✓ Branch 0 taken 43 times.
✓ Branch 1 taken 2154214 times.
 2154257 || (command_type == kSetCleanupPolicy)) {
1382 4301634 const int desc_length = command_buffer[num_commands].desc_length;
1383 4301634 ReadPipe(quota_mgr->pipe_lru_[0],
1384
1/2
✓ Branch 1 taken 4301634 times.
✗ Branch 2 not taken.
 4301634 &description_buffer[kMaxDescription * num_commands],
1385 desc_length);
1386 }
1387
1388 // The protocol revision is returned immediately
1389
2/2
✓ Branch 0 taken 43 times.
✓ Branch 1 taken 6455805 times.
 6455848 if (command_type == kGetProtocolRevision) {
1390
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 const int return_pipe = quota_mgr->BindReturnPipe(
1391 command_buffer[num_commands].return_pipe);
1392
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 43 times.
 43 if (return_pipe < 0)
1393 ✗ continue;
1394
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 WritePipe(return_pipe, &quota_mgr->kProtocolRevision,
1395 sizeof(quota_mgr->kProtocolRevision));
1396
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 quota_mgr->UnbindReturnPipe(return_pipe);
1397 43 continue;
1398 43 }
1399
1400 // Register a new mountpoint
1401
2/2
✓ Branch 0 taken 43 times.
✓ Branch 1 taken 6455762 times.
 6455805 if (command_type == kRegisterMountpoint) {
1402 43 const std::string mountpoint(&description_buffer[num_commands*kMaxDescription],
1403
1/2
✓ Branch 2 taken 43 times.
✗ Branch 3 not taken.
 43 command_buffer[num_commands].desc_length);
1404
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 quota_mgr->mountpoints_.push_back(mountpoint);
1405
1/2
✓ Branch 2 taken 43 times.
✗ Branch 3 not taken.
 43 LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
1406 "Mountpoint %s registered in the group", mountpoint.c_str());
1407 43 continue;
1408 43 }
1409
1410 // Set Cleanup Policy
1411
2/2
✓ Branch 0 taken 43 times.
✓ Branch 1 taken 6455719 times.
 6455762 if (command_type == kSetCleanupPolicy) {
1412 43 quota_mgr->cleanup_unused_first_ = (description_buffer[num_commands * kMaxDescription] == 'S') ? true : false;
1413 43 continue;
1414 }
1415 // Mountpoints are returned immediately
1416
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6455719 times.
 6455719 if (command_type == kGetMountpoints) {
1417 ✗ const int return_pipe = quota_mgr->BindReturnPipe(
1418 command_buffer[num_commands].return_pipe);
1419 ✗ if (return_pipe < 0)
1420 ✗ continue;
1421
1422 ✗ std::string mps;
1423 ✗ for (auto it = quota_mgr->mountpoints_.begin();
1424 ✗ it != quota_mgr->mountpoints_.end();
1425 ✗ ++it) {
1426 ✗ mps += *it + "\n";
1427 }
1428 ✗ size_t mp_size = mps.size();
1429 ✗ WritePipe(return_pipe, &mp_size, sizeof(size_t));
1430 ✗ WritePipe(return_pipe, mps.c_str(), mp_size);
1431 ✗ quota_mgr->UnbindReturnPipe(return_pipe);
1432 ✗ continue;
1433 }
1434
1435 // Group hashes are returned immediately
1436
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6455719 times.
 6455719 if (command_type == kGetGroupHashes) {
1437 ✗ const int return_pipe = quota_mgr->BindReturnPipe(
1438 command_buffer[num_commands].return_pipe);
1439 ✗ if (return_pipe < 0)
1440 ✗ continue;
1441
1442 ✗ std::vector<shash::Short> gh = quota_mgr->CollectAllOpenHashes();
1443 ✗ std::string result;
1444 ✗ for (auto it = gh.begin(); it != gh.end(); ++it) {
1445 ✗ result += (*it).ToString() + "\n";
1446 }
1447 ✗ size_t result_size = result.size();
1448 ✗ WritePipe(return_pipe, &result_size, sizeof(size_t));
1449 ✗ WritePipe(return_pipe, result.c_str(), result_size);
1450 ✗ quota_mgr->UnbindReturnPipe(return_pipe);
1451 ✗ continue;
1452 }
1453
1454 // The cleanup rate is returned immediately
1455
2/2
✓ Branch 0 taken 172 times.
✓ Branch 1 taken 6455547 times.
 6455719 if (command_type == kCleanupRate) {
1456
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 const int return_pipe = quota_mgr->BindReturnPipe(
1457 command_buffer[num_commands].return_pipe);
1458
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 172 times.
 172 if (return_pipe < 0)
1459 ✗ continue;
1460 const uint64_t
1461 172 period_s = size; // use the size field to transmit the period
1462
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 uint64_t rate = quota_mgr->cleanup_recorder_.GetNoTicks(period_s);
1463
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 WritePipe(return_pipe, &rate, sizeof(rate));
1464
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 quota_mgr->UnbindReturnPipe(return_pipe);
1465 172 continue;
1466 172 }
1467
1468
2/2
✓ Branch 0 taken 43 times.
✓ Branch 1 taken 6455504 times.
 6455547 if (command_type == kSetLimit) {
1469
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 const int return_pipe = quota_mgr->BindReturnPipe(
1470 command_buffer[num_commands].return_pipe);
1471
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 43 times.
 43 if (return_pipe < 0)
1472 ✗ continue;
1473 43 quota_mgr->limit_ = size; // use the size field to transmit the size
1474 43 quota_mgr->cleanup_threshold_ = size / 2;
1475
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
1476 "Quota limit set to %lu / threshold %lu", quota_mgr->limit_,
1477 quota_mgr->cleanup_threshold_);
1478 43 bool ret = true;
1479
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 WritePipe(return_pipe, &ret, sizeof(ret));
1480
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 quota_mgr->UnbindReturnPipe(return_pipe);
1481 43 continue;
1482 43 }
1483
1484 // Reservations are handled immediately and "out of band"
1485
2/2
✓ Branch 0 taken 602 times.
✓ Branch 1 taken 6454902 times.
 6455504 if (command_type == kReserve) {
1486 602 bool success = true;
1487
1/2
✓ Branch 1 taken 602 times.
✗ Branch 2 not taken.
 602 const int return_pipe = quota_mgr->BindReturnPipe(
1488 command_buffer[num_commands].return_pipe);
1489
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 602 times.
 602 if (return_pipe < 0)
1490 ✗ continue;
1491
1492
1/2
✓ Branch 1 taken 602 times.
✗ Branch 2 not taken.
 602 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1493
1/2
✓ Branch 1 taken 602 times.
✗ Branch 2 not taken.
 602 const string hash_str(hash.ToString());
1494
1/2
✓ Branch 2 taken 602 times.
✗ Branch 3 not taken.
 602 LogCvmfs(kLogQuota, kLogDebug, "reserve %lu bytes for %s", size,
1495 hash_str.c_str());
1496
1497
1/2
✓ Branch 1 taken 602 times.
✗ Branch 2 not taken.
 602 if (quota_mgr->pinned_chunks_.find(hash)
1498
2/2
✓ Branch 2 taken 516 times.
✓ Branch 3 taken 86 times.
 1204 == quota_mgr->pinned_chunks_.end()) {
1499
2/2
✓ Branch 0 taken 43 times.
✓ Branch 1 taken 473 times.
 516 if ((quota_mgr->pinned_ + size) > quota_mgr->cleanup_threshold_) {
1500
1/2
✓ Branch 2 taken 43 times.
✗ Branch 3 not taken.
 43 LogCvmfs(kLogQuota, kLogDebug,
1501 "failed to insert %s (pinned), no space", hash_str.c_str());
1502 43 success = false;
1503 } else {
1504
1/2
✓ Branch 1 taken 473 times.
✗ Branch 2 not taken.
 473 quota_mgr->pinned_chunks_[hash] = size;
1505 473 quota_mgr->pinned_ += size;
1506
1/2
✓ Branch 1 taken 473 times.
✗ Branch 2 not taken.
 473 quota_mgr->CheckHighPinWatermark();
1507 }
1508 }
1509
1510
1/2
✓ Branch 1 taken 602 times.
✗ Branch 2 not taken.
 602 WritePipe(return_pipe, &success, sizeof(success));
1511
1/2
✓ Branch 1 taken 602 times.
✗ Branch 2 not taken.
 602 quota_mgr->UnbindReturnPipe(return_pipe);
1512 602 continue;
1513 602 }
1514
1515 // Back channels are also handled out of band
1516
2/2
✓ Branch 0 taken 172 times.
✓ Branch 1 taken 6454730 times.
 6454902 if (command_type == kRegisterBackChannel) {
1517
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 const int return_pipe = quota_mgr->BindReturnPipe(
1518 command_buffer[num_commands].return_pipe);
1519
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 172 times.
 172 if (return_pipe < 0)
1520 ✗ continue;
1521
1522
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 quota_mgr->UnlinkReturnPipe(command_buffer[num_commands].return_pipe);
1523
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 Block2Nonblock(return_pipe); // back channels are opportunistic
1524
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 shash::Md5 hash;
1525 172 memcpy(hash.digest, command_buffer[num_commands].digest,
1526 172 shash::kDigestSizes[shash::kMd5]);
1527
1528 172 quota_mgr->LockBackChannels();
1529 const map<shash::Md5, int>::const_iterator
1530
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 iter = quota_mgr->back_channels_.find(hash);
1531
1/2
✗ Branch 3 not taken.
✓ Branch 4 taken 172 times.
 172 if (iter != quota_mgr->back_channels_.end()) {
1532 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
1533 ✗ "closing left-over back channel %s", hash.ToString().c_str());
1534 ✗ close(iter->second);
1535 }
1536
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 quota_mgr->back_channels_[hash] = return_pipe;
1537 172 quota_mgr->UnlockBackChannels();
1538
1539 172 char success = 'S';
1540
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 WritePipe(return_pipe, &success, sizeof(success));
1541
1/2
✓ Branch 2 taken 172 times.
✗ Branch 3 not taken.
 172 LogCvmfs(kLogQuota, kLogDebug, "register back channel %s on fd %d",
1542
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 344 hash.ToString().c_str(), return_pipe);
1543
1544 172 continue;
1545 172 }
1546
1547
2/2
✓ Branch 0 taken 86 times.
✓ Branch 1 taken 6454644 times.
 6454730 if (command_type == kUnregisterBackChannel) {
1548
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 86 shash::Md5 hash;
1549 86 memcpy(hash.digest, command_buffer[num_commands].digest,
1550 86 shash::kDigestSizes[shash::kMd5]);
1551
1552 86 quota_mgr->LockBackChannels();
1553 const map<shash::Md5, int>::iterator iter = quota_mgr->back_channels_
1554
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 86 .find(hash);
1555
1/2
✓ Branch 2 taken 86 times.
✗ Branch 3 not taken.
 86 if (iter != quota_mgr->back_channels_.end()) {
1556
1/2
✓ Branch 2 taken 86 times.
✗ Branch 3 not taken.
 86 LogCvmfs(kLogQuota, kLogDebug, "closing back channel %s",
1557
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 172 hash.ToString().c_str());
1558
1/2
✓ Branch 2 taken 86 times.
✗ Branch 3 not taken.
 86 close(iter->second);
1559
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 86 quota_mgr->back_channels_.erase(iter);
1560 } else {
1561 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
1562 ✗ "did not find back channel %s", hash.ToString().c_str());
1563 }
1564 86 quota_mgr->UnlockBackChannels();
1565
1566 86 continue;
1567 86 }
1568
1569 // Unpinnings are also handled immediately with respect to the pinned gauge
1570
2/2
✓ Branch 0 taken 86 times.
✓ Branch 1 taken 6454558 times.
 6454644 if (command_type == kUnpin) {
1571
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 86 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1572
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 86 const string hash_str(hash.ToString());
1573
1574 const map<shash::Any, uint64_t>::iterator iter = quota_mgr->pinned_chunks_
1575
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 86 .find(hash);
1576
1/2
✓ Branch 2 taken 86 times.
✗ Branch 3 not taken.
 86 if (iter != quota_mgr->pinned_chunks_.end()) {
1577 86 quota_mgr->pinned_ -= iter->second;
1578
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 86 quota_mgr->pinned_chunks_.erase(iter);
1579 // It can happen that files get pinned that were removed from the cache
1580 // (see cache.cc). We fix this at this point, where we remove such
1581 // entries from the cache database.
1582
2/4
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 86 times.
✗ Branch 5 not taken.
 172 if (!FileExists(quota_mgr->cache_dir_ + "/"
1583
4/6
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 86 times.
✗ Branch 5 not taken.
✓ Branch 9 taken 43 times.
✓ Branch 10 taken 43 times.
 258 + hash.MakePathWithoutSuffix())) {
1584
1/2
✓ Branch 2 taken 43 times.
✗ Branch 3 not taken.
 43 LogCvmfs(kLogQuota, kLogDebug,
1585 "remove orphaned pinned hash %s from cache database",
1586 hash_str.c_str());
1587
1/2
✓ Branch 2 taken 43 times.
✗ Branch 3 not taken.
 43 sqlite3_bind_text(quota_mgr->stmt_size_, 1, &hash_str[0],
1588 43 hash_str.length(), SQLITE_STATIC);
1589 int retval;
1590
2/4
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 43 times.
✗ Branch 4 not taken.
 43 if ((retval = sqlite3_step(quota_mgr->stmt_size_)) == SQLITE_ROW) {
1591
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 const uint64_t size = sqlite3_column_int64(quota_mgr->stmt_size_,
1592 43 0);
1593
1/2
✓ Branch 2 taken 43 times.
✗ Branch 3 not taken.
 43 sqlite3_bind_text(quota_mgr->stmt_rm_, 1, &(hash_str[0]),
1594 43 hash_str.length(), SQLITE_STATIC);
1595
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 retval = sqlite3_step(quota_mgr->stmt_rm_);
1596
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 43 if ((retval == SQLITE_DONE) || (retval == SQLITE_OK)) {
1597 43 quota_mgr->gauge_ -= size;
1598 } else {
1599 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1600 "failed to delete %s (%d)", hash_str.c_str(), retval);
1601 }
1602
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 sqlite3_reset(quota_mgr->stmt_rm_);
1603 }
1604
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 sqlite3_reset(quota_mgr->stmt_size_);
1605 }
1606 } else {
1607 ✗ LogCvmfs(kLogQuota, kLogDebug, "this chunk was not pinned");
1608 }
1609 86 }
1610
1611 // Immediate commands trigger flushing of the buffer
1612 6454644 const bool immediate_command = (command_type == kCleanup)
1613
2/2
✓ Branch 0 taken 6453268 times.
✓ Branch 1 taken 989 times.
 6454257 || (command_type == kList)
1614
2/2
✓ Branch 0 taken 6452924 times.
✓ Branch 1 taken 344 times.
 6453268 || (command_type == kListPinned)
1615
2/2
✓ Branch 0 taken 6452795 times.
✓ Branch 1 taken 129 times.
 6452924 || (command_type == kListCatalogs)
1616
2/2
✓ Branch 0 taken 6452666 times.
✓ Branch 1 taken 129 times.
 6452795 || (command_type == kListVolatile)
1617
2/2
✓ Branch 0 taken 6452537 times.
✓ Branch 1 taken 129 times.
 6452666 || (command_type == kRemove)
1618
2/2
✓ Branch 0 taken 6451763 times.
✓ Branch 1 taken 774 times.
 6452537 || (command_type == kStatus)
1619
1/2
✓ Branch 0 taken 6451763 times.
✗ Branch 1 not taken.
 6451763 || (command_type == kLimits)
1620
3/4
✓ Branch 0 taken 6454257 times.
✓ Branch 1 taken 387 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 6451763 times.
 12908901 || (command_type == kPid);
1621
2/2
✓ Branch 0 taken 6451763 times.
✓ Branch 1 taken 2881 times.
 6454644 if (!immediate_command)
1622 6451763 num_commands++;
1623
1624
4/4
✓ Branch 0 taken 6253103 times.
✓ Branch 1 taken 201541 times.
✓ Branch 2 taken 2881 times.
✓ Branch 3 taken 6250222 times.
 6454644 if ((num_commands == kCommandBufferSize) || immediate_command) {
1625
1/2
✓ Branch 1 taken 204422 times.
✗ Branch 2 not taken.
 204422 quota_mgr->ProcessCommandBunch(num_commands, command_buffer,
1626 description_buffer);
1627
2/2
✓ Branch 0 taken 201541 times.
✓ Branch 1 taken 2881 times.
 204422 if (!immediate_command)
1628 201541 num_commands = 0;
1629 }
1630
1631
2/2
✓ Branch 0 taken 2881 times.
✓ Branch 1 taken 6451763 times.
 6454644 if (immediate_command) {
1632 // Process cleanup, listings
1633
1/2
✓ Branch 1 taken 2881 times.
✗ Branch 2 not taken.
 2881 const int return_pipe = quota_mgr->BindReturnPipe(
1634 command_buffer[num_commands].return_pipe);
1635
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2881 times.
 2881 if (return_pipe < 0) {
1636 ✗ num_commands = 0;
1637 ✗ continue;
1638 }
1639
1640 int retval;
1641 2881 sqlite3_stmt *this_stmt_list = NULL;
1642
7/10
✓ Branch 0 taken 129 times.
✓ Branch 1 taken 387 times.
✓ Branch 2 taken 989 times.
✓ Branch 3 taken 344 times.
✓ Branch 4 taken 129 times.
✓ Branch 5 taken 129 times.
✓ Branch 6 taken 774 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
 2881 switch (command_type) {
1643 129 case kRemove: {
1644
1/2
✓ Branch 1 taken 129 times.
✗ Branch 2 not taken.
 129 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1645
1/2
✓ Branch 1 taken 129 times.
✗ Branch 2 not taken.
 129 const string hash_str = hash.ToString();
1646
1/2
✓ Branch 2 taken 129 times.
✗ Branch 3 not taken.
 129 LogCvmfs(kLogQuota, kLogDebug, "manually removing %s",
1647 hash_str.c_str());
1648 129 bool success = false;
1649
1650
1/2
✓ Branch 2 taken 129 times.
✗ Branch 3 not taken.
 129 sqlite3_bind_text(quota_mgr->stmt_size_, 1, &hash_str[0],
1651 129 hash_str.length(), SQLITE_STATIC);
1652 int retval;
1653
3/4
✓ Branch 1 taken 129 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 86 times.
✓ Branch 4 taken 43 times.
 129 if ((retval = sqlite3_step(quota_mgr->stmt_size_)) == SQLITE_ROW) {
1654
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 86 const uint64_t size = sqlite3_column_int64(quota_mgr->stmt_size_,
1655 86 0);
1656
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 86 const uint64_t is_pinned = sqlite3_column_int64(
1657 86 quota_mgr->stmt_size_, 1);
1658
1659
1/2
✓ Branch 2 taken 86 times.
✗ Branch 3 not taken.
 86 sqlite3_bind_text(quota_mgr->stmt_rm_, 1, &(hash_str[0]),
1660 86 hash_str.length(), SQLITE_STATIC);
1661
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 86 retval = sqlite3_step(quota_mgr->stmt_rm_);
1662
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 86 if ((retval == SQLITE_DONE) || (retval == SQLITE_OK)) {
1663 86 success = true;
1664 86 quota_mgr->gauge_ -= size;
1665
2/2
✓ Branch 0 taken 43 times.
✓ Branch 1 taken 43 times.
 86 if (is_pinned) {
1666
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 quota_mgr->pinned_chunks_.erase(hash);
1667 43 quota_mgr->pinned_ -= size;
1668 }
1669 } else {
1670 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1671 "failed to delete %s (%d)", hash_str.c_str(), retval);
1672 }
1673
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 86 sqlite3_reset(quota_mgr->stmt_rm_);
1674 } else {
1675 // File does not exist
1676 43 success = true;
1677 }
1678
1/2
✓ Branch 1 taken 129 times.
✗ Branch 2 not taken.
 129 sqlite3_reset(quota_mgr->stmt_size_);
1679
1680
1/2
✓ Branch 1 taken 129 times.
✗ Branch 2 not taken.
 129 WritePipe(return_pipe, &success, sizeof(success));
1681 129 break;
1682 129 }
1683 387 case kCleanup:
1684
1/2
✓ Branch 1 taken 387 times.
✗ Branch 2 not taken.
 387 retval = quota_mgr->DoCleanup(size);
1685
1/2
✓ Branch 1 taken 387 times.
✗ Branch 2 not taken.
 387 WritePipe(return_pipe, &retval, sizeof(retval));
1686 387 break;
1687 989 case kList:
1688
1/2
✓ Branch 0 taken 989 times.
✗ Branch 1 not taken.
 989 if (!this_stmt_list)
1689 989 this_stmt_list = quota_mgr->stmt_list_;
1690 case kListPinned:
1691
2/2
✓ Branch 0 taken 344 times.
✓ Branch 1 taken 989 times.
 1333 if (!this_stmt_list)
1692 344 this_stmt_list = quota_mgr->stmt_list_pinned_;
1693 case kListCatalogs:
1694
2/2
✓ Branch 0 taken 129 times.
✓ Branch 1 taken 1333 times.
 1462 if (!this_stmt_list)
1695 129 this_stmt_list = quota_mgr->stmt_list_catalogs_;
1696 case kListVolatile:
1697
2/2
✓ Branch 0 taken 129 times.
✓ Branch 1 taken 1462 times.
 1591 if (!this_stmt_list)
1698 129 this_stmt_list = quota_mgr->stmt_list_volatile_;
1699
1700 // Pipe back the list, one by one
1701 int length;
1702
3/4
✓ Branch 1 taken 4303354 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 4301763 times.
✓ Branch 4 taken 1591 times.
 4303354 while (sqlite3_step(this_stmt_list) == SQLITE_ROW) {
1703
1/2
✓ Branch 2 taken 4301763 times.
✗ Branch 3 not taken.
 4301763 string path = "(NULL)";
1704
2/4
✓ Branch 1 taken 4301763 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 4301763 times.
✗ Branch 4 not taken.
 4301763 if (sqlite3_column_type(this_stmt_list, 0) != SQLITE_NULL) {
1705 8603526 path = string(reinterpret_cast<const char *>(
1706
2/4
✓ Branch 1 taken 4301763 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 4301763 times.
✗ Branch 5 not taken.
 4301763 sqlite3_column_text(this_stmt_list, 0)));
1707 }
1708 4301763 length = path.length();
1709
1/2
✓ Branch 1 taken 4301763 times.
✗ Branch 2 not taken.
 4301763 WritePipe(return_pipe, &length, sizeof(length));
1710
1/2
✓ Branch 0 taken 4301763 times.
✗ Branch 1 not taken.
 4301763 if (length > 0)
1711
2/4
✓ Branch 1 taken 4301763 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 4301763 times.
✗ Branch 5 not taken.
 4301763 WritePipe(return_pipe, &path[0], length);
1712 4301763 }
1713 1591 length = -1;
1714
1/2
✓ Branch 1 taken 1591 times.
✗ Branch 2 not taken.
 1591 WritePipe(return_pipe, &length, sizeof(length));
1715
1/2
✓ Branch 1 taken 1591 times.
✗ Branch 2 not taken.
 1591 sqlite3_reset(this_stmt_list);
1716 1591 break;
1717 774 case kStatus:
1718
1/2
✓ Branch 1 taken 774 times.
✗ Branch 2 not taken.
 774 WritePipe(return_pipe, &quota_mgr->gauge_, sizeof(quota_mgr->gauge_));
1719
1/2
✓ Branch 1 taken 774 times.
✗ Branch 2 not taken.
 774 WritePipe(return_pipe, &quota_mgr->pinned_,
1720 sizeof(quota_mgr->pinned_));
1721 774 break;
1722 ✗ case kLimits:
1723 ✗ WritePipe(return_pipe, &quota_mgr->limit_, sizeof(quota_mgr->limit_));
1724 ✗ WritePipe(return_pipe, &quota_mgr->cleanup_threshold_,
1725 sizeof(quota_mgr->cleanup_threshold_));
1726 ✗ break;
1727 ✗ case kPid: {
1728 ✗ pid_t pid = getpid();
1729 ✗ WritePipe(return_pipe, &pid, sizeof(pid));
1730 ✗ break;
1731 }
1732 ✗ default:
1733 ✗ PANIC(NULL); // other types are handled by the bunch processor
1734 }
1735
1/2
✓ Branch 1 taken 2881 times.
✗ Branch 2 not taken.
 2881 quota_mgr->UnbindReturnPipe(return_pipe);
1736 2881 num_commands = 0;
1737 }
1738 }
1739
1740
1/2
✓ Branch 1 taken 1462 times.
✗ Branch 2 not taken.
 1462 LogCvmfs(kLogQuota, kLogDebug, "stopping cache manager (%d)", errno);
1741
1/2
✓ Branch 1 taken 1462 times.
✗ Branch 2 not taken.
 1462 close(quota_mgr->pipe_lru_[0]);
1742
1/2
✓ Branch 1 taken 1462 times.
✗ Branch 2 not taken.
 1462 quota_mgr->ProcessCommandBunch(num_commands, command_buffer,
1743 description_buffer);
1744
1745 // Unpin
1746 1462 command_buffer[0].command_type = kTouch;
1747 1462 for (map<shash::Any, uint64_t>::const_iterator
1748 1462 i = quota_mgr->pinned_chunks_.begin(),
1749 1462 iEnd = quota_mgr->pinned_chunks_.end();
1750
2/2
✓ Branch 1 taken 430 times.
✓ Branch 2 taken 1462 times.
 1892 i != iEnd;
1751 430 ++i) {
1752
1/2
✓ Branch 2 taken 430 times.
✗ Branch 3 not taken.
 430 command_buffer[0].StoreHash(i->first);
1753
1/2
✓ Branch 1 taken 430 times.
✗ Branch 2 not taken.
 430 quota_mgr->ProcessCommandBunch(1, command_buffer, description_buffer);
1754 }
1755
1756 1462 return NULL;
1757 }
1758
1759
1760 4042 void PosixQuotaManager::MakeReturnPipe(int pipe[2]) {
1761
2/2
✓ Branch 0 taken 3913 times.
✓ Branch 1 taken 129 times.
 4042 if (!shared_) {
1762 3913 MakePipe(pipe);
1763 3913 return;
1764 }
1765
1766 // Create FIFO in cache directory, store path name (number) in pipe write end
1767 129 int i = 0;
1768 int retval;
1769 do {
1770
2/4
✓ Branch 2 taken 172 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 172 times.
✗ Branch 6 not taken.
 172 retval = mkfifo((workspace_dir_ + "/pipe" + StringifyInt(i)).c_str(), 0600);
1771 172 pipe[1] = i;
1772 172 i++;
1773
3/4
✓ Branch 0 taken 43 times.
✓ Branch 1 taken 129 times.
✓ Branch 2 taken 43 times.
✗ Branch 3 not taken.
 172 } while ((retval == -1) && (errno == EEXIST));
1774
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 129 times.
 129 assert(retval == 0);
1775
1776 // Connect reader's end
1777
3/6
✓ Branch 2 taken 129 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 129 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 129 times.
✗ Branch 10 not taken.
 129 pipe[0] = open((workspace_dir_ + "/pipe" + StringifyInt(pipe[1])).c_str(),
1778 O_RDONLY | O_NONBLOCK);
1779
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 129 times.
 129 assert(pipe[0] >= 0);
1780 129 Nonblock2Block(pipe[0]);
1781 }
1782
1783
1784 3274 void PosixQuotaManager::ParseDirectories(const std::string cache_workspace,
1785 std::string *cache_dir,
1786 std::string *workspace_dir) {
1787
1/2
✓ Branch 1 taken 3274 times.
✗ Branch 2 not taken.
 3274 vector<string> dir_tokens(SplitString(cache_workspace, ':'));
1788
2/3
✓ Branch 1 taken 3188 times.
✓ Branch 2 taken 86 times.
✗ Branch 3 not taken.
 3274 switch (dir_tokens.size()) {
1789 3188 case 1:
1790
2/4
✓ Branch 2 taken 3188 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 3188 times.
✗ Branch 6 not taken.
 3188 *cache_dir = *workspace_dir = dir_tokens[0];
1791 3188 break;
1792 86 case 2:
1793
1/2
✓ Branch 2 taken 86 times.
✗ Branch 3 not taken.
 86 *cache_dir = dir_tokens[0];
1794
1/2
✓ Branch 2 taken 86 times.
✗ Branch 3 not taken.
 86 *workspace_dir = dir_tokens[1];
1795 86 break;
1796 ✗ default:
1797 ✗ PANIC(NULL);
1798 }
1799 3274 }
1800
1801 43 void PosixQuotaManager::SkipEviction(const EvictCandidate &candidate) {
1802 43 bool res = true;
1803
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 std::string hash_str = candidate.hash.ToString();
1804
1/2
✓ Branch 2 taken 43 times.
✗ Branch 3 not taken.
 43 LogCvmfs(kLogQuota, kLogDebug, "Exclude %s from eviction", hash_str.c_str());
1805
2/4
✓ Branch 2 taken 43 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 43 times.
✗ Branch 6 not taken.
 43 sqlite3_bind_text(stmt_block_, 1, &hash_str[0], hash_str.length(),
1806 SQLITE_STATIC);
1807
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 res = (sqlite3_step(stmt_block_) == SQLITE_DONE);
1808
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 sqlite3_reset(stmt_block_);
1809
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 43 times.
 43 assert(res);
1810 43 }
1811
1812 /**
1813 * Immediately inserts a new pinned catalog. Does cache cleanup if necessary.
1814 *
1815 * \return True on success, false otherwise
1816 */
1817 1363 bool PosixQuotaManager::Pin(const shash::Any &hash,
1818 const uint64_t size,
1819 const string &description,
1820 const bool is_catalog) {
1821
3/4
✓ Branch 0 taken 86 times.
✓ Branch 1 taken 1277 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 86 times.
 1363 assert((size > 0) || !is_catalog);
1822
1823
1/2
✓ Branch 1 taken 1363 times.
✗ Branch 2 not taken.
 1363 const string hash_str = hash.ToString();
1824
1/2
✓ Branch 3 taken 1363 times.
✗ Branch 4 not taken.
 1363 LogCvmfs(kLogQuota, kLogDebug, "pin into lru %s, path %s", hash_str.c_str(),
1825 description.c_str());
1826
1827 // Has to run when not yet spawned (cvmfs initialization)
1828
2/2
✓ Branch 0 taken 761 times.
✓ Branch 1 taken 602 times.
 1363 if (!spawned_) {
1829 // Code duplication here
1830
3/4
✓ Branch 2 taken 761 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 449 times.
✓ Branch 6 taken 312 times.
 761 if (pinned_chunks_.find(hash) == pinned_chunks_.end()) {
1831
2/2
✓ Branch 0 taken 86 times.
✓ Branch 1 taken 363 times.
 449 if (pinned_ + size > cleanup_threshold_) {
1832
1/2
✓ Branch 2 taken 86 times.
✗ Branch 3 not taken.
 86 LogCvmfs(kLogQuota, kLogDebug, "failed to insert %s (pinned), no space",
1833 hash_str.c_str());
1834 86 return false;
1835 } else {
1836
1/2
✓ Branch 1 taken 363 times.
✗ Branch 2 not taken.
 363 pinned_chunks_[hash] = size;
1837 363 pinned_ += size;
1838
1/2
✓ Branch 1 taken 363 times.
✗ Branch 2 not taken.
 363 CheckHighPinWatermark();
1839 }
1840 }
1841
1/2
✓ Branch 1 taken 675 times.
✗ Branch 2 not taken.
 675 const bool exists = Contains(hash_str);
1842
4/4
✓ Branch 0 taken 363 times.
✓ Branch 1 taken 312 times.
✓ Branch 2 taken 43 times.
✓ Branch 3 taken 320 times.
 675 if (!exists && (gauge_ + size > limit_)) {
1843
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 LogCvmfs(kLogQuota, kLogDebug, "over limit, gauge %lu, file size %lu",
1844 gauge_, size);
1845
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 const int retval = DoCleanup(cleanup_threshold_);
1846
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 43 times.
 43 assert(retval != 0);
1847 }
1848
1/2
✓ Branch 3 taken 675 times.
✗ Branch 4 not taken.
 675 sqlite3_bind_text(stmt_new_, 1, &hash_str[0], hash_str.length(),
1849 SQLITE_STATIC);
1850
1/2
✓ Branch 1 taken 675 times.
✗ Branch 2 not taken.
 675 sqlite3_bind_int64(stmt_new_, 2, size);
1851
1/2
✓ Branch 1 taken 675 times.
✗ Branch 2 not taken.
 675 sqlite3_bind_int64(stmt_new_, 3, seq_++);
1852
1/2
✓ Branch 3 taken 675 times.
✗ Branch 4 not taken.
 675 sqlite3_bind_text(stmt_new_, 4, &description[0], description.length(),
1853 SQLITE_STATIC);
1854
3/4
✓ Branch 0 taken 546 times.
✓ Branch 1 taken 129 times.
✓ Branch 3 taken 675 times.
✗ Branch 4 not taken.
 675 sqlite3_bind_int64(stmt_new_, 5, is_catalog ? kFileCatalog : kFileRegular);
1855
1/2
✓ Branch 1 taken 675 times.
✗ Branch 2 not taken.
 675 sqlite3_bind_int64(stmt_new_, 6, 1);
1856
1/2
✓ Branch 1 taken 675 times.
✗ Branch 2 not taken.
 675 const int retval = sqlite3_step(stmt_new_);
1857
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 675 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 675 assert((retval == SQLITE_DONE) || (retval == SQLITE_OK));
1858
1/2
✓ Branch 1 taken 675 times.
✗ Branch 2 not taken.
 675 sqlite3_reset(stmt_new_);
1859
2/2
✓ Branch 0 taken 363 times.
✓ Branch 1 taken 312 times.
 675 if (!exists)
1860 363 gauge_ += size;
1861 675 return true;
1862 }
1863
1864 int pipe_reserve[2];
1865
1/2
✓ Branch 1 taken 602 times.
✗ Branch 2 not taken.
 602 MakeReturnPipe(pipe_reserve);
1866
1867 602 LruCommand cmd;
1868 602 cmd.command_type = kReserve;
1869 602 cmd.SetSize(size);
1870
1/2
✓ Branch 1 taken 602 times.
✗ Branch 2 not taken.
 602 cmd.StoreHash(hash);
1871 602 cmd.return_pipe = pipe_reserve[1];
1872
1/2
✓ Branch 1 taken 602 times.
✗ Branch 2 not taken.
 602 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
1873 bool result;
1874
1/2
✓ Branch 1 taken 602 times.
✗ Branch 2 not taken.
 602 ManagedReadHalfPipe(pipe_reserve[0], &result, sizeof(result));
1875
1/2
✓ Branch 1 taken 602 times.
✗ Branch 2 not taken.
 602 CloseReturnPipe(pipe_reserve);
1876
1877
2/2
✓ Branch 0 taken 43 times.
✓ Branch 1 taken 559 times.
 602 if (!result)
1878 43 return false;
1879
3/4
✓ Branch 0 taken 86 times.
✓ Branch 1 taken 473 times.
✓ Branch 3 taken 559 times.
✗ Branch 4 not taken.
 559 DoInsert(hash, size, description, is_catalog ? kPin : kPinRegular);
1880
1881 559 return true;
1882 1363 }
1883
1884
1885 3188 PosixQuotaManager::PosixQuotaManager(const uint64_t limit,
1886 const uint64_t cleanup_threshold,
1887 3188 const string &cache_workspace)
1888 3188 : shared_(false)
1889 3188 , spawned_(false)
1890 3188 , limit_(limit)
1891 3188 , cleanup_threshold_(cleanup_threshold)
1892 3188 , gauge_(0)
1893 3188 , pinned_(0)
1894 3188 , seq_(0)
1895 3188 , cache_dir_() // initialized in body
1896 3188 , workspace_dir_() // initialized in body
1897 3188 , fd_lock_cachedb_(-1)
1898 3188 , async_delete_(true)
1899 3188 , cachemgr_pid_(0)
1900 3188 , database_(NULL)
1901 3188 , stmt_touch_(NULL)
1902 3188 , stmt_unpin_(NULL)
1903 3188 , stmt_block_(NULL)
1904 3188 , stmt_unblock_(NULL)
1905 3188 , stmt_new_(NULL)
1906 3188 , stmt_lru_(NULL)
1907 3188 , stmt_size_(NULL)
1908 3188 , stmt_rm_(NULL)
1909 3188 , stmt_rm_batch_(NULL)
1910 3188 , stmt_list_(NULL)
1911 3188 , stmt_list_pinned_(NULL)
1912 3188 , stmt_list_catalogs_(NULL)
1913 3188 , stmt_list_volatile_(NULL)
1914 3188 , initialized_(false)
1915 6376 , cleanup_unused_first_(false) {
1916
2/4
✓ Branch 1 taken 3188 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3188 times.
✗ Branch 5 not taken.
 3188 ParseDirectories(cache_workspace, &cache_dir_, &workspace_dir_);
1917 3188 pipe_lru_[0] = pipe_lru_[1] = -1;
1918
1/2
✓ Branch 1 taken 3188 times.
✗ Branch 2 not taken.
 3188 cleanup_recorder_.AddRecorder(1, 90); // last 1.5 min with second resolution
1919 // last 1.5 h with minute resolution
1920
1/2
✓ Branch 1 taken 3188 times.
✗ Branch 2 not taken.
 3188 cleanup_recorder_.AddRecorder(60, 90 * 60);
1921 // last 18 hours with 20 min resolution
1922
1/2
✓ Branch 1 taken 3188 times.
✗ Branch 2 not taken.
 3188 cleanup_recorder_.AddRecorder(20 * 60, 60 * 60 * 18);
1923 // last 4 days with hour resolution
1924
1/2
✓ Branch 1 taken 3188 times.
✗ Branch 2 not taken.
 3188 cleanup_recorder_.AddRecorder(60 * 60, 60 * 60 * 24 * 4);
1925
1926 3188 lock_open_files_ = reinterpret_cast<pthread_mutex_t *>(
1927 3188 smalloc(sizeof(pthread_mutex_t)));
1928 3188 }
1929
1930
1931 12748 PosixQuotaManager::~PosixQuotaManager() {
1932 6374 free(lock_open_files_);
1933
1934
2/2
✓ Branch 0 taken 129 times.
✓ Branch 1 taken 3058 times.
 6374 if (!initialized_)
1935 258 return;
1936
1937
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3058 times.
 6116 if (shared_) {
1938 // Most of cleanup is done elsewhen by shared cache manager
1939 ✗ close(pipe_lru_[1]);
1940 ✗ return;
1941 }
1942
1943
2/2
✓ Branch 0 taken 1462 times.
✓ Branch 1 taken 1596 times.
 6116 if (spawned_) {
1944 2924 char fin = 0;
1945 2924 WritePipe(pipe_lru_[1], &fin, 1);
1946 2924 close(pipe_lru_[1]);
1947 2924 pthread_join(thread_lru_, NULL);
1948 } else {
1949 3192 ClosePipe(pipe_lru_);
1950 }
1951
1952 6116 CloseDatabase();
1953
14/14
✓ Branch 1 taken 3058 times.
✓ Branch 2 taken 129 times.
✓ Branch 4 taken 3058 times.
✓ Branch 5 taken 129 times.
✓ Branch 7 taken 3058 times.
✓ Branch 8 taken 129 times.
✓ Branch 10 taken 3058 times.
✓ Branch 11 taken 129 times.
✓ Branch 13 taken 3058 times.
✓ Branch 14 taken 129 times.
✓ Branch 16 taken 3058 times.
✓ Branch 17 taken 129 times.
✓ Branch 19 taken 3058 times.
✓ Branch 20 taken 129 times.
 14296 }
1954
1955
1956 206314 void PosixQuotaManager::ProcessCommandBunch(const unsigned num,
1957 const LruCommand *commands,
1958 const char *descriptions) {
1959 206314 int retval = sqlite3_exec(database_, "BEGIN", NULL, NULL, NULL);
1960
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 206314 times.
 206314 assert(retval == SQLITE_OK);
1961
1962
2/2
✓ Branch 0 taken 6452193 times.
✓ Branch 1 taken 206314 times.
 6658507 for (unsigned i = 0; i < num; ++i) {
1963
1/2
✓ Branch 1 taken 6452193 times.
✗ Branch 2 not taken.
 6452193 const shash::Any hash = commands[i].RetrieveHash();
1964
1/2
✓ Branch 1 taken 6452193 times.
✗ Branch 2 not taken.
 6452193 const string hash_str = hash.ToString();
1965 6452193 const unsigned size = commands[i].GetSize();
1966
1/2
✓ Branch 1 taken 6452193 times.
✗ Branch 2 not taken.
 6452193 LogCvmfs(kLogQuota, kLogDebug, "processing %s (%d)", hash_str.c_str(),
1967 6452193 commands[i].command_type);
1968
1969 bool exists;
1970
3/4
✓ Branch 0 taken 2150559 times.
✓ Branch 1 taken 86 times.
✓ Branch 2 taken 4301548 times.
✗ Branch 3 not taken.
 6452193 switch (commands[i].command_type) {
1971 2150559 case kTouch:
1972
1/2
✓ Branch 1 taken 2150559 times.
✗ Branch 2 not taken.
 2150559 sqlite3_bind_int64(stmt_touch_, 1, seq_++);
1973
1/2
✓ Branch 3 taken 2150559 times.
✗ Branch 4 not taken.
 2150559 sqlite3_bind_text(stmt_touch_, 2, &hash_str[0], hash_str.length(),
1974 SQLITE_STATIC);
1975
1/2
✓ Branch 1 taken 2150559 times.
✗ Branch 2 not taken.
 2150559 retval = sqlite3_step(stmt_touch_);
1976
1/2
✓ Branch 1 taken 2150559 times.
✗ Branch 2 not taken.
 2150559 LogCvmfs(kLogQuota, kLogDebug, "touching %s (%ld): %d",
1977 2150559 hash_str.c_str(), seq_ - 1, retval);
1978
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2150559 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 2150559 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
1979 ✗ PANIC(kLogSyslogErr, "failed to update %s in cachedb, error %d",
1980 hash_str.c_str(), retval);
1981 }
1982
1/2
✓ Branch 1 taken 2150559 times.
✗ Branch 2 not taken.
 2150559 sqlite3_reset(stmt_touch_);
1983 2150559 break;
1984 86 case kUnpin:
1985
1/2
✓ Branch 3 taken 86 times.
✗ Branch 4 not taken.
 86 sqlite3_bind_text(stmt_unpin_, 1, &hash_str[0], hash_str.length(),
1986 SQLITE_STATIC);
1987
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 86 retval = sqlite3_step(stmt_unpin_);
1988
1/2
✓ Branch 2 taken 86 times.
✗ Branch 3 not taken.
 86 LogCvmfs(kLogQuota, kLogDebug, "unpinning %s: %d", hash_str.c_str(),
1989 retval);
1990
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 86 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
1991 ✗ PANIC(kLogSyslogErr, "failed to unpin %s in cachedb, error %d",
1992 hash_str.c_str(), retval);
1993 }
1994
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 86 sqlite3_reset(stmt_unpin_);
1995 86 break;
1996 4301548 case kPin:
1997 case kPinRegular:
1998 case kInsert:
1999 case kInsertVolatile:
2000 // It could already be in, check
2001
1/2
✓ Branch 1 taken 4301548 times.
✗ Branch 2 not taken.
 4301548 exists = Contains(hash_str);
2002
2003 // Cleanup, move to trash and unlink
2004
3/4
✓ Branch 0 taken 4301333 times.
✓ Branch 1 taken 215 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 4301333 times.
 4301548 if (!exists && (gauge_ + size > limit_)) {
2005 ✗ LogCvmfs(kLogQuota, kLogDebug, "over limit, gauge %lu, file size %u",
2006 gauge_, size);
2007 ✗ retval = DoCleanup(cleanup_threshold_);
2008 ✗ assert(retval != 0);
2009 }
2010
2011 // Insert or replace
2012
1/2
✓ Branch 3 taken 4301548 times.
✗ Branch 4 not taken.
 4301548 sqlite3_bind_text(stmt_new_, 1, &hash_str[0], hash_str.length(),
2013 SQLITE_STATIC);
2014
1/2
✓ Branch 1 taken 4301548 times.
✗ Branch 2 not taken.
 4301548 sqlite3_bind_int64(stmt_new_, 2, size);
2015
2/2
✓ Branch 0 taken 172 times.
✓ Branch 1 taken 4301376 times.
 4301548 if (commands[i].command_type == kInsertVolatile) {
2016
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 sqlite3_bind_int64(stmt_new_, 3, (seq_++) | kVolatileFlag);
2017 } else {
2018
1/2
✓ Branch 1 taken 4301376 times.
✗ Branch 2 not taken.
 4301376 sqlite3_bind_int64(stmt_new_, 3, seq_++);
2019 }
2020 4301548 sqlite3_bind_text(stmt_new_, 4, &descriptions[i * kMaxDescription],
2021
1/2
✓ Branch 1 taken 4301548 times.
✗ Branch 2 not taken.
 4301548 commands[i].desc_length, SQLITE_STATIC);
2022
1/2
✓ Branch 1 taken 4301548 times.
✗ Branch 2 not taken.
 4301548 sqlite3_bind_int64(
2023 stmt_new_, 5,
2024
2/2
✓ Branch 0 taken 86 times.
✓ Branch 1 taken 4301462 times.
 4301548 (commands[i].command_type == kPin) ? kFileCatalog : kFileRegular);
2025
1/2
✓ Branch 1 taken 4301548 times.
✗ Branch 2 not taken.
 4301548 sqlite3_bind_int64(stmt_new_, 6,
2026
2/2
✓ Branch 0 taken 4301462 times.
✓ Branch 1 taken 86 times.
 4301548 ((commands[i].command_type == kPin)
2027
2/2
✓ Branch 0 taken 473 times.
✓ Branch 1 taken 4300989 times.
 4301462 || (commands[i].command_type == kPinRegular))
2028 ? 1
2029 : 0);
2030
1/2
✓ Branch 1 taken 4301548 times.
✗ Branch 2 not taken.
 4301548 retval = sqlite3_step(stmt_new_);
2031
1/2
✓ Branch 1 taken 4301548 times.
✗ Branch 2 not taken.
 4301548 LogCvmfs(kLogQuota, kLogDebug, "insert or replace %s, method %d: %d",
2032 4301548 hash_str.c_str(), commands[i].command_type, retval);
2033
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 4301548 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 4301548 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
2034 ✗ PANIC(kLogSyslogErr, "failed to insert %s in cachedb, error %d",
2035 hash_str.c_str(), retval);
2036 }
2037
1/2
✓ Branch 1 taken 4301548 times.
✗ Branch 2 not taken.
 4301548 sqlite3_reset(stmt_new_);
2038
2039
2/2
✓ Branch 0 taken 4301333 times.
✓ Branch 1 taken 215 times.
 4301548 if (!exists)
2040 4301333 gauge_ += size;
2041 4301548 break;
2042 ✗ default:
2043 // other types should have been taken care of by event loop
2044 ✗ PANIC(NULL);
2045 }
2046 6452193 }
2047
2048 206314 retval = sqlite3_exec(database_, "COMMIT", NULL, NULL, NULL);
2049
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 206314 times.
 206314 if (retval != SQLITE_OK) {
2050 ✗ PANIC(kLogSyslogErr, "failed to commit to cachedb, error %d", retval);
2051 }
2052 206314 }
2053
2054
2055 3102 bool PosixQuotaManager::RebuildDatabase() {
2056 3102 bool result = false;
2057 3102 string sql;
2058 3102 sqlite3_stmt *stmt_select = NULL;
2059 3102 sqlite3_stmt *stmt_insert = NULL;
2060 int sqlerr;
2061 3102 int seq = 0;
2062 char hex[4];
2063 struct stat info;
2064 platform_dirent64 *d;
2065 3102 DIR *dirp = NULL;
2066 3102 string path;
2067
2068
1/2
✓ Branch 1 taken 3102 times.
✗ Branch 2 not taken.
 3102 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug, "re-building cache database");
2069
2070 // Empty cache catalog and fscache
2071
1/2
✓ Branch 1 taken 3102 times.
✗ Branch 2 not taken.
 3102 sql = "DELETE FROM cache_catalog; DELETE FROM fscache;";
2072
1/2
✓ Branch 2 taken 3102 times.
✗ Branch 3 not taken.
 3102 sqlerr = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
2073
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3102 times.
 3102 if (sqlerr != SQLITE_OK) {
2074 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not clear cache database");
2075 ✗ goto build_return;
2076 }
2077
2078 3102 gauge_ = 0;
2079
2080 // Insert files from cache sub-directories 00 - ff
2081 // TODO(jblomer): fs_traversal
2082
1/2
✓ Branch 1 taken 3102 times.
✗ Branch 2 not taken.
 3102 sqlite3_prepare_v2(database_,
2083 "INSERT INTO fscache (sha1, size, actime) "
2084 "VALUES (:sha1, :s, :t);",
2085 -1, &stmt_insert, NULL);
2086
2087
2/2
✓ Branch 0 taken 761217 times.
✓ Branch 1 taken 2973 times.
 764190 for (int i = 0; i <= 0xff; i++) {
2088 761217 snprintf(hex, sizeof(hex), "%02x", i);
2089
3/6
✓ Branch 2 taken 761217 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 761217 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 761217 times.
✗ Branch 9 not taken.
 761217 path = cache_dir_ + "/" + string(hex);
2090
3/4
✓ Branch 2 taken 761217 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 129 times.
✓ Branch 5 taken 761088 times.
 761217 if ((dirp = opendir(path.c_str())) == NULL) {
2091
1/2
✓ Branch 2 taken 129 times.
✗ Branch 3 not taken.
 129 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
2092 "failed to open directory %s (tmpwatch interfering?)",
2093 path.c_str());
2094 129 goto build_return;
2095 }
2096
3/4
✓ Branch 1 taken 2283350 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1522262 times.
✓ Branch 4 taken 761088 times.
 2283350 while ((d = platform_readdir(dirp)) != NULL) {
2097
3/6
✓ Branch 2 taken 1522262 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1522262 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1522262 times.
✗ Branch 9 not taken.
 3044524 const string file_path = path + "/" + string(d->d_name);
2098
1/2
✓ Branch 2 taken 1522262 times.
✗ Branch 3 not taken.
 1522262 if (stat(file_path.c_str(), &info) == 0) {
2099
2/2
✓ Branch 0 taken 1522176 times.
✓ Branch 1 taken 86 times.
 1522262 if (!S_ISREG(info.st_mode))
2100 1522219 continue;
2101
2/2
✓ Branch 0 taken 43 times.
✓ Branch 1 taken 43 times.
 86 if (info.st_size == 0) {
2102
1/2
✓ Branch 2 taken 43 times.
✗ Branch 3 not taken.
 43 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug,
2103 "removing empty file %s during automatic cache db rebuild",
2104 file_path.c_str());
2105 43 unlink(file_path.c_str());
2106 43 continue;
2107 }
2108
2109
3/6
✓ Branch 2 taken 43 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 43 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 43 times.
✗ Branch 10 not taken.
 86 string hash = string(hex) + string(d->d_name);
2110
1/2
✓ Branch 3 taken 43 times.
✗ Branch 4 not taken.
 43 sqlite3_bind_text(stmt_insert, 1, hash.data(), hash.length(),
2111 SQLITE_STATIC);
2112
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 sqlite3_bind_int64(stmt_insert, 2, info.st_size);
2113
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 sqlite3_bind_int64(stmt_insert, 3, info.st_atime);
2114
2/4
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 43 times.
 43 if (sqlite3_step(stmt_insert) != SQLITE_DONE) {
2115 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not insert into temp table");
2116 ✗ goto build_return;
2117 }
2118
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 sqlite3_reset(stmt_insert);
2119
2120 43 gauge_ += info.st_size;
2121
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 } else {
2122 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not stat %s", file_path.c_str());
2123 }
2124
2/3
✓ Branch 1 taken 43 times.
✓ Branch 2 taken 1522219 times.
✗ Branch 3 not taken.
 1522262 }
2125
1/2
✓ Branch 1 taken 761088 times.
✗ Branch 2 not taken.
 761088 closedir(dirp);
2126 761088 dirp = NULL;
2127 }
2128
1/2
✓ Branch 1 taken 2973 times.
✗ Branch 2 not taken.
 2973 sqlite3_finalize(stmt_insert);
2129 2973 stmt_insert = NULL;
2130
2131 // Transfer from temp table in cache catalog
2132
1/2
✓ Branch 1 taken 2973 times.
✗ Branch 2 not taken.
 2973 sqlite3_prepare_v2(database_,
2133 "SELECT sha1, size FROM fscache ORDER BY actime;", -1,
2134 &stmt_select, NULL);
2135
1/2
✓ Branch 1 taken 2973 times.
✗ Branch 2 not taken.
 2973 sqlite3_prepare_v2(
2136 database_,
2137 "INSERT INTO cache_catalog (sha1, size, acseq, path, type, pinned) "
2138 "VALUES (:sha1, :s, :seq, 'unknown (automatic rebuild)', :t, 0);",
2139 -1, &stmt_insert, NULL);
2140
3/4
✓ Branch 1 taken 3016 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 43 times.
✓ Branch 4 taken 2973 times.
 3016 while (sqlite3_step(stmt_select) == SQLITE_ROW) {
2141 const string hash = string(
2142
2/4
✓ Branch 2 taken 43 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 43 times.
✗ Branch 6 not taken.
 43 reinterpret_cast<const char *>(sqlite3_column_text(stmt_select, 0)));
2143
1/2
✓ Branch 3 taken 43 times.
✗ Branch 4 not taken.
 43 sqlite3_bind_text(stmt_insert, 1, &hash[0], hash.length(), SQLITE_STATIC);
2144
2/4
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 43 times.
✗ Branch 5 not taken.
 43 sqlite3_bind_int64(stmt_insert, 2, sqlite3_column_int64(stmt_select, 1));
2145
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 sqlite3_bind_int64(stmt_insert, 3, seq++);
2146 // Might also be a catalog (information is lost)
2147
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 sqlite3_bind_int64(stmt_insert, 4, kFileRegular);
2148
2149
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 const int retval = sqlite3_step(stmt_insert);
2150
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 43 times.
 43 if (retval != SQLITE_DONE) {
2151 // If the file system hosting the cache is full, we'll likely notice here
2152 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
2153 "could not insert into cache catalog (%d - %s)", retval,
2154 sqlite3_errstr(retval));
2155 ✗ goto build_return;
2156 }
2157
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 sqlite3_reset(stmt_insert);
2158
1/2
✓ Branch 1 taken 43 times.
✗ Branch 2 not taken.
 43 }
2159
2160 // Delete temporary table
2161
1/2
✓ Branch 1 taken 2973 times.
✗ Branch 2 not taken.
 2973 sql = "DELETE FROM fscache;";
2162
1/2
✓ Branch 2 taken 2973 times.
✗ Branch 3 not taken.
 2973 sqlerr = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
2163
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2973 times.
 2973 if (sqlerr != SQLITE_OK) {
2164 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not clear temporary table (%d)",
2165 sqlerr);
2166 ✗ goto build_return;
2167 }
2168
2169 2973 seq_ = seq;
2170 2973 result = true;
2171
1/2
✓ Branch 1 taken 2973 times.
✗ Branch 2 not taken.
 2973 LogCvmfs(kLogQuota, kLogDebug,
2172 "rebuilding finished, sequence %" PRIu64 ", gauge %" PRIu64, seq_,
2173 gauge_);
2174
2175 3102 build_return:
2176
1/2
✓ Branch 0 taken 3102 times.
✗ Branch 1 not taken.
 3102 if (stmt_insert)
2177
1/2
✓ Branch 1 taken 3102 times.
✗ Branch 2 not taken.
 3102 sqlite3_finalize(stmt_insert);
2178
2/2
✓ Branch 0 taken 2973 times.
✓ Branch 1 taken 129 times.
 3102 if (stmt_select)
2179
1/2
✓ Branch 1 taken 2973 times.
✗ Branch 2 not taken.
 2973 sqlite3_finalize(stmt_select);
2180
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3102 times.
 3102 if (dirp)
2181 ✗ closedir(dirp);
2182 3102 return result;
2183 3102 }
2184
2185
2186 /**
2187 * Register a channel that allows the cache manager to trigger action to its
2188 * clients. Currently used for releasing pinned catalogs.
2189 */
2190 172 void PosixQuotaManager::RegisterBackChannel(int back_channel[2],
2191 const string &channel_id) {
2192
1/2
✓ Branch 0 taken 172 times.
✗ Branch 1 not taken.
 172 if (protocol_revision_ >= 1) {
2193
1/2
✓ Branch 2 taken 172 times.
✗ Branch 3 not taken.
 172 shash::Md5 hash = shash::Md5(shash::AsciiPtr(channel_id));
2194
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 MakeReturnPipe(back_channel);
2195
2196 172 LruCommand cmd;
2197 172 cmd.command_type = kRegisterBackChannel;
2198 172 cmd.return_pipe = back_channel[1];
2199 // Not StoreHash(). This is an MD5 hash.
2200 172 memcpy(cmd.digest, hash.digest, hash.GetDigestSize());
2201
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2202
2203 char success;
2204
1/2
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
 172 ManagedReadHalfPipe(back_channel[0], &success, sizeof(success));
2205 // At this point, the named FIFO is unlinked, so don't use CloseReturnPipe
2206
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 172 times.
 172 if (success != 'S') {
2207 ✗ PANIC(kLogDebug | kLogSyslogErr,
2208 "failed to register quota back channel (%c)", success);
2209 }
2210 } else {
2211 // Dummy pipe to return valid file descriptors
2212 ✗ MakePipe(back_channel);
2213 }
2214 172 }
2215
2216
2217 /**
2218 * Removes a chunk from cache, if it exists.
2219 */
2220 129 void PosixQuotaManager::Remove(const shash::Any &hash) {
2221
1/2
✓ Branch 1 taken 129 times.
✗ Branch 2 not taken.
 129 const string hash_str = hash.ToString();
2222
2223 int pipe_remove[2];
2224
1/2
✓ Branch 1 taken 129 times.
✗ Branch 2 not taken.
 129 MakeReturnPipe(pipe_remove);
2225
2226 129 LruCommand cmd;
2227 129 cmd.command_type = kRemove;
2228 129 cmd.return_pipe = pipe_remove[1];
2229
1/2
✓ Branch 1 taken 129 times.
✗ Branch 2 not taken.
 129 cmd.StoreHash(hash);
2230
1/2
✓ Branch 1 taken 129 times.
✗ Branch 2 not taken.
 129 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2231
2232 bool success;
2233
1/2
✓ Branch 1 taken 129 times.
✗ Branch 2 not taken.
 129 ManagedReadHalfPipe(pipe_remove[0], &success, sizeof(success));
2234
1/2
✓ Branch 1 taken 129 times.
✗ Branch 2 not taken.
 129 CloseReturnPipe(pipe_remove);
2235
2236
3/6
✓ Branch 1 taken 129 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 129 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 129 times.
✗ Branch 8 not taken.
 129 unlink((cache_dir_ + "/" + hash.MakePathWithoutSuffix()).c_str());
2237 129 }
2238
2239
2240 1548 void PosixQuotaManager::Spawn() {
2241
2/2
✓ Branch 0 taken 86 times.
✓ Branch 1 taken 1462 times.
 1548 if (spawned_)
2242 86 return;
2243
2244 1462 if (pthread_create(&thread_lru_, NULL, MainCommandServer,
2245 static_cast<void *>(this))
2246
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1462 times.
 1462 != 0) {
2247 ✗ PANIC(kLogDebug, "could not create lru thread");
2248 }
2249
2250 1462 spawned_ = true;
2251 }
2252
2253
2254 /**
2255 * Updates the sequence number of the file specified by the hash.
2256 */
2257 2150410 void PosixQuotaManager::Touch(const shash::Any &hash) {
2258 2150410 LruCommand cmd;
2259 2150410 cmd.command_type = kTouch;
2260
1/2
✓ Branch 1 taken 2150410 times.
✗ Branch 2 not taken.
 2150410 cmd.StoreHash(hash);
2261
1/2
✓ Branch 1 taken 2150410 times.
✗ Branch 2 not taken.
 2150410 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2262 2150410 }
2263
2264
2265 3784 void PosixQuotaManager::UnbindReturnPipe(int pipe_wronly) {
2266
2/2
✓ Branch 0 taken 43 times.
✓ Branch 1 taken 3741 times.
 3784 if (shared_)
2267 43 close(pipe_wronly);
2268 3784 }
2269
2270
2271 301 void PosixQuotaManager::UnlinkReturnPipe(int pipe_wronly) {
2272
2/2
✓ Branch 0 taken 129 times.
✓ Branch 1 taken 172 times.
 301 if (shared_)
2273
2/4
✓ Branch 2 taken 129 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 129 times.
✗ Branch 6 not taken.
 129 unlink((workspace_dir_ + "/pipe" + StringifyInt(pipe_wronly)).c_str());
2274 301 }
2275
2276
2277 289 void PosixQuotaManager::Unpin(const shash::Any &hash) {
2278
2/4
✓ Branch 1 taken 289 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 289 times.
✗ Branch 6 not taken.
 289 LogCvmfs(kLogQuota, kLogDebug, "Unpin %s", hash.ToString().c_str());
2279
2280 289 LruCommand cmd;
2281 289 cmd.command_type = kUnpin;
2282
1/2
✓ Branch 1 taken 289 times.
✗ Branch 2 not taken.
 289 cmd.StoreHash(hash);
2283
1/2
✓ Branch 1 taken 289 times.
✗ Branch 2 not taken.
 289 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2284 289 }
2285
2286
2287 86 void PosixQuotaManager::UnregisterBackChannel(int back_channel[2],
2288 const string &channel_id) {
2289
1/2
✓ Branch 0 taken 86 times.
✗ Branch 1 not taken.
 86 if (protocol_revision_ >= 1) {
2290
1/2
✓ Branch 2 taken 86 times.
✗ Branch 3 not taken.
 86 shash::Md5 hash = shash::Md5(shash::AsciiPtr(channel_id));
2291
2292 86 LruCommand cmd;
2293 86 cmd.command_type = kUnregisterBackChannel;
2294 // Not StoreHash(). This is an MD5 hash.
2295 86 memcpy(cmd.digest, hash.digest, hash.GetDigestSize());
2296
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 86 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2297
2298 // Writer's end will be closed by cache manager, FIFO is already unlinked
2299
1/2
✓ Branch 1 taken 86 times.
✗ Branch 2 not taken.
 86 close(back_channel[0]);
2300 } else {
2301 ✗ ClosePipe(back_channel);
2302 }
2303 86 }
2304
2305 4305676 void PosixQuotaManager::ManagedReadHalfPipe(int fd, void *buf, size_t nbyte) {
2306
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4305676 times.
 4305676 const unsigned timeout_ms = cachemgr_pid_ ? 1000 : 0;
2307 4305676 bool result = false;
2308 do {
2309 4305676 result = ReadHalfPipe(fd, buf, nbyte, timeout_ms);
2310 // try only as long as the cachemgr is still alive
2311
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 4305676 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 4305676 times.
 4305676 } while (!result && getpgid(cachemgr_pid_) >= 0);
2312
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4305676 times.
 4305676 if (!result) {
2313 ✗ PANIC(kLogStderr,
2314 "Error: quota manager could not read from cachemanager pipe");
2315 }
2316 4305676 }
2317
2318 ✗ void *PosixQuotaManager::CollectMountpointsHashes(void *data) {
2319 #ifndef __APPLE__
2320 ✗ pthread_setname_np(pthread_self(), "hash_collector");
2321 ✗ auto *handler = static_cast<CollectorHandler *>(data);
2322
2323 ✗ const std::string mountpoint = handler->mp[handler->i];
2324 ✗ ssize_t n = getxattr(mountpoint.c_str(), "user.list_open_hashes", nullptr, 0);
2325 ✗ if (n < 0) {
2326 ✗ pthread_exit(nullptr);
2327 }
2328 ✗ std::vector<char> buf((size_t)n);
2329 ✗ n = getxattr(mountpoint.c_str(), "user.list_open_hashes", buf.data(),
2330 buf.size());
2331 ✗ if (n < 0) {
2332 ✗ pthread_exit(nullptr);
2333 }
2334
2335 ✗ std::vector<std::string> hash_strs;
2336 ✗ std::string hash_str;
2337 ✗ for (const char c : buf) {
2338 ✗ if (c == '\n') {
2339 ✗ hash_strs.push_back(hash_str);
2340 ✗ hash_str.clear();
2341 } else {
2342 ✗ hash_str += c;
2343 }
2344 }
2345 ✗ const MutexLockGuard lock_guard(handler->l);
2346 ✗ for (auto hash_str : hash_strs) {
2347 ✗ handler->of.push_back(
2348 ✗ shash::Short(shash::MkFromHexPtr(shash::HexPtr(hash_str))));
2349 }
2350 #endif
2351 ✗ pthread_exit(nullptr);
2352 }
2353
2354 ✗ std::vector<shash::Short> PosixQuotaManager::CollectAllOpenHashes() {
2355 ✗ std::vector<CollectorHandler *> handlers;
2356 ✗ std::vector<pthread_t *> threads;
2357 ✗ open_files_.clear();
2358 #ifndef __APPLE__
2359 ✗ auto &&a_after_b = [](const struct timespec a, const struct timespec b) {
2360 ✗ return (a.tv_sec > b.tv_sec) ? true : false;
2361 };
2362
2363 ✗ for (size_t i = 0; i < mountpoints_.size(); ++i) {
2364 ✗ handlers.push_back(
2365 ✗ new CollectorHandler{open_files_, mountpoints_, lock_open_files_, i});
2366 ✗ threads.push_back(new pthread_t);
2367 }
2368
2369 ✗ const int retval = pthread_mutex_init(lock_open_files_, NULL);
2370 ✗ assert(retval == 0);
2371
2372 ✗ for (size_t i = 0; i < mountpoints_.size(); ++i) {
2373 ✗ pthread_create(threads[i], nullptr, CollectMountpointsHashes, handlers[i]);
2374 }
2375
2376 ✗ std::vector<bool> joined(handlers.size(), false);
2377 struct timespec reference, current;
2378 ✗ clock_gettime(CLOCK_REALTIME, &reference);
2379 ✗ clock_gettime(CLOCK_REALTIME, &current);
2380 ✗ reference.tv_sec += 10; // Give 10sec for hash collection
2381 ✗ size_t i = 0;
2382 ✗ while (
2383 ✗ (not std::all_of(joined.begin(), joined.end(), [](bool b) { return b; }))
2384 ✗ and a_after_b(reference, current)) {
2385 // as long as there are still threads that haven't joined yet
2386 // and for 10 seconds
2387 ✗ if (not joined[i]) {
2388 ✗ const int s = pthread_tryjoin_np(*threads[i], NULL);
2389 ✗ if (s == 0) {
2390 ✗ joined[i] = true;
2391 }
2392 }
2393 ✗ ++i;
2394 ✗ i = i % handlers.size();
2395 ✗ clock_gettime(CLOCK_REALTIME, &current);
2396 }
2397
2398 ✗ for (size_t i = 0; i < handlers.size(); ++i) {
2399 ✗ delete handlers[i];
2400 }
2401
2402 ✗ pthread_mutex_destroy(lock_open_files_);
2403 #endif
2404 ✗ return open_files_;
2405 }
2406
2407