GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/quota_posix.cc
Date: 2026-10-11 02:40:02
Exec Total Coverage
Lines: 987 1424 69.3%
Branches: 769 1964 39.2%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * This module implements a "managed local cache".
5 * This way, we are able to track access times of files in the cache
6 * and remove files based on least recently used strategy.
7 *
8 * We setup another SQLite catalog, a "cache catalog", that helps us
9 * in the bookkeeping of files, file sizes and access times.
10 *
11 * We might choose to not manage the local cache. This is indicated
12 * by limit == 0 and everything succeeds in that case.
13 */
14
15 #define __STDC_LIMIT_MACROS
16
17
18 #include "quota_posix.h"
19
20 #include <dirent.h>
21 #include <errno.h>
22 #include <fcntl.h>
23 #include <inttypes.h>
24 #include <pthread.h>
25 #include <signal.h>
26 #include <stdint.h>
27 #include <sys/dir.h>
28 #include <sys/stat.h>
29 #include <sys/xattr.h>
30
31 #ifndef __APPLE__
32 #include <sys/statfs.h>
33 #endif
34 #include <sys/statvfs.h>
35 #include <sys/types.h>
36 #include <sys/wait.h>
37 #include <unistd.h>
38
39 #include <algorithm>
40 #include <cassert>
41 #include <cstdio>
42 #include <cstdlib>
43 #include <cstring>
44 #include <limits>
45 #include <map>
46 #include <memory>
47 #include <set>
48 #include <string>
49 #include <vector>
50
51 #include "crypto/hash.h"
52 #include "duplex_sqlite3.h"
53 #include "monitor.h"
54 #include "statistics.h"
55 #include "util/capabilities.h"
56 #include "util/concurrency.h"
57 #include "util/exception.h"
58 #include "util/logging.h"
59 #include "util/posix.h"
60 #include "util/smalloc.h"
61 #include "util/string.h"
62
63 using namespace std; // NOLINT
64
65
66 1786 int PosixQuotaManager::BindReturnPipe(int pipe_wronly) {
67
2/2
✓ Branch 0 taken 1748 times.
✓ Branch 1 taken 38 times.
 1786 if (!shared_)
68 1748 return pipe_wronly;
69
70 // Connect writer's end
71
1/2
✓ Branch 2 taken 38 times.
✗ Branch 3 not taken.
 38 const int result = open(
72
2/4
✓ Branch 2 taken 38 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 38 times.
✗ Branch 6 not taken.
 76 (workspace_dir_ + "/pipe" + StringifyInt(pipe_wronly)).c_str(),
73 O_WRONLY | O_NONBLOCK);
74
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 19 times.
 38 if (result >= 0) {
75 19 Nonblock2Block(result);
76 } else {
77 19 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
78 19 "failed to bind return pipe (%d)", errno);
79 }
80 38 return result;
81 }
82
83
84 638 void PosixQuotaManager::CheckHighPinWatermark() {
85 638 const uint64_t watermark = kHighPinWatermark * cleanup_threshold_ / 100;
86
3/4
✓ Branch 0 taken 638 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 57 times.
✓ Branch 3 taken 581 times.
 638 if ((cleanup_threshold_ > 0) && (pinned_ > watermark)) {
87 57 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
88 "high watermark of pinned files (%" PRIu64 "M > %" PRIu64 "M)",
89 57 pinned_ / (1024 * 1024), watermark / (1024 * 1024));
90
2/4
✓ Branch 2 taken 57 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 57 times.
✗ Branch 6 not taken.
 57 BroadcastBackchannels("R"); // clients: please release pinned catalogs
91 }
92 638 }
93
94
95 ✗ void PosixQuotaManager::CleanupPipes() {
96 ✗ DIR *dirp = opendir(workspace_dir_.c_str());
97 ✗ assert(dirp != NULL);
98
99 platform_dirent64 *dent;
100 ✗ bool found_leftovers = false;
101 ✗ while ((dent = platform_readdir(dirp)) != NULL) {
102 ✗ const string name = dent->d_name;
103 ✗ const string path = workspace_dir_ + "/" + name;
104 platform_stat64 info;
105 ✗ const int retval = platform_stat(path.c_str(), &info);
106 ✗ if (retval != 0)
107 ✗ continue;
108
109 // CVMFS FIFOs are prefixed with "pipe"
110 ✗ if (S_ISFIFO(info.st_mode) && (name.substr(0, 4) == "pipe")) {
111 ✗ if (!found_leftovers) {
112 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogWarn,
113 "removing left-over FIFOs from cache directory");
114 }
115 ✗ found_leftovers = true;
116 ✗ unlink(path.c_str());
117 }
118 }
119 ✗ closedir(dirp);
120 }
121
122
123 /**
124 * Cleans up in data cache, until cache size is below leave_size.
125 * The actual unlinking is done in a separate process (fork).
126 *
127 * \return True on success, false otherwise
128 */
129 171 bool PosixQuotaManager::Cleanup(const uint64_t leave_size) {
130
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 171 times.
 171 if (!spawned_)
131 ✗ return DoCleanup(leave_size);
132
133 bool result;
134 int pipe_cleanup[2];
135
1/2
✓ Branch 1 taken 171 times.
✗ Branch 2 not taken.
 171 MakeReturnPipe(pipe_cleanup);
136
137 171 LruCommand cmd;
138 171 cmd.command_type = kCleanup;
139 171 cmd.size = leave_size;
140 171 cmd.return_pipe = pipe_cleanup[1];
141
142
1/2
✓ Branch 1 taken 171 times.
✗ Branch 2 not taken.
 171 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
143
1/2
✓ Branch 1 taken 171 times.
✗ Branch 2 not taken.
 171 ManagedReadHalfPipe(pipe_cleanup[0], &result, sizeof(result));
144
1/2
✓ Branch 1 taken 171 times.
✗ Branch 2 not taken.
 171 CloseReturnPipe(pipe_cleanup);
145
146 171 return result;
147 }
148
149
150 2619 void PosixQuotaManager::CloseDatabase() {
151
1/2
✓ Branch 0 taken 2619 times.
✗ Branch 1 not taken.
 2619 if (stmt_list_catalogs_)
152 2619 sqlite3_finalize(stmt_list_catalogs_);
153
1/2
✓ Branch 0 taken 2619 times.
✗ Branch 1 not taken.
 2619 if (stmt_list_pinned_)
154 2619 sqlite3_finalize(stmt_list_pinned_);
155
1/2
✓ Branch 0 taken 2619 times.
✗ Branch 1 not taken.
 2619 if (stmt_list_volatile_)
156 2619 sqlite3_finalize(stmt_list_volatile_);
157
1/2
✓ Branch 0 taken 2619 times.
✗ Branch 1 not taken.
 2619 if (stmt_list_)
158 2619 sqlite3_finalize(stmt_list_);
159
1/2
✓ Branch 0 taken 2619 times.
✗ Branch 1 not taken.
 2619 if (stmt_lru_)
160 2619 sqlite3_finalize(stmt_lru_);
161
1/2
✓ Branch 0 taken 2619 times.
✗ Branch 1 not taken.
 2619 if (stmt_rm_)
162 2619 sqlite3_finalize(stmt_rm_);
163
1/2
✓ Branch 0 taken 2619 times.
✗ Branch 1 not taken.
 2619 if (stmt_rm_batch_)
164 2619 sqlite3_finalize(stmt_rm_batch_);
165
1/2
✓ Branch 0 taken 2619 times.
✗ Branch 1 not taken.
 2619 if (stmt_size_)
166 2619 sqlite3_finalize(stmt_size_);
167
1/2
✓ Branch 0 taken 2619 times.
✗ Branch 1 not taken.
 2619 if (stmt_touch_)
168 2619 sqlite3_finalize(stmt_touch_);
169
1/2
✓ Branch 0 taken 2619 times.
✗ Branch 1 not taken.
 2619 if (stmt_unpin_)
170 2619 sqlite3_finalize(stmt_unpin_);
171
1/2
✓ Branch 0 taken 2619 times.
✗ Branch 1 not taken.
 2619 if (stmt_block_)
172 2619 sqlite3_finalize(stmt_block_);
173
1/2
✓ Branch 0 taken 2619 times.
✗ Branch 1 not taken.
 2619 if (stmt_unblock_)
174 2619 sqlite3_finalize(stmt_unblock_);
175
1/2
✓ Branch 0 taken 2619 times.
✗ Branch 1 not taken.
 2619 if (stmt_new_)
176 2619 sqlite3_finalize(stmt_new_);
177
1/2
✓ Branch 0 taken 2619 times.
✗ Branch 1 not taken.
 2619 if (database_)
178 2619 sqlite3_close(database_);
179 2619 UnlockFile(fd_lock_cachedb_);
180
181 2619 stmt_list_catalogs_ = NULL;
182 2619 stmt_list_pinned_ = NULL;
183 2619 stmt_list_volatile_ = NULL;
184 2619 stmt_list_ = NULL;
185 2619 stmt_rm_ = NULL;
186 2619 stmt_rm_batch_ = NULL;
187 2619 stmt_size_ = NULL;
188 2619 stmt_touch_ = NULL;
189 2619 stmt_unpin_ = NULL;
190 2619 stmt_block_ = NULL;
191 2619 stmt_unblock_ = NULL;
192 2619 stmt_new_ = NULL;
193 2619 database_ = NULL;
194
195 2619 pinned_chunks_.clear();
196 2619 }
197
198
199 1691 void PosixQuotaManager::CloseReturnPipe(int pipe[2]) {
200
2/2
✓ Branch 0 taken 38 times.
✓ Branch 1 taken 1653 times.
 1691 if (shared_) {
201 38 close(pipe[0]);
202 38 UnlinkReturnPipe(pipe[1]);
203 } else {
204 1653 ClosePipe(pipe);
205 }
206 1691 }
207
208
209 1901506 bool PosixQuotaManager::Contains(const string &hash_str) {
210 1901506 bool result = false;
211
212 1901506 sqlite3_bind_text(stmt_size_, 1, &hash_str[0], hash_str.length(),
213 SQLITE_STATIC);
214
2/2
✓ Branch 1 taken 469 times.
✓ Branch 2 taken 1901037 times.
 1901506 if (sqlite3_step(stmt_size_) == SQLITE_ROW)
215 469 result = true;
216 1901506 sqlite3_reset(stmt_size_);
217 1901506 LogCvmfs(kLogQuota, kLogDebug, "contains %s returns %d", hash_str.c_str(),
218 result);
219
220 1901506 return result;
221 }
222
223
224 2582 void PosixQuotaManager::CheckFreeSpace() {
225
3/4
✓ Branch 0 taken 2582 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 19 times.
✓ Branch 3 taken 2563 times.
 2582 if ((limit_ == 0) || (gauge_ >= limit_))
226 19 return;
227
228 struct statvfs vfs_info;
229
1/2
✓ Branch 1 taken 2563 times.
✗ Branch 2 not taken.
 2563 const int retval = statvfs((cache_dir_ + "/cachedb").c_str(), &vfs_info);
230
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2563 times.
 2563 if (retval != 0) {
231 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
232 "failed to query %s for free space (%d)", cache_dir_.c_str(),
233 ✗ errno);
234 ✗ return;
235 }
236 2563 const int64_t free_space_byte = vfs_info.f_bavail * vfs_info.f_bsize;
237
1/2
✓ Branch 1 taken 2563 times.
✗ Branch 2 not taken.
 2563 LogCvmfs(kLogQuota, kLogDebug, "free space: %" PRId64 " MB",
238 free_space_byte / (1024 * 1024));
239
240 2563 const int64_t required_byte = limit_ - gauge_;
241
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2563 times.
 2563 if (free_space_byte < required_byte) {
242 ✗ LogCvmfs(kLogQuota, kLogSyslogWarn,
243 "too little free space on the file system hosting the cache,"
244 " %" PRId64 " MB available",
245 free_space_byte / (1024 * 1024));
246 }
247 }
248
249
250 2639 PosixQuotaManager *PosixQuotaManager::Create(const string &cache_workspace,
251 const uint64_t limit,
252 const uint64_t cleanup_threshold,
253 const bool rebuild_database) {
254
2/2
✓ Branch 0 taken 38 times.
✓ Branch 1 taken 2601 times.
 2639 if (cleanup_threshold >= limit) {
255 38 LogCvmfs(kLogQuota, kLogDebug,
256 "invalid parameters: limit %" PRIu64 ", "
257 "cleanup_threshold %" PRIu64,
258 limit, cleanup_threshold);
259 38 return NULL;
260 }
261
262 PosixQuotaManager *quota_manager = new PosixQuotaManager(
263
1/2
✓ Branch 2 taken 2601 times.
✗ Branch 3 not taken.
 2601 limit, cleanup_threshold, cache_workspace);
264
265 // Initialize cache catalog
266
2/2
✓ Branch 1 taken 19 times.
✓ Branch 2 taken 2582 times.
 2601 if (!quota_manager->InitDatabase(rebuild_database)) {
267
1/2
✓ Branch 0 taken 19 times.
✗ Branch 1 not taken.
 19 delete quota_manager;
268 19 return NULL;
269 }
270 2582 quota_manager->CheckFreeSpace();
271 2582 MakePipe(quota_manager->pipe_lru_);
272
273 2582 quota_manager->protocol_revision_ = kProtocolRevision;
274 2582 quota_manager->initialized_ = true;
275 2582 return quota_manager;
276 }
277
278
279 /**
280 * Connects to a running shared local quota manager. Creates one if necessary.
281 */
282 38 PosixQuotaManager *PosixQuotaManager::CreateShared(
283 const std::string &exe_path,
284 const std::string &cache_workspace,
285 const uint64_t limit,
286 const uint64_t cleanup_threshold,
287 bool foreground) {
288 38 string cache_dir;
289 38 string workspace_dir;
290
2/4
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 38 times.
✗ Branch 5 not taken.
 38 ParseDirectories(cache_workspace, &cache_dir, &workspace_dir);
291
292 pid_t new_cachemgr_pid;
293
294 // Create lock file: only one fuse client at a time
295
2/4
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 38 times.
✗ Branch 5 not taken.
 38 const int fd_lockfile = LockFile(workspace_dir + "/lock_cachemgr");
296
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 19 times.
 38 if (fd_lockfile < 0) {
297
1/2
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
 19 LogCvmfs(kLogQuota, kLogDebug, "could not open lock file %s (%d)",
298
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 38 (workspace_dir + "/lock_cachemgr").c_str(), errno);
299 19 return NULL;
300 }
301
302 PosixQuotaManager *quota_mgr = new PosixQuotaManager(limit, cleanup_threshold,
303
2/4
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 19 times.
✗ Branch 5 not taken.
 19 cache_workspace);
304 19 quota_mgr->shared_ = true;
305 19 quota_mgr->spawned_ = true;
306
307 // Try to connect to pipe
308
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 const string fifo_path = workspace_dir + "/cachemgr";
309
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 LogCvmfs(kLogQuota, kLogDebug, "trying to connect to existing pipe");
310
1/2
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
 19 quota_mgr->pipe_lru_[1] = open(fifo_path.c_str(), O_WRONLY | O_NONBLOCK);
311
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19 times.
 19 if (quota_mgr->pipe_lru_[1] >= 0) {
312 ✗ const int fd_lockfile_rw = open((workspace_dir + "/lock_cachemgr").c_str(),
313 O_RDWR, 0600);
314 ✗ unsigned lockfile_magicnumber = 0;
315 ✗ const ssize_t result_mn = SafeRead(fd_lockfile_rw, &lockfile_magicnumber,
316 sizeof(lockfile_magicnumber));
317 ✗ const ssize_t result = SafeRead(fd_lockfile_rw, &new_cachemgr_pid,
318 sizeof(new_cachemgr_pid));
319 ✗ close(fd_lockfile_rw);
320
321 ✗ if ((lockfile_magicnumber != kLockFileMagicNumber) || (result < 0)
322 ✗ || (result_mn < 0)
323 ✗ || (static_cast<size_t>(result) < sizeof(new_cachemgr_pid))) {
324 ✗ if (result != 0) {
325 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
326 "could not read cache manager pid from lockfile");
327 ✗ UnlockFile(fd_lockfile);
328 ✗ delete quota_mgr;
329 ✗ return NULL;
330 } else {
331 // support reload from old versions of the cache manager
332 // lock file is empty in this case, try a plain ReadHalfPipe to get pid
333 ✗ quota_mgr->SetCacheMgrPid(quota_mgr->GetPid());
334 }
335 } else {
336 ✗ quota_mgr->SetCacheMgrPid(new_cachemgr_pid);
337 }
338
339
340 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to existing cache manager pipe");
341 ✗ quota_mgr->initialized_ = true;
342 ✗ Nonblock2Block(quota_mgr->pipe_lru_[1]);
343 ✗ UnlockFile(fd_lockfile);
344 ✗ quota_mgr->GetLimits(&quota_mgr->limit_, &quota_mgr->cleanup_threshold_);
345 ✗ LogCvmfs(kLogQuota, kLogDebug,
346 "received limit %" PRIu64 ", threshold %" PRIu64,
347 quota_mgr->limit_, quota_mgr->cleanup_threshold_);
348 ✗ if (FileExists(workspace_dir + "/cachemgr.protocol")) {
349 ✗ quota_mgr->protocol_revision_ = quota_mgr->GetProtocolRevision();
350 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected protocol revision %u",
351 quota_mgr->protocol_revision_);
352 } else {
353 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to ancient cache manager");
354 }
355 ✗ return quota_mgr;
356 }
357 19 const int connect_error = errno;
358
359 // Lock file: let existing cache manager finish first
360
2/4
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 19 times.
✗ Branch 5 not taken.
 19 const int fd_lockfile_fifo = LockFile(workspace_dir + "/lock_cachemgr.fifo");
361
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19 times.
 19 if (fd_lockfile_fifo < 0) {
362 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not open lock file %s (%d)",
363 ✗ (workspace_dir + "/lock_cachemgr.fifo").c_str(), errno);
364 ✗ UnlockFile(fd_lockfile);
365 ✗ delete quota_mgr;
366 ✗ return NULL;
367 }
368
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 UnlockFile(fd_lockfile_fifo);
369
370
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19 times.
 19 if (connect_error == ENXIO) {
371 ✗ LogCvmfs(kLogQuota, kLogDebug, "left-over FIFO found, unlinking");
372 ✗ unlink(fifo_path.c_str());
373 }
374
375 // Creating a new FIFO for the cache manager (to be bound later)
376 19 int retval = mkfifo(fifo_path.c_str(), 0600);
377
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19 times.
 19 if (retval != 0) {
378 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to create cache manager FIFO (%d)",
379 ✗ errno);
380 ✗ UnlockFile(fd_lockfile);
381 ✗ delete quota_mgr;
382 ✗ return NULL;
383 }
384
385 // Create new cache manager
386 int pipe_boot[2];
387 int pipe_handshake[2];
388
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 MakePipe(pipe_boot);
389
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 MakePipe(pipe_handshake);
390
391 19 vector<string> command_line;
392
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 command_line.push_back(exe_path);
393
2/4
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 19 times.
✗ Branch 6 not taken.
 19 command_line.push_back("__cachemgr__");
394
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 command_line.push_back(cache_workspace);
395
2/4
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 19 times.
✗ Branch 5 not taken.
 19 command_line.push_back(StringifyInt(pipe_boot[1]));
396
2/4
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 19 times.
✗ Branch 5 not taken.
 19 command_line.push_back(StringifyInt(pipe_handshake[0]));
397
2/4
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 19 times.
✗ Branch 5 not taken.
 19 command_line.push_back(StringifyInt(limit));
398
2/4
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 19 times.
✗ Branch 5 not taken.
 19 command_line.push_back(StringifyInt(cleanup_threshold));
399 // do not propagate foreground in order to reliably get pid from exec
400 // instead, daemonize right here
401
2/4
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 19 times.
✗ Branch 5 not taken.
 19 command_line.push_back(StringifyInt(true)); // foreground
402
3/6
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 19 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 19 times.
✗ Branch 8 not taken.
 19 command_line.push_back(StringifyInt(GetLogSyslogLevel()));
403
3/6
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 19 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 19 times.
✗ Branch 8 not taken.
 19 command_line.push_back(StringifyInt(GetLogSyslogFacility()));
404
5/14
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 19 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 19 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 19 times.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 19 times.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
 19 command_line.push_back(GetLogDebugFile() + ":" + GetLogMicroSyslog());
405
406 19 set<int> preserve_filedes;
407
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 preserve_filedes.insert(0);
408
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 preserve_filedes.insert(1);
409
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 preserve_filedes.insert(2);
410
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 preserve_filedes.insert(pipe_boot[1]);
411
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 preserve_filedes.insert(pipe_handshake[0]);
412
413
1/2
✓ Branch 0 taken 19 times.
✗ Branch 1 not taken.
 19 if (foreground) {
414
1/2
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
 19 retval = ManagedExec(command_line, preserve_filedes, map<int, int>(),
415 /*drop_credentials*/ false,
416 /*clear_env*/ false,
417 /*double_fork*/ true, &new_cachemgr_pid);
418 } else {
419 ✗ retval = ExecAsDaemon(command_line, &new_cachemgr_pid);
420 }
421
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19 times.
 19 if (!retval) {
422 ✗ UnlockFile(fd_lockfile);
423 ✗ ClosePipe(pipe_boot);
424 ✗ ClosePipe(pipe_handshake);
425 ✗ delete quota_mgr;
426 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to start cache manager");
427 ✗ return NULL;
428 }
429
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 LogCvmfs(kLogQuota, kLogDebug,
430 "new cache manager pid: %d protocol revision %d", new_cachemgr_pid,
431 QuotaManager::kProtocolRevision);
432 19 quota_mgr->SetCacheMgrPid(new_cachemgr_pid);
433
2/4
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 19 times.
✗ Branch 6 not taken.
 19 const int fd_lockfile_rw = open((workspace_dir + "/lock_cachemgr").c_str(),
434 O_RDWR | O_TRUNC, 0600);
435 19 const unsigned magic_number = PosixQuotaManager::kLockFileMagicNumber;
436
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 const bool result_mn = SafeWrite(fd_lockfile_rw, &magic_number,
437 sizeof(magic_number));
438
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 const bool result = SafeWrite(fd_lockfile_rw, &new_cachemgr_pid,
439 sizeof(new_cachemgr_pid));
440
2/4
✓ Branch 0 taken 19 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 19 times.
 19 if (!result || !result_mn) {
441 ✗ PANIC(kLogSyslogErr, "could not write cache manager pid to lockfile");
442 }
443
444
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 close(fd_lockfile_rw);
445 // Wait for cache manager to be ready
446
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 close(pipe_boot[1]);
447
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 close(pipe_handshake[0]);
448 char buf;
449
2/4
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 19 times.
✗ Branch 4 not taken.
 19 if (read(pipe_boot[0], &buf, 1) != 1) {
450
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 UnlockFile(fd_lockfile);
451
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 close(pipe_boot[0]);
452
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 close(pipe_handshake[1]);
453
1/2
✓ Branch 0 taken 19 times.
✗ Branch 1 not taken.
 19 delete quota_mgr;
454
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
455 "cache manager did not start");
456 19 return NULL;
457 }
458 ✗ close(pipe_boot[0]);
459
460 // Connect write end
461 ✗ quota_mgr->pipe_lru_[1] = open(fifo_path.c_str(), O_WRONLY | O_NONBLOCK);
462 ✗ if (quota_mgr->pipe_lru_[1] < 0) {
463 ✗ LogCvmfs(kLogQuota, kLogDebug,
464 ✗ "failed to connect to newly created FIFO (%d)", errno);
465 ✗ close(pipe_handshake[1]);
466 ✗ UnlockFile(fd_lockfile);
467 ✗ delete quota_mgr;
468 ✗ return NULL;
469 }
470
471 // Finalize handshake
472 ✗ buf = 'C';
473 ✗ if (write(pipe_handshake[1], &buf, 1) != 1) {
474 ✗ UnlockFile(fd_lockfile);
475 ✗ close(pipe_handshake[1]);
476 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not finalize handshake");
477 ✗ delete quota_mgr;
478 ✗ return NULL;
479 }
480 ✗ close(pipe_handshake[1]);
481
482 ✗ Nonblock2Block(quota_mgr->pipe_lru_[1]);
483 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to a new cache manager");
484 ✗ quota_mgr->protocol_revision_ = kProtocolRevision;
485
486 ✗ UnlockFile(fd_lockfile);
487
488 ✗ quota_mgr->initialized_ = true;
489 ✗ quota_mgr->GetLimits(&quota_mgr->limit_, &quota_mgr->cleanup_threshold_);
490 ✗ LogCvmfs(kLogQuota, kLogDebug,
491 "received limit %" PRIu64 ", "
492 "threshold %" PRIu64,
493 quota_mgr->limit_, quota_mgr->cleanup_threshold_);
494 ✗ return quota_mgr;
495 38 }
496
497
498 190 bool PosixQuotaManager::DoCleanup(const uint64_t leave_size) {
499
2/2
✓ Branch 0 taken 38 times.
✓ Branch 1 taken 152 times.
 190 if (gauge_ <= leave_size)
500 38 return true;
501
502 // TODO(jblomer) transaction
503
1/2
✓ Branch 1 taken 152 times.
✗ Branch 2 not taken.
 152 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug,
504 "clean up cache until at most %lu KB is used", leave_size / 1024);
505
1/2
✓ Branch 1 taken 152 times.
✗ Branch 2 not taken.
 152 LogCvmfs(kLogQuota, kLogDebug, "gauge %" PRIu64, gauge_);
506
1/2
✓ Branch 1 taken 152 times.
✗ Branch 2 not taken.
 152 cleanup_recorder_.Tick();
507
508 bool result;
509 152 vector<string> trash;
510
511 // Note that volatile files start counting from the smallest int64 number:
512 // the absolute sequence number with the first bit set in two's complement.
513 // So -1 can be a marker that will never appear in the database.
514 152 int64_t max_acseq = -1;
515 152 std::vector<EvictCandidate> lru_ordered_open;
516
517 do {
518
1/2
✓ Branch 1 taken 1102 times.
✗ Branch 2 not taken.
 1102 sqlite3_reset(stmt_lru_);
519
3/4
✓ Branch 0 taken 152 times.
✓ Branch 1 taken 950 times.
✓ Branch 3 taken 1102 times.
✗ Branch 4 not taken.
 1254 sqlite3_bind_int64(stmt_lru_, 1,
520 152 (max_acseq == -1) ? std::numeric_limits<int64_t>::min()
521 : (max_acseq + 1));
522
523 1102 std::vector<EvictCandidate> candidates;
524
1/2
✓ Branch 1 taken 1102 times.
✗ Branch 2 not taken.
 1102 candidates.reserve(kEvictBatchSize);
525 1102 string hash_str;
526 1102 unsigned i = 0;
527
3/4
✓ Branch 1 taken 989292 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 988190 times.
✓ Branch 4 taken 1102 times.
 989292 while (sqlite3_step(stmt_lru_) == SQLITE_ROW) {
528 hash_str = reinterpret_cast<const char *>(
529
2/4
✓ Branch 1 taken 988190 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 988190 times.
✗ Branch 5 not taken.
 988190 sqlite3_column_text(stmt_lru_, 0));
530
1/2
✓ Branch 2 taken 988190 times.
✗ Branch 3 not taken.
 988190 LogCvmfs(kLogQuota, kLogDebug, "add %s to candidates for eviction",
531 hash_str.c_str());
532
1/2
✓ Branch 1 taken 988190 times.
✗ Branch 2 not taken.
 988190 candidates.push_back(
533
1/2
✓ Branch 1 taken 988190 times.
✗ Branch 2 not taken.
 988190 EvictCandidate(shash::MkFromHexPtr(shash::HexPtr(hash_str)),
534 988190 sqlite3_column_int64(stmt_lru_, 1),
535
2/4
✓ Branch 1 taken 988190 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 988190 times.
✗ Branch 5 not taken.
 988190 sqlite3_column_int64(stmt_lru_, 2)));
536 988190 i++;
537 }
538
2/2
✓ Branch 1 taken 19 times.
✓ Branch 2 taken 1083 times.
 1102 if (candidates.empty()) {
539
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 LogCvmfs(kLogQuota, kLogDebug, "no more entries to evict");
540 19 break;
541 }
542
543 1083 const unsigned N = candidates.size();
544
545 1083 open_files_.clear();
546
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1083 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 2166 open_files_ = (cleanup_unused_first_) ? CollectAllOpenHashes()
547 1083 : std::vector<shash::Short>();
548
549
2/2
✓ Branch 0 taken 950171 times.
✓ Branch 1 taken 950 times.
 951121 for (i = 0; i < N; ++i) {
550 // That's a critical condition. We must not delete a not yet inserted
551 // pinned file as it is already reserved (but will be inserted later).
552 // Instead, set the pin bit in the db to not run into an endless loop
553
1/2
✓ Branch 2 taken 950171 times.
✗ Branch 3 not taken.
 950171 const bool is_pinned = pinned_chunks_.find(candidates[i].hash)
554 1900342 != pinned_chunks_.end();
555
556 // Avoid evicting open files hopping there are enough more recently used
557 // files to satisfy the cleanup request
558 /*
559 const bool is_open = std::find_if(
560 open_files_.begin(), open_files_.end(),
561 [&candidates, &i](const auto &elem) -> bool
562 { return elem.Collide(candidates[i].hash);
563 })
564 != open_files_.end();
565 */
566 950171 bool is_open = false;
567
1/2
✗ Branch 4 not taken.
✓ Branch 5 taken 950171 times.
 950171 for (auto it = open_files_.begin(); it != open_files_.end(); ++it) {
568 ✗ if (it->Collide(candidates[i].hash)) {
569 ✗ is_open = true;
570 ✗ break;
571 }
572 }
573
574
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 950152 times.
 950171 if (is_pinned) {
575
1/2
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
 19 SkipEviction(candidates[i]);
576 19 continue;
577 }
578
579
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 950152 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 950152 if (cleanup_unused_first_ and is_open) {
580 ✗ SkipEviction(candidates[i]);
581 ✗ lru_ordered_open.push_back(candidates[i]);
582 ✗ continue;
583 }
584
585
2/4
✓ Branch 1 taken 950152 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 950152 times.
✗ Branch 5 not taken.
 1900304 trash.push_back(cache_dir_ + "/"
586
2/4
✓ Branch 2 taken 950152 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 950152 times.
✗ Branch 6 not taken.
 2850456 + candidates[i].hash.MakePathWithoutSuffix());
587 950152 gauge_ -= candidates[i].size;
588 950152 max_acseq = candidates[i].acseq;
589
1/2
✓ Branch 2 taken 950152 times.
✗ Branch 3 not taken.
 950152 LogCvmfs(kLogQuota, kLogDebug, "lru cleanup %s, new gauge %" PRIu64,
590
1/2
✓ Branch 2 taken 950152 times.
✗ Branch 3 not taken.
 1900304 candidates[i].hash.ToString().c_str(), gauge_);
591
592
2/2
✓ Branch 0 taken 133 times.
✓ Branch 1 taken 950019 times.
 950152 if (gauge_ <= leave_size)
593 133 break;
594 }
595
6/6
✓ Branch 1 taken 1083 times.
✓ Branch 2 taken 19 times.
✓ Branch 4 taken 1083 times.
✓ Branch 5 taken 19 times.
✓ Branch 6 taken 950 times.
✓ Branch 7 taken 133 times.
 2204 } while (gauge_ > leave_size);
596
597
1/2
✓ Branch 0 taken 152 times.
✗ Branch 1 not taken.
 152 if (max_acseq != -1) {
598
1/2
✓ Branch 1 taken 152 times.
✗ Branch 2 not taken.
 152 sqlite3_bind_int64(stmt_rm_batch_, 1, max_acseq);
599
1/2
✓ Branch 1 taken 152 times.
✗ Branch 2 not taken.
 152 result = (sqlite3_step(stmt_rm_batch_) == SQLITE_DONE);
600
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 152 times.
 152 assert(result);
601
1/2
✓ Branch 1 taken 152 times.
✗ Branch 2 not taken.
 152 sqlite3_reset(stmt_rm_batch_);
602
603
1/2
✓ Branch 1 taken 152 times.
✗ Branch 2 not taken.
 152 result = (sqlite3_step(stmt_unblock_) == SQLITE_DONE);
604
1/2
✓ Branch 1 taken 152 times.
✗ Branch 2 not taken.
 152 sqlite3_reset(stmt_unblock_);
605
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 152 times.
 152 assert(result);
606 }
607
608
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 152 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 152 times.
 152 while (!lru_ordered_open.empty() and gauge_ > leave_size) {
609 // cleanup files in use
610 ✗ auto &candidate = lru_ordered_open[0];
611 ✗ trash.push_back(cache_dir_ + "/" + candidate.hash.MakePathWithoutSuffix());
612 ✗ gauge_ -= candidate.size;
613 ✗ max_acseq = candidate.acseq;
614 ✗ LogCvmfs(kLogQuota, kLogDebug, "lru cleanup %s, new gauge %" PRIu64,
615 ✗ candidate.hash.ToString().c_str(), gauge_);
616 ✗ lru_ordered_open.erase(lru_ordered_open.begin());
617 }
618
619
2/4
✓ Branch 1 taken 152 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 152 times.
 152 if (!EmptyTrash(trash))
620 ✗ return false;
621
622
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 133 times.
 152 if (gauge_ > leave_size) {
623
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
624 "request to clean until %" PRIu64 ", "
625 "but effective gauge is %" PRIu64,
626 leave_size, gauge_);
627 19 return false;
628 }
629 133 return true;
630 152 }
631
632 152 bool PosixQuotaManager::EmptyTrash(const std::vector<std::string> &trash) {
633
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 152 times.
 152 if (trash.empty())
634 ✗ return true;
635
636
2/2
✓ Branch 0 taken 114 times.
✓ Branch 1 taken 38 times.
 152 if (async_delete_) {
637 // Double fork avoids zombie, forked removal process must not flush file
638 // buffers
639 pid_t pid;
640 int statloc;
641
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 114 times.
 114 if ((pid = fork()) == 0) {
642 // TODO(jblomer): eviciting files in the cache should perhaps become a
643 // thread. This would also allow to block the chunks and prevent the
644 // race with re-insertion. Then again, a thread can block umount.
645 #ifndef DEBUGMSG
646 ✗ CloseAllFildes(std::set<int>());
647 #endif
648 ✗ if (fork() == 0) {
649 ✗ for (unsigned i = 0, iEnd = trash.size(); i < iEnd; ++i) {
650 ✗ LogCvmfs(kLogQuota, kLogDebug, "unlink %s", trash[i].c_str());
651 ✗ unlink(trash[i].c_str());
652 }
653 ✗ _exit(0);
654 }
655 ✗ _exit(0);
656 } else {
657
1/2
✓ Branch 0 taken 114 times.
✗ Branch 1 not taken.
 114 if (pid > 0)
658
1/2
✓ Branch 1 taken 114 times.
✗ Branch 2 not taken.
 114 waitpid(pid, &statloc, 0);
659 else
660 ✗ return false;
661 }
662 } else { // !async_delete_
663
2/2
✓ Branch 1 taken 57 times.
✓ Branch 2 taken 38 times.
 95 for (unsigned i = 0, iEnd = trash.size(); i < iEnd; ++i) {
664 57 LogCvmfs(kLogQuota, kLogDebug, "unlink %s", trash[i].c_str());
665 57 unlink(trash[i].c_str());
666 }
667 }
668 152 return true;
669 }
670
671
672 1900926 void PosixQuotaManager::DoInsert(const shash::Any &hash,
673 const uint64_t size,
674 const string &description,
675 const CommandType command_type) {
676
1/2
✓ Branch 1 taken 1900926 times.
✗ Branch 2 not taken.
 1900926 const string hash_str = hash.ToString();
677
1/2
✓ Branch 3 taken 1900926 times.
✗ Branch 4 not taken.
 1900926 LogCvmfs(kLogQuota, kLogDebug, "insert into lru %s, path %s, method %d",
678 hash_str.c_str(), description.c_str(), command_type);
679 1900926 const unsigned desc_length = (description.length() > kMaxDescription)
680 ? kMaxDescription
681
1/2
✓ Branch 0 taken 1900926 times.
✗ Branch 1 not taken.
 1900926 : description.length();
682
683 LruCommand *cmd = reinterpret_cast<LruCommand *>(
684 1900926 alloca(sizeof(LruCommand) + desc_length));
685 1900926 new (cmd) LruCommand;
686 1900926 cmd->command_type = command_type;
687 1900926 cmd->SetSize(size);
688
1/2
✓ Branch 1 taken 1900926 times.
✗ Branch 2 not taken.
 1900926 cmd->StoreHash(hash);
689 1900926 cmd->desc_length = desc_length;
690 1900926 memcpy(reinterpret_cast<char *>(cmd) + sizeof(LruCommand), &description[0],
691 desc_length);
692
1/2
✓ Branch 1 taken 1900926 times.
✗ Branch 2 not taken.
 1900926 WritePipe(pipe_lru_[1], cmd, sizeof(LruCommand) + desc_length);
693 1900926 }
694
695
696 703 vector<string> PosixQuotaManager::DoList(const CommandType list_command) {
697 703 vector<string> result;
698
699 int pipe_list[2];
700
1/2
✓ Branch 1 taken 703 times.
✗ Branch 2 not taken.
 703 MakeReturnPipe(pipe_list);
701 char description_buffer[kMaxDescription];
702
703 703 LruCommand cmd;
704 703 cmd.command_type = list_command;
705 703 cmd.return_pipe = pipe_list[1];
706
1/2
✓ Branch 1 taken 703 times.
✗ Branch 2 not taken.
 703 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
707
708 int length;
709 do {
710
1/2
✓ Branch 1 taken 1901482 times.
✗ Branch 2 not taken.
 1901482 ManagedReadHalfPipe(pipe_list[0], &length, sizeof(length));
711
2/2
✓ Branch 0 taken 1900779 times.
✓ Branch 1 taken 703 times.
 1901482 if (length > 0) {
712
1/2
✓ Branch 1 taken 1900779 times.
✗ Branch 2 not taken.
 1900779 ReadPipe(pipe_list[0], description_buffer, length);
713
2/4
✓ Branch 2 taken 1900779 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1900779 times.
✗ Branch 6 not taken.
 1900779 result.push_back(string(description_buffer, length));
714 }
715
2/2
✓ Branch 0 taken 1900779 times.
✓ Branch 1 taken 703 times.
 1901482 } while (length >= 0);
716
717
1/2
✓ Branch 1 taken 703 times.
✗ Branch 2 not taken.
 703 CloseReturnPipe(pipe_list);
718 1406 return result;
719 }
720
721
722 1480 uint64_t PosixQuotaManager::GetCapacity() {
723
1/2
✓ Branch 0 taken 1480 times.
✗ Branch 1 not taken.
 1480 if (limit_ != (uint64_t)(-1))
724 1480 return limit_;
725
726 // Unrestricted cache, look at free space on cache dir fs
727 struct statfs info;
728 ✗ if (statfs(".", &info) == 0) {
729 ✗ return info.f_bavail * info.f_bsize;
730 } else {
731 ✗ LogCvmfs(kLogQuota, kLogSyslogErr | kLogDebug,
732 ✗ "failed to query file system info of cache (%d)", errno);
733 ✗ return limit_;
734 }
735 }
736
737
738 ✗ void PosixQuotaManager::GetLimits(uint64_t *limit,
739 uint64_t *cleanup_threshold) {
740 int pipe_limits[2];
741 ✗ MakeReturnPipe(pipe_limits);
742
743 ✗ LruCommand cmd;
744 ✗ cmd.command_type = kLimits;
745 ✗ cmd.return_pipe = pipe_limits[1];
746 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
747 ✗ ManagedReadHalfPipe(pipe_limits[0], limit, sizeof(*limit));
748 ✗ ReadPipe(pipe_limits[0], cleanup_threshold, sizeof(*cleanup_threshold));
749 ✗ CloseReturnPipe(pipe_limits);
750 }
751
752
753 /**
754 * Since we only cleanup until cleanup_threshold, we can only add
755 * files smaller than limit-cleanup_threshold.
756 */
757 205 uint64_t PosixQuotaManager::GetMaxFileSize() {
758 205 return limit_ - cleanup_threshold_;
759 }
760
761
762 19 pid_t PosixQuotaManager::GetPid() {
763
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 19 if (!shared_ || !spawned_) {
764 19 return getpid();
765 }
766 ✗ if (cachemgr_pid_) {
767 ✗ return cachemgr_pid_;
768 }
769
770 pid_t result;
771 int pipe_pid[2];
772 ✗ MakeReturnPipe(pipe_pid);
773
774 ✗ LruCommand cmd;
775 ✗ cmd.command_type = kPid;
776 ✗ cmd.return_pipe = pipe_pid[1];
777 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
778 ✗ ReadHalfPipe(pipe_pid[0], &result, sizeof(result));
779 ✗ CloseReturnPipe(pipe_pid);
780 ✗ return result;
781 }
782
783
784 19 uint32_t PosixQuotaManager::GetProtocolRevision() {
785 int pipe_revision[2];
786
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 MakeReturnPipe(pipe_revision);
787
788 19 LruCommand cmd;
789 19 cmd.command_type = kGetProtocolRevision;
790 19 cmd.return_pipe = pipe_revision[1];
791
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
792
793 uint32_t revision;
794
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 ManagedReadHalfPipe(pipe_revision[0], &revision, sizeof(revision));
795
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 CloseReturnPipe(pipe_revision);
796 19 return revision;
797 }
798
799 1480 void PosixQuotaManager::SetCleanupPolicy(bool cleanup_unused_first) {
800
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1480 times.
 1480 if (protocol_revision_ < 3)
801 ✗ return;
802
803
3/4
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 1461 times.
✓ Branch 3 taken 1480 times.
✗ Branch 4 not taken.
 1480 LogCvmfs(
804 kLogQuota, kLogDebug, "Set cleanup policy to %s",
805 (cleanup_unused_first) ? "cleanup unused files first." : "vanilla lru.");
806
807
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 1461 times.
 1480 char policy = (cleanup_unused_first) ? 'S' : 'R'; // S: smart, R: regular;
808
809 LruCommand *cmd = reinterpret_cast<LruCommand *>(
810 1480 alloca(sizeof(LruCommand) + sizeof(policy)));
811 1480 new (cmd) LruCommand;
812 1480 cmd->command_type = kSetCleanupPolicy;
813 1480 cmd->desc_length = sizeof(policy);
814 1480 memcpy(reinterpret_cast<char *>(cmd) + sizeof(LruCommand), &policy,
815 sizeof(policy));
816
1/2
✓ Branch 1 taken 1480 times.
✗ Branch 2 not taken.
 1480 WritePipe(pipe_lru_[1], cmd, sizeof(LruCommand) + sizeof(policy));
817 }
818
819 19 void PosixQuotaManager::RegisterMountpoint(const std::string &mountpoint) {
820
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19 times.
 19 if (protocol_revision_ < 3)
821 ✗ return;
822
823 19 LogCvmfs(kLogQuota, kLogDebug, "Register Mountpoint %s", mountpoint.c_str());
824
825 19 const unsigned desc_length = (mountpoint.size() > kMaxDescription)
826 ? kMaxDescription
827
1/2
✓ Branch 0 taken 19 times.
✗ Branch 1 not taken.
 19 : mountpoint.size();
828 LruCommand *cmd = reinterpret_cast<LruCommand *>(
829 19 alloca(sizeof(LruCommand) + desc_length));
830 19 new (cmd) LruCommand;
831 19 cmd->command_type = kRegisterMountpoint;
832 19 cmd->desc_length = desc_length;
833 19 memcpy(reinterpret_cast<char *>(cmd) + sizeof(LruCommand), mountpoint.data(),
834 desc_length);
835 19 WritePipe(pipe_lru_[1], cmd, sizeof(LruCommand) + desc_length);
836 }
837
838 19 std::string PosixQuotaManager::ReadPipeString(int fd, size_t size) {
839
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19 times.
 19 if (size == 0)
840 ✗ return "";
841
842
1/2
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
 19 std::vector<char> buf(size);
843
1/2
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
 19 ManagedReadHalfPipe(fd, buf.data(), size);
844
1/2
✓ Branch 3 taken 19 times.
✗ Branch 4 not taken.
 19 return std::string(buf.data(), size);
845 19 }
846
847 ✗ std::string PosixQuotaManager::GetMountpoints() {
848 ✗ if (protocol_revision_ < 3)
849 ✗ return "";
850
851 int pipe_mp[2];
852 ✗ MakeReturnPipe(pipe_mp);
853
854 ✗ LruCommand cmd;
855 ✗ cmd.command_type = kGetMountpoints;
856 ✗ cmd.return_pipe = pipe_mp[1];
857 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
858 ✗ size_t mp_str_size = 0;
859 ✗ ManagedReadHalfPipe(pipe_mp[0], &mp_str_size, sizeof(size_t));
860 ✗ const std::string result = ReadPipeString(pipe_mp[0], mp_str_size);
861 ✗ CloseReturnPipe(pipe_mp);
862 ✗ return result;
863 }
864
865 ✗ std::string PosixQuotaManager::GetGroupHashes() {
866 ✗ if (protocol_revision_ < 3)
867 ✗ return "";
868
869 int pipe_gh[2];
870 ✗ MakeReturnPipe(pipe_gh);
871
872 ✗ LruCommand cmd;
873 ✗ cmd.command_type = kGetGroupHashes;
874 ✗ cmd.return_pipe = pipe_gh[1];
875 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
876 ✗ size_t mp_str_size = 0;
877 ✗ ManagedReadHalfPipe(pipe_gh[0], &mp_str_size, sizeof(size_t));
878 ✗ const std::string result = ReadPipeString(pipe_gh[0], mp_str_size);
879 ✗ CloseReturnPipe(pipe_gh);
880 ✗ return result;
881 }
882
883 /**
884 * Queries the shared local hard disk quota manager.
885 */
886 342 void PosixQuotaManager::GetSharedStatus(uint64_t *gauge, uint64_t *pinned) {
887 int pipe_status[2];
888
1/2
✓ Branch 1 taken 342 times.
✗ Branch 2 not taken.
 342 MakeReturnPipe(pipe_status);
889
890 342 LruCommand cmd;
891 342 cmd.command_type = kStatus;
892 342 cmd.return_pipe = pipe_status[1];
893
1/2
✓ Branch 1 taken 342 times.
✗ Branch 2 not taken.
 342 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
894
1/2
✓ Branch 1 taken 342 times.
✗ Branch 2 not taken.
 342 ManagedReadHalfPipe(pipe_status[0], gauge, sizeof(*gauge));
895
1/2
✓ Branch 1 taken 342 times.
✗ Branch 2 not taken.
 342 ReadPipe(pipe_status[0], pinned, sizeof(*pinned));
896
1/2
✓ Branch 1 taken 342 times.
✗ Branch 2 not taken.
 342 CloseReturnPipe(pipe_status);
897 342 }
898
899 19 bool PosixQuotaManager::SetSharedLimit(uint64_t limit) {
900 int pipe_set_limit[2];
901 bool result;
902
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 MakeReturnPipe(pipe_set_limit);
903
904 19 LruCommand cmd;
905 19 cmd.command_type = kSetLimit;
906 19 cmd.size = limit;
907 19 cmd.return_pipe = pipe_set_limit[1];
908
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
909
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 ReadHalfPipe(pipe_set_limit[0], &result, sizeof(result));
910
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 CloseReturnPipe(pipe_set_limit);
911 19 return result;
912 }
913
914
915 19 bool PosixQuotaManager::SetLimit(uint64_t size) {
916
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19 times.
 19 if (!spawned_) {
917 ✗ limit_ = size;
918 ✗ cleanup_threshold_ = size / 2;
919 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
920 "Quota limit set to %lu / threshold %lu", limit_,
921 cleanup_threshold_);
922 ✗ return true;
923 }
924 19 return SetSharedLimit(size);
925 }
926
927 3226 uint64_t PosixQuotaManager::GetSize() {
928
2/2
✓ Branch 0 taken 2922 times.
✓ Branch 1 taken 304 times.
 3226 if (!spawned_)
929 2922 return gauge_;
930 uint64_t gauge, size_pinned;
931
1/2
✓ Branch 1 taken 304 times.
✗ Branch 2 not taken.
 304 GetSharedStatus(&gauge, &size_pinned);
932 304 return gauge;
933 }
934
935
936 38 uint64_t PosixQuotaManager::GetSizePinned() {
937
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 38 times.
 38 if (!spawned_)
938 ✗ return pinned_;
939 uint64_t gauge, size_pinned;
940
1/2
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
 38 GetSharedStatus(&gauge, &size_pinned);
941 38 return size_pinned;
942 }
943
944
945 76 uint64_t PosixQuotaManager::GetCleanupRate(uint64_t period_s) {
946
2/4
✓ Branch 0 taken 76 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 76 times.
 76 if (!spawned_ || (protocol_revision_ < 2))
947 ✗ return 0;
948 uint64_t cleanup_rate;
949
950 int pipe_cleanup_rate[2];
951
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 76 MakeReturnPipe(pipe_cleanup_rate);
952 76 LruCommand cmd;
953 76 cmd.command_type = kCleanupRate;
954 76 cmd.size = period_s;
955 76 cmd.return_pipe = pipe_cleanup_rate[1];
956
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 76 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
957
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 76 ManagedReadHalfPipe(pipe_cleanup_rate[0], &cleanup_rate,
958 sizeof(cleanup_rate));
959
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 76 CloseReturnPipe(pipe_cleanup_rate);
960
961 76 return cleanup_rate;
962 }
963
964
965 2696 bool PosixQuotaManager::InitDatabase(const bool rebuild_database) {
966 2696 string sql;
967 sqlite3_stmt *stmt;
968
969
2/4
✓ Branch 1 taken 2696 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2696 times.
✗ Branch 5 not taken.
 2696 fd_lock_cachedb_ = LockFile(workspace_dir_ + "/lock_cachedb");
970
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 2677 times.
 2696 if (fd_lock_cachedb_ < 0) {
971
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 LogCvmfs(kLogQuota, kLogDebug, "failed to create cachedb lock");
972 19 return false;
973 }
974
975 2677 bool retry = false;
976
1/2
✓ Branch 1 taken 2677 times.
✗ Branch 2 not taken.
 2677 const string db_file = cache_dir_ + "/cachedb";
977
2/2
✓ Branch 0 taken 2564 times.
✓ Branch 1 taken 113 times.
 2677 if (rebuild_database) {
978
1/2
✓ Branch 2 taken 113 times.
✗ Branch 3 not taken.
 113 LogCvmfs(kLogQuota, kLogDebug, "rebuild database, unlinking existing (%s)",
979 db_file.c_str());
980 113 unlink(db_file.c_str());
981
1/2
✓ Branch 1 taken 113 times.
✗ Branch 2 not taken.
 113 unlink((db_file + "-journal").c_str());
982 }
983
984 2564 init_recover:
985
1/2
✓ Branch 2 taken 2677 times.
✗ Branch 3 not taken.
 2677 int err = sqlite3_open(db_file.c_str(), &database_);
986
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2677 times.
 2677 if (err != SQLITE_OK) {
987 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not open cache database (%d)", err);
988 ✗ goto init_database_fail;
989 }
990 // TODO(reneme): make this a `QuotaDatabase : public sqlite::Database`
991 sql = "PRAGMA synchronous=0; PRAGMA locking_mode=EXCLUSIVE; "
992 "PRAGMA auto_vacuum=1; "
993 "CREATE TABLE IF NOT EXISTS cache_catalog (sha1 TEXT, size INTEGER, "
994 " acseq INTEGER, path TEXT, type INTEGER, pinned INTEGER, "
995 "CONSTRAINT pk_cache_catalog PRIMARY KEY (sha1)); "
996 "CREATE UNIQUE INDEX IF NOT EXISTS idx_cache_catalog_acseq "
997 " ON cache_catalog (acseq); "
998 "CREATE TEMP TABLE fscache (sha1 TEXT, size INTEGER, actime INTEGER, "
999 "CONSTRAINT pk_fscache PRIMARY KEY (sha1)); "
1000 "CREATE INDEX idx_fscache_actime ON fscache (actime); "
1001 "CREATE TABLE IF NOT EXISTS properties (key TEXT, value TEXT, "
1002
1/2
✓ Branch 1 taken 2677 times.
✗ Branch 2 not taken.
 2677 " CONSTRAINT pk_properties PRIMARY KEY(key));";
1003
1/2
✓ Branch 2 taken 2677 times.
✗ Branch 3 not taken.
 2677 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1004
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2677 times.
 2677 if (err != SQLITE_OK) {
1005 ✗ if (!retry) {
1006 ✗ retry = true;
1007 ✗ sqlite3_close(database_);
1008 ✗ unlink(db_file.c_str());
1009 ✗ unlink((db_file + "-journal").c_str());
1010 ✗ LogCvmfs(kLogQuota, kLogSyslogWarn,
1011 "LRU database corrupted, re-building");
1012 ✗ goto init_recover;
1013 }
1014 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
1015 sql.c_str());
1016 ✗ goto init_database_fail;
1017 }
1018
1019 // If this an old cache catalog,
1020 // add and initialize new columns to cache_catalog
1021 sql = "ALTER TABLE cache_catalog ADD type INTEGER; "
1022
1/2
✓ Branch 1 taken 2677 times.
✗ Branch 2 not taken.
 2677 "ALTER TABLE cache_catalog ADD pinned INTEGER";
1023
1/2
✓ Branch 2 taken 2677 times.
✗ Branch 3 not taken.
 2677 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1024
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2677 times.
 2677 if (err == SQLITE_OK) {
1025 ✗ sql = "UPDATE cache_catalog SET type=" + StringifyInt(kFileRegular) + ";";
1026 ✗ err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1027 ✗ if (err != SQLITE_OK) {
1028 ✗ LogCvmfs(kLogQuota, kLogDebug,
1029 "could not init cache database (failed: %s)", sql.c_str());
1030 ✗ goto init_database_fail;
1031 }
1032 }
1033
1034 // Set pinned back
1035
1/2
✓ Branch 1 taken 2677 times.
✗ Branch 2 not taken.
 2677 sql = "UPDATE cache_catalog SET pinned=0;";
1036
1/2
✓ Branch 2 taken 2677 times.
✗ Branch 3 not taken.
 2677 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1037
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2677 times.
 2677 if (err != SQLITE_OK) {
1038 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
1039 sql.c_str());
1040 ✗ goto init_database_fail;
1041 }
1042
1043 // Set schema version
1044 sql = "INSERT OR REPLACE INTO properties (key, value) "
1045
1/2
✓ Branch 1 taken 2677 times.
✗ Branch 2 not taken.
 2677 "VALUES ('schema', '1.0')";
1046
1/2
✓ Branch 2 taken 2677 times.
✗ Branch 3 not taken.
 2677 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1047
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2677 times.
 2677 if (err != SQLITE_OK) {
1048 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
1049 sql.c_str());
1050 ✗ goto init_database_fail;
1051 }
1052
1053 // If cache catalog is empty, recreate from file system
1054
1/2
✓ Branch 1 taken 2677 times.
✗ Branch 2 not taken.
 2677 sql = "SELECT count(*) FROM cache_catalog;";
1055
1/2
✓ Branch 2 taken 2677 times.
✗ Branch 3 not taken.
 2677 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1056
2/4
✓ Branch 1 taken 2677 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2677 times.
✗ Branch 4 not taken.
 2677 if (sqlite3_step(stmt) == SQLITE_ROW) {
1057
6/8
✓ Branch 1 taken 2677 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 76 times.
✓ Branch 4 taken 2601 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 76 times.
✓ Branch 7 taken 2601 times.
✓ Branch 8 taken 76 times.
 2677 if ((sqlite3_column_int64(stmt, 0)) == 0 || rebuild_database) {
1058
1/2
✓ Branch 1 taken 2601 times.
✗ Branch 2 not taken.
 2601 LogCvmfs(kLogCvmfs, kLogDebug,
1059 "CernVM-FS: building lru cache database...");
1060
3/4
✓ Branch 1 taken 2601 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 57 times.
✓ Branch 4 taken 2544 times.
 2601 if (!RebuildDatabase()) {
1061
1/2
✓ Branch 1 taken 57 times.
✗ Branch 2 not taken.
 57 LogCvmfs(kLogQuota, kLogDebug,
1062 "could not build cache database from file system");
1063
1/2
✓ Branch 1 taken 57 times.
✗ Branch 2 not taken.
 57 sqlite3_finalize(stmt);
1064 57 goto init_database_fail;
1065 }
1066 }
1067
1/2
✓ Branch 1 taken 2620 times.
✗ Branch 2 not taken.
 2620 sqlite3_finalize(stmt);
1068 } else {
1069 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not select on cache catalog");
1070 ✗ sqlite3_finalize(stmt);
1071 ✗ goto init_database_fail;
1072 }
1073
1074 // How many bytes do we already have in cache?
1075
1/2
✓ Branch 1 taken 2620 times.
✗ Branch 2 not taken.
 2620 sql = "SELECT sum(size) FROM cache_catalog;";
1076
1/2
✓ Branch 2 taken 2620 times.
✗ Branch 3 not taken.
 2620 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1077
2/4
✓ Branch 1 taken 2620 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2620 times.
✗ Branch 4 not taken.
 2620 if (sqlite3_step(stmt) == SQLITE_ROW) {
1078
1/2
✓ Branch 1 taken 2620 times.
✗ Branch 2 not taken.
 2620 gauge_ = sqlite3_column_int64(stmt, 0);
1079 } else {
1080 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not determine cache size");
1081 ✗ sqlite3_finalize(stmt);
1082 ✗ goto init_database_fail;
1083 }
1084
1/2
✓ Branch 1 taken 2620 times.
✗ Branch 2 not taken.
 2620 sqlite3_finalize(stmt);
1085
1086 // Highest seq-no?
1087
1/2
✓ Branch 1 taken 2620 times.
✗ Branch 2 not taken.
 2620 sql = "SELECT coalesce(max(acseq & (~(1<<63))), 0) FROM cache_catalog;";
1088
1/2
✓ Branch 2 taken 2620 times.
✗ Branch 3 not taken.
 2620 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1089
2/4
✓ Branch 1 taken 2620 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2620 times.
✗ Branch 4 not taken.
 2620 if (sqlite3_step(stmt) == SQLITE_ROW) {
1090
1/2
✓ Branch 1 taken 2620 times.
✗ Branch 2 not taken.
 2620 seq_ = sqlite3_column_int64(stmt, 0) + 1;
1091 } else {
1092 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not determine highest seq-no");
1093 ✗ sqlite3_finalize(stmt);
1094 ✗ goto init_database_fail;
1095 }
1096
1/2
✓ Branch 1 taken 2620 times.
✗ Branch 2 not taken.
 2620 sqlite3_finalize(stmt);
1097
1098 // Prepare touch, new, remove statements
1099
1/2
✓ Branch 1 taken 2620 times.
✗ Branch 2 not taken.
 2620 sqlite3_prepare_v2(database_,
1100 "UPDATE cache_catalog SET acseq=:seq | (acseq&(1<<63)) "
1101 "WHERE sha1=:sha1;",
1102 -1, &stmt_touch_, NULL);
1103
1/2
✓ Branch 1 taken 2620 times.
✗ Branch 2 not taken.
 2620 sqlite3_prepare_v2(database_,
1104 "UPDATE cache_catalog SET pinned=0 "
1105 "WHERE sha1=:sha1;",
1106 -1, &stmt_unpin_, NULL);
1107
1/2
✓ Branch 1 taken 2620 times.
✗ Branch 2 not taken.
 2620 sqlite3_prepare_v2(database_,
1108 "UPDATE cache_catalog SET pinned=2 "
1109 "WHERE sha1=:sha1;",
1110 -1, &stmt_block_, NULL);
1111
1/2
✓ Branch 1 taken 2620 times.
✗ Branch 2 not taken.
 2620 sqlite3_prepare_v2(database_,
1112 "UPDATE cache_catalog SET pinned=1 "
1113 "WHERE pinned=2;",
1114 -1, &stmt_unblock_, NULL);
1115
1/2
✓ Branch 1 taken 2620 times.
✗ Branch 2 not taken.
 2620 sqlite3_prepare_v2(database_,
1116 "INSERT OR REPLACE INTO cache_catalog "
1117 "(sha1, size, acseq, path, type, pinned) "
1118 "VALUES (:sha1, :s, :seq, :p, :t, :pin);",
1119 -1, &stmt_new_, NULL);
1120
1/2
✓ Branch 1 taken 2620 times.
✗ Branch 2 not taken.
 2620 sqlite3_prepare_v2(database_,
1121 "SELECT size, pinned FROM cache_catalog WHERE sha1=:sha1;",
1122 -1, &stmt_size_, NULL);
1123
1/2
✓ Branch 1 taken 2620 times.
✗ Branch 2 not taken.
 2620 sqlite3_prepare_v2(database_, "DELETE FROM cache_catalog WHERE sha1=:sha1;",
1124 -1, &stmt_rm_, NULL);
1125
1/2
✓ Branch 1 taken 2620 times.
✗ Branch 2 not taken.
 2620 sqlite3_prepare_v2(database_,
1126 "DELETE FROM cache_catalog WHERE acseq<=:a AND pinned<>2;",
1127 -1, &stmt_rm_batch_, NULL);
1128
1/2
✓ Branch 2 taken 2620 times.
✗ Branch 3 not taken.
 2620 sqlite3_prepare_v2(database_,
1129
1/2
✓ Branch 2 taken 2620 times.
✗ Branch 3 not taken.
 5240 (std::string("SELECT sha1, size, acseq FROM cache_catalog "
1130 "WHERE pinned<>2 AND acseq>=:a "
1131 "ORDER BY acseq ASC "
1132 "LIMIT ")
1133
3/6
✓ Branch 1 taken 2620 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2620 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 2620 times.
✗ Branch 8 not taken.
 10480 + StringifyInt(kEvictBatchSize) + ";")
1134 .c_str(),
1135 -1, &stmt_lru_, NULL);
1136
1/2
✓ Branch 2 taken 2620 times.
✗ Branch 3 not taken.
 2620 sqlite3_prepare_v2(database_,
1137 ("SELECT path FROM cache_catalog WHERE type="
1138
3/6
✓ Branch 1 taken 2620 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2620 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 2620 times.
✗ Branch 8 not taken.
 5240 + StringifyInt(kFileRegular) + ";")
1139 .c_str(),
1140 -1, &stmt_list_, NULL);
1141
1/2
✓ Branch 1 taken 2620 times.
✗ Branch 2 not taken.
 2620 sqlite3_prepare_v2(database_,
1142 "SELECT path FROM cache_catalog WHERE pinned<>0;", -1,
1143 &stmt_list_pinned_, NULL);
1144
1/2
✓ Branch 1 taken 2620 times.
✗ Branch 2 not taken.
 2620 sqlite3_prepare_v2(database_,
1145 "SELECT path FROM cache_catalog WHERE acseq < 0;", -1,
1146 &stmt_list_volatile_, NULL);
1147
1/2
✓ Branch 2 taken 2620 times.
✗ Branch 3 not taken.
 2620 sqlite3_prepare_v2(database_,
1148 ("SELECT path FROM cache_catalog WHERE type="
1149
3/6
✓ Branch 1 taken 2620 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2620 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 2620 times.
✗ Branch 8 not taken.
 5240 + StringifyInt(kFileCatalog) + ";")
1150 .c_str(),
1151 -1, &stmt_list_catalogs_, NULL);
1152 2620 return true;
1153
1154 57 init_database_fail:
1155
1/2
✓ Branch 1 taken 57 times.
✗ Branch 2 not taken.
 57 sqlite3_close(database_);
1156 57 database_ = NULL;
1157
1/2
✓ Branch 1 taken 57 times.
✗ Branch 2 not taken.
 57 UnlockFile(fd_lock_cachedb_);
1158 57 return false;
1159 2696 }
1160
1161
1162 /**
1163 * Inserts a new file into cache catalog. This file gets a new,
1164 * highest sequence number. Does cache cleanup if necessary.
1165 */
1166 1900603 void PosixQuotaManager::Insert(const shash::Any &any_hash,
1167 const uint64_t size,
1168 const string &description) {
1169 1900603 DoInsert(any_hash, size, description, kInsert);
1170 1900603 }
1171
1172
1173 /**
1174 * Inserts a new file into cache catalog. This file is marked as volatile
1175 * and gets a new highest sequence number with the first bit set. Cache cleanup
1176 * treats these files with priority.
1177 */
1178 76 void PosixQuotaManager::InsertVolatile(const shash::Any &any_hash,
1179 const uint64_t size,
1180 const string &description) {
1181 76 DoInsert(any_hash, size, description, kInsertVolatile);
1182 76 }
1183
1184
1185 /**
1186 * Lists all path names from the cache db.
1187 */
1188 437 vector<string> PosixQuotaManager::List() { return DoList(kList); }
1189
1190
1191 /**
1192 * Lists all pinned files from the cache db.
1193 */
1194 152 vector<string> PosixQuotaManager::ListPinned() { return DoList(kListPinned); }
1195
1196
1197 /**
1198 * Lists all sqlite catalog files from the cache db.
1199 */
1200 57 vector<string> PosixQuotaManager::ListCatalogs() {
1201 57 return DoList(kListCatalogs);
1202 }
1203
1204
1205 /**
1206 * Lists only files flagged as volatile (priority removal)
1207 */
1208 57 vector<string> PosixQuotaManager::ListVolatile() {
1209 57 return DoList(kListVolatile);
1210 }
1211
1212
1213 /**
1214 * Entry point for the shared cache manager process
1215 */
1216 ✗ int PosixQuotaManager::MainCacheManager(int argc, char **argv) {
1217 ✗ LogCvmfs(kLogQuota, kLogDebug, "starting quota manager");
1218 int retval;
1219
1220 ✗ PosixQuotaManager shared_manager(0, 0, "");
1221 ✗ shared_manager.shared_ = true;
1222 ✗ shared_manager.spawned_ = true;
1223 ✗ shared_manager.pinned_ = 0;
1224
1225 // Process command line arguments
1226 ✗ ParseDirectories(string(argv[2]),
1227 &shared_manager.cache_dir_,
1228 &shared_manager.workspace_dir_);
1229 ✗ const int pipe_boot = String2Int64(argv[3]);
1230 ✗ const int pipe_handshake = String2Int64(argv[4]);
1231 ✗ shared_manager.limit_ = String2Int64(argv[5]);
1232 ✗ shared_manager.cleanup_threshold_ = String2Int64(argv[6]);
1233 ✗ const int foreground = String2Int64(argv[7]);
1234 ✗ const int syslog_level = String2Int64(argv[8]);
1235 ✗ const int syslog_facility = String2Int64(argv[9]);
1236 ✗ vector<string> logfiles = SplitString(argv[10], ':');
1237
1238 ✗ SetLogSyslogLevel(syslog_level);
1239 ✗ SetLogSyslogFacility(syslog_facility);
1240 ✗ if ((logfiles.size() > 0) && (logfiles[0] != ""))
1241 ✗ SetLogDebugFile(logfiles[0] + ".cachemgr");
1242 ✗ if (logfiles.size() > 1)
1243 ✗ SetLogMicroSyslog(logfiles[1]);
1244
1245 ✗ if (!foreground)
1246 ✗ Daemonize();
1247
1248 ✗ if ((geteuid() != 0) && SetuidCapabilityPermitted()) {
1249 // Permanently drop credentials
1250 ✗ const std::vector<cap_value_t> nocaps;
1251 ✗ if (!ClearPermittedCapabilities(nocaps, nocaps))
1252 ✗ PANIC(kLogStderr | kLogSyslogErr,
1253 "Failed to clear quota manager capabilities");
1254 // Leave this process ptraceable
1255 ✗ if (!platform_set_dumpable())
1256 ✗ PANIC(kLogStderr | kLogSyslogErr,
1257 "Failed to make quota manager process ptraceable");
1258 // but without core dumps
1259 ✗ if (!SetLimitCore(0))
1260 ✗ PANIC(kLogStderr | kLogSyslogErr,
1261 "Failed to disable quota manager core dumps");
1262 }
1263
1264 const std::unique_ptr<Watchdog> watchdog(
1265 ✗ Watchdog::Create(NULL, false /* needs_read_environ */));
1266 ✗ assert(watchdog.get() != nullptr);
1267 ✗ watchdog->Spawn("./stacktrace.cachemgr");
1268
1269 // Initialize pipe, open non-blocking as cvmfs is not yet connected
1270 ✗ const int fd_lockfile_fifo = LockFile(shared_manager.workspace_dir_
1271 ✗ + "/lock_cachemgr.fifo");
1272 ✗ if (fd_lockfile_fifo < 0) {
1273 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1274 "could not open lock file "
1275 "%s (%d)",
1276 ✗ (shared_manager.workspace_dir_ + "/lock_cachemgr.fifo").c_str(),
1277 ✗ errno);
1278 ✗ return 1;
1279 }
1280 ✗ const string crash_guard = shared_manager.cache_dir_ + "/cachemgr.running";
1281 ✗ const bool rebuild = FileExists(crash_guard);
1282 ✗ retval = open(crash_guard.c_str(), O_RDONLY | O_CREAT, 0600);
1283 ✗ if (retval < 0) {
1284 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1285 "failed to create shared cache manager crash guard");
1286 ✗ UnlockFile(fd_lockfile_fifo);
1287 ✗ return 1;
1288 }
1289 ✗ close(retval);
1290
1291 // Redirect SQlite temp directory to cache (global variable)
1292 ✗ const string tmp_dir = shared_manager.workspace_dir_;
1293 ✗ sqlite3_temp_directory = static_cast<char *>(
1294 ✗ sqlite3_malloc(tmp_dir.length() + 1));
1295 ✗ snprintf(sqlite3_temp_directory, tmp_dir.length() + 1, "%s", tmp_dir.c_str());
1296
1297 // Cleanup leftover named pipes
1298 ✗ shared_manager.CleanupPipes();
1299
1300 ✗ if (!shared_manager.InitDatabase(rebuild)) {
1301 ✗ UnlockFile(fd_lockfile_fifo);
1302 ✗ return 1;
1303 }
1304 ✗ shared_manager.CheckFreeSpace();
1305
1306 // Save protocol revision to file. If the file is not found, it indicates
1307 // to the client that the cache manager is from times before the protocol
1308 // was versioned.
1309 const string protocol_revision_path = shared_manager.workspace_dir_
1310 ✗ + "/cachemgr.protocol";
1311 ✗ retval = open(protocol_revision_path.c_str(), O_WRONLY | O_CREAT, 0600);
1312 ✗ if (retval < 0) {
1313 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1314 ✗ "failed to open protocol revision file (%d)", errno);
1315 ✗ UnlockFile(fd_lockfile_fifo);
1316 ✗ return 1;
1317 }
1318 ✗ const string revision = StringifyInt(kProtocolRevision);
1319 ✗ const int written = write(retval, revision.data(), revision.length());
1320 ✗ close(retval);
1321 ✗ if ((written < 0) || static_cast<unsigned>(written) != revision.length()) {
1322 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1323 ✗ "failed to write protocol revision (%d)", errno);
1324 ✗ UnlockFile(fd_lockfile_fifo);
1325 ✗ return 1;
1326 }
1327
1328 ✗ const string fifo_path = shared_manager.workspace_dir_ + "/cachemgr";
1329 ✗ shared_manager.pipe_lru_[0] = open(fifo_path.c_str(), O_RDONLY | O_NONBLOCK);
1330 ✗ if (shared_manager.pipe_lru_[0] < 0) {
1331 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to listen on FIFO %s (%d)",
1332 ✗ fifo_path.c_str(), errno);
1333 ✗ UnlockFile(fd_lockfile_fifo);
1334 ✗ return 1;
1335 }
1336 ✗ Nonblock2Block(shared_manager.pipe_lru_[0]);
1337 ✗ LogCvmfs(kLogQuota, kLogDebug, "shared cache manager listening");
1338
1339 ✗ char buf = 'C';
1340 ✗ WritePipe(pipe_boot, &buf, 1);
1341 ✗ close(pipe_boot);
1342
1343 ✗ ReadPipe(pipe_handshake, &buf, 1);
1344 ✗ close(pipe_handshake);
1345 ✗ LogCvmfs(kLogQuota, kLogDebug, "shared cache manager handshake done");
1346
1347 // Ensure that broken pipes from clients do not kill the cache manager
1348 ✗ signal(SIGPIPE, SIG_IGN);
1349 // Don't let Ctrl-C ungracefully kill interactive session
1350 ✗ signal(SIGINT, SIG_IGN);
1351
1352 ✗ shared_manager.MainCommandServer(&shared_manager);
1353 ✗ unlink(fifo_path.c_str());
1354 ✗ unlink(protocol_revision_path.c_str());
1355 ✗ shared_manager.CloseDatabase();
1356 ✗ unlink(crash_guard.c_str());
1357 ✗ UnlockFile(fd_lockfile_fifo);
1358
1359 ✗ if (sqlite3_temp_directory) {
1360 ✗ sqlite3_free(sqlite3_temp_directory);
1361 ✗ sqlite3_temp_directory = NULL;
1362 }
1363
1364 ✗ return 0;
1365 }
1366
1367
1368 646 void *PosixQuotaManager::MainCommandServer(void *data) {
1369 646 PosixQuotaManager *quota_mgr = static_cast<PosixQuotaManager *>(data);
1370
1371
1/2
✓ Branch 1 taken 646 times.
✗ Branch 2 not taken.
 646 LogCvmfs(kLogQuota, kLogDebug, "starting quota manager");
1372
1/2
✓ Branch 1 taken 646 times.
✗ Branch 2 not taken.
 646 sqlite3_soft_heap_limit(quota_mgr->kSqliteMemPerThread);
1373
1374
2/2
✓ Branch 1 taken 20672 times.
✓ Branch 2 taken 646 times.
 21318 LruCommand command_buffer[kCommandBufferSize];
1375 char description_buffer[kCommandBufferSize * kMaxDescription];
1376 646 unsigned num_commands = 0;
1377
1378
1/2
✓ Branch 1 taken 2853230 times.
✗ Branch 2 not taken.
 2853230 while (read(quota_mgr->pipe_lru_[0], &command_buffer[num_commands],
1379 sizeof(command_buffer[0]))
1380
2/2
✓ Branch 0 taken 2852584 times.
✓ Branch 1 taken 646 times.
 2853230 == sizeof(command_buffer[0])) {
1381 2852584 const CommandType command_type = command_buffer[num_commands].command_type;
1382
1/2
✓ Branch 1 taken 2852584 times.
✗ Branch 2 not taken.
 2852584 LogCvmfs(kLogQuota, kLogDebug, "received command %d", command_type);
1383 2852584 const uint64_t size = command_buffer[num_commands].GetSize();
1384
1385 // Inserts and pins come with a description (usually a path)
1386
4/4
✓ Branch 0 taken 952223 times.
✓ Branch 1 taken 1900361 times.
✓ Branch 2 taken 952147 times.
✓ Branch 3 taken 76 times.
 2852584 if ((command_type == kInsert) || (command_type == kInsertVolatile)
1387
4/4
✓ Branch 0 taken 952109 times.
✓ Branch 1 taken 38 times.
✓ Branch 2 taken 951900 times.
✓ Branch 3 taken 209 times.
 952147 || (command_type == kPin) || (command_type == kPinRegular)
1388
2/2
✓ Branch 0 taken 951881 times.
✓ Branch 1 taken 19 times.
 951900 || (command_type == kRegisterMountpoint)
1389
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 951862 times.
 951881 || (command_type == kSetCleanupPolicy)) {
1390 1900722 const int desc_length = command_buffer[num_commands].desc_length;
1391 1900722 ReadPipe(quota_mgr->pipe_lru_[0],
1392
1/2
✓ Branch 1 taken 1900722 times.
✗ Branch 2 not taken.
 1900722 &description_buffer[kMaxDescription * num_commands],
1393 desc_length);
1394 }
1395
1396 // The protocol revision is returned immediately
1397
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 2852565 times.
 2852584 if (command_type == kGetProtocolRevision) {
1398
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 const int return_pipe = quota_mgr->BindReturnPipe(
1399 command_buffer[num_commands].return_pipe);
1400
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19 times.
 19 if (return_pipe < 0)
1401 ✗ continue;
1402
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 WritePipe(return_pipe, &quota_mgr->kProtocolRevision,
1403 sizeof(quota_mgr->kProtocolRevision));
1404
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 quota_mgr->UnbindReturnPipe(return_pipe);
1405 19 continue;
1406 19 }
1407
1408 // Register a new mountpoint
1409
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 2852546 times.
 2852565 if (command_type == kRegisterMountpoint) {
1410 const std::string mountpoint(
1411 19 &description_buffer[num_commands * kMaxDescription],
1412
1/2
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
 19 command_buffer[num_commands].desc_length);
1413
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 quota_mgr->mountpoints_.push_back(mountpoint);
1414
1/2
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
 19 LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
1415 "Mountpoint %s registered in the group", mountpoint.c_str());
1416 19 continue;
1417 19 }
1418
1419 // Set Cleanup Policy
1420
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 2852527 times.
 2852546 if (command_type == kSetCleanupPolicy) {
1421 19 quota_mgr->cleanup_unused_first_ = (description_buffer[num_commands
1422 19 * kMaxDescription]
1423 == 'S')
1424 19 ? true
1425 : false;
1426 19 continue;
1427 }
1428 // Mountpoints are returned immediately
1429
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2852527 times.
 2852527 if (command_type == kGetMountpoints) {
1430 ✗ const int return_pipe = quota_mgr->BindReturnPipe(
1431 command_buffer[num_commands].return_pipe);
1432 ✗ if (return_pipe < 0)
1433 ✗ continue;
1434
1435 ✗ std::string mps;
1436 ✗ for (auto it = quota_mgr->mountpoints_.begin();
1437 ✗ it != quota_mgr->mountpoints_.end();
1438 ✗ ++it) {
1439 ✗ mps += *it + "\n";
1440 }
1441 ✗ size_t mp_size = mps.size();
1442 ✗ WritePipe(return_pipe, &mp_size, sizeof(size_t));
1443 ✗ WritePipe(return_pipe, mps.c_str(), mp_size);
1444 ✗ quota_mgr->UnbindReturnPipe(return_pipe);
1445 ✗ continue;
1446 }
1447
1448 // Group hashes are returned immediately
1449
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2852527 times.
 2852527 if (command_type == kGetGroupHashes) {
1450 ✗ const int return_pipe = quota_mgr->BindReturnPipe(
1451 command_buffer[num_commands].return_pipe);
1452 ✗ if (return_pipe < 0)
1453 ✗ continue;
1454
1455 ✗ std::vector<shash::Short> gh = quota_mgr->CollectAllOpenHashes();
1456 ✗ std::string result;
1457 ✗ for (auto it = gh.begin(); it != gh.end(); ++it) {
1458 ✗ result += (*it).ToString() + "\n";
1459 }
1460 ✗ size_t result_size = result.size();
1461 ✗ WritePipe(return_pipe, &result_size, sizeof(size_t));
1462 ✗ WritePipe(return_pipe, result.c_str(), result_size);
1463 ✗ quota_mgr->UnbindReturnPipe(return_pipe);
1464 ✗ continue;
1465 }
1466
1467 // The cleanup rate is returned immediately
1468
2/2
✓ Branch 0 taken 76 times.
✓ Branch 1 taken 2852451 times.
 2852527 if (command_type == kCleanupRate) {
1469
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 76 const int return_pipe = quota_mgr->BindReturnPipe(
1470 command_buffer[num_commands].return_pipe);
1471
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 76 times.
 76 if (return_pipe < 0)
1472 ✗ continue;
1473 const uint64_t
1474 76 period_s = size; // use the size field to transmit the period
1475
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 76 uint64_t rate = quota_mgr->cleanup_recorder_.GetNoTicks(period_s);
1476
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 76 WritePipe(return_pipe, &rate, sizeof(rate));
1477
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 76 quota_mgr->UnbindReturnPipe(return_pipe);
1478 76 continue;
1479 76 }
1480
1481
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 2852432 times.
 2852451 if (command_type == kSetLimit) {
1482
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 const int return_pipe = quota_mgr->BindReturnPipe(
1483 command_buffer[num_commands].return_pipe);
1484
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19 times.
 19 if (return_pipe < 0)
1485 ✗ continue;
1486 19 quota_mgr->limit_ = size; // use the size field to transmit the size
1487 19 quota_mgr->cleanup_threshold_ = size / 2;
1488
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
1489 "Quota limit set to %lu / threshold %lu", quota_mgr->limit_,
1490 quota_mgr->cleanup_threshold_);
1491 19 bool ret = true;
1492
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 WritePipe(return_pipe, &ret, sizeof(ret));
1493
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 quota_mgr->UnbindReturnPipe(return_pipe);
1494 19 continue;
1495 19 }
1496
1497 // Reservations are handled immediately and "out of band"
1498
2/2
✓ Branch 0 taken 266 times.
✓ Branch 1 taken 2852166 times.
 2852432 if (command_type == kReserve) {
1499 266 bool success = true;
1500
1/2
✓ Branch 1 taken 266 times.
✗ Branch 2 not taken.
 266 const int return_pipe = quota_mgr->BindReturnPipe(
1501 command_buffer[num_commands].return_pipe);
1502
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 266 times.
 266 if (return_pipe < 0)
1503 ✗ continue;
1504
1505
1/2
✓ Branch 1 taken 266 times.
✗ Branch 2 not taken.
 266 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1506
1/2
✓ Branch 1 taken 266 times.
✗ Branch 2 not taken.
 266 const string hash_str(hash.ToString());
1507
1/2
✓ Branch 2 taken 266 times.
✗ Branch 3 not taken.
 266 LogCvmfs(kLogQuota, kLogDebug, "reserve %lu bytes for %s", size,
1508 hash_str.c_str());
1509
1510
1/2
✓ Branch 1 taken 266 times.
✗ Branch 2 not taken.
 266 if (quota_mgr->pinned_chunks_.find(hash)
1511
2/2
✓ Branch 2 taken 228 times.
✓ Branch 3 taken 38 times.
 532 == quota_mgr->pinned_chunks_.end()) {
1512
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 209 times.
 228 if ((quota_mgr->pinned_ + size) > quota_mgr->cleanup_threshold_) {
1513
1/2
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
 19 LogCvmfs(kLogQuota, kLogDebug,
1514 "failed to insert %s (pinned), no space", hash_str.c_str());
1515 19 success = false;
1516 } else {
1517
1/2
✓ Branch 1 taken 209 times.
✗ Branch 2 not taken.
 209 quota_mgr->pinned_chunks_[hash] = size;
1518 209 quota_mgr->pinned_ += size;
1519
1/2
✓ Branch 1 taken 209 times.
✗ Branch 2 not taken.
 209 quota_mgr->CheckHighPinWatermark();
1520 }
1521 }
1522
1523
1/2
✓ Branch 1 taken 266 times.
✗ Branch 2 not taken.
 266 WritePipe(return_pipe, &success, sizeof(success));
1524
1/2
✓ Branch 1 taken 266 times.
✗ Branch 2 not taken.
 266 quota_mgr->UnbindReturnPipe(return_pipe);
1525 266 continue;
1526 266 }
1527
1528 // Back channels are also handled out of band
1529
2/2
✓ Branch 0 taken 76 times.
✓ Branch 1 taken 2852090 times.
 2852166 if (command_type == kRegisterBackChannel) {
1530
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 76 const int return_pipe = quota_mgr->BindReturnPipe(
1531 command_buffer[num_commands].return_pipe);
1532
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 76 times.
 76 if (return_pipe < 0)
1533 ✗ continue;
1534
1535
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 76 quota_mgr->UnlinkReturnPipe(command_buffer[num_commands].return_pipe);
1536
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 76 Block2Nonblock(return_pipe); // back channels are opportunistic
1537
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 76 shash::Md5 hash;
1538 76 memcpy(hash.digest, command_buffer[num_commands].digest,
1539 76 shash::kDigestSizes[shash::kMd5]);
1540
1541 76 quota_mgr->LockBackChannels();
1542 const map<shash::Md5, int>::const_iterator
1543
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 76 iter = quota_mgr->back_channels_.find(hash);
1544
1/2
✗ Branch 3 not taken.
✓ Branch 4 taken 76 times.
 76 if (iter != quota_mgr->back_channels_.end()) {
1545 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
1546 ✗ "closing left-over back channel %s", hash.ToString().c_str());
1547 ✗ close(iter->second);
1548 }
1549
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 76 quota_mgr->back_channels_[hash] = return_pipe;
1550 76 quota_mgr->UnlockBackChannels();
1551
1552 76 char success = 'S';
1553
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 76 WritePipe(return_pipe, &success, sizeof(success));
1554
1/2
✓ Branch 2 taken 76 times.
✗ Branch 3 not taken.
 76 LogCvmfs(kLogQuota, kLogDebug, "register back channel %s on fd %d",
1555
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 152 hash.ToString().c_str(), return_pipe);
1556
1557 76 continue;
1558 76 }
1559
1560
2/2
✓ Branch 0 taken 38 times.
✓ Branch 1 taken 2852052 times.
 2852090 if (command_type == kUnregisterBackChannel) {
1561
1/2
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
 38 shash::Md5 hash;
1562 38 memcpy(hash.digest, command_buffer[num_commands].digest,
1563 38 shash::kDigestSizes[shash::kMd5]);
1564
1565 38 quota_mgr->LockBackChannels();
1566 const map<shash::Md5, int>::iterator iter = quota_mgr->back_channels_
1567
1/2
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
 38 .find(hash);
1568
1/2
✓ Branch 2 taken 38 times.
✗ Branch 3 not taken.
 38 if (iter != quota_mgr->back_channels_.end()) {
1569
1/2
✓ Branch 2 taken 38 times.
✗ Branch 3 not taken.
 38 LogCvmfs(kLogQuota, kLogDebug, "closing back channel %s",
1570
1/2
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
 76 hash.ToString().c_str());
1571
1/2
✓ Branch 2 taken 38 times.
✗ Branch 3 not taken.
 38 close(iter->second);
1572
1/2
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
 38 quota_mgr->back_channels_.erase(iter);
1573 } else {
1574 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
1575 ✗ "did not find back channel %s", hash.ToString().c_str());
1576 }
1577 38 quota_mgr->UnlockBackChannels();
1578
1579 38 continue;
1580 38 }
1581
1582 // Unpinnings are also handled immediately with respect to the pinned gauge
1583
2/2
✓ Branch 0 taken 38 times.
✓ Branch 1 taken 2852014 times.
 2852052 if (command_type == kUnpin) {
1584
1/2
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
 38 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1585
1/2
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
 38 const string hash_str(hash.ToString());
1586
1587 const map<shash::Any, uint64_t>::iterator iter = quota_mgr->pinned_chunks_
1588
1/2
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
 38 .find(hash);
1589
1/2
✓ Branch 2 taken 38 times.
✗ Branch 3 not taken.
 38 if (iter != quota_mgr->pinned_chunks_.end()) {
1590 38 quota_mgr->pinned_ -= iter->second;
1591
1/2
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
 38 quota_mgr->pinned_chunks_.erase(iter);
1592 // It can happen that files get pinned that were removed from the cache
1593 // (see cache.cc). We fix this at this point, where we remove such
1594 // entries from the cache database.
1595
2/4
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 38 times.
✗ Branch 5 not taken.
 76 if (!FileExists(quota_mgr->cache_dir_ + "/"
1596
4/6
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 38 times.
✗ Branch 5 not taken.
✓ Branch 9 taken 19 times.
✓ Branch 10 taken 19 times.
 114 + hash.MakePathWithoutSuffix())) {
1597
1/2
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
 19 LogCvmfs(kLogQuota, kLogDebug,
1598 "remove orphaned pinned hash %s from cache database",
1599 hash_str.c_str());
1600
1/2
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
 19 sqlite3_bind_text(quota_mgr->stmt_size_, 1, &hash_str[0],
1601 19 hash_str.length(), SQLITE_STATIC);
1602 int retval;
1603
2/4
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 19 times.
✗ Branch 4 not taken.
 19 if ((retval = sqlite3_step(quota_mgr->stmt_size_)) == SQLITE_ROW) {
1604
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 const uint64_t size = sqlite3_column_int64(quota_mgr->stmt_size_,
1605 19 0);
1606
1/2
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
 19 sqlite3_bind_text(quota_mgr->stmt_rm_, 1, &(hash_str[0]),
1607 19 hash_str.length(), SQLITE_STATIC);
1608
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 retval = sqlite3_step(quota_mgr->stmt_rm_);
1609
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 19 if ((retval == SQLITE_DONE) || (retval == SQLITE_OK)) {
1610 19 quota_mgr->gauge_ -= size;
1611 } else {
1612 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1613 "failed to delete %s (%d)", hash_str.c_str(), retval);
1614 }
1615
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 sqlite3_reset(quota_mgr->stmt_rm_);
1616 }
1617
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 sqlite3_reset(quota_mgr->stmt_size_);
1618 }
1619 } else {
1620 ✗ LogCvmfs(kLogQuota, kLogDebug, "this chunk was not pinned");
1621 }
1622 38 }
1623
1624 // Immediate commands trigger flushing of the buffer
1625 2852052 const bool immediate_command = (command_type == kCleanup)
1626
2/2
✓ Branch 0 taken 2851444 times.
✓ Branch 1 taken 437 times.
 2851881 || (command_type == kList)
1627
2/2
✓ Branch 0 taken 2851292 times.
✓ Branch 1 taken 152 times.
 2851444 || (command_type == kListPinned)
1628
2/2
✓ Branch 0 taken 2851235 times.
✓ Branch 1 taken 57 times.
 2851292 || (command_type == kListCatalogs)
1629
2/2
✓ Branch 0 taken 2851178 times.
✓ Branch 1 taken 57 times.
 2851235 || (command_type == kListVolatile)
1630
2/2
✓ Branch 0 taken 2851121 times.
✓ Branch 1 taken 57 times.
 2851178 || (command_type == kRemove)
1631
2/2
✓ Branch 0 taken 2850779 times.
✓ Branch 1 taken 342 times.
 2851121 || (command_type == kStatus)
1632
1/2
✓ Branch 0 taken 2850779 times.
✗ Branch 1 not taken.
 2850779 || (command_type == kLimits)
1633
3/4
✓ Branch 0 taken 2851881 times.
✓ Branch 1 taken 171 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2850779 times.
 5703933 || (command_type == kPid);
1634
2/2
✓ Branch 0 taken 2850779 times.
✓ Branch 1 taken 1273 times.
 2852052 if (!immediate_command)
1635 2850779 num_commands++;
1636
1637
4/4
✓ Branch 0 taken 2762999 times.
✓ Branch 1 taken 89053 times.
✓ Branch 2 taken 1273 times.
✓ Branch 3 taken 2761726 times.
 2852052 if ((num_commands == kCommandBufferSize) || immediate_command) {
1638
1/2
✓ Branch 1 taken 90326 times.
✗ Branch 2 not taken.
 90326 quota_mgr->ProcessCommandBunch(num_commands, command_buffer,
1639 description_buffer);
1640
2/2
✓ Branch 0 taken 89053 times.
✓ Branch 1 taken 1273 times.
 90326 if (!immediate_command)
1641 89053 num_commands = 0;
1642 }
1643
1644
2/2
✓ Branch 0 taken 1273 times.
✓ Branch 1 taken 2850779 times.
 2852052 if (immediate_command) {
1645 // Process cleanup, listings
1646
1/2
✓ Branch 1 taken 1273 times.
✗ Branch 2 not taken.
 1273 const int return_pipe = quota_mgr->BindReturnPipe(
1647 command_buffer[num_commands].return_pipe);
1648
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1273 times.
 1273 if (return_pipe < 0) {
1649 ✗ num_commands = 0;
1650 ✗ continue;
1651 }
1652
1653 int retval;
1654 1273 sqlite3_stmt *this_stmt_list = NULL;
1655
7/10
✓ Branch 0 taken 57 times.
✓ Branch 1 taken 171 times.
✓ Branch 2 taken 437 times.
✓ Branch 3 taken 152 times.
✓ Branch 4 taken 57 times.
✓ Branch 5 taken 57 times.
✓ Branch 6 taken 342 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
 1273 switch (command_type) {
1656 57 case kRemove: {
1657
1/2
✓ Branch 1 taken 57 times.
✗ Branch 2 not taken.
 57 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1658
1/2
✓ Branch 1 taken 57 times.
✗ Branch 2 not taken.
 57 const string hash_str = hash.ToString();
1659
1/2
✓ Branch 2 taken 57 times.
✗ Branch 3 not taken.
 57 LogCvmfs(kLogQuota, kLogDebug, "manually removing %s",
1660 hash_str.c_str());
1661 57 bool success = false;
1662
1663
1/2
✓ Branch 2 taken 57 times.
✗ Branch 3 not taken.
 57 sqlite3_bind_text(quota_mgr->stmt_size_, 1, &hash_str[0],
1664 57 hash_str.length(), SQLITE_STATIC);
1665 int retval;
1666
3/4
✓ Branch 1 taken 57 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 38 times.
✓ Branch 4 taken 19 times.
 57 if ((retval = sqlite3_step(quota_mgr->stmt_size_)) == SQLITE_ROW) {
1667
1/2
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
 38 const uint64_t size = sqlite3_column_int64(quota_mgr->stmt_size_,
1668 38 0);
1669
1/2
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
 38 const uint64_t is_pinned = sqlite3_column_int64(
1670 38 quota_mgr->stmt_size_, 1);
1671
1672
1/2
✓ Branch 2 taken 38 times.
✗ Branch 3 not taken.
 38 sqlite3_bind_text(quota_mgr->stmt_rm_, 1, &(hash_str[0]),
1673 38 hash_str.length(), SQLITE_STATIC);
1674
1/2
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
 38 retval = sqlite3_step(quota_mgr->stmt_rm_);
1675
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 38 if ((retval == SQLITE_DONE) || (retval == SQLITE_OK)) {
1676 38 success = true;
1677 38 quota_mgr->gauge_ -= size;
1678
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 19 times.
 38 if (is_pinned) {
1679
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 quota_mgr->pinned_chunks_.erase(hash);
1680 19 quota_mgr->pinned_ -= size;
1681 }
1682 } else {
1683 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1684 "failed to delete %s (%d)", hash_str.c_str(), retval);
1685 }
1686
1/2
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
 38 sqlite3_reset(quota_mgr->stmt_rm_);
1687 } else {
1688 // File does not exist
1689 19 success = true;
1690 }
1691
1/2
✓ Branch 1 taken 57 times.
✗ Branch 2 not taken.
 57 sqlite3_reset(quota_mgr->stmt_size_);
1692
1693
1/2
✓ Branch 1 taken 57 times.
✗ Branch 2 not taken.
 57 WritePipe(return_pipe, &success, sizeof(success));
1694 57 break;
1695 57 }
1696 171 case kCleanup:
1697
1/2
✓ Branch 1 taken 171 times.
✗ Branch 2 not taken.
 171 retval = quota_mgr->DoCleanup(size);
1698
1/2
✓ Branch 1 taken 171 times.
✗ Branch 2 not taken.
 171 WritePipe(return_pipe, &retval, sizeof(retval));
1699 171 break;
1700 437 case kList:
1701
1/2
✓ Branch 0 taken 437 times.
✗ Branch 1 not taken.
 437 if (!this_stmt_list)
1702 437 this_stmt_list = quota_mgr->stmt_list_;
1703 case kListPinned:
1704
2/2
✓ Branch 0 taken 152 times.
✓ Branch 1 taken 437 times.
 589 if (!this_stmt_list)
1705 152 this_stmt_list = quota_mgr->stmt_list_pinned_;
1706 case kListCatalogs:
1707
2/2
✓ Branch 0 taken 57 times.
✓ Branch 1 taken 589 times.
 646 if (!this_stmt_list)
1708 57 this_stmt_list = quota_mgr->stmt_list_catalogs_;
1709 case kListVolatile:
1710
2/2
✓ Branch 0 taken 57 times.
✓ Branch 1 taken 646 times.
 703 if (!this_stmt_list)
1711 57 this_stmt_list = quota_mgr->stmt_list_volatile_;
1712
1713 // Pipe back the list, one by one
1714 int length;
1715
3/4
✓ Branch 1 taken 1901482 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1900779 times.
✓ Branch 4 taken 703 times.
 1901482 while (sqlite3_step(this_stmt_list) == SQLITE_ROW) {
1716
1/2
✓ Branch 2 taken 1900779 times.
✗ Branch 3 not taken.
 1900779 string path = "(NULL)";
1717
2/4
✓ Branch 1 taken 1900779 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1900779 times.
✗ Branch 4 not taken.
 1900779 if (sqlite3_column_type(this_stmt_list, 0) != SQLITE_NULL) {
1718 3801558 path = string(reinterpret_cast<const char *>(
1719
2/4
✓ Branch 1 taken 1900779 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1900779 times.
✗ Branch 5 not taken.
 1900779 sqlite3_column_text(this_stmt_list, 0)));
1720 }
1721 1900779 length = path.length();
1722
1/2
✓ Branch 1 taken 1900779 times.
✗ Branch 2 not taken.
 1900779 WritePipe(return_pipe, &length, sizeof(length));
1723
1/2
✓ Branch 0 taken 1900779 times.
✗ Branch 1 not taken.
 1900779 if (length > 0)
1724
2/4
✓ Branch 1 taken 1900779 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1900779 times.
✗ Branch 5 not taken.
 1900779 WritePipe(return_pipe, &path[0], length);
1725 1900779 }
1726 703 length = -1;
1727
1/2
✓ Branch 1 taken 703 times.
✗ Branch 2 not taken.
 703 WritePipe(return_pipe, &length, sizeof(length));
1728
1/2
✓ Branch 1 taken 703 times.
✗ Branch 2 not taken.
 703 sqlite3_reset(this_stmt_list);
1729 703 break;
1730 342 case kStatus:
1731
1/2
✓ Branch 1 taken 342 times.
✗ Branch 2 not taken.
 342 WritePipe(return_pipe, &quota_mgr->gauge_, sizeof(quota_mgr->gauge_));
1732
1/2
✓ Branch 1 taken 342 times.
✗ Branch 2 not taken.
 342 WritePipe(return_pipe, &quota_mgr->pinned_,
1733 sizeof(quota_mgr->pinned_));
1734 342 break;
1735 ✗ case kLimits:
1736 ✗ WritePipe(return_pipe, &quota_mgr->limit_, sizeof(quota_mgr->limit_));
1737 ✗ WritePipe(return_pipe, &quota_mgr->cleanup_threshold_,
1738 sizeof(quota_mgr->cleanup_threshold_));
1739 ✗ break;
1740 ✗ case kPid: {
1741 ✗ pid_t pid = getpid();
1742 ✗ WritePipe(return_pipe, &pid, sizeof(pid));
1743 ✗ break;
1744 }
1745 ✗ default:
1746 ✗ PANIC(NULL); // other types are handled by the bunch processor
1747 }
1748
1/2
✓ Branch 1 taken 1273 times.
✗ Branch 2 not taken.
 1273 quota_mgr->UnbindReturnPipe(return_pipe);
1749 1273 num_commands = 0;
1750 }
1751 }
1752
1753
1/2
✓ Branch 1 taken 646 times.
✗ Branch 2 not taken.
 646 LogCvmfs(kLogQuota, kLogDebug, "stopping cache manager (%d)", errno);
1754
1/2
✓ Branch 1 taken 646 times.
✗ Branch 2 not taken.
 646 close(quota_mgr->pipe_lru_[0]);
1755
1/2
✓ Branch 1 taken 646 times.
✗ Branch 2 not taken.
 646 quota_mgr->ProcessCommandBunch(num_commands, command_buffer,
1756 description_buffer);
1757
1758 // Unpin
1759 646 command_buffer[0].command_type = kTouch;
1760 646 for (map<shash::Any, uint64_t>::const_iterator
1761 646 i = quota_mgr->pinned_chunks_.begin(),
1762 646 iEnd = quota_mgr->pinned_chunks_.end();
1763
2/2
✓ Branch 1 taken 190 times.
✓ Branch 2 taken 646 times.
 836 i != iEnd;
1764 190 ++i) {
1765
1/2
✓ Branch 2 taken 190 times.
✗ Branch 3 not taken.
 190 command_buffer[0].StoreHash(i->first);
1766
1/2
✓ Branch 1 taken 190 times.
✗ Branch 2 not taken.
 190 quota_mgr->ProcessCommandBunch(1, command_buffer, description_buffer);
1767 }
1768
1769 646 return NULL;
1770 }
1771
1772
1773 1786 void PosixQuotaManager::MakeReturnPipe(int pipe[2]) {
1774
2/2
✓ Branch 0 taken 1729 times.
✓ Branch 1 taken 57 times.
 1786 if (!shared_) {
1775 1729 MakePipe(pipe);
1776 1729 return;
1777 }
1778
1779 // Create FIFO in cache directory, store path name (number) in pipe write end
1780 57 int i = 0;
1781 int retval;
1782 do {
1783
2/4
✓ Branch 2 taken 76 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 76 times.
✗ Branch 6 not taken.
 76 retval = mkfifo((workspace_dir_ + "/pipe" + StringifyInt(i)).c_str(), 0600);
1784 76 pipe[1] = i;
1785 76 i++;
1786
3/4
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 57 times.
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
 76 } while ((retval == -1) && (errno == EEXIST));
1787
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 57 times.
 57 assert(retval == 0);
1788
1789 // Connect reader's end
1790
3/6
✓ Branch 2 taken 57 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 57 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 57 times.
✗ Branch 10 not taken.
 57 pipe[0] = open((workspace_dir_ + "/pipe" + StringifyInt(pipe[1])).c_str(),
1791 O_RDONLY | O_NONBLOCK);
1792
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 57 times.
 57 assert(pipe[0] >= 0);
1793 57 Nonblock2Block(pipe[0]);
1794 }
1795
1796
1797 2677 void PosixQuotaManager::ParseDirectories(const std::string cache_workspace,
1798 std::string *cache_dir,
1799 std::string *workspace_dir) {
1800
1/2
✓ Branch 1 taken 2677 times.
✗ Branch 2 not taken.
 2677 vector<string> dir_tokens(SplitString(cache_workspace, ':'));
1801
2/3
✓ Branch 1 taken 2639 times.
✓ Branch 2 taken 38 times.
✗ Branch 3 not taken.
 2677 switch (dir_tokens.size()) {
1802 2639 case 1:
1803
2/4
✓ Branch 2 taken 2639 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2639 times.
✗ Branch 6 not taken.
 2639 *cache_dir = *workspace_dir = dir_tokens[0];
1804 2639 break;
1805 38 case 2:
1806
1/2
✓ Branch 2 taken 38 times.
✗ Branch 3 not taken.
 38 *cache_dir = dir_tokens[0];
1807
1/2
✓ Branch 2 taken 38 times.
✗ Branch 3 not taken.
 38 *workspace_dir = dir_tokens[1];
1808 38 break;
1809 ✗ default:
1810 ✗ PANIC(NULL);
1811 }
1812 2677 }
1813
1814 19 void PosixQuotaManager::SkipEviction(const EvictCandidate &candidate) {
1815 19 bool res = true;
1816
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 std::string hash_str = candidate.hash.ToString();
1817
1/2
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
 19 LogCvmfs(kLogQuota, kLogDebug, "Exclude %s from eviction", hash_str.c_str());
1818
2/4
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 19 times.
✗ Branch 6 not taken.
 19 sqlite3_bind_text(stmt_block_, 1, &hash_str[0], hash_str.length(),
1819 SQLITE_STATIC);
1820
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 res = (sqlite3_step(stmt_block_) == SQLITE_DONE);
1821
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 sqlite3_reset(stmt_block_);
1822
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19 times.
 19 assert(res);
1823 19 }
1824
1825 /**
1826 * Immediately inserts a new pinned catalog. Does cache cleanup if necessary.
1827 *
1828 * \return True on success, false otherwise
1829 */
1830 1069 bool PosixQuotaManager::Pin(const shash::Any &hash,
1831 const uint64_t size,
1832 const string &description,
1833 const bool is_catalog) {
1834
3/4
✓ Branch 0 taken 38 times.
✓ Branch 1 taken 1031 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 38 times.
 1069 assert((size > 0) || !is_catalog);
1835
1836
1/2
✓ Branch 1 taken 1069 times.
✗ Branch 2 not taken.
 1069 const string hash_str = hash.ToString();
1837
1/2
✓ Branch 3 taken 1069 times.
✗ Branch 4 not taken.
 1069 LogCvmfs(kLogQuota, kLogDebug, "pin into lru %s, path %s", hash_str.c_str(),
1838 description.c_str());
1839
1840 // Has to run when not yet spawned (cvmfs initialization)
1841
2/2
✓ Branch 0 taken 803 times.
✓ Branch 1 taken 266 times.
 1069 if (!spawned_) {
1842 // Code duplication here
1843
3/4
✓ Branch 2 taken 803 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 467 times.
✓ Branch 6 taken 336 times.
 803 if (pinned_chunks_.find(hash) == pinned_chunks_.end()) {
1844
2/2
✓ Branch 0 taken 38 times.
✓ Branch 1 taken 429 times.
 467 if (pinned_ + size > cleanup_threshold_) {
1845
1/2
✓ Branch 2 taken 38 times.
✗ Branch 3 not taken.
 38 LogCvmfs(kLogQuota, kLogDebug, "failed to insert %s (pinned), no space",
1846 hash_str.c_str());
1847 38 return false;
1848 } else {
1849
1/2
✓ Branch 1 taken 429 times.
✗ Branch 2 not taken.
 429 pinned_chunks_[hash] = size;
1850 429 pinned_ += size;
1851
1/2
✓ Branch 1 taken 429 times.
✗ Branch 2 not taken.
 429 CheckHighPinWatermark();
1852 }
1853 }
1854
1/2
✓ Branch 1 taken 765 times.
✗ Branch 2 not taken.
 765 const bool exists = Contains(hash_str);
1855
4/4
✓ Branch 0 taken 429 times.
✓ Branch 1 taken 336 times.
✓ Branch 2 taken 19 times.
✓ Branch 3 taken 410 times.
 765 if (!exists && (gauge_ + size > limit_)) {
1856
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 LogCvmfs(kLogQuota, kLogDebug, "over limit, gauge %lu, file size %lu",
1857 gauge_, size);
1858
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 const int retval = DoCleanup(cleanup_threshold_);
1859
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19 times.
 19 assert(retval != 0);
1860 }
1861
1/2
✓ Branch 3 taken 765 times.
✗ Branch 4 not taken.
 765 sqlite3_bind_text(stmt_new_, 1, &hash_str[0], hash_str.length(),
1862 SQLITE_STATIC);
1863
1/2
✓ Branch 1 taken 765 times.
✗ Branch 2 not taken.
 765 sqlite3_bind_int64(stmt_new_, 2, size);
1864
1/2
✓ Branch 1 taken 765 times.
✗ Branch 2 not taken.
 765 sqlite3_bind_int64(stmt_new_, 3, seq_++);
1865
1/2
✓ Branch 3 taken 765 times.
✗ Branch 4 not taken.
 765 sqlite3_bind_text(stmt_new_, 4, &description[0], description.length(),
1866 SQLITE_STATIC);
1867
3/4
✓ Branch 0 taken 708 times.
✓ Branch 1 taken 57 times.
✓ Branch 3 taken 765 times.
✗ Branch 4 not taken.
 765 sqlite3_bind_int64(stmt_new_, 5, is_catalog ? kFileCatalog : kFileRegular);
1868
1/2
✓ Branch 1 taken 765 times.
✗ Branch 2 not taken.
 765 sqlite3_bind_int64(stmt_new_, 6, 1);
1869
1/2
✓ Branch 1 taken 765 times.
✗ Branch 2 not taken.
 765 const int retval = sqlite3_step(stmt_new_);
1870
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 765 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 765 assert((retval == SQLITE_DONE) || (retval == SQLITE_OK));
1871
1/2
✓ Branch 1 taken 765 times.
✗ Branch 2 not taken.
 765 sqlite3_reset(stmt_new_);
1872
2/2
✓ Branch 0 taken 429 times.
✓ Branch 1 taken 336 times.
 765 if (!exists)
1873 429 gauge_ += size;
1874 765 return true;
1875 }
1876
1877 int pipe_reserve[2];
1878
1/2
✓ Branch 1 taken 266 times.
✗ Branch 2 not taken.
 266 MakeReturnPipe(pipe_reserve);
1879
1880 266 LruCommand cmd;
1881 266 cmd.command_type = kReserve;
1882 266 cmd.SetSize(size);
1883
1/2
✓ Branch 1 taken 266 times.
✗ Branch 2 not taken.
 266 cmd.StoreHash(hash);
1884 266 cmd.return_pipe = pipe_reserve[1];
1885
1/2
✓ Branch 1 taken 266 times.
✗ Branch 2 not taken.
 266 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
1886 bool result;
1887
1/2
✓ Branch 1 taken 266 times.
✗ Branch 2 not taken.
 266 ManagedReadHalfPipe(pipe_reserve[0], &result, sizeof(result));
1888
1/2
✓ Branch 1 taken 266 times.
✗ Branch 2 not taken.
 266 CloseReturnPipe(pipe_reserve);
1889
1890
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 247 times.
 266 if (!result)
1891 19 return false;
1892
3/4
✓ Branch 0 taken 38 times.
✓ Branch 1 taken 209 times.
✓ Branch 3 taken 247 times.
✗ Branch 4 not taken.
 247 DoInsert(hash, size, description, is_catalog ? kPin : kPinRegular);
1893
1894 247 return true;
1895 1069 }
1896
1897
1898 2639 PosixQuotaManager::PosixQuotaManager(const uint64_t limit,
1899 const uint64_t cleanup_threshold,
1900 2639 const string &cache_workspace)
1901 2639 : shared_(false)
1902 2639 , spawned_(false)
1903 2639 , limit_(limit)
1904 2639 , cleanup_threshold_(cleanup_threshold)
1905 2639 , gauge_(0)
1906 2639 , pinned_(0)
1907 2639 , seq_(0)
1908 2639 , cache_dir_() // initialized in body
1909 2639 , workspace_dir_() // initialized in body
1910 2639 , fd_lock_cachedb_(-1)
1911 2639 , async_delete_(true)
1912 2639 , cachemgr_pid_(0)
1913 2639 , database_(NULL)
1914 2639 , stmt_touch_(NULL)
1915 2639 , stmt_unpin_(NULL)
1916 2639 , stmt_block_(NULL)
1917 2639 , stmt_unblock_(NULL)
1918 2639 , stmt_new_(NULL)
1919 2639 , stmt_lru_(NULL)
1920 2639 , stmt_size_(NULL)
1921 2639 , stmt_rm_(NULL)
1922 2639 , stmt_rm_batch_(NULL)
1923 2639 , stmt_list_(NULL)
1924 2639 , stmt_list_pinned_(NULL)
1925 2639 , stmt_list_catalogs_(NULL)
1926 2639 , stmt_list_volatile_(NULL)
1927 2639 , initialized_(false)
1928 5278 , cleanup_unused_first_(false) {
1929
2/4
✓ Branch 1 taken 2639 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2639 times.
✗ Branch 5 not taken.
 2639 ParseDirectories(cache_workspace, &cache_dir_, &workspace_dir_);
1930 2639 pipe_lru_[0] = pipe_lru_[1] = -1;
1931
1/2
✓ Branch 1 taken 2639 times.
✗ Branch 2 not taken.
 2639 cleanup_recorder_.AddRecorder(1, 90); // last 1.5 min with second resolution
1932 // last 1.5 h with minute resolution
1933
1/2
✓ Branch 1 taken 2639 times.
✗ Branch 2 not taken.
 2639 cleanup_recorder_.AddRecorder(60, 90 * 60);
1934 // last 18 hours with 20 min resolution
1935
1/2
✓ Branch 1 taken 2639 times.
✗ Branch 2 not taken.
 2639 cleanup_recorder_.AddRecorder(20 * 60, 60 * 60 * 18);
1936 // last 4 days with hour resolution
1937
1/2
✓ Branch 1 taken 2639 times.
✗ Branch 2 not taken.
 2639 cleanup_recorder_.AddRecorder(60 * 60, 60 * 60 * 24 * 4);
1938
1939 2639 lock_open_files_ = reinterpret_cast<pthread_mutex_t *>(
1940 2639 smalloc(sizeof(pthread_mutex_t)));
1941 2639 }
1942
1943
1944 10552 PosixQuotaManager::~PosixQuotaManager() {
1945 5276 free(lock_open_files_);
1946
1947
2/2
✓ Branch 0 taken 57 times.
✓ Branch 1 taken 2581 times.
 5276 if (!initialized_)
1948 114 return;
1949
1950
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2581 times.
 5162 if (shared_) {
1951 // Most of cleanup is done elsewhen by shared cache manager
1952 ✗ close(pipe_lru_[1]);
1953 ✗ return;
1954 }
1955
1956
2/2
✓ Branch 0 taken 646 times.
✓ Branch 1 taken 1935 times.
 5162 if (spawned_) {
1957 1292 char fin = 0;
1958 1292 WritePipe(pipe_lru_[1], &fin, 1);
1959 1292 close(pipe_lru_[1]);
1960 1292 pthread_join(thread_lru_, NULL);
1961 } else {
1962 3870 ClosePipe(pipe_lru_);
1963 }
1964
1965 5162 CloseDatabase();
1966
14/14
✓ Branch 1 taken 2581 times.
✓ Branch 2 taken 57 times.
✓ Branch 4 taken 2581 times.
✓ Branch 5 taken 57 times.
✓ Branch 7 taken 2581 times.
✓ Branch 8 taken 57 times.
✓ Branch 10 taken 2581 times.
✓ Branch 11 taken 57 times.
✓ Branch 13 taken 2581 times.
✓ Branch 14 taken 57 times.
✓ Branch 16 taken 2581 times.
✓ Branch 17 taken 57 times.
✓ Branch 19 taken 2581 times.
✓ Branch 20 taken 57 times.
 11236 }
1967
1968
1969 91162 void PosixQuotaManager::ProcessCommandBunch(const unsigned num,
1970 const LruCommand *commands,
1971 const char *descriptions) {
1972 91162 int retval = sqlite3_exec(database_, "BEGIN", NULL, NULL, NULL);
1973
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 91162 times.
 91162 assert(retval == SQLITE_OK);
1974
1975
2/2
✓ Branch 0 taken 2850969 times.
✓ Branch 1 taken 91162 times.
 2942131 for (unsigned i = 0; i < num; ++i) {
1976
1/2
✓ Branch 1 taken 2850969 times.
✗ Branch 2 not taken.
 2850969 const shash::Any hash = commands[i].RetrieveHash();
1977
1/2
✓ Branch 1 taken 2850969 times.
✗ Branch 2 not taken.
 2850969 const string hash_str = hash.ToString();
1978 2850969 const unsigned size = commands[i].GetSize();
1979
1/2
✓ Branch 1 taken 2850969 times.
✗ Branch 2 not taken.
 2850969 LogCvmfs(kLogQuota, kLogDebug, "processing %s (%d)", hash_str.c_str(),
1980 2850969 commands[i].command_type);
1981
1982 bool exists;
1983
3/4
✓ Branch 0 taken 950247 times.
✓ Branch 1 taken 38 times.
✓ Branch 2 taken 1900684 times.
✗ Branch 3 not taken.
 2850969 switch (commands[i].command_type) {
1984 950247 case kTouch:
1985
1/2
✓ Branch 1 taken 950247 times.
✗ Branch 2 not taken.
 950247 sqlite3_bind_int64(stmt_touch_, 1, seq_++);
1986
1/2
✓ Branch 3 taken 950247 times.
✗ Branch 4 not taken.
 950247 sqlite3_bind_text(stmt_touch_, 2, &hash_str[0], hash_str.length(),
1987 SQLITE_STATIC);
1988
1/2
✓ Branch 1 taken 950247 times.
✗ Branch 2 not taken.
 950247 retval = sqlite3_step(stmt_touch_);
1989
1/2
✓ Branch 1 taken 950247 times.
✗ Branch 2 not taken.
 950247 LogCvmfs(kLogQuota, kLogDebug, "touching %s (%ld): %d",
1990 950247 hash_str.c_str(), seq_ - 1, retval);
1991
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 950247 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 950247 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
1992 ✗ PANIC(kLogSyslogErr, "failed to update %s in cachedb, error %d",
1993 hash_str.c_str(), retval);
1994 }
1995
1/2
✓ Branch 1 taken 950247 times.
✗ Branch 2 not taken.
 950247 sqlite3_reset(stmt_touch_);
1996 950247 break;
1997 38 case kUnpin:
1998
1/2
✓ Branch 3 taken 38 times.
✗ Branch 4 not taken.
 38 sqlite3_bind_text(stmt_unpin_, 1, &hash_str[0], hash_str.length(),
1999 SQLITE_STATIC);
2000
1/2
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
 38 retval = sqlite3_step(stmt_unpin_);
2001
1/2
✓ Branch 2 taken 38 times.
✗ Branch 3 not taken.
 38 LogCvmfs(kLogQuota, kLogDebug, "unpinning %s: %d", hash_str.c_str(),
2002 retval);
2003
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 38 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
2004 ✗ PANIC(kLogSyslogErr, "failed to unpin %s in cachedb, error %d",
2005 hash_str.c_str(), retval);
2006 }
2007
1/2
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
 38 sqlite3_reset(stmt_unpin_);
2008 38 break;
2009 1900684 case kPin:
2010 case kPinRegular:
2011 case kInsert:
2012 case kInsertVolatile:
2013 // It could already be in, check
2014
1/2
✓ Branch 1 taken 1900684 times.
✗ Branch 2 not taken.
 1900684 exists = Contains(hash_str);
2015
2016 // Cleanup, move to trash and unlink
2017
3/4
✓ Branch 0 taken 1900589 times.
✓ Branch 1 taken 95 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1900589 times.
 1900684 if (!exists && (gauge_ + size > limit_)) {
2018 ✗ LogCvmfs(kLogQuota, kLogDebug, "over limit, gauge %lu, file size %u",
2019 gauge_, size);
2020 ✗ retval = DoCleanup(cleanup_threshold_);
2021 ✗ assert(retval != 0);
2022 }
2023
2024 // Insert or replace
2025
1/2
✓ Branch 3 taken 1900684 times.
✗ Branch 4 not taken.
 1900684 sqlite3_bind_text(stmt_new_, 1, &hash_str[0], hash_str.length(),
2026 SQLITE_STATIC);
2027
1/2
✓ Branch 1 taken 1900684 times.
✗ Branch 2 not taken.
 1900684 sqlite3_bind_int64(stmt_new_, 2, size);
2028
2/2
✓ Branch 0 taken 76 times.
✓ Branch 1 taken 1900608 times.
 1900684 if (commands[i].command_type == kInsertVolatile) {
2029
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 76 sqlite3_bind_int64(stmt_new_, 3, (seq_++) | kVolatileFlag);
2030 } else {
2031
1/2
✓ Branch 1 taken 1900608 times.
✗ Branch 2 not taken.
 1900608 sqlite3_bind_int64(stmt_new_, 3, seq_++);
2032 }
2033 1900684 sqlite3_bind_text(stmt_new_, 4, &descriptions[i * kMaxDescription],
2034
1/2
✓ Branch 1 taken 1900684 times.
✗ Branch 2 not taken.
 1900684 commands[i].desc_length, SQLITE_STATIC);
2035
1/2
✓ Branch 1 taken 1900684 times.
✗ Branch 2 not taken.
 1900684 sqlite3_bind_int64(
2036 stmt_new_, 5,
2037
2/2
✓ Branch 0 taken 38 times.
✓ Branch 1 taken 1900646 times.
 1900684 (commands[i].command_type == kPin) ? kFileCatalog : kFileRegular);
2038
1/2
✓ Branch 1 taken 1900684 times.
✗ Branch 2 not taken.
 1900684 sqlite3_bind_int64(stmt_new_, 6,
2039
2/2
✓ Branch 0 taken 1900646 times.
✓ Branch 1 taken 38 times.
 1900684 ((commands[i].command_type == kPin)
2040
2/2
✓ Branch 0 taken 209 times.
✓ Branch 1 taken 1900437 times.
 1900646 || (commands[i].command_type == kPinRegular))
2041 ? 1
2042 : 0);
2043
1/2
✓ Branch 1 taken 1900684 times.
✗ Branch 2 not taken.
 1900684 retval = sqlite3_step(stmt_new_);
2044
1/2
✓ Branch 1 taken 1900684 times.
✗ Branch 2 not taken.
 1900684 LogCvmfs(kLogQuota, kLogDebug, "insert or replace %s, method %d: %d",
2045 1900684 hash_str.c_str(), commands[i].command_type, retval);
2046
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1900684 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 1900684 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
2047 ✗ PANIC(kLogSyslogErr, "failed to insert %s in cachedb, error %d",
2048 hash_str.c_str(), retval);
2049 }
2050
1/2
✓ Branch 1 taken 1900684 times.
✗ Branch 2 not taken.
 1900684 sqlite3_reset(stmt_new_);
2051
2052
2/2
✓ Branch 0 taken 1900589 times.
✓ Branch 1 taken 95 times.
 1900684 if (!exists)
2053 1900589 gauge_ += size;
2054 1900684 break;
2055 ✗ default:
2056 // other types should have been taken care of by event loop
2057 ✗ PANIC(NULL);
2058 }
2059 2850969 }
2060
2061 91162 retval = sqlite3_exec(database_, "COMMIT", NULL, NULL, NULL);
2062
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 91162 times.
 91162 if (retval != SQLITE_OK) {
2063 ✗ PANIC(kLogSyslogErr, "failed to commit to cachedb, error %d", retval);
2064 }
2065 91162 }
2066
2067
2068 2601 bool PosixQuotaManager::RebuildDatabase() {
2069 2601 bool result = false;
2070 2601 string sql;
2071 2601 sqlite3_stmt *stmt_select = NULL;
2072 2601 sqlite3_stmt *stmt_insert = NULL;
2073 int sqlerr;
2074 2601 int seq = 0;
2075 char hex[4];
2076 struct stat info;
2077 platform_dirent64 *d;
2078 2601 DIR *dirp = NULL;
2079 2601 string path;
2080
2081
1/2
✓ Branch 1 taken 2601 times.
✗ Branch 2 not taken.
 2601 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug, "re-building cache database");
2082
2083 // Empty cache catalog and fscache
2084
1/2
✓ Branch 1 taken 2601 times.
✗ Branch 2 not taken.
 2601 sql = "DELETE FROM cache_catalog; DELETE FROM fscache;";
2085
1/2
✓ Branch 2 taken 2601 times.
✗ Branch 3 not taken.
 2601 sqlerr = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
2086
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2601 times.
 2601 if (sqlerr != SQLITE_OK) {
2087 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not clear cache database");
2088 ✗ goto build_return;
2089 }
2090
2091 2601 gauge_ = 0;
2092
2093 // Insert files from cache sub-directories 00 - ff
2094 // TODO(jblomer): fs_traversal
2095
1/2
✓ Branch 1 taken 2601 times.
✗ Branch 2 not taken.
 2601 sqlite3_prepare_v2(database_,
2096 "INSERT INTO fscache (sha1, size, actime) "
2097 "VALUES (:sha1, :s, :t);",
2098 -1, &stmt_insert, NULL);
2099
2100
2/2
✓ Branch 0 taken 651321 times.
✓ Branch 1 taken 2544 times.
 653865 for (int i = 0; i <= 0xff; i++) {
2101 651321 snprintf(hex, sizeof(hex), "%02x", i);
2102
3/6
✓ Branch 2 taken 651321 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 651321 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 651321 times.
✗ Branch 9 not taken.
 651321 path = cache_dir_ + "/" + string(hex);
2103
3/4
✓ Branch 2 taken 651321 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 57 times.
✓ Branch 5 taken 651264 times.
 651321 if ((dirp = opendir(path.c_str())) == NULL) {
2104
1/2
✓ Branch 2 taken 57 times.
✗ Branch 3 not taken.
 57 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
2105 "failed to open directory %s (tmpwatch interfering?)",
2106 path.c_str());
2107 57 goto build_return;
2108 }
2109
3/4
✓ Branch 1 taken 1953830 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1302566 times.
✓ Branch 4 taken 651264 times.
 1953830 while ((d = platform_readdir(dirp)) != NULL) {
2110
3/6
✓ Branch 2 taken 1302566 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1302566 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1302566 times.
✗ Branch 9 not taken.
 2605132 const string file_path = path + "/" + string(d->d_name);
2111
1/2
✓ Branch 2 taken 1302566 times.
✗ Branch 3 not taken.
 1302566 if (stat(file_path.c_str(), &info) == 0) {
2112
2/2
✓ Branch 0 taken 1302528 times.
✓ Branch 1 taken 38 times.
 1302566 if (!S_ISREG(info.st_mode))
2113 1302547 continue;
2114
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 19 times.
 38 if (info.st_size == 0) {
2115
1/2
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
 19 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug,
2116 "removing empty file %s during automatic cache db rebuild",
2117 file_path.c_str());
2118 19 unlink(file_path.c_str());
2119 19 continue;
2120 }
2121
2122
3/6
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 19 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 19 times.
✗ Branch 10 not taken.
 38 string hash = string(hex) + string(d->d_name);
2123
1/2
✓ Branch 3 taken 19 times.
✗ Branch 4 not taken.
 19 sqlite3_bind_text(stmt_insert, 1, hash.data(), hash.length(),
2124 SQLITE_STATIC);
2125
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 sqlite3_bind_int64(stmt_insert, 2, info.st_size);
2126
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 sqlite3_bind_int64(stmt_insert, 3, info.st_atime);
2127
2/4
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 19 times.
 19 if (sqlite3_step(stmt_insert) != SQLITE_DONE) {
2128 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not insert into temp table");
2129 ✗ goto build_return;
2130 }
2131
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 sqlite3_reset(stmt_insert);
2132
2133 19 gauge_ += info.st_size;
2134
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 } else {
2135 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not stat %s", file_path.c_str());
2136 }
2137
2/3
✓ Branch 1 taken 19 times.
✓ Branch 2 taken 1302547 times.
✗ Branch 3 not taken.
 1302566 }
2138
1/2
✓ Branch 1 taken 651264 times.
✗ Branch 2 not taken.
 651264 closedir(dirp);
2139 651264 dirp = NULL;
2140 }
2141
1/2
✓ Branch 1 taken 2544 times.
✗ Branch 2 not taken.
 2544 sqlite3_finalize(stmt_insert);
2142 2544 stmt_insert = NULL;
2143
2144 // Transfer from temp table in cache catalog
2145
1/2
✓ Branch 1 taken 2544 times.
✗ Branch 2 not taken.
 2544 sqlite3_prepare_v2(database_,
2146 "SELECT sha1, size FROM fscache ORDER BY actime;", -1,
2147 &stmt_select, NULL);
2148
1/2
✓ Branch 1 taken 2544 times.
✗ Branch 2 not taken.
 2544 sqlite3_prepare_v2(
2149 database_,
2150 "INSERT INTO cache_catalog (sha1, size, acseq, path, type, pinned) "
2151 "VALUES (:sha1, :s, :seq, 'unknown (automatic rebuild)', :t, 0);",
2152 -1, &stmt_insert, NULL);
2153
3/4
✓ Branch 1 taken 2563 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 19 times.
✓ Branch 4 taken 2544 times.
 2563 while (sqlite3_step(stmt_select) == SQLITE_ROW) {
2154 const string hash = string(
2155
2/4
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 19 times.
✗ Branch 6 not taken.
 19 reinterpret_cast<const char *>(sqlite3_column_text(stmt_select, 0)));
2156
1/2
✓ Branch 3 taken 19 times.
✗ Branch 4 not taken.
 19 sqlite3_bind_text(stmt_insert, 1, &hash[0], hash.length(), SQLITE_STATIC);
2157
2/4
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 19 times.
✗ Branch 5 not taken.
 19 sqlite3_bind_int64(stmt_insert, 2, sqlite3_column_int64(stmt_select, 1));
2158
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 sqlite3_bind_int64(stmt_insert, 3, seq++);
2159 // Might also be a catalog (information is lost)
2160
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 sqlite3_bind_int64(stmt_insert, 4, kFileRegular);
2161
2162
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 const int retval = sqlite3_step(stmt_insert);
2163
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19 times.
 19 if (retval != SQLITE_DONE) {
2164 // If the file system hosting the cache is full, we'll likely notice here
2165 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
2166 "could not insert into cache catalog (%d - %s)", retval,
2167 sqlite3_errstr(retval));
2168 ✗ goto build_return;
2169 }
2170
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 sqlite3_reset(stmt_insert);
2171
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 }
2172
2173 // Delete temporary table
2174
1/2
✓ Branch 1 taken 2544 times.
✗ Branch 2 not taken.
 2544 sql = "DELETE FROM fscache;";
2175
1/2
✓ Branch 2 taken 2544 times.
✗ Branch 3 not taken.
 2544 sqlerr = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
2176
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2544 times.
 2544 if (sqlerr != SQLITE_OK) {
2177 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not clear temporary table (%d)",
2178 sqlerr);
2179 ✗ goto build_return;
2180 }
2181
2182 2544 seq_ = seq;
2183 2544 result = true;
2184
1/2
✓ Branch 1 taken 2544 times.
✗ Branch 2 not taken.
 2544 LogCvmfs(kLogQuota, kLogDebug,
2185 "rebuilding finished, sequence %" PRIu64 ", gauge %" PRIu64, seq_,
2186 gauge_);
2187
2188 2601 build_return:
2189
1/2
✓ Branch 0 taken 2601 times.
✗ Branch 1 not taken.
 2601 if (stmt_insert)
2190
1/2
✓ Branch 1 taken 2601 times.
✗ Branch 2 not taken.
 2601 sqlite3_finalize(stmt_insert);
2191
2/2
✓ Branch 0 taken 2544 times.
✓ Branch 1 taken 57 times.
 2601 if (stmt_select)
2192
1/2
✓ Branch 1 taken 2544 times.
✗ Branch 2 not taken.
 2544 sqlite3_finalize(stmt_select);
2193
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2601 times.
 2601 if (dirp)
2194 ✗ closedir(dirp);
2195 2601 return result;
2196 2601 }
2197
2198
2199 /**
2200 * Register a channel that allows the cache manager to trigger action to its
2201 * clients. Currently used for releasing pinned catalogs.
2202 */
2203 76 void PosixQuotaManager::RegisterBackChannel(int back_channel[2],
2204 const string &channel_id) {
2205
1/2
✓ Branch 0 taken 76 times.
✗ Branch 1 not taken.
 76 if (protocol_revision_ >= 1) {
2206
1/2
✓ Branch 2 taken 76 times.
✗ Branch 3 not taken.
 76 shash::Md5 hash = shash::Md5(shash::AsciiPtr(channel_id));
2207
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 76 MakeReturnPipe(back_channel);
2208
2209 76 LruCommand cmd;
2210 76 cmd.command_type = kRegisterBackChannel;
2211 76 cmd.return_pipe = back_channel[1];
2212 // Not StoreHash(). This is an MD5 hash.
2213 76 memcpy(cmd.digest, hash.digest, hash.GetDigestSize());
2214
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 76 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2215
2216 char success;
2217
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 76 ManagedReadHalfPipe(back_channel[0], &success, sizeof(success));
2218 // At this point, the named FIFO is unlinked, so don't use CloseReturnPipe
2219
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 76 times.
 76 if (success != 'S') {
2220 ✗ PANIC(kLogDebug | kLogSyslogErr,
2221 "failed to register quota back channel (%c)", success);
2222 }
2223 } else {
2224 // Dummy pipe to return valid file descriptors
2225 ✗ MakePipe(back_channel);
2226 }
2227 76 }
2228
2229
2230 /**
2231 * Removes a chunk from cache, if it exists.
2232 */
2233 57 void PosixQuotaManager::Remove(const shash::Any &hash) {
2234
1/2
✓ Branch 1 taken 57 times.
✗ Branch 2 not taken.
 57 const string hash_str = hash.ToString();
2235
2236 int pipe_remove[2];
2237
1/2
✓ Branch 1 taken 57 times.
✗ Branch 2 not taken.
 57 MakeReturnPipe(pipe_remove);
2238
2239 57 LruCommand cmd;
2240 57 cmd.command_type = kRemove;
2241 57 cmd.return_pipe = pipe_remove[1];
2242
1/2
✓ Branch 1 taken 57 times.
✗ Branch 2 not taken.
 57 cmd.StoreHash(hash);
2243
1/2
✓ Branch 1 taken 57 times.
✗ Branch 2 not taken.
 57 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2244
2245 bool success;
2246
1/2
✓ Branch 1 taken 57 times.
✗ Branch 2 not taken.
 57 ManagedReadHalfPipe(pipe_remove[0], &success, sizeof(success));
2247
1/2
✓ Branch 1 taken 57 times.
✗ Branch 2 not taken.
 57 CloseReturnPipe(pipe_remove);
2248
2249
3/6
✓ Branch 1 taken 57 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 57 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 57 times.
✗ Branch 8 not taken.
 57 unlink((cache_dir_ + "/" + hash.MakePathWithoutSuffix()).c_str());
2250 57 }
2251
2252
2253 684 void PosixQuotaManager::Spawn() {
2254
2/2
✓ Branch 0 taken 38 times.
✓ Branch 1 taken 646 times.
 684 if (spawned_)
2255 38 return;
2256
2257 646 if (pthread_create(&thread_lru_, NULL, MainCommandServer,
2258 static_cast<void *>(this))
2259
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 646 times.
 646 != 0) {
2260 ✗ PANIC(kLogDebug, "could not create lru thread");
2261 }
2262
2263 646 spawned_ = true;
2264 }
2265
2266
2267 /**
2268 * Updates the sequence number of the file specified by the hash.
2269 */
2270 950485 void PosixQuotaManager::Touch(const shash::Any &hash) {
2271 950485 LruCommand cmd;
2272 950485 cmd.command_type = kTouch;
2273
1/2
✓ Branch 1 taken 950485 times.
✗ Branch 2 not taken.
 950485 cmd.StoreHash(hash);
2274
1/2
✓ Branch 1 taken 950485 times.
✗ Branch 2 not taken.
 950485 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2275 950485 }
2276
2277
2278 1672 void PosixQuotaManager::UnbindReturnPipe(int pipe_wronly) {
2279
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 1653 times.
 1672 if (shared_)
2280 19 close(pipe_wronly);
2281 1672 }
2282
2283
2284 133 void PosixQuotaManager::UnlinkReturnPipe(int pipe_wronly) {
2285
2/2
✓ Branch 0 taken 57 times.
✓ Branch 1 taken 76 times.
 133 if (shared_)
2286
2/4
✓ Branch 2 taken 57 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 57 times.
✗ Branch 6 not taken.
 57 unlink((workspace_dir_ + "/pipe" + StringifyInt(pipe_wronly)).c_str());
2287 133 }
2288
2289
2290 613 void PosixQuotaManager::Unpin(const shash::Any &hash) {
2291
2/4
✓ Branch 1 taken 613 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 613 times.
✗ Branch 6 not taken.
 613 LogCvmfs(kLogQuota, kLogDebug, "Unpin %s", hash.ToString().c_str());
2292
2293 613 LruCommand cmd;
2294 613 cmd.command_type = kUnpin;
2295
1/2
✓ Branch 1 taken 613 times.
✗ Branch 2 not taken.
 613 cmd.StoreHash(hash);
2296
1/2
✓ Branch 1 taken 613 times.
✗ Branch 2 not taken.
 613 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2297 613 }
2298
2299
2300 38 void PosixQuotaManager::UnregisterBackChannel(int back_channel[2],
2301 const string &channel_id) {
2302
1/2
✓ Branch 0 taken 38 times.
✗ Branch 1 not taken.
 38 if (protocol_revision_ >= 1) {
2303
1/2
✓ Branch 2 taken 38 times.
✗ Branch 3 not taken.
 38 shash::Md5 hash = shash::Md5(shash::AsciiPtr(channel_id));
2304
2305 38 LruCommand cmd;
2306 38 cmd.command_type = kUnregisterBackChannel;
2307 // Not StoreHash(). This is an MD5 hash.
2308 38 memcpy(cmd.digest, hash.digest, hash.GetDigestSize());
2309
1/2
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
 38 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2310
2311 // Writer's end will be closed by cache manager, FIFO is already unlinked
2312
1/2
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
 38 close(back_channel[0]);
2313 } else {
2314 ✗ ClosePipe(back_channel);
2315 }
2316 38 }
2317
2318 1902508 void PosixQuotaManager::ManagedReadHalfPipe(int fd, void *buf, size_t nbyte) {
2319
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1902508 times.
 1902508 const unsigned timeout_ms = cachemgr_pid_ ? 1000 : 0;
2320 1902508 bool result = false;
2321 do {
2322 1902508 result = ReadHalfPipe(fd, buf, nbyte, timeout_ms);
2323 // try only as long as the cachemgr is still alive
2324
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 1902508 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 1902508 times.
 1902508 } while (!result && getpgid(cachemgr_pid_) >= 0);
2325
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1902508 times.
 1902508 if (!result) {
2326 ✗ PANIC(kLogStderr,
2327 "Error: quota manager could not read from cachemanager pipe");
2328 }
2329 1902508 }
2330
2331 ✗ void *PosixQuotaManager::CollectMountpointsHashes(void *data) {
2332 #ifndef __APPLE__
2333 ✗ pthread_setname_np(pthread_self(), "hash_collector");
2334 ✗ auto *handler = static_cast<CollectorHandler *>(data);
2335
2336 ✗ const std::string mountpoint = handler->mp[handler->i];
2337 ✗ ssize_t n = getxattr(mountpoint.c_str(), "user.list_open_hashes", nullptr, 0);
2338 ✗ if (n < 0) {
2339 ✗ pthread_exit(nullptr);
2340 }
2341 ✗ std::vector<char> buf((size_t)n);
2342 ✗ n = getxattr(mountpoint.c_str(), "user.list_open_hashes", buf.data(),
2343 buf.size());
2344 ✗ if (n < 0) {
2345 ✗ pthread_exit(nullptr);
2346 }
2347
2348 ✗ std::vector<std::string> hash_strs;
2349 ✗ std::string hash_str;
2350 ✗ for (const char c : buf) {
2351 ✗ if (c == '\n') {
2352 ✗ hash_strs.push_back(hash_str);
2353 ✗ hash_str.clear();
2354 } else {
2355 ✗ hash_str += c;
2356 }
2357 }
2358 ✗ const MutexLockGuard lock_guard(handler->l);
2359 ✗ for (auto hash_str : hash_strs) {
2360 ✗ handler->of.push_back(
2361 ✗ shash::Short(shash::MkFromHexPtr(shash::HexPtr(hash_str))));
2362 }
2363 #endif
2364 ✗ pthread_exit(nullptr);
2365 }
2366
2367 ✗ std::vector<shash::Short> PosixQuotaManager::CollectAllOpenHashes() {
2368 ✗ std::vector<CollectorHandler *> handlers;
2369 ✗ std::vector<pthread_t *> threads;
2370 ✗ open_files_.clear();
2371 #ifndef __APPLE__
2372 ✗ auto &&a_after_b = [](const struct timespec a, const struct timespec b) {
2373 ✗ return (a.tv_sec > b.tv_sec) ? true : false;
2374 };
2375
2376 ✗ for (size_t i = 0; i < mountpoints_.size(); ++i) {
2377 ✗ handlers.push_back(
2378 ✗ new CollectorHandler{open_files_, mountpoints_, lock_open_files_, i});
2379 ✗ threads.push_back(new pthread_t);
2380 }
2381
2382 ✗ const int retval = pthread_mutex_init(lock_open_files_, NULL);
2383 ✗ assert(retval == 0);
2384
2385 ✗ for (size_t i = 0; i < mountpoints_.size(); ++i) {
2386 ✗ pthread_create(threads[i], nullptr, CollectMountpointsHashes, handlers[i]);
2387 }
2388
2389 ✗ std::vector<bool> joined(handlers.size(), false);
2390 struct timespec reference, current;
2391 ✗ clock_gettime(CLOCK_REALTIME, &reference);
2392 ✗ clock_gettime(CLOCK_REALTIME, &current);
2393 ✗ reference.tv_sec += 10; // Give 10sec for hash collection
2394 ✗ size_t i = 0;
2395 ✗ while (
2396 ✗ (not std::all_of(joined.begin(), joined.end(), [](bool b) { return b; }))
2397 ✗ and a_after_b(reference, current)) {
2398 // as long as there are still threads that haven't joined yet
2399 // and for 10 seconds
2400 ✗ if (not joined[i]) {
2401 ✗ const int s = pthread_tryjoin_np(*threads[i], NULL);
2402 ✗ if (s == 0) {
2403 ✗ joined[i] = true;
2404 }
2405 }
2406 ✗ ++i;
2407 ✗ i = i % handlers.size();
2408 ✗ clock_gettime(CLOCK_REALTIME, &current);
2409 }
2410
2411 ✗ for (size_t i = 0; i < handlers.size(); ++i) {
2412 ✗ delete handlers[i];
2413 }
2414
2415 ✗ pthread_mutex_destroy(lock_open_files_);
2416 #endif
2417 ✗ return open_files_;
2418 }
2419