GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/quota_posix.cc
Date: 2026-04-05 02:35:23
Exec Total Coverage
Lines: 954 1399 68.2%
Branches: 750 1954 38.4%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * This module implements a "managed local cache".
5 * This way, we are able to track access times of files in the cache
6 * and remove files based on least recently used strategy.
7 *
8 * We setup another SQLite catalog, a "cache catalog", that helps us
9 * in the bookkeeping of files, file sizes and access times.
10 *
11 * We might choose to not manage the local cache. This is indicated
12 * by limit == 0 and everything succeeds in that case.
13 */
14
15 #define __STDC_LIMIT_MACROS
16 #define __STDC_FORMAT_MACROS
17
18
19 #include "quota_posix.h"
20
21 #include <dirent.h>
22 #include <errno.h>
23 #include <fcntl.h>
24 #include <inttypes.h>
25 #include <pthread.h>
26 #include <signal.h>
27 #include <stdint.h>
28 #include <sys/dir.h>
29 #include <sys/stat.h>
30 #include <sys/xattr.h>
31
32 #ifndef __APPLE__
33 #include <sys/statfs.h>
34 #endif
35 #include <sys/statvfs.h>
36 #include <sys/types.h>
37 #include <sys/wait.h>
38 #include <unistd.h>
39
40 #include <algorithm>
41 #include <cassert>
42 #include <cstdio>
43 #include <cstdlib>
44 #include <cstring>
45 #include <limits>
46 #include <map>
47 #include <set>
48 #include <string>
49 #include <vector>
50
51 #include "crypto/hash.h"
52 #include "duplex_sqlite3.h"
53 #include "monitor.h"
54 #include "statistics.h"
55 #include "util/capabilities.h"
56 #include "util/concurrency.h"
57 #include "util/exception.h"
58 #include "util/logging.h"
59 #include "util/platform.h"
60 #include "util/pointer.h"
61 #include "util/posix.h"
62 #include "util/smalloc.h"
63 #include "util/string.h"
64
65 using namespace std; // NOLINT
66
67
68 1012 int PosixQuotaManager::BindReturnPipe(int pipe_wronly) {
69
2/2
✓ Branch 0 taken 990 times.
✓ Branch 1 taken 22 times.
 1012 if (!shared_)
70 990 return pipe_wronly;
71
72 // Connect writer's end
73
1/2
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
 22 const int result = open(
74
2/4
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 22 times.
✗ Branch 6 not taken.
 44 (workspace_dir_ + "/pipe" + StringifyInt(pipe_wronly)).c_str(),
75 O_WRONLY | O_NONBLOCK);
76
2/2
✓ Branch 0 taken 11 times.
✓ Branch 1 taken 11 times.
 22 if (result >= 0) {
77 11 Nonblock2Block(result);
78 } else {
79 11 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
80 11 "failed to bind return pipe (%d)", errno);
81 }
82 22 return result;
83 }
84
85
86 539 void PosixQuotaManager::CheckHighPinWatermark() {
87 539 const uint64_t watermark = kHighPinWatermark * cleanup_threshold_ / 100;
88
3/4
✓ Branch 0 taken 539 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 33 times.
✓ Branch 3 taken 506 times.
 539 if ((cleanup_threshold_ > 0) && (pinned_ > watermark)) {
89 33 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
90 "high watermark of pinned files (%" PRIu64 "M > %" PRIu64 "M)",
91 33 pinned_ / (1024 * 1024), watermark / (1024 * 1024));
92
2/4
✓ Branch 2 taken 33 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 33 times.
✗ Branch 6 not taken.
 33 BroadcastBackchannels("R"); // clients: please release pinned catalogs
93 }
94 539 }
95
96
97 ✗ void PosixQuotaManager::CleanupPipes() {
98 ✗ DIR *dirp = opendir(workspace_dir_.c_str());
99 ✗ assert(dirp != NULL);
100
101 platform_dirent64 *dent;
102 ✗ bool found_leftovers = false;
103 ✗ while ((dent = platform_readdir(dirp)) != NULL) {
104 ✗ const string name = dent->d_name;
105 ✗ const string path = workspace_dir_ + "/" + name;
106 platform_stat64 info;
107 ✗ const int retval = platform_stat(path.c_str(), &info);
108 ✗ if (retval != 0)
109 ✗ continue;
110 ✗ if (S_ISFIFO(info.st_mode) && (name.substr(0, 4) == "pipe")) {
111 ✗ if (!found_leftovers) {
112 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogWarn,
113 "removing left-over FIFOs from cache directory");
114 }
115 ✗ found_leftovers = true;
116 ✗ unlink(path.c_str());
117 }
118 }
119 ✗ closedir(dirp);
120 }
121
122
123 /**
124 * Cleans up in data cache, until cache size is below leave_size.
125 * The actual unlinking is done in a separate process (fork).
126 *
127 * \return True on success, false otherwise
128 */
129 99 bool PosixQuotaManager::Cleanup(const uint64_t leave_size) {
130
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 99 times.
 99 if (!spawned_)
131 ✗ return DoCleanup(leave_size);
132
133 bool result;
134 int pipe_cleanup[2];
135
1/2
✓ Branch 1 taken 99 times.
✗ Branch 2 not taken.
 99 MakeReturnPipe(pipe_cleanup);
136
137 99 LruCommand cmd;
138 99 cmd.command_type = kCleanup;
139 99 cmd.size = leave_size;
140 99 cmd.return_pipe = pipe_cleanup[1];
141
142
1/2
✓ Branch 1 taken 99 times.
✗ Branch 2 not taken.
 99 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
143
1/2
✓ Branch 1 taken 99 times.
✗ Branch 2 not taken.
 99 ManagedReadHalfPipe(pipe_cleanup[0], &result, sizeof(result));
144
1/2
✓ Branch 1 taken 99 times.
✗ Branch 2 not taken.
 99 CloseReturnPipe(pipe_cleanup);
145
146 99 return result;
147 }
148
149
150 1818 void PosixQuotaManager::CloseDatabase() {
151
1/2
✓ Branch 0 taken 1818 times.
✗ Branch 1 not taken.
 1818 if (stmt_list_catalogs_)
152 1818 sqlite3_finalize(stmt_list_catalogs_);
153
1/2
✓ Branch 0 taken 1818 times.
✗ Branch 1 not taken.
 1818 if (stmt_list_pinned_)
154 1818 sqlite3_finalize(stmt_list_pinned_);
155
1/2
✓ Branch 0 taken 1818 times.
✗ Branch 1 not taken.
 1818 if (stmt_list_volatile_)
156 1818 sqlite3_finalize(stmt_list_volatile_);
157
1/2
✓ Branch 0 taken 1818 times.
✗ Branch 1 not taken.
 1818 if (stmt_list_)
158 1818 sqlite3_finalize(stmt_list_);
159
1/2
✓ Branch 0 taken 1818 times.
✗ Branch 1 not taken.
 1818 if (stmt_lru_)
160 1818 sqlite3_finalize(stmt_lru_);
161
1/2
✓ Branch 0 taken 1818 times.
✗ Branch 1 not taken.
 1818 if (stmt_rm_)
162 1818 sqlite3_finalize(stmt_rm_);
163
1/2
✓ Branch 0 taken 1818 times.
✗ Branch 1 not taken.
 1818 if (stmt_rm_batch_)
164 1818 sqlite3_finalize(stmt_rm_batch_);
165
1/2
✓ Branch 0 taken 1818 times.
✗ Branch 1 not taken.
 1818 if (stmt_size_)
166 1818 sqlite3_finalize(stmt_size_);
167
1/2
✓ Branch 0 taken 1818 times.
✗ Branch 1 not taken.
 1818 if (stmt_touch_)
168 1818 sqlite3_finalize(stmt_touch_);
169
1/2
✓ Branch 0 taken 1818 times.
✗ Branch 1 not taken.
 1818 if (stmt_unpin_)
170 1818 sqlite3_finalize(stmt_unpin_);
171
1/2
✓ Branch 0 taken 1818 times.
✗ Branch 1 not taken.
 1818 if (stmt_block_)
172 1818 sqlite3_finalize(stmt_block_);
173
1/2
✓ Branch 0 taken 1818 times.
✗ Branch 1 not taken.
 1818 if (stmt_unblock_)
174 1818 sqlite3_finalize(stmt_unblock_);
175
1/2
✓ Branch 0 taken 1818 times.
✗ Branch 1 not taken.
 1818 if (stmt_new_)
176 1818 sqlite3_finalize(stmt_new_);
177
1/2
✓ Branch 0 taken 1818 times.
✗ Branch 1 not taken.
 1818 if (database_)
178 1818 sqlite3_close(database_);
179 1818 UnlockFile(fd_lock_cachedb_);
180
181 1818 stmt_list_catalogs_ = NULL;
182 1818 stmt_list_pinned_ = NULL;
183 1818 stmt_list_volatile_ = NULL;
184 1818 stmt_list_ = NULL;
185 1818 stmt_rm_ = NULL;
186 1818 stmt_rm_batch_ = NULL;
187 1818 stmt_size_ = NULL;
188 1818 stmt_touch_ = NULL;
189 1818 stmt_unpin_ = NULL;
190 1818 stmt_block_ = NULL;
191 1818 stmt_unblock_ = NULL;
192 1818 stmt_new_ = NULL;
193 1818 database_ = NULL;
194
195 1818 pinned_chunks_.clear();
196 1818 }
197
198
199 957 void PosixQuotaManager::CloseReturnPipe(int pipe[2]) {
200
2/2
✓ Branch 0 taken 22 times.
✓ Branch 1 taken 935 times.
 957 if (shared_) {
201 22 close(pipe[0]);
202 22 UnlinkReturnPipe(pipe[1]);
203 } else {
204 935 ClosePipe(pipe);
205 }
206 957 }
207
208
209 1101138 bool PosixQuotaManager::Contains(const string &hash_str) {
210 1101138 bool result = false;
211
212 1101138 sqlite3_bind_text(stmt_size_, 1, &hash_str[0], hash_str.length(),
213 SQLITE_STATIC);
214
2/2
✓ Branch 1 taken 390 times.
✓ Branch 2 taken 1100748 times.
 1101138 if (sqlite3_step(stmt_size_) == SQLITE_ROW)
215 390 result = true;
216 1101138 sqlite3_reset(stmt_size_);
217 1101138 LogCvmfs(kLogQuota, kLogDebug, "contains %s returns %d", hash_str.c_str(),
218 result);
219
220 1101138 return result;
221 }
222
223
224 1797 void PosixQuotaManager::CheckFreeSpace() {
225
3/4
✓ Branch 0 taken 1797 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 11 times.
✓ Branch 3 taken 1786 times.
 1797 if ((limit_ == 0) || (gauge_ >= limit_))
226 11 return;
227
228 struct statvfs vfs_info;
229
1/2
✓ Branch 1 taken 1786 times.
✗ Branch 2 not taken.
 1786 const int retval = statvfs((cache_dir_ + "/cachedb").c_str(), &vfs_info);
230
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1786 times.
 1786 if (retval != 0) {
231 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
232 "failed to query %s for free space (%d)", cache_dir_.c_str(),
233 ✗ errno);
234 ✗ return;
235 }
236 1786 const int64_t free_space_byte = vfs_info.f_bavail * vfs_info.f_bsize;
237
1/2
✓ Branch 1 taken 1786 times.
✗ Branch 2 not taken.
 1786 LogCvmfs(kLogQuota, kLogDebug, "free space: %" PRId64 " MB",
238 free_space_byte / (1024 * 1024));
239
240 1786 const int64_t required_byte = limit_ - gauge_;
241
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1786 times.
 1786 if (free_space_byte < required_byte) {
242 ✗ LogCvmfs(kLogQuota, kLogSyslogWarn,
243 "too little free space on the file system hosting the cache,"
244 " %" PRId64 " MB available",
245 free_space_byte / (1024 * 1024));
246 }
247 }
248
249
250 1830 PosixQuotaManager *PosixQuotaManager::Create(const string &cache_workspace,
251 const uint64_t limit,
252 const uint64_t cleanup_threshold,
253 const bool rebuild_database) {
254
2/2
✓ Branch 0 taken 22 times.
✓ Branch 1 taken 1808 times.
 1830 if (cleanup_threshold >= limit) {
255 22 LogCvmfs(kLogQuota, kLogDebug,
256 "invalid parameters: limit %" PRIu64 ", "
257 "cleanup_threshold %" PRIu64,
258 limit, cleanup_threshold);
259 22 return NULL;
260 }
261
262 PosixQuotaManager *quota_manager = new PosixQuotaManager(
263
1/2
✓ Branch 2 taken 1808 times.
✗ Branch 3 not taken.
 1808 limit, cleanup_threshold, cache_workspace);
264
265 // Initialize cache catalog
266
2/2
✓ Branch 1 taken 11 times.
✓ Branch 2 taken 1797 times.
 1808 if (!quota_manager->InitDatabase(rebuild_database)) {
267
1/2
✓ Branch 0 taken 11 times.
✗ Branch 1 not taken.
 11 delete quota_manager;
268 11 return NULL;
269 }
270 1797 quota_manager->CheckFreeSpace();
271 1797 MakePipe(quota_manager->pipe_lru_);
272
273 1797 quota_manager->protocol_revision_ = kProtocolRevision;
274 1797 quota_manager->initialized_ = true;
275 1797 return quota_manager;
276 }
277
278
279 /**
280 * Connects to a running shared local quota manager. Creates one if necessary.
281 */
282 22 PosixQuotaManager *PosixQuotaManager::CreateShared(
283 const std::string &exe_path,
284 const std::string &cache_workspace,
285 const uint64_t limit,
286 const uint64_t cleanup_threshold,
287 bool foreground) {
288 22 string cache_dir;
289 22 string workspace_dir;
290
2/4
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 22 times.
✗ Branch 5 not taken.
 22 ParseDirectories(cache_workspace, &cache_dir, &workspace_dir);
291
292 pid_t new_cachemgr_pid;
293
294 // Create lock file: only one fuse client at a time
295
2/4
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 22 times.
✗ Branch 5 not taken.
 22 const int fd_lockfile = LockFile(workspace_dir + "/lock_cachemgr");
296
2/2
✓ Branch 0 taken 11 times.
✓ Branch 1 taken 11 times.
 22 if (fd_lockfile < 0) {
297
1/2
✓ Branch 2 taken 11 times.
✗ Branch 3 not taken.
 11 LogCvmfs(kLogQuota, kLogDebug, "could not open lock file %s (%d)",
298
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 22 (workspace_dir + "/lock_cachemgr").c_str(), errno);
299 11 return NULL;
300 }
301
302 PosixQuotaManager *quota_mgr = new PosixQuotaManager(limit, cleanup_threshold,
303
2/4
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 11 times.
✗ Branch 5 not taken.
 11 cache_workspace);
304 11 quota_mgr->shared_ = true;
305 11 quota_mgr->spawned_ = true;
306
307 // Try to connect to pipe
308
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 const string fifo_path = workspace_dir + "/cachemgr";
309
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 LogCvmfs(kLogQuota, kLogDebug, "trying to connect to existing pipe");
310
1/2
✓ Branch 2 taken 11 times.
✗ Branch 3 not taken.
 11 quota_mgr->pipe_lru_[1] = open(fifo_path.c_str(), O_WRONLY | O_NONBLOCK);
311
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11 times.
 11 if (quota_mgr->pipe_lru_[1] >= 0) {
312 ✗ const int fd_lockfile_rw = open((workspace_dir + "/lock_cachemgr").c_str(),
313 O_RDWR, 0600);
314 ✗ unsigned lockfile_magicnumber = 0;
315 ✗ const ssize_t result_mn = SafeRead(fd_lockfile_rw, &lockfile_magicnumber,
316 sizeof(lockfile_magicnumber));
317 ✗ const ssize_t result = SafeRead(fd_lockfile_rw, &new_cachemgr_pid,
318 sizeof(new_cachemgr_pid));
319 ✗ close(fd_lockfile_rw);
320
321 ✗ if ((lockfile_magicnumber != kLockFileMagicNumber) || (result < 0)
322 ✗ || (result_mn < 0)
323 ✗ || (static_cast<size_t>(result) < sizeof(new_cachemgr_pid))) {
324 ✗ if (result != 0) {
325 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
326 "could not read cache manager pid from lockfile");
327 ✗ UnlockFile(fd_lockfile);
328 ✗ delete quota_mgr;
329 ✗ return NULL;
330 } else {
331 // support reload from old versions of the cache manager
332 // lock file is empty in this case, try a plain ReadHalfPipe to get pid
333 ✗ quota_mgr->SetCacheMgrPid(quota_mgr->GetPid());
334 }
335 } else {
336 ✗ quota_mgr->SetCacheMgrPid(new_cachemgr_pid);
337 }
338
339
340 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to existing cache manager pipe");
341 ✗ quota_mgr->initialized_ = true;
342 ✗ Nonblock2Block(quota_mgr->pipe_lru_[1]);
343 ✗ UnlockFile(fd_lockfile);
344 ✗ quota_mgr->GetLimits(&quota_mgr->limit_, &quota_mgr->cleanup_threshold_);
345 ✗ LogCvmfs(kLogQuota, kLogDebug,
346 "received limit %" PRIu64 ", threshold %" PRIu64,
347 quota_mgr->limit_, quota_mgr->cleanup_threshold_);
348 ✗ if (FileExists(workspace_dir + "/cachemgr.protocol")) {
349 ✗ quota_mgr->protocol_revision_ = quota_mgr->GetProtocolRevision();
350 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected protocol revision %u",
351 quota_mgr->protocol_revision_);
352 } else {
353 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to ancient cache manager");
354 }
355 ✗ return quota_mgr;
356 }
357 11 const int connect_error = errno;
358
359 // Lock file: let existing cache manager finish first
360
2/4
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 11 times.
✗ Branch 5 not taken.
 11 const int fd_lockfile_fifo = LockFile(workspace_dir + "/lock_cachemgr.fifo");
361
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11 times.
 11 if (fd_lockfile_fifo < 0) {
362 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not open lock file %s (%d)",
363 ✗ (workspace_dir + "/lock_cachemgr.fifo").c_str(), errno);
364 ✗ UnlockFile(fd_lockfile);
365 ✗ delete quota_mgr;
366 ✗ return NULL;
367 }
368
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 UnlockFile(fd_lockfile_fifo);
369
370
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11 times.
 11 if (connect_error == ENXIO) {
371 ✗ LogCvmfs(kLogQuota, kLogDebug, "left-over FIFO found, unlinking");
372 ✗ unlink(fifo_path.c_str());
373 }
374
375 // Creating a new FIFO for the cache manager (to be bound later)
376 11 int retval = mkfifo(fifo_path.c_str(), 0600);
377
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11 times.
 11 if (retval != 0) {
378 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to create cache manager FIFO (%d)",
379 ✗ errno);
380 ✗ UnlockFile(fd_lockfile);
381 ✗ delete quota_mgr;
382 ✗ return NULL;
383 }
384
385 // Create new cache manager
386 int pipe_boot[2];
387 int pipe_handshake[2];
388
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 MakePipe(pipe_boot);
389
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 MakePipe(pipe_handshake);
390
391 11 vector<string> command_line;
392
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 command_line.push_back(exe_path);
393
2/4
✓ Branch 2 taken 11 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 11 times.
✗ Branch 6 not taken.
 11 command_line.push_back("__cachemgr__");
394
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 command_line.push_back(cache_workspace);
395
2/4
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 11 times.
✗ Branch 5 not taken.
 11 command_line.push_back(StringifyInt(pipe_boot[1]));
396
2/4
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 11 times.
✗ Branch 5 not taken.
 11 command_line.push_back(StringifyInt(pipe_handshake[0]));
397
2/4
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 11 times.
✗ Branch 5 not taken.
 11 command_line.push_back(StringifyInt(limit));
398
2/4
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 11 times.
✗ Branch 5 not taken.
 11 command_line.push_back(StringifyInt(cleanup_threshold));
399 // do not propagate foreground in order to reliably get pid from exec
400 // instead, daemonize right here
401
2/4
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 11 times.
✗ Branch 5 not taken.
 11 command_line.push_back(StringifyInt(true)); // foreground
402
3/6
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 11 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 11 times.
✗ Branch 8 not taken.
 11 command_line.push_back(StringifyInt(GetLogSyslogLevel()));
403
3/6
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 11 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 11 times.
✗ Branch 8 not taken.
 11 command_line.push_back(StringifyInt(GetLogSyslogFacility()));
404
5/14
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 11 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 11 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 11 times.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 11 times.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
 11 command_line.push_back(GetLogDebugFile() + ":" + GetLogMicroSyslog());
405
406 11 set<int> preserve_filedes;
407
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 preserve_filedes.insert(0);
408
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 preserve_filedes.insert(1);
409
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 preserve_filedes.insert(2);
410
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 preserve_filedes.insert(pipe_boot[1]);
411
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 preserve_filedes.insert(pipe_handshake[0]);
412
413
1/2
✓ Branch 0 taken 11 times.
✗ Branch 1 not taken.
 11 if (foreground) {
414
1/2
✓ Branch 2 taken 11 times.
✗ Branch 3 not taken.
 11 retval = ManagedExec(command_line, preserve_filedes, map<int, int>(),
415 /*drop_credentials*/ false,
416 /*clear_env*/ false,
417 /*double_fork*/ true, &new_cachemgr_pid);
418 } else {
419 ✗ retval = ExecAsDaemon(command_line, &new_cachemgr_pid);
420 }
421
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11 times.
 11 if (!retval) {
422 ✗ UnlockFile(fd_lockfile);
423 ✗ ClosePipe(pipe_boot);
424 ✗ ClosePipe(pipe_handshake);
425 ✗ delete quota_mgr;
426 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to start cache manager");
427 ✗ return NULL;
428 }
429
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 LogCvmfs(kLogQuota, kLogDebug,
430 "new cache manager pid: %d protocol revision %d",
431 new_cachemgr_pid, QuotaManager::kProtocolRevision);
432 11 quota_mgr->SetCacheMgrPid(new_cachemgr_pid);
433
2/4
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 11 times.
✗ Branch 6 not taken.
 11 const int fd_lockfile_rw = open((workspace_dir + "/lock_cachemgr").c_str(),
434 O_RDWR | O_TRUNC, 0600);
435 11 const unsigned magic_number = PosixQuotaManager::kLockFileMagicNumber;
436
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 const bool result_mn = SafeWrite(fd_lockfile_rw, &magic_number,
437 sizeof(magic_number));
438
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 const bool result = SafeWrite(fd_lockfile_rw, &new_cachemgr_pid,
439 sizeof(new_cachemgr_pid));
440
2/4
✓ Branch 0 taken 11 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 11 times.
 11 if (!result || !result_mn) {
441 ✗ PANIC(kLogSyslogErr, "could not write cache manager pid to lockfile");
442 }
443
444
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 close(fd_lockfile_rw);
445 // Wait for cache manager to be ready
446
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 close(pipe_boot[1]);
447
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 close(pipe_handshake[0]);
448 char buf;
449
2/4
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 11 times.
✗ Branch 4 not taken.
 11 if (read(pipe_boot[0], &buf, 1) != 1) {
450
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 UnlockFile(fd_lockfile);
451
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 close(pipe_boot[0]);
452
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 close(pipe_handshake[1]);
453
1/2
✓ Branch 0 taken 11 times.
✗ Branch 1 not taken.
 11 delete quota_mgr;
454
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
455 "cache manager did not start");
456 11 return NULL;
457 }
458 ✗ close(pipe_boot[0]);
459
460 // Connect write end
461 ✗ quota_mgr->pipe_lru_[1] = open(fifo_path.c_str(), O_WRONLY | O_NONBLOCK);
462 ✗ if (quota_mgr->pipe_lru_[1] < 0) {
463 ✗ LogCvmfs(kLogQuota, kLogDebug,
464 ✗ "failed to connect to newly created FIFO (%d)", errno);
465 ✗ close(pipe_handshake[1]);
466 ✗ UnlockFile(fd_lockfile);
467 ✗ delete quota_mgr;
468 ✗ return NULL;
469 }
470
471 // Finalize handshake
472 ✗ buf = 'C';
473 ✗ if (write(pipe_handshake[1], &buf, 1) != 1) {
474 ✗ UnlockFile(fd_lockfile);
475 ✗ close(pipe_handshake[1]);
476 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not finalize handshake");
477 ✗ delete quota_mgr;
478 ✗ return NULL;
479 }
480 ✗ close(pipe_handshake[1]);
481
482 ✗ Nonblock2Block(quota_mgr->pipe_lru_[1]);
483 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to a new cache manager");
484 ✗ quota_mgr->protocol_revision_ = kProtocolRevision;
485
486 ✗ UnlockFile(fd_lockfile);
487
488 ✗ quota_mgr->initialized_ = true;
489 ✗ quota_mgr->GetLimits(&quota_mgr->limit_, &quota_mgr->cleanup_threshold_);
490 ✗ LogCvmfs(kLogQuota, kLogDebug,
491 "received limit %" PRIu64 ", "
492 "threshold %" PRIu64,
493 quota_mgr->limit_, quota_mgr->cleanup_threshold_);
494 ✗ return quota_mgr;
495 22 }
496
497
498 110 bool PosixQuotaManager::DoCleanup(const uint64_t leave_size) {
499
2/2
✓ Branch 0 taken 22 times.
✓ Branch 1 taken 88 times.
 110 if (gauge_ <= leave_size)
500 22 return true;
501
502 // TODO(jblomer) transaction
503
1/2
✓ Branch 1 taken 88 times.
✗ Branch 2 not taken.
 88 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug,
504 "clean up cache until at most %lu KB is used", leave_size / 1024);
505
1/2
✓ Branch 1 taken 88 times.
✗ Branch 2 not taken.
 88 LogCvmfs(kLogQuota, kLogDebug, "gauge %" PRIu64, gauge_);
506
1/2
✓ Branch 1 taken 88 times.
✗ Branch 2 not taken.
 88 cleanup_recorder_.Tick();
507
508 bool result;
509 88 vector<string> trash;
510
511 // Note that volatile files start counting from the smallest int64 number:
512 // the absolute sequence number with the first bit set in two's complement.
513 // So -1 can be a marker that will never appear in the database.
514 88 int64_t max_acseq = -1;
515 88 std::vector<EvictCandidate> lru_ordered_open;
516
517 do {
518
1/2
✓ Branch 1 taken 638 times.
✗ Branch 2 not taken.
 638 sqlite3_reset(stmt_lru_);
519
3/4
✓ Branch 0 taken 88 times.
✓ Branch 1 taken 550 times.
✓ Branch 3 taken 638 times.
✗ Branch 4 not taken.
 726 sqlite3_bind_int64(stmt_lru_, 1,
520 88 (max_acseq == -1) ? std::numeric_limits<int64_t>::min()
521 : (max_acseq + 1));
522
523 638 std::vector<EvictCandidate> candidates;
524
1/2
✓ Branch 1 taken 638 times.
✗ Branch 2 not taken.
 638 candidates.reserve(kEvictBatchSize);
525 638 string hash_str;
526 638 unsigned i = 0;
527
3/4
✓ Branch 1 taken 572748 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 572110 times.
✓ Branch 4 taken 638 times.
 572748 while (sqlite3_step(stmt_lru_) == SQLITE_ROW) {
528 hash_str = reinterpret_cast<const char *>(
529
2/4
✓ Branch 1 taken 572110 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 572110 times.
✗ Branch 5 not taken.
 572110 sqlite3_column_text(stmt_lru_, 0));
530
1/2
✓ Branch 2 taken 572110 times.
✗ Branch 3 not taken.
 572110 LogCvmfs(kLogQuota, kLogDebug, "add %s to candidates for eviction",
531 hash_str.c_str());
532
1/2
✓ Branch 1 taken 572110 times.
✗ Branch 2 not taken.
 572110 candidates.push_back(
533
1/2
✓ Branch 1 taken 572110 times.
✗ Branch 2 not taken.
 572110 EvictCandidate(shash::MkFromHexPtr(shash::HexPtr(hash_str)),
534 572110 sqlite3_column_int64(stmt_lru_, 1),
535
2/4
✓ Branch 1 taken 572110 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 572110 times.
✗ Branch 5 not taken.
 572110 sqlite3_column_int64(stmt_lru_, 2)));
536 572110 i++;
537 }
538
2/2
✓ Branch 1 taken 11 times.
✓ Branch 2 taken 627 times.
 638 if (candidates.empty()) {
539
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 LogCvmfs(kLogQuota, kLogDebug, "no more entries to evict");
540 11 break;
541 }
542
543 627 const unsigned N = candidates.size();
544
545 627 open_files_.clear();
546
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 627 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 1254 open_files_ = (cleanup_unused_first_) ? CollectAllOpenHashes()
547 627 : std::vector<shash::Short>();
548
549
2/2
✓ Branch 0 taken 550099 times.
✓ Branch 1 taken 550 times.
 550649 for (i = 0; i < N; ++i) {
550 // That's a critical condition. We must not delete a not yet inserted
551 // pinned file as it is already reserved (but will be inserted later).
552 // Instead, set the pin bit in the db to not run into an endless loop
553
1/2
✓ Branch 2 taken 550099 times.
✗ Branch 3 not taken.
 550099 const bool is_pinned = pinned_chunks_.find(candidates[i].hash)
554 1100198 != pinned_chunks_.end();
555
556 // Avoid evicting open files hopping there are enough more recently used
557 // files to satisfy the cleanup request
558 /*
559 const bool is_open = std::find_if(
560 open_files_.begin(), open_files_.end(),
561 [&candidates, &i](const auto &elem) -> bool {
562 return elem.Collide(candidates[i].hash);
563 })
564 != open_files_.end();
565 */
566 550099 bool is_open=false;
567
1/2
✗ Branch 4 not taken.
✓ Branch 5 taken 550099 times.
 550099 for(auto it=open_files_.begin();it!=open_files_.end();++it){
568 ✗ if (it->Collide(candidates[i].hash)){is_open=true; break;}
569 }
570
571
2/2
✓ Branch 0 taken 11 times.
✓ Branch 1 taken 550088 times.
 550099 if (is_pinned) {
572
1/2
✓ Branch 2 taken 11 times.
✗ Branch 3 not taken.
 11 SkipEviction(candidates[i]);
573 11 continue;
574 }
575
576
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 550088 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 550088 if (cleanup_unused_first_ and is_open) {
577 ✗ SkipEviction(candidates[i]);
578 ✗ lru_ordered_open.push_back(candidates[i]);
579 ✗ continue;
580 }
581
582
2/4
✓ Branch 1 taken 550088 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 550088 times.
✗ Branch 5 not taken.
 1100176 trash.push_back(cache_dir_ + "/"
583
2/4
✓ Branch 2 taken 550088 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 550088 times.
✗ Branch 6 not taken.
 1650264 + candidates[i].hash.MakePathWithoutSuffix());
584 550088 gauge_ -= candidates[i].size;
585 550088 max_acseq = candidates[i].acseq;
586
1/2
✓ Branch 2 taken 550088 times.
✗ Branch 3 not taken.
 550088 LogCvmfs(kLogQuota, kLogDebug, "lru cleanup %s, new gauge %" PRIu64,
587
1/2
✓ Branch 2 taken 550088 times.
✗ Branch 3 not taken.
 1100176 candidates[i].hash.ToString().c_str(), gauge_);
588
589
2/2
✓ Branch 0 taken 77 times.
✓ Branch 1 taken 550011 times.
 550088 if (gauge_ <= leave_size)
590 77 break;
591 }
592
6/6
✓ Branch 1 taken 627 times.
✓ Branch 2 taken 11 times.
✓ Branch 4 taken 627 times.
✓ Branch 5 taken 11 times.
✓ Branch 6 taken 550 times.
✓ Branch 7 taken 77 times.
 1276 } while (gauge_ > leave_size);
593
594
1/2
✓ Branch 0 taken 88 times.
✗ Branch 1 not taken.
 88 if (max_acseq != -1) {
595
1/2
✓ Branch 1 taken 88 times.
✗ Branch 2 not taken.
 88 sqlite3_bind_int64(stmt_rm_batch_, 1, max_acseq);
596
1/2
✓ Branch 1 taken 88 times.
✗ Branch 2 not taken.
 88 result = (sqlite3_step(stmt_rm_batch_) == SQLITE_DONE);
597
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 88 times.
 88 assert(result);
598
1/2
✓ Branch 1 taken 88 times.
✗ Branch 2 not taken.
 88 sqlite3_reset(stmt_rm_batch_);
599
600
1/2
✓ Branch 1 taken 88 times.
✗ Branch 2 not taken.
 88 result = (sqlite3_step(stmt_unblock_) == SQLITE_DONE);
601
1/2
✓ Branch 1 taken 88 times.
✗ Branch 2 not taken.
 88 sqlite3_reset(stmt_unblock_);
602
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 88 times.
 88 assert(result);
603 }
604
605
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 88 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 88 times.
 88 while (!lru_ordered_open.empty() and gauge_ > leave_size) {
606 // cleanup files in use
607 ✗ auto &candidate = lru_ordered_open[0];
608 ✗ trash.push_back(cache_dir_ + "/" + candidate.hash.MakePathWithoutSuffix());
609 ✗ gauge_ -= candidate.size;
610 ✗ max_acseq = candidate.acseq;
611 ✗ LogCvmfs(kLogQuota, kLogDebug, "lru cleanup %s, new gauge %" PRIu64,
612 ✗ candidate.hash.ToString().c_str(), gauge_);
613 ✗ lru_ordered_open.erase(lru_ordered_open.begin());
614 }
615
616
2/4
✓ Branch 1 taken 88 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 88 times.
 88 if (!EmptyTrash(trash))
617 ✗ return false;
618
619
2/2
✓ Branch 0 taken 11 times.
✓ Branch 1 taken 77 times.
 88 if (gauge_ > leave_size) {
620
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
621 "request to clean until %" PRIu64 ", "
622 "but effective gauge is %" PRIu64,
623 leave_size, gauge_);
624 11 return false;
625 }
626 77 return true;
627 88 }
628
629 88 bool PosixQuotaManager::EmptyTrash(const std::vector<std::string> &trash) {
630
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 88 times.
 88 if (trash.empty())
631 ✗ return true;
632
633
2/2
✓ Branch 0 taken 66 times.
✓ Branch 1 taken 22 times.
 88 if (async_delete_) {
634 // Double fork avoids zombie, forked removal process must not flush file
635 // buffers
636 pid_t pid;
637 int statloc;
638
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 66 times.
 66 if ((pid = fork()) == 0) {
639 // TODO(jblomer): eviciting files in the cache should perhaps become a
640 // thread. This would also allow to block the chunks and prevent the
641 // race with re-insertion. Then again, a thread can block umount.
642 #ifndef DEBUGMSG
643 ✗ CloseAllFildes(std::set<int>());
644 #endif
645 ✗ if (fork() == 0) {
646 ✗ for (unsigned i = 0, iEnd = trash.size(); i < iEnd; ++i) {
647 ✗ LogCvmfs(kLogQuota, kLogDebug, "unlink %s", trash[i].c_str());
648 ✗ unlink(trash[i].c_str());
649 }
650 ✗ _exit(0);
651 }
652 ✗ _exit(0);
653 } else {
654
1/2
✓ Branch 0 taken 66 times.
✗ Branch 1 not taken.
 66 if (pid > 0)
655
1/2
✓ Branch 1 taken 66 times.
✗ Branch 2 not taken.
 66 waitpid(pid, &statloc, 0);
656 else
657 ✗ return false;
658 }
659 } else { // !async_delete_
660
2/2
✓ Branch 1 taken 33 times.
✓ Branch 2 taken 22 times.
 55 for (unsigned i = 0, iEnd = trash.size(); i < iEnd; ++i) {
661 33 LogCvmfs(kLogQuota, kLogDebug, "unlink %s", trash[i].c_str());
662 33 unlink(trash[i].c_str());
663 }
664 }
665 88 return true;
666 }
667
668
669 1100621 void PosixQuotaManager::DoInsert(const shash::Any &hash,
670 const uint64_t size,
671 const string &description,
672 const CommandType command_type) {
673
1/2
✓ Branch 1 taken 1100621 times.
✗ Branch 2 not taken.
 1100621 const string hash_str = hash.ToString();
674
1/2
✓ Branch 3 taken 1100621 times.
✗ Branch 4 not taken.
 1100621 LogCvmfs(kLogQuota, kLogDebug, "insert into lru %s, path %s, method %d",
675 hash_str.c_str(), description.c_str(), command_type);
676 1100621 const unsigned desc_length = (description.length() > kMaxDescription)
677 ? kMaxDescription
678
1/2
✓ Branch 0 taken 1100621 times.
✗ Branch 1 not taken.
 1100621 : description.length();
679
680 LruCommand *cmd = reinterpret_cast<LruCommand *>(
681 1100621 alloca(sizeof(LruCommand) + desc_length));
682 1100621 new (cmd) LruCommand;
683 1100621 cmd->command_type = command_type;
684 1100621 cmd->SetSize(size);
685
1/2
✓ Branch 1 taken 1100621 times.
✗ Branch 2 not taken.
 1100621 cmd->StoreHash(hash);
686 1100621 cmd->desc_length = desc_length;
687 1100621 memcpy(reinterpret_cast<char *>(cmd) + sizeof(LruCommand), &description[0],
688 desc_length);
689
1/2
✓ Branch 1 taken 1100621 times.
✗ Branch 2 not taken.
 1100621 WritePipe(pipe_lru_[1], cmd, sizeof(LruCommand) + desc_length);
690 1100621 }
691
692
693 385 vector<string> PosixQuotaManager::DoList(const CommandType list_command) {
694 385 vector<string> result;
695
696 int pipe_list[2];
697
1/2
✓ Branch 1 taken 385 times.
✗ Branch 2 not taken.
 385 MakeReturnPipe(pipe_list);
698 char description_buffer[kMaxDescription];
699
700 385 LruCommand cmd;
701 385 cmd.command_type = list_command;
702 385 cmd.return_pipe = pipe_list[1];
703
1/2
✓ Branch 1 taken 385 times.
✗ Branch 2 not taken.
 385 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
704
705 int length;
706 do {
707
1/2
✓ Branch 1 taken 1100814 times.
✗ Branch 2 not taken.
 1100814 ManagedReadHalfPipe(pipe_list[0], &length, sizeof(length));
708
2/2
✓ Branch 0 taken 1100429 times.
✓ Branch 1 taken 385 times.
 1100814 if (length > 0) {
709
1/2
✓ Branch 1 taken 1100429 times.
✗ Branch 2 not taken.
 1100429 ReadPipe(pipe_list[0], description_buffer, length);
710
2/4
✓ Branch 2 taken 1100429 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1100429 times.
✗ Branch 6 not taken.
 1100429 result.push_back(string(description_buffer, length));
711 }
712
2/2
✓ Branch 0 taken 1100429 times.
✓ Branch 1 taken 385 times.
 1100814 } while (length >= 0);
713
714
1/2
✓ Branch 1 taken 385 times.
✗ Branch 2 not taken.
 385 CloseReturnPipe(pipe_list);
715 770 return result;
716 }
717
718
719 1225 uint64_t PosixQuotaManager::GetCapacity() {
720
1/2
✓ Branch 0 taken 1225 times.
✗ Branch 1 not taken.
 1225 if (limit_ != (uint64_t)(-1))
721 1225 return limit_;
722
723 // Unrestricted cache, look at free space on cache dir fs
724 struct statfs info;
725 ✗ if (statfs(".", &info) == 0) {
726 ✗ return info.f_bavail * info.f_bsize;
727 } else {
728 ✗ LogCvmfs(kLogQuota, kLogSyslogErr | kLogDebug,
729 ✗ "failed to query file system info of cache (%d)", errno);
730 ✗ return limit_;
731 }
732 }
733
734
735 ✗ void PosixQuotaManager::GetLimits(uint64_t *limit,
736 uint64_t *cleanup_threshold) {
737 int pipe_limits[2];
738 ✗ MakeReturnPipe(pipe_limits);
739
740 ✗ LruCommand cmd;
741 ✗ cmd.command_type = kLimits;
742 ✗ cmd.return_pipe = pipe_limits[1];
743 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
744 ✗ ManagedReadHalfPipe(pipe_limits[0], limit, sizeof(*limit));
745 ✗ ReadPipe(pipe_limits[0], cleanup_threshold, sizeof(*cleanup_threshold));
746 ✗ CloseReturnPipe(pipe_limits);
747 }
748
749
750 /**
751 * Since we only cleanup until cleanup_threshold, we can only add
752 * files smaller than limit-cleanup_threshold.
753 */
754 200 uint64_t PosixQuotaManager::GetMaxFileSize() {
755 200 return limit_ - cleanup_threshold_;
756 }
757
758
759 11 pid_t PosixQuotaManager::GetPid() {
760
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 11 if (!shared_ || !spawned_) {
761 11 return getpid();
762 }
763 ✗ if (cachemgr_pid_) {
764 ✗ return cachemgr_pid_;
765 }
766
767 pid_t result;
768 int pipe_pid[2];
769 ✗ MakeReturnPipe(pipe_pid);
770
771 ✗ LruCommand cmd;
772 ✗ cmd.command_type = kPid;
773 ✗ cmd.return_pipe = pipe_pid[1];
774 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
775 ✗ ReadHalfPipe(pipe_pid[0], &result, sizeof(result));
776 ✗ CloseReturnPipe(pipe_pid);
777 ✗ return result;
778 }
779
780
781 11 uint32_t PosixQuotaManager::GetProtocolRevision() {
782 int pipe_revision[2];
783
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 MakeReturnPipe(pipe_revision);
784
785 11 LruCommand cmd;
786 11 cmd.command_type = kGetProtocolRevision;
787 11 cmd.return_pipe = pipe_revision[1];
788
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
789
790 uint32_t revision;
791
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 ManagedReadHalfPipe(pipe_revision[0], &revision, sizeof(revision));
792
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 CloseReturnPipe(pipe_revision);
793 11 return revision;
794 }
795
796 1214 void PosixQuotaManager::SetCleanupPolicy(bool cleanup_unused_first) {
797
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1214 times.
 1214 if (protocol_revision_ < 3) return;
798
799 1214 LruCommand cmd;
800 1214 cmd.command_type = kSetCleanupPolicy;
801
1/2
✓ Branch 1 taken 1214 times.
✗ Branch 2 not taken.
 1214 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
802
803
1/2
✓ Branch 1 taken 1214 times.
✗ Branch 2 not taken.
 1214 WritePipe(pipe_lru_[1], &cleanup_unused_first, sizeof(bool));
804 }
805
806 ✗ void PosixQuotaManager::RegisterMountpoint(const std::string &mountpoint) {
807 ✗ if (protocol_revision_ < 3) return;
808
809 ✗ LruCommand cmd;
810 ✗ cmd.command_type = kRegisterMountpoint;
811 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
812
813 ✗ size_t mp_size = mountpoint.size();
814 ✗ WritePipe(pipe_lru_[1], &mp_size, sizeof(size_t));
815 ✗ WritePipe(pipe_lru_[1], mountpoint.data(), mp_size);
816 }
817
818 ✗ std::string PosixQuotaManager::GetMountpoints() {
819 ✗ if (protocol_revision_ < 3) return "";
820
821 int pipe_mp[2];
822 ✗ MakeReturnPipe(pipe_mp);
823
824 ✗ LruCommand cmd;
825 ✗ cmd.command_type = kGetMountpoints;
826 ✗ cmd.return_pipe = pipe_mp[1];
827 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
828 ✗ size_t mp_str_size = 0;
829 ✗ ManagedReadHalfPipe(pipe_mp[0], &mp_str_size, sizeof(size_t));
830 ✗ char *buf = (char *)malloc(mp_str_size * sizeof(char));
831 ✗ ManagedReadHalfPipe(pipe_mp[0], buf, mp_str_size);
832 ✗ return std::string(buf);
833 }
834
835 ✗ std::string PosixQuotaManager::GetGroupHashes() {
836 ✗ if (protocol_revision_ < 3) return "";
837
838 int pipe_gh[2];
839 ✗ MakeReturnPipe(pipe_gh);
840
841 ✗ LruCommand cmd;
842 ✗ cmd.command_type = kGetGroupHashes;
843 ✗ cmd.return_pipe = pipe_gh[1];
844 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
845 ✗ size_t mp_str_size = 0;
846 ✗ ManagedReadHalfPipe(pipe_gh[0], &mp_str_size, sizeof(size_t));
847 ✗ char *buf = (char *)malloc(mp_str_size * sizeof(char));
848 ✗ ManagedReadHalfPipe(pipe_gh[0], buf, mp_str_size);
849 ✗ return std::string(buf);
850 }
851
852 /**
853 * Queries the shared local hard disk quota manager.
854 */
855 198 void PosixQuotaManager::GetSharedStatus(uint64_t *gauge, uint64_t *pinned) {
856 int pipe_status[2];
857
1/2
✓ Branch 1 taken 198 times.
✗ Branch 2 not taken.
 198 MakeReturnPipe(pipe_status);
858
859 198 LruCommand cmd;
860 198 cmd.command_type = kStatus;
861 198 cmd.return_pipe = pipe_status[1];
862
1/2
✓ Branch 1 taken 198 times.
✗ Branch 2 not taken.
 198 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
863
1/2
✓ Branch 1 taken 198 times.
✗ Branch 2 not taken.
 198 ManagedReadHalfPipe(pipe_status[0], gauge, sizeof(*gauge));
864
1/2
✓ Branch 1 taken 198 times.
✗ Branch 2 not taken.
 198 ReadPipe(pipe_status[0], pinned, sizeof(*pinned));
865
1/2
✓ Branch 1 taken 198 times.
✗ Branch 2 not taken.
 198 CloseReturnPipe(pipe_status);
866 198 }
867
868 11 bool PosixQuotaManager::SetSharedLimit(uint64_t limit) {
869 int pipe_set_limit[2];
870 bool result;
871
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 MakeReturnPipe(pipe_set_limit);
872
873 11 LruCommand cmd;
874 11 cmd.command_type = kSetLimit;
875 11 cmd.size = limit;
876 11 cmd.return_pipe = pipe_set_limit[1];
877
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
878
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 ReadHalfPipe(pipe_set_limit[0], &result, sizeof(result));
879
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 CloseReturnPipe(pipe_set_limit);
880 11 return result;
881 }
882
883
884 11 bool PosixQuotaManager::SetLimit(uint64_t size) {
885
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11 times.
 11 if (!spawned_) {
886 ✗ limit_ = size;
887 ✗ cleanup_threshold_ = size / 2;
888 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
889 "Quota limit set to %lu / threshold %lu", limit_,
890 cleanup_threshold_);
891 ✗ return true;
892 }
893 11 return SetSharedLimit(size);
894 }
895
896 2604 uint64_t PosixQuotaManager::GetSize() {
897
2/2
✓ Branch 0 taken 2428 times.
✓ Branch 1 taken 176 times.
 2604 if (!spawned_)
898 2428 return gauge_;
899 uint64_t gauge, size_pinned;
900
1/2
✓ Branch 1 taken 176 times.
✗ Branch 2 not taken.
 176 GetSharedStatus(&gauge, &size_pinned);
901 176 return gauge;
902 }
903
904
905 22 uint64_t PosixQuotaManager::GetSizePinned() {
906
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 22 times.
 22 if (!spawned_)
907 ✗ return pinned_;
908 uint64_t gauge, size_pinned;
909
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 GetSharedStatus(&gauge, &size_pinned);
910 22 return size_pinned;
911 }
912
913
914 44 uint64_t PosixQuotaManager::GetCleanupRate(uint64_t period_s) {
915
2/4
✓ Branch 0 taken 44 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 44 times.
 44 if (!spawned_ || (protocol_revision_ < 2))
916 ✗ return 0;
917 uint64_t cleanup_rate;
918
919 int pipe_cleanup_rate[2];
920
1/2
✓ Branch 1 taken 44 times.
✗ Branch 2 not taken.
 44 MakeReturnPipe(pipe_cleanup_rate);
921 44 LruCommand cmd;
922 44 cmd.command_type = kCleanupRate;
923 44 cmd.size = period_s;
924 44 cmd.return_pipe = pipe_cleanup_rate[1];
925
1/2
✓ Branch 1 taken 44 times.
✗ Branch 2 not taken.
 44 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
926
1/2
✓ Branch 1 taken 44 times.
✗ Branch 2 not taken.
 44 ManagedReadHalfPipe(pipe_cleanup_rate[0], &cleanup_rate,
927 sizeof(cleanup_rate));
928
1/2
✓ Branch 1 taken 44 times.
✗ Branch 2 not taken.
 44 CloseReturnPipe(pipe_cleanup_rate);
929
930 44 return cleanup_rate;
931 }
932
933
934 1863 bool PosixQuotaManager::InitDatabase(const bool rebuild_database) {
935 1863 string sql;
936 sqlite3_stmt *stmt;
937
938
2/4
✓ Branch 1 taken 1863 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1863 times.
✗ Branch 5 not taken.
 1863 fd_lock_cachedb_ = LockFile(workspace_dir_ + "/lock_cachedb");
939
2/2
✓ Branch 0 taken 11 times.
✓ Branch 1 taken 1852 times.
 1863 if (fd_lock_cachedb_ < 0) {
940
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 LogCvmfs(kLogQuota, kLogDebug, "failed to create cachedb lock");
941 11 return false;
942 }
943
944 1852 bool retry = false;
945
1/2
✓ Branch 1 taken 1852 times.
✗ Branch 2 not taken.
 1852 const string db_file = cache_dir_ + "/cachedb";
946
2/2
✓ Branch 0 taken 1761 times.
✓ Branch 1 taken 91 times.
 1852 if (rebuild_database) {
947
1/2
✓ Branch 2 taken 91 times.
✗ Branch 3 not taken.
 91 LogCvmfs(kLogQuota, kLogDebug, "rebuild database, unlinking existing (%s)",
948 db_file.c_str());
949 91 unlink(db_file.c_str());
950
1/2
✓ Branch 1 taken 91 times.
✗ Branch 2 not taken.
 91 unlink((db_file + "-journal").c_str());
951 }
952
953 1761 init_recover:
954
1/2
✓ Branch 2 taken 1852 times.
✗ Branch 3 not taken.
 1852 int err = sqlite3_open(db_file.c_str(), &database_);
955
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1852 times.
 1852 if (err != SQLITE_OK) {
956 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not open cache database (%d)", err);
957 ✗ goto init_database_fail;
958 }
959 // TODO(reneme): make this a `QuotaDatabase : public sqlite::Database`
960 sql = "PRAGMA synchronous=0; PRAGMA locking_mode=EXCLUSIVE; "
961 "PRAGMA auto_vacuum=1; "
962 "CREATE TABLE IF NOT EXISTS cache_catalog (sha1 TEXT, size INTEGER, "
963 " acseq INTEGER, path TEXT, type INTEGER, pinned INTEGER, "
964 "CONSTRAINT pk_cache_catalog PRIMARY KEY (sha1)); "
965 "CREATE UNIQUE INDEX IF NOT EXISTS idx_cache_catalog_acseq "
966 " ON cache_catalog (acseq); "
967 "CREATE TEMP TABLE fscache (sha1 TEXT, size INTEGER, actime INTEGER, "
968 "CONSTRAINT pk_fscache PRIMARY KEY (sha1)); "
969 "CREATE INDEX idx_fscache_actime ON fscache (actime); "
970 "CREATE TABLE IF NOT EXISTS properties (key TEXT, value TEXT, "
971
1/2
✓ Branch 1 taken 1852 times.
✗ Branch 2 not taken.
 1852 " CONSTRAINT pk_properties PRIMARY KEY(key));";
972
1/2
✓ Branch 2 taken 1852 times.
✗ Branch 3 not taken.
 1852 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
973
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1852 times.
 1852 if (err != SQLITE_OK) {
974 ✗ if (!retry) {
975 ✗ retry = true;
976 ✗ sqlite3_close(database_);
977 ✗ unlink(db_file.c_str());
978 ✗ unlink((db_file + "-journal").c_str());
979 ✗ LogCvmfs(kLogQuota, kLogSyslogWarn,
980 "LRU database corrupted, re-building");
981 ✗ goto init_recover;
982 }
983 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
984 sql.c_str());
985 ✗ goto init_database_fail;
986 }
987
988 // If this an old cache catalog,
989 // add and initialize new columns to cache_catalog
990 sql = "ALTER TABLE cache_catalog ADD type INTEGER; "
991
1/2
✓ Branch 1 taken 1852 times.
✗ Branch 2 not taken.
 1852 "ALTER TABLE cache_catalog ADD pinned INTEGER";
992
1/2
✓ Branch 2 taken 1852 times.
✗ Branch 3 not taken.
 1852 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
993
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1852 times.
 1852 if (err == SQLITE_OK) {
994 ✗ sql = "UPDATE cache_catalog SET type=" + StringifyInt(kFileRegular) + ";";
995 ✗ err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
996 ✗ if (err != SQLITE_OK) {
997 ✗ LogCvmfs(kLogQuota, kLogDebug,
998 "could not init cache database (failed: %s)", sql.c_str());
999 ✗ goto init_database_fail;
1000 }
1001 }
1002
1003 // Set pinned back
1004
1/2
✓ Branch 1 taken 1852 times.
✗ Branch 2 not taken.
 1852 sql = "UPDATE cache_catalog SET pinned=0;";
1005
1/2
✓ Branch 2 taken 1852 times.
✗ Branch 3 not taken.
 1852 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1006
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1852 times.
 1852 if (err != SQLITE_OK) {
1007 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
1008 sql.c_str());
1009 ✗ goto init_database_fail;
1010 }
1011
1012 // Set schema version
1013 sql = "INSERT OR REPLACE INTO properties (key, value) "
1014
1/2
✓ Branch 1 taken 1852 times.
✗ Branch 2 not taken.
 1852 "VALUES ('schema', '1.0')";
1015
1/2
✓ Branch 2 taken 1852 times.
✗ Branch 3 not taken.
 1852 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1016
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1852 times.
 1852 if (err != SQLITE_OK) {
1017 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
1018 sql.c_str());
1019 ✗ goto init_database_fail;
1020 }
1021
1022 // If cache catalog is empty, recreate from file system
1023
1/2
✓ Branch 1 taken 1852 times.
✗ Branch 2 not taken.
 1852 sql = "SELECT count(*) FROM cache_catalog;";
1024
1/2
✓ Branch 2 taken 1852 times.
✗ Branch 3 not taken.
 1852 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1025
2/4
✓ Branch 1 taken 1852 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1852 times.
✗ Branch 4 not taken.
 1852 if (sqlite3_step(stmt) == SQLITE_ROW) {
1026
6/8
✓ Branch 1 taken 1852 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 44 times.
✓ Branch 4 taken 1808 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 44 times.
✓ Branch 7 taken 1808 times.
✓ Branch 8 taken 44 times.
 1852 if ((sqlite3_column_int64(stmt, 0)) == 0 || rebuild_database) {
1027
1/2
✓ Branch 1 taken 1808 times.
✗ Branch 2 not taken.
 1808 LogCvmfs(kLogCvmfs, kLogDebug,
1028 "CernVM-FS: building lru cache database...");
1029
3/4
✓ Branch 1 taken 1808 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 33 times.
✓ Branch 4 taken 1775 times.
 1808 if (!RebuildDatabase()) {
1030
1/2
✓ Branch 1 taken 33 times.
✗ Branch 2 not taken.
 33 LogCvmfs(kLogQuota, kLogDebug,
1031 "could not build cache database from file system");
1032
1/2
✓ Branch 1 taken 33 times.
✗ Branch 2 not taken.
 33 sqlite3_finalize(stmt);
1033 33 goto init_database_fail;
1034 }
1035 }
1036
1/2
✓ Branch 1 taken 1819 times.
✗ Branch 2 not taken.
 1819 sqlite3_finalize(stmt);
1037 } else {
1038 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not select on cache catalog");
1039 ✗ sqlite3_finalize(stmt);
1040 ✗ goto init_database_fail;
1041 }
1042
1043 // How many bytes do we already have in cache?
1044
1/2
✓ Branch 1 taken 1819 times.
✗ Branch 2 not taken.
 1819 sql = "SELECT sum(size) FROM cache_catalog;";
1045
1/2
✓ Branch 2 taken 1819 times.
✗ Branch 3 not taken.
 1819 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1046
2/4
✓ Branch 1 taken 1819 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1819 times.
✗ Branch 4 not taken.
 1819 if (sqlite3_step(stmt) == SQLITE_ROW) {
1047
1/2
✓ Branch 1 taken 1819 times.
✗ Branch 2 not taken.
 1819 gauge_ = sqlite3_column_int64(stmt, 0);
1048 } else {
1049 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not determine cache size");
1050 ✗ sqlite3_finalize(stmt);
1051 ✗ goto init_database_fail;
1052 }
1053
1/2
✓ Branch 1 taken 1819 times.
✗ Branch 2 not taken.
 1819 sqlite3_finalize(stmt);
1054
1055 // Highest seq-no?
1056
1/2
✓ Branch 1 taken 1819 times.
✗ Branch 2 not taken.
 1819 sql = "SELECT coalesce(max(acseq & (~(1<<63))), 0) FROM cache_catalog;";
1057
1/2
✓ Branch 2 taken 1819 times.
✗ Branch 3 not taken.
 1819 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1058
2/4
✓ Branch 1 taken 1819 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1819 times.
✗ Branch 4 not taken.
 1819 if (sqlite3_step(stmt) == SQLITE_ROW) {
1059
1/2
✓ Branch 1 taken 1819 times.
✗ Branch 2 not taken.
 1819 seq_ = sqlite3_column_int64(stmt, 0) + 1;
1060 } else {
1061 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not determine highest seq-no");
1062 ✗ sqlite3_finalize(stmt);
1063 ✗ goto init_database_fail;
1064 }
1065
1/2
✓ Branch 1 taken 1819 times.
✗ Branch 2 not taken.
 1819 sqlite3_finalize(stmt);
1066
1067 // Prepare touch, new, remove statements
1068
1/2
✓ Branch 1 taken 1819 times.
✗ Branch 2 not taken.
 1819 sqlite3_prepare_v2(database_,
1069 "UPDATE cache_catalog SET acseq=:seq | (acseq&(1<<63)) "
1070 "WHERE sha1=:sha1;",
1071 -1, &stmt_touch_, NULL);
1072
1/2
✓ Branch 1 taken 1819 times.
✗ Branch 2 not taken.
 1819 sqlite3_prepare_v2(database_,
1073 "UPDATE cache_catalog SET pinned=0 "
1074 "WHERE sha1=:sha1;",
1075 -1, &stmt_unpin_, NULL);
1076
1/2
✓ Branch 1 taken 1819 times.
✗ Branch 2 not taken.
 1819 sqlite3_prepare_v2(database_,
1077 "UPDATE cache_catalog SET pinned=2 "
1078 "WHERE sha1=:sha1;",
1079 -1, &stmt_block_, NULL);
1080
1/2
✓ Branch 1 taken 1819 times.
✗ Branch 2 not taken.
 1819 sqlite3_prepare_v2(database_,
1081 "UPDATE cache_catalog SET pinned=1 "
1082 "WHERE pinned=2;",
1083 -1, &stmt_unblock_, NULL);
1084
1/2
✓ Branch 1 taken 1819 times.
✗ Branch 2 not taken.
 1819 sqlite3_prepare_v2(database_,
1085 "INSERT OR REPLACE INTO cache_catalog "
1086 "(sha1, size, acseq, path, type, pinned) "
1087 "VALUES (:sha1, :s, :seq, :p, :t, :pin);",
1088 -1, &stmt_new_, NULL);
1089
1/2
✓ Branch 1 taken 1819 times.
✗ Branch 2 not taken.
 1819 sqlite3_prepare_v2(database_,
1090 "SELECT size, pinned FROM cache_catalog WHERE sha1=:sha1;",
1091 -1, &stmt_size_, NULL);
1092
1/2
✓ Branch 1 taken 1819 times.
✗ Branch 2 not taken.
 1819 sqlite3_prepare_v2(database_, "DELETE FROM cache_catalog WHERE sha1=:sha1;",
1093 -1, &stmt_rm_, NULL);
1094
1/2
✓ Branch 1 taken 1819 times.
✗ Branch 2 not taken.
 1819 sqlite3_prepare_v2(database_,
1095 "DELETE FROM cache_catalog WHERE acseq<=:a AND pinned<>2;",
1096 -1, &stmt_rm_batch_, NULL);
1097
1/2
✓ Branch 2 taken 1819 times.
✗ Branch 3 not taken.
 1819 sqlite3_prepare_v2(database_,
1098
1/2
✓ Branch 2 taken 1819 times.
✗ Branch 3 not taken.
 3638 (std::string("SELECT sha1, size, acseq FROM cache_catalog "
1099 "WHERE pinned<>2 AND acseq>=:a "
1100 "ORDER BY acseq ASC "
1101 "LIMIT ")
1102
3/6
✓ Branch 1 taken 1819 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1819 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 1819 times.
✗ Branch 8 not taken.
 7276 + StringifyInt(kEvictBatchSize) + ";")
1103 .c_str(),
1104 -1, &stmt_lru_, NULL);
1105
1/2
✓ Branch 2 taken 1819 times.
✗ Branch 3 not taken.
 1819 sqlite3_prepare_v2(database_,
1106 ("SELECT path FROM cache_catalog WHERE type="
1107
3/6
✓ Branch 1 taken 1819 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1819 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 1819 times.
✗ Branch 8 not taken.
 3638 + StringifyInt(kFileRegular) + ";")
1108 .c_str(),
1109 -1, &stmt_list_, NULL);
1110
1/2
✓ Branch 1 taken 1819 times.
✗ Branch 2 not taken.
 1819 sqlite3_prepare_v2(database_,
1111 "SELECT path FROM cache_catalog WHERE pinned<>0;", -1,
1112 &stmt_list_pinned_, NULL);
1113
1/2
✓ Branch 1 taken 1819 times.
✗ Branch 2 not taken.
 1819 sqlite3_prepare_v2(database_,
1114 "SELECT path FROM cache_catalog WHERE acseq < 0;", -1,
1115 &stmt_list_volatile_, NULL);
1116
1/2
✓ Branch 2 taken 1819 times.
✗ Branch 3 not taken.
 1819 sqlite3_prepare_v2(database_,
1117 ("SELECT path FROM cache_catalog WHERE type="
1118
3/6
✓ Branch 1 taken 1819 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1819 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 1819 times.
✗ Branch 8 not taken.
 3638 + StringifyInt(kFileCatalog) + ";")
1119 .c_str(),
1120 -1, &stmt_list_catalogs_, NULL);
1121 1819 return true;
1122
1123 33 init_database_fail:
1124
1/2
✓ Branch 1 taken 33 times.
✗ Branch 2 not taken.
 33 sqlite3_close(database_);
1125 33 database_ = NULL;
1126
1/2
✓ Branch 1 taken 33 times.
✗ Branch 2 not taken.
 33 UnlockFile(fd_lock_cachedb_);
1127 33 return false;
1128 1863 }
1129
1130
1131 /**
1132 * Inserts a new file into cache catalog. This file gets a new,
1133 * highest sequence number. Does cache cleanup if necessary.
1134 */
1135 1100434 void PosixQuotaManager::Insert(const shash::Any &any_hash,
1136 const uint64_t size,
1137 const string &description) {
1138 1100434 DoInsert(any_hash, size, description, kInsert);
1139 1100434 }
1140
1141
1142 /**
1143 * Inserts a new file into cache catalog. This file is marked as volatile
1144 * and gets a new highest sequence number with the first bit set. Cache cleanup
1145 * treats these files with priority.
1146 */
1147 44 void PosixQuotaManager::InsertVolatile(const shash::Any &any_hash,
1148 const uint64_t size,
1149 const string &description) {
1150 44 DoInsert(any_hash, size, description, kInsertVolatile);
1151 44 }
1152
1153
1154 /**
1155 * Lists all path names from the cache db.
1156 */
1157 231 vector<string> PosixQuotaManager::List() { return DoList(kList); }
1158
1159
1160 /**
1161 * Lists all pinned files from the cache db.
1162 */
1163 88 vector<string> PosixQuotaManager::ListPinned() { return DoList(kListPinned); }
1164
1165
1166 /**
1167 * Lists all sqlite catalog files from the cache db.
1168 */
1169 33 vector<string> PosixQuotaManager::ListCatalogs() {
1170 33 return DoList(kListCatalogs);
1171 }
1172
1173
1174 /**
1175 * Lists only files flagged as volatile (priority removal)
1176 */
1177 33 vector<string> PosixQuotaManager::ListVolatile() {
1178 33 return DoList(kListVolatile);
1179 }
1180
1181
1182 /**
1183 * Entry point for the shared cache manager process
1184 */
1185 ✗ int PosixQuotaManager::MainCacheManager(int argc, char **argv) {
1186 ✗ LogCvmfs(kLogQuota, kLogDebug, "starting quota manager");
1187 int retval;
1188
1189 ✗ PosixQuotaManager shared_manager(0, 0, "");
1190 ✗ shared_manager.shared_ = true;
1191 ✗ shared_manager.spawned_ = true;
1192 ✗ shared_manager.pinned_ = 0;
1193
1194 // Process command line arguments
1195 ✗ ParseDirectories(string(argv[2]),
1196 &shared_manager.cache_dir_,
1197 &shared_manager.workspace_dir_);
1198 ✗ const int pipe_boot = String2Int64(argv[3]);
1199 ✗ const int pipe_handshake = String2Int64(argv[4]);
1200 ✗ shared_manager.limit_ = String2Int64(argv[5]);
1201 ✗ shared_manager.cleanup_threshold_ = String2Int64(argv[6]);
1202 ✗ const int foreground = String2Int64(argv[7]);
1203 ✗ const int syslog_level = String2Int64(argv[8]);
1204 ✗ const int syslog_facility = String2Int64(argv[9]);
1205 ✗ vector<string> logfiles = SplitString(argv[10], ':');
1206
1207 ✗ SetLogSyslogLevel(syslog_level);
1208 ✗ SetLogSyslogFacility(syslog_facility);
1209 ✗ if ((logfiles.size() > 0) && (logfiles[0] != ""))
1210 ✗ SetLogDebugFile(logfiles[0] + ".cachemgr");
1211 ✗ if (logfiles.size() > 1)
1212 ✗ SetLogMicroSyslog(logfiles[1]);
1213
1214 ✗ if (!foreground)
1215 ✗ Daemonize();
1216
1217 ✗ if ((geteuid() != 0) && SetuidCapabilityPermitted()) {
1218 // Permanently drop credentials
1219 ✗ const std::vector<cap_value_t> nocaps;
1220 ✗ assert(ClearPermittedCapabilities(nocaps, nocaps));
1221 // Leave this process ptraceable
1222 ✗ assert(platform_set_dumpable());
1223 // but without core dumps
1224 ✗ assert(SetLimitCore(0));
1225 }
1226
1227 ✗ const UniquePtr<Watchdog> watchdog(Watchdog::Create(NULL, false /* needs_read_environ */));
1228 ✗ assert(watchdog.IsValid());
1229 ✗ watchdog->Spawn("./stacktrace.cachemgr");
1230
1231 // Initialize pipe, open non-blocking as cvmfs is not yet connected
1232 ✗ const int fd_lockfile_fifo = LockFile(shared_manager.workspace_dir_
1233 ✗ + "/lock_cachemgr.fifo");
1234 ✗ if (fd_lockfile_fifo < 0) {
1235 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1236 "could not open lock file "
1237 "%s (%d)",
1238 ✗ (shared_manager.workspace_dir_ + "/lock_cachemgr.fifo").c_str(),
1239 ✗ errno);
1240 ✗ return 1;
1241 }
1242 ✗ const string crash_guard = shared_manager.cache_dir_ + "/cachemgr.running";
1243 ✗ const bool rebuild = FileExists(crash_guard);
1244 ✗ retval = open(crash_guard.c_str(), O_RDONLY | O_CREAT, 0600);
1245 ✗ if (retval < 0) {
1246 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1247 "failed to create shared cache manager crash guard");
1248 ✗ UnlockFile(fd_lockfile_fifo);
1249 ✗ return 1;
1250 }
1251 ✗ close(retval);
1252
1253 // Redirect SQlite temp directory to cache (global variable)
1254 ✗ const string tmp_dir = shared_manager.workspace_dir_;
1255 ✗ sqlite3_temp_directory = static_cast<char *>(
1256 ✗ sqlite3_malloc(tmp_dir.length() + 1));
1257 ✗ snprintf(sqlite3_temp_directory, tmp_dir.length() + 1, "%s", tmp_dir.c_str());
1258
1259 // Cleanup leftover named pipes
1260 ✗ shared_manager.CleanupPipes();
1261
1262 ✗ if (!shared_manager.InitDatabase(rebuild)) {
1263 ✗ UnlockFile(fd_lockfile_fifo);
1264 ✗ return 1;
1265 }
1266 ✗ shared_manager.CheckFreeSpace();
1267
1268 // Save protocol revision to file. If the file is not found, it indicates
1269 // to the client that the cache manager is from times before the protocol
1270 // was versioned.
1271 const string protocol_revision_path = shared_manager.workspace_dir_
1272 ✗ + "/cachemgr.protocol";
1273 ✗ retval = open(protocol_revision_path.c_str(), O_WRONLY | O_CREAT, 0600);
1274 ✗ if (retval < 0) {
1275 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1276 ✗ "failed to open protocol revision file (%d)", errno);
1277 ✗ UnlockFile(fd_lockfile_fifo);
1278 ✗ return 1;
1279 }
1280 ✗ const string revision = StringifyInt(kProtocolRevision);
1281 ✗ const int written = write(retval, revision.data(), revision.length());
1282 ✗ close(retval);
1283 ✗ if ((written < 0) || static_cast<unsigned>(written) != revision.length()) {
1284 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1285 ✗ "failed to write protocol revision (%d)", errno);
1286 ✗ UnlockFile(fd_lockfile_fifo);
1287 ✗ return 1;
1288 }
1289
1290 ✗ const string fifo_path = shared_manager.workspace_dir_ + "/cachemgr";
1291 ✗ shared_manager.pipe_lru_[0] = open(fifo_path.c_str(), O_RDONLY | O_NONBLOCK);
1292 ✗ if (shared_manager.pipe_lru_[0] < 0) {
1293 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to listen on FIFO %s (%d)",
1294 ✗ fifo_path.c_str(), errno);
1295 ✗ UnlockFile(fd_lockfile_fifo);
1296 ✗ return 1;
1297 }
1298 ✗ Nonblock2Block(shared_manager.pipe_lru_[0]);
1299 ✗ LogCvmfs(kLogQuota, kLogDebug, "shared cache manager listening");
1300
1301 ✗ char buf = 'C';
1302 ✗ WritePipe(pipe_boot, &buf, 1);
1303 ✗ close(pipe_boot);
1304
1305 ✗ ReadPipe(pipe_handshake, &buf, 1);
1306 ✗ close(pipe_handshake);
1307 ✗ LogCvmfs(kLogQuota, kLogDebug, "shared cache manager handshake done");
1308
1309 // Ensure that broken pipes from clients do not kill the cache manager
1310 ✗ signal(SIGPIPE, SIG_IGN);
1311 // Don't let Ctrl-C ungracefully kill interactive session
1312 ✗ signal(SIGINT, SIG_IGN);
1313
1314 ✗ shared_manager.MainCommandServer(&shared_manager);
1315 ✗ unlink(fifo_path.c_str());
1316 ✗ unlink(protocol_revision_path.c_str());
1317 ✗ shared_manager.CloseDatabase();
1318 ✗ unlink(crash_guard.c_str());
1319 ✗ UnlockFile(fd_lockfile_fifo);
1320
1321 ✗ if (sqlite3_temp_directory) {
1322 ✗ sqlite3_free(sqlite3_temp_directory);
1323 ✗ sqlite3_temp_directory = NULL;
1324 }
1325
1326 ✗ return 0;
1327 }
1328
1329
1330 341 void *PosixQuotaManager::MainCommandServer(void *data) {
1331 341 PosixQuotaManager *quota_mgr = static_cast<PosixQuotaManager *>(data);
1332
1333
1/2
✓ Branch 1 taken 341 times.
✗ Branch 2 not taken.
 341 LogCvmfs(kLogQuota, kLogDebug, "starting quota manager");
1334
1/2
✓ Branch 1 taken 341 times.
✗ Branch 2 not taken.
 341 sqlite3_soft_heap_limit(quota_mgr->kSqliteMemPerThread);
1335
1336
2/2
✓ Branch 1 taken 10912 times.
✓ Branch 2 taken 341 times.
 11253 LruCommand command_buffer[kCommandBufferSize];
1337 char description_buffer[kCommandBufferSize * kMaxDescription];
1338 341 unsigned num_commands = 0;
1339
1340
1/2
✓ Branch 1 taken 1651771 times.
✗ Branch 2 not taken.
 1651771 while (read(quota_mgr->pipe_lru_[0], &command_buffer[num_commands],
1341 sizeof(command_buffer[0]))
1342
2/2
✓ Branch 0 taken 1651430 times.
✓ Branch 1 taken 341 times.
 1651771 == sizeof(command_buffer[0])) {
1343 1651430 const CommandType command_type = command_buffer[num_commands].command_type;
1344
1/2
✓ Branch 1 taken 1651430 times.
✗ Branch 2 not taken.
 1651430 LogCvmfs(kLogQuota, kLogDebug, "received command %d", command_type);
1345 1651430 const uint64_t size = command_buffer[num_commands].GetSize();
1346
1347 // Inserts and pins come with a description (usually a path)
1348
4/4
✓ Branch 0 taken 551243 times.
✓ Branch 1 taken 1100187 times.
✓ Branch 2 taken 551199 times.
✓ Branch 3 taken 44 times.
 1651430 if ((command_type == kInsert) || (command_type == kInsertVolatile)
1349
4/4
✓ Branch 0 taken 551177 times.
✓ Branch 1 taken 22 times.
✓ Branch 2 taken 121 times.
✓ Branch 3 taken 551056 times.
 551199 || (command_type == kPin) || (command_type == kPinRegular)) {
1350 1100374 const int desc_length = command_buffer[num_commands].desc_length;
1351 1100374 ReadPipe(quota_mgr->pipe_lru_[0],
1352
1/2
✓ Branch 1 taken 1100374 times.
✗ Branch 2 not taken.
 1100374 &description_buffer[kMaxDescription * num_commands],
1353 desc_length);
1354 }
1355
1356 // The protocol revision is returned immediately
1357
2/2
✓ Branch 0 taken 11 times.
✓ Branch 1 taken 1651419 times.
 1651430 if (command_type == kGetProtocolRevision) {
1358
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 const int return_pipe = quota_mgr->BindReturnPipe(
1359 command_buffer[num_commands].return_pipe);
1360
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11 times.
 11 if (return_pipe < 0)
1361 ✗ continue;
1362
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 WritePipe(return_pipe, &quota_mgr->kProtocolRevision,
1363 sizeof(quota_mgr->kProtocolRevision));
1364
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 quota_mgr->UnbindReturnPipe(return_pipe);
1365 11 continue;
1366 11 }
1367
1368 // Register a new mountpoint
1369
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1651419 times.
 1651419 if (command_type == kRegisterMountpoint) {
1370 ✗ size_t mp_size = 0;
1371 ✗ ReadPipe(quota_mgr->pipe_lru_[0], &mp_size, sizeof(size_t));
1372 ✗ std::string mountpoint(mp_size, '\0');
1373 ✗ ReadPipe(quota_mgr->pipe_lru_[0], (void *)mountpoint.data(), mp_size);
1374 ✗ quota_mgr->mountpoints_.push_back(mountpoint);
1375 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
1376 "Mountpoint %s registered in the group", mountpoint.c_str());
1377 ✗ continue;
1378 }
1379
1380 // Set Cleanup Policy
1381
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1651419 times.
 1651419 if (command_type == kSetCleanupPolicy) {
1382 ✗ bool policy = false;
1383 ✗ ReadPipe(quota_mgr->pipe_lru_[0], &policy, sizeof(bool));
1384 ✗ quota_mgr->cleanup_unused_first_ = policy;
1385 ✗ continue;
1386 }
1387 // Mountpoints are returned immediately
1388
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1651419 times.
 1651419 if (command_type == kGetMountpoints) {
1389 ✗ const int return_pipe = quota_mgr->BindReturnPipe(
1390 command_buffer[num_commands].return_pipe);
1391 ✗ if (return_pipe < 0)
1392 ✗ continue;
1393
1394 ✗ std::string mps;
1395 ✗ for (auto it = quota_mgr->mountpoints_.begin();
1396 ✗ it != quota_mgr->mountpoints_.end();
1397 ✗ ++it) {
1398 ✗ mps += *it + "\n";
1399 }
1400 ✗ size_t mp_size = mps.size();
1401 ✗ WritePipe(return_pipe, &mp_size, sizeof(size_t));
1402 ✗ WritePipe(return_pipe, mps.c_str(), mp_size);
1403 ✗ quota_mgr->UnbindReturnPipe(return_pipe);
1404 ✗ continue;
1405 }
1406
1407 // Group hashes are returned immediately
1408
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1651419 times.
 1651419 if (command_type == kGetGroupHashes) {
1409 ✗ const int return_pipe = quota_mgr->BindReturnPipe(
1410 command_buffer[num_commands].return_pipe);
1411 ✗ if (return_pipe < 0)
1412 ✗ continue;
1413
1414 ✗ std::vector<shash::Short> gh = quota_mgr->CollectAllOpenHashes();
1415 ✗ std::string result;
1416 ✗ for (auto it = gh.begin(); it != gh.end(); ++it) {
1417 ✗ result += (*it).ToString() + "\n";
1418 }
1419 ✗ size_t result_size = result.size();
1420 ✗ WritePipe(return_pipe, &result_size, sizeof(size_t));
1421 ✗ WritePipe(return_pipe, result.c_str(), result_size);
1422 ✗ quota_mgr->UnbindReturnPipe(return_pipe);
1423 ✗ continue;
1424 }
1425
1426 // The cleanup rate is returned immediately
1427
2/2
✓ Branch 0 taken 44 times.
✓ Branch 1 taken 1651375 times.
 1651419 if (command_type == kCleanupRate) {
1428
1/2
✓ Branch 1 taken 44 times.
✗ Branch 2 not taken.
 44 const int return_pipe = quota_mgr->BindReturnPipe(
1429 command_buffer[num_commands].return_pipe);
1430
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 44 times.
 44 if (return_pipe < 0)
1431 ✗ continue;
1432 const uint64_t
1433 44 period_s = size; // use the size field to transmit the period
1434
1/2
✓ Branch 1 taken 44 times.
✗ Branch 2 not taken.
 44 uint64_t rate = quota_mgr->cleanup_recorder_.GetNoTicks(period_s);
1435
1/2
✓ Branch 1 taken 44 times.
✗ Branch 2 not taken.
 44 WritePipe(return_pipe, &rate, sizeof(rate));
1436
1/2
✓ Branch 1 taken 44 times.
✗ Branch 2 not taken.
 44 quota_mgr->UnbindReturnPipe(return_pipe);
1437 44 continue;
1438 44 }
1439
1440
2/2
✓ Branch 0 taken 11 times.
✓ Branch 1 taken 1651364 times.
 1651375 if (command_type == kSetLimit) {
1441
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 const int return_pipe = quota_mgr->BindReturnPipe(
1442 command_buffer[num_commands].return_pipe);
1443
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11 times.
 11 if (return_pipe < 0)
1444 ✗ continue;
1445 11 quota_mgr->limit_ = size; // use the size field to transmit the size
1446 11 quota_mgr->cleanup_threshold_ = size / 2;
1447
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
1448 "Quota limit set to %lu / threshold %lu", quota_mgr->limit_,
1449 quota_mgr->cleanup_threshold_);
1450 11 bool ret = true;
1451
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 WritePipe(return_pipe, &ret, sizeof(ret));
1452
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 quota_mgr->UnbindReturnPipe(return_pipe);
1453 11 continue;
1454 11 }
1455
1456 // Reservations are handled immediately and "out of band"
1457
2/2
✓ Branch 0 taken 154 times.
✓ Branch 1 taken 1651210 times.
 1651364 if (command_type == kReserve) {
1458 154 bool success = true;
1459
1/2
✓ Branch 1 taken 154 times.
✗ Branch 2 not taken.
 154 const int return_pipe = quota_mgr->BindReturnPipe(
1460 command_buffer[num_commands].return_pipe);
1461
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 154 times.
 154 if (return_pipe < 0)
1462 ✗ continue;
1463
1464
1/2
✓ Branch 1 taken 154 times.
✗ Branch 2 not taken.
 154 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1465
1/2
✓ Branch 1 taken 154 times.
✗ Branch 2 not taken.
 154 const string hash_str(hash.ToString());
1466
1/2
✓ Branch 2 taken 154 times.
✗ Branch 3 not taken.
 154 LogCvmfs(kLogQuota, kLogDebug, "reserve %lu bytes for %s", size,
1467 hash_str.c_str());
1468
1469
1/2
✓ Branch 1 taken 154 times.
✗ Branch 2 not taken.
 154 if (quota_mgr->pinned_chunks_.find(hash)
1470
2/2
✓ Branch 2 taken 132 times.
✓ Branch 3 taken 22 times.
 308 == quota_mgr->pinned_chunks_.end()) {
1471
2/2
✓ Branch 0 taken 11 times.
✓ Branch 1 taken 121 times.
 132 if ((quota_mgr->pinned_ + size) > quota_mgr->cleanup_threshold_) {
1472
1/2
✓ Branch 2 taken 11 times.
✗ Branch 3 not taken.
 11 LogCvmfs(kLogQuota, kLogDebug,
1473 "failed to insert %s (pinned), no space", hash_str.c_str());
1474 11 success = false;
1475 } else {
1476
1/2
✓ Branch 1 taken 121 times.
✗ Branch 2 not taken.
 121 quota_mgr->pinned_chunks_[hash] = size;
1477 121 quota_mgr->pinned_ += size;
1478
1/2
✓ Branch 1 taken 121 times.
✗ Branch 2 not taken.
 121 quota_mgr->CheckHighPinWatermark();
1479 }
1480 }
1481
1482
1/2
✓ Branch 1 taken 154 times.
✗ Branch 2 not taken.
 154 WritePipe(return_pipe, &success, sizeof(success));
1483
1/2
✓ Branch 1 taken 154 times.
✗ Branch 2 not taken.
 154 quota_mgr->UnbindReturnPipe(return_pipe);
1484 154 continue;
1485 154 }
1486
1487 // Back channels are also handled out of band
1488
2/2
✓ Branch 0 taken 44 times.
✓ Branch 1 taken 1651166 times.
 1651210 if (command_type == kRegisterBackChannel) {
1489
1/2
✓ Branch 1 taken 44 times.
✗ Branch 2 not taken.
 44 const int return_pipe = quota_mgr->BindReturnPipe(
1490 command_buffer[num_commands].return_pipe);
1491
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 44 times.
 44 if (return_pipe < 0)
1492 ✗ continue;
1493
1494
1/2
✓ Branch 1 taken 44 times.
✗ Branch 2 not taken.
 44 quota_mgr->UnlinkReturnPipe(command_buffer[num_commands].return_pipe);
1495
1/2
✓ Branch 1 taken 44 times.
✗ Branch 2 not taken.
 44 Block2Nonblock(return_pipe); // back channels are opportunistic
1496
1/2
✓ Branch 1 taken 44 times.
✗ Branch 2 not taken.
 44 shash::Md5 hash;
1497 44 memcpy(hash.digest, command_buffer[num_commands].digest,
1498 44 shash::kDigestSizes[shash::kMd5]);
1499
1500 44 quota_mgr->LockBackChannels();
1501 const map<shash::Md5, int>::const_iterator
1502
1/2
✓ Branch 1 taken 44 times.
✗ Branch 2 not taken.
 44 iter = quota_mgr->back_channels_.find(hash);
1503
1/2
✗ Branch 3 not taken.
✓ Branch 4 taken 44 times.
 44 if (iter != quota_mgr->back_channels_.end()) {
1504 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
1505 ✗ "closing left-over back channel %s", hash.ToString().c_str());
1506 ✗ close(iter->second);
1507 }
1508
1/2
✓ Branch 1 taken 44 times.
✗ Branch 2 not taken.
 44 quota_mgr->back_channels_[hash] = return_pipe;
1509 44 quota_mgr->UnlockBackChannels();
1510
1511 44 char success = 'S';
1512
1/2
✓ Branch 1 taken 44 times.
✗ Branch 2 not taken.
 44 WritePipe(return_pipe, &success, sizeof(success));
1513
1/2
✓ Branch 2 taken 44 times.
✗ Branch 3 not taken.
 44 LogCvmfs(kLogQuota, kLogDebug, "register back channel %s on fd %d",
1514
1/2
✓ Branch 1 taken 44 times.
✗ Branch 2 not taken.
 88 hash.ToString().c_str(), return_pipe);
1515
1516 44 continue;
1517 44 }
1518
1519
2/2
✓ Branch 0 taken 22 times.
✓ Branch 1 taken 1651144 times.
 1651166 if (command_type == kUnregisterBackChannel) {
1520
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 shash::Md5 hash;
1521 22 memcpy(hash.digest, command_buffer[num_commands].digest,
1522 22 shash::kDigestSizes[shash::kMd5]);
1523
1524 22 quota_mgr->LockBackChannels();
1525 const map<shash::Md5, int>::iterator iter = quota_mgr->back_channels_
1526
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 .find(hash);
1527
1/2
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
 22 if (iter != quota_mgr->back_channels_.end()) {
1528
1/2
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
 22 LogCvmfs(kLogQuota, kLogDebug, "closing back channel %s",
1529
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 44 hash.ToString().c_str());
1530
1/2
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
 22 close(iter->second);
1531
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 quota_mgr->back_channels_.erase(iter);
1532 } else {
1533 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
1534 ✗ "did not find back channel %s", hash.ToString().c_str());
1535 }
1536 22 quota_mgr->UnlockBackChannels();
1537
1538 22 continue;
1539 22 }
1540
1541 // Unpinnings are also handled immediately with respect to the pinned gauge
1542
2/2
✓ Branch 0 taken 22 times.
✓ Branch 1 taken 1651122 times.
 1651144 if (command_type == kUnpin) {
1543
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1544
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 const string hash_str(hash.ToString());
1545
1546 const map<shash::Any, uint64_t>::iterator iter = quota_mgr->pinned_chunks_
1547
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 .find(hash);
1548
1/2
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
 22 if (iter != quota_mgr->pinned_chunks_.end()) {
1549 22 quota_mgr->pinned_ -= iter->second;
1550
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 quota_mgr->pinned_chunks_.erase(iter);
1551 // It can happen that files get pinned that were removed from the cache
1552 // (see cache.cc). We fix this at this point, where we remove such
1553 // entries from the cache database.
1554
2/4
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 22 times.
✗ Branch 5 not taken.
 44 if (!FileExists(quota_mgr->cache_dir_ + "/"
1555
4/6
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 22 times.
✗ Branch 5 not taken.
✓ Branch 9 taken 11 times.
✓ Branch 10 taken 11 times.
 66 + hash.MakePathWithoutSuffix())) {
1556
1/2
✓ Branch 2 taken 11 times.
✗ Branch 3 not taken.
 11 LogCvmfs(kLogQuota, kLogDebug,
1557 "remove orphaned pinned hash %s from cache database",
1558 hash_str.c_str());
1559
1/2
✓ Branch 2 taken 11 times.
✗ Branch 3 not taken.
 11 sqlite3_bind_text(quota_mgr->stmt_size_, 1, &hash_str[0],
1560 11 hash_str.length(), SQLITE_STATIC);
1561 int retval;
1562
2/4
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 11 times.
✗ Branch 4 not taken.
 11 if ((retval = sqlite3_step(quota_mgr->stmt_size_)) == SQLITE_ROW) {
1563
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 const uint64_t size = sqlite3_column_int64(quota_mgr->stmt_size_,
1564 11 0);
1565
1/2
✓ Branch 2 taken 11 times.
✗ Branch 3 not taken.
 11 sqlite3_bind_text(quota_mgr->stmt_rm_, 1, &(hash_str[0]),
1566 11 hash_str.length(), SQLITE_STATIC);
1567
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 retval = sqlite3_step(quota_mgr->stmt_rm_);
1568
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 11 if ((retval == SQLITE_DONE) || (retval == SQLITE_OK)) {
1569 11 quota_mgr->gauge_ -= size;
1570 } else {
1571 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1572 "failed to delete %s (%d)", hash_str.c_str(), retval);
1573 }
1574
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 sqlite3_reset(quota_mgr->stmt_rm_);
1575 }
1576
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 sqlite3_reset(quota_mgr->stmt_size_);
1577 }
1578 } else {
1579 ✗ LogCvmfs(kLogQuota, kLogDebug, "this chunk was not pinned");
1580 }
1581 22 }
1582
1583 // Immediate commands trigger flushing of the buffer
1584 1651144 const bool immediate_command = (command_type == kCleanup)
1585
2/2
✓ Branch 0 taken 1650814 times.
✓ Branch 1 taken 231 times.
 1651045 || (command_type == kList)
1586
2/2
✓ Branch 0 taken 1650726 times.
✓ Branch 1 taken 88 times.
 1650814 || (command_type == kListPinned)
1587
2/2
✓ Branch 0 taken 1650693 times.
✓ Branch 1 taken 33 times.
 1650726 || (command_type == kListCatalogs)
1588
2/2
✓ Branch 0 taken 1650660 times.
✓ Branch 1 taken 33 times.
 1650693 || (command_type == kListVolatile)
1589
2/2
✓ Branch 0 taken 1650627 times.
✓ Branch 1 taken 33 times.
 1650660 || (command_type == kRemove)
1590
2/2
✓ Branch 0 taken 1650429 times.
✓ Branch 1 taken 198 times.
 1650627 || (command_type == kStatus)
1591
1/2
✓ Branch 0 taken 1650429 times.
✗ Branch 1 not taken.
 1650429 || (command_type == kLimits)
1592
3/4
✓ Branch 0 taken 1651045 times.
✓ Branch 1 taken 99 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1650429 times.
 3302189 || (command_type == kPid);
1593
2/2
✓ Branch 0 taken 1650429 times.
✓ Branch 1 taken 715 times.
 1651144 if (!immediate_command)
1594 1650429 num_commands++;
1595
1596
4/4
✓ Branch 0 taken 1599587 times.
✓ Branch 1 taken 51557 times.
✓ Branch 2 taken 715 times.
✓ Branch 3 taken 1598872 times.
 1651144 if ((num_commands == kCommandBufferSize) || immediate_command) {
1597
1/2
✓ Branch 1 taken 52272 times.
✗ Branch 2 not taken.
 52272 quota_mgr->ProcessCommandBunch(num_commands, command_buffer,
1598 description_buffer);
1599
2/2
✓ Branch 0 taken 51557 times.
✓ Branch 1 taken 715 times.
 52272 if (!immediate_command)
1600 51557 num_commands = 0;
1601 }
1602
1603
2/2
✓ Branch 0 taken 715 times.
✓ Branch 1 taken 1650429 times.
 1651144 if (immediate_command) {
1604 // Process cleanup, listings
1605
1/2
✓ Branch 1 taken 715 times.
✗ Branch 2 not taken.
 715 const int return_pipe = quota_mgr->BindReturnPipe(
1606 command_buffer[num_commands].return_pipe);
1607
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 715 times.
 715 if (return_pipe < 0) {
1608 ✗ num_commands = 0;
1609 ✗ continue;
1610 }
1611
1612 int retval;
1613 715 sqlite3_stmt *this_stmt_list = NULL;
1614
7/10
✓ Branch 0 taken 33 times.
✓ Branch 1 taken 99 times.
✓ Branch 2 taken 231 times.
✓ Branch 3 taken 88 times.
✓ Branch 4 taken 33 times.
✓ Branch 5 taken 33 times.
✓ Branch 6 taken 198 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
 715 switch (command_type) {
1615 33 case kRemove: {
1616
1/2
✓ Branch 1 taken 33 times.
✗ Branch 2 not taken.
 33 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1617
1/2
✓ Branch 1 taken 33 times.
✗ Branch 2 not taken.
 33 const string hash_str = hash.ToString();
1618
1/2
✓ Branch 2 taken 33 times.
✗ Branch 3 not taken.
 33 LogCvmfs(kLogQuota, kLogDebug, "manually removing %s",
1619 hash_str.c_str());
1620 33 bool success = false;
1621
1622
1/2
✓ Branch 2 taken 33 times.
✗ Branch 3 not taken.
 33 sqlite3_bind_text(quota_mgr->stmt_size_, 1, &hash_str[0],
1623 33 hash_str.length(), SQLITE_STATIC);
1624 int retval;
1625
3/4
✓ Branch 1 taken 33 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 22 times.
✓ Branch 4 taken 11 times.
 33 if ((retval = sqlite3_step(quota_mgr->stmt_size_)) == SQLITE_ROW) {
1626
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 const uint64_t size = sqlite3_column_int64(quota_mgr->stmt_size_,
1627 22 0);
1628
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 const uint64_t is_pinned = sqlite3_column_int64(
1629 22 quota_mgr->stmt_size_, 1);
1630
1631
1/2
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
 22 sqlite3_bind_text(quota_mgr->stmt_rm_, 1, &(hash_str[0]),
1632 22 hash_str.length(), SQLITE_STATIC);
1633
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 retval = sqlite3_step(quota_mgr->stmt_rm_);
1634
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 22 if ((retval == SQLITE_DONE) || (retval == SQLITE_OK)) {
1635 22 success = true;
1636 22 quota_mgr->gauge_ -= size;
1637
2/2
✓ Branch 0 taken 11 times.
✓ Branch 1 taken 11 times.
 22 if (is_pinned) {
1638
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 quota_mgr->pinned_chunks_.erase(hash);
1639 11 quota_mgr->pinned_ -= size;
1640 }
1641 } else {
1642 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1643 "failed to delete %s (%d)", hash_str.c_str(), retval);
1644 }
1645
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 sqlite3_reset(quota_mgr->stmt_rm_);
1646 } else {
1647 // File does not exist
1648 11 success = true;
1649 }
1650
1/2
✓ Branch 1 taken 33 times.
✗ Branch 2 not taken.
 33 sqlite3_reset(quota_mgr->stmt_size_);
1651
1652
1/2
✓ Branch 1 taken 33 times.
✗ Branch 2 not taken.
 33 WritePipe(return_pipe, &success, sizeof(success));
1653 33 break;
1654 33 }
1655 99 case kCleanup:
1656
1/2
✓ Branch 1 taken 99 times.
✗ Branch 2 not taken.
 99 retval = quota_mgr->DoCleanup(size);
1657
1/2
✓ Branch 1 taken 99 times.
✗ Branch 2 not taken.
 99 WritePipe(return_pipe, &retval, sizeof(retval));
1658 99 break;
1659 231 case kList:
1660
1/2
✓ Branch 0 taken 231 times.
✗ Branch 1 not taken.
 231 if (!this_stmt_list)
1661 231 this_stmt_list = quota_mgr->stmt_list_;
1662 case kListPinned:
1663
2/2
✓ Branch 0 taken 88 times.
✓ Branch 1 taken 231 times.
 319 if (!this_stmt_list)
1664 88 this_stmt_list = quota_mgr->stmt_list_pinned_;
1665 case kListCatalogs:
1666
2/2
✓ Branch 0 taken 33 times.
✓ Branch 1 taken 319 times.
 352 if (!this_stmt_list)
1667 33 this_stmt_list = quota_mgr->stmt_list_catalogs_;
1668 case kListVolatile:
1669
2/2
✓ Branch 0 taken 33 times.
✓ Branch 1 taken 352 times.
 385 if (!this_stmt_list)
1670 33 this_stmt_list = quota_mgr->stmt_list_volatile_;
1671
1672 // Pipe back the list, one by one
1673 int length;
1674
3/4
✓ Branch 1 taken 1100814 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1100429 times.
✓ Branch 4 taken 385 times.
 1100814 while (sqlite3_step(this_stmt_list) == SQLITE_ROW) {
1675
1/2
✓ Branch 2 taken 1100429 times.
✗ Branch 3 not taken.
 1100429 string path = "(NULL)";
1676
2/4
✓ Branch 1 taken 1100429 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1100429 times.
✗ Branch 4 not taken.
 1100429 if (sqlite3_column_type(this_stmt_list, 0) != SQLITE_NULL) {
1677 2200858 path = string(reinterpret_cast<const char *>(
1678
2/4
✓ Branch 1 taken 1100429 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1100429 times.
✗ Branch 5 not taken.
 1100429 sqlite3_column_text(this_stmt_list, 0)));
1679 }
1680 1100429 length = path.length();
1681
1/2
✓ Branch 1 taken 1100429 times.
✗ Branch 2 not taken.
 1100429 WritePipe(return_pipe, &length, sizeof(length));
1682
1/2
✓ Branch 0 taken 1100429 times.
✗ Branch 1 not taken.
 1100429 if (length > 0)
1683
2/4
✓ Branch 1 taken 1100429 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1100429 times.
✗ Branch 5 not taken.
 1100429 WritePipe(return_pipe, &path[0], length);
1684 1100429 }
1685 385 length = -1;
1686
1/2
✓ Branch 1 taken 385 times.
✗ Branch 2 not taken.
 385 WritePipe(return_pipe, &length, sizeof(length));
1687
1/2
✓ Branch 1 taken 385 times.
✗ Branch 2 not taken.
 385 sqlite3_reset(this_stmt_list);
1688 385 break;
1689 198 case kStatus:
1690
1/2
✓ Branch 1 taken 198 times.
✗ Branch 2 not taken.
 198 WritePipe(return_pipe, &quota_mgr->gauge_, sizeof(quota_mgr->gauge_));
1691
1/2
✓ Branch 1 taken 198 times.
✗ Branch 2 not taken.
 198 WritePipe(return_pipe, &quota_mgr->pinned_,
1692 sizeof(quota_mgr->pinned_));
1693 198 break;
1694 ✗ case kLimits:
1695 ✗ WritePipe(return_pipe, &quota_mgr->limit_, sizeof(quota_mgr->limit_));
1696 ✗ WritePipe(return_pipe, &quota_mgr->cleanup_threshold_,
1697 sizeof(quota_mgr->cleanup_threshold_));
1698 ✗ break;
1699 ✗ case kPid: {
1700 ✗ pid_t pid = getpid();
1701 ✗ WritePipe(return_pipe, &pid, sizeof(pid));
1702 ✗ break;
1703 }
1704 ✗ default:
1705 ✗ PANIC(NULL); // other types are handled by the bunch processor
1706 }
1707
1/2
✓ Branch 1 taken 715 times.
✗ Branch 2 not taken.
 715 quota_mgr->UnbindReturnPipe(return_pipe);
1708 715 num_commands = 0;
1709 }
1710 }
1711
1712
1/2
✓ Branch 1 taken 341 times.
✗ Branch 2 not taken.
 341 LogCvmfs(kLogQuota, kLogDebug, "stopping cache manager (%d)", errno);
1713
1/2
✓ Branch 1 taken 341 times.
✗ Branch 2 not taken.
 341 close(quota_mgr->pipe_lru_[0]);
1714
1/2
✓ Branch 1 taken 341 times.
✗ Branch 2 not taken.
 341 quota_mgr->ProcessCommandBunch(num_commands, command_buffer,
1715 description_buffer);
1716
1717 // Unpin
1718 341 command_buffer[0].command_type = kTouch;
1719 341 for (map<shash::Any, uint64_t>::const_iterator
1720 341 i = quota_mgr->pinned_chunks_.begin(),
1721 341 iEnd = quota_mgr->pinned_chunks_.end();
1722
2/2
✓ Branch 1 taken 110 times.
✓ Branch 2 taken 341 times.
 451 i != iEnd;
1723 110 ++i) {
1724
1/2
✓ Branch 2 taken 110 times.
✗ Branch 3 not taken.
 110 command_buffer[0].StoreHash(i->first);
1725
1/2
✓ Branch 1 taken 110 times.
✗ Branch 2 not taken.
 110 quota_mgr->ProcessCommandBunch(1, command_buffer, description_buffer);
1726 }
1727
1728 341 return NULL;
1729 }
1730
1731
1732 1012 void PosixQuotaManager::MakeReturnPipe(int pipe[2]) {
1733
2/2
✓ Branch 0 taken 979 times.
✓ Branch 1 taken 33 times.
 1012 if (!shared_) {
1734 979 MakePipe(pipe);
1735 979 return;
1736 }
1737
1738 // Create FIFO in cache directory, store path name (number) in pipe write end
1739 33 int i = 0;
1740 int retval;
1741 do {
1742
2/4
✓ Branch 2 taken 44 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 44 times.
✗ Branch 6 not taken.
 44 retval = mkfifo((workspace_dir_ + "/pipe" + StringifyInt(i)).c_str(), 0600);
1743 44 pipe[1] = i;
1744 44 i++;
1745
3/4
✓ Branch 0 taken 11 times.
✓ Branch 1 taken 33 times.
✓ Branch 2 taken 11 times.
✗ Branch 3 not taken.
 44 } while ((retval == -1) && (errno == EEXIST));
1746
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 33 times.
 33 assert(retval == 0);
1747
1748 // Connect reader's end
1749
3/6
✓ Branch 2 taken 33 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 33 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 33 times.
✗ Branch 10 not taken.
 33 pipe[0] = open((workspace_dir_ + "/pipe" + StringifyInt(pipe[1])).c_str(),
1750 O_RDONLY | O_NONBLOCK);
1751
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 33 times.
 33 assert(pipe[0] >= 0);
1752 33 Nonblock2Block(pipe[0]);
1753 }
1754
1755
1756 1852 void PosixQuotaManager::ParseDirectories(const std::string cache_workspace,
1757 std::string *cache_dir,
1758 std::string *workspace_dir) {
1759
1/2
✓ Branch 1 taken 1852 times.
✗ Branch 2 not taken.
 1852 vector<string> dir_tokens(SplitString(cache_workspace, ':'));
1760
2/3
✓ Branch 1 taken 1830 times.
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
 1852 switch (dir_tokens.size()) {
1761 1830 case 1:
1762
2/4
✓ Branch 2 taken 1830 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1830 times.
✗ Branch 6 not taken.
 1830 *cache_dir = *workspace_dir = dir_tokens[0];
1763 1830 break;
1764 22 case 2:
1765
1/2
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
 22 *cache_dir = dir_tokens[0];
1766
1/2
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
 22 *workspace_dir = dir_tokens[1];
1767 22 break;
1768 ✗ default:
1769 ✗ PANIC(NULL);
1770 }
1771 1852 }
1772
1773 11 void PosixQuotaManager::SkipEviction(const EvictCandidate &candidate) {
1774 11 bool res = true;
1775
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 std::string hash_str = candidate.hash.ToString();
1776
1/2
✓ Branch 2 taken 11 times.
✗ Branch 3 not taken.
 11 LogCvmfs(kLogQuota, kLogDebug, "Exclude %s from eviction", hash_str.c_str());
1777
2/4
✓ Branch 2 taken 11 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 11 times.
✗ Branch 6 not taken.
 11 sqlite3_bind_text(stmt_block_, 1, &hash_str[0], hash_str.length(),
1778 SQLITE_STATIC);
1779
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 res = (sqlite3_step(stmt_block_) == SQLITE_DONE);
1780
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 sqlite3_reset(stmt_block_);
1781
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11 times.
 11 assert(res);
1782 11 }
1783
1784 /**
1785 * Immediately inserts a new pinned catalog. Does cache cleanup if necessary.
1786 *
1787 * \return True on success, false otherwise
1788 */
1789 907 bool PosixQuotaManager::Pin(const shash::Any &hash,
1790 const uint64_t size,
1791 const string &description,
1792 const bool is_catalog) {
1793
3/4
✓ Branch 0 taken 22 times.
✓ Branch 1 taken 885 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 22 times.
 907 assert((size > 0) || !is_catalog);
1794
1795
1/2
✓ Branch 1 taken 907 times.
✗ Branch 2 not taken.
 907 const string hash_str = hash.ToString();
1796
1/2
✓ Branch 3 taken 907 times.
✗ Branch 4 not taken.
 907 LogCvmfs(kLogQuota, kLogDebug, "pin into lru %s, path %s", hash_str.c_str(),
1797 description.c_str());
1798
1799 // Has to run when not yet spawned (cvmfs initialization)
1800
2/2
✓ Branch 0 taken 753 times.
✓ Branch 1 taken 154 times.
 907 if (!spawned_) {
1801 // Code duplication here
1802
3/4
✓ Branch 2 taken 753 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 440 times.
✓ Branch 6 taken 313 times.
 753 if (pinned_chunks_.find(hash) == pinned_chunks_.end()) {
1803
2/2
✓ Branch 0 taken 22 times.
✓ Branch 1 taken 418 times.
 440 if (pinned_ + size > cleanup_threshold_) {
1804
1/2
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
 22 LogCvmfs(kLogQuota, kLogDebug, "failed to insert %s (pinned), no space",
1805 hash_str.c_str());
1806 22 return false;
1807 } else {
1808
1/2
✓ Branch 1 taken 418 times.
✗ Branch 2 not taken.
 418 pinned_chunks_[hash] = size;
1809 418 pinned_ += size;
1810
1/2
✓ Branch 1 taken 418 times.
✗ Branch 2 not taken.
 418 CheckHighPinWatermark();
1811 }
1812 }
1813
1/2
✓ Branch 1 taken 731 times.
✗ Branch 2 not taken.
 731 const bool exists = Contains(hash_str);
1814
4/4
✓ Branch 0 taken 418 times.
✓ Branch 1 taken 313 times.
✓ Branch 2 taken 11 times.
✓ Branch 3 taken 407 times.
 731 if (!exists && (gauge_ + size > limit_)) {
1815
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 LogCvmfs(kLogQuota, kLogDebug, "over limit, gauge %lu, file size %lu",
1816 gauge_, size);
1817
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 const int retval = DoCleanup(cleanup_threshold_);
1818
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11 times.
 11 assert(retval != 0);
1819 }
1820
1/2
✓ Branch 3 taken 731 times.
✗ Branch 4 not taken.
 731 sqlite3_bind_text(stmt_new_, 1, &hash_str[0], hash_str.length(),
1821 SQLITE_STATIC);
1822
1/2
✓ Branch 1 taken 731 times.
✗ Branch 2 not taken.
 731 sqlite3_bind_int64(stmt_new_, 2, size);
1823
1/2
✓ Branch 1 taken 731 times.
✗ Branch 2 not taken.
 731 sqlite3_bind_int64(stmt_new_, 3, seq_++);
1824
1/2
✓ Branch 3 taken 731 times.
✗ Branch 4 not taken.
 731 sqlite3_bind_text(stmt_new_, 4, &description[0], description.length(),
1825 SQLITE_STATIC);
1826
3/4
✓ Branch 0 taken 698 times.
✓ Branch 1 taken 33 times.
✓ Branch 3 taken 731 times.
✗ Branch 4 not taken.
 731 sqlite3_bind_int64(stmt_new_, 5, is_catalog ? kFileCatalog : kFileRegular);
1827
1/2
✓ Branch 1 taken 731 times.
✗ Branch 2 not taken.
 731 sqlite3_bind_int64(stmt_new_, 6, 1);
1828
1/2
✓ Branch 1 taken 731 times.
✗ Branch 2 not taken.
 731 const int retval = sqlite3_step(stmt_new_);
1829
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 731 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 731 assert((retval == SQLITE_DONE) || (retval == SQLITE_OK));
1830
1/2
✓ Branch 1 taken 731 times.
✗ Branch 2 not taken.
 731 sqlite3_reset(stmt_new_);
1831
2/2
✓ Branch 0 taken 418 times.
✓ Branch 1 taken 313 times.
 731 if (!exists)
1832 418 gauge_ += size;
1833 731 return true;
1834 }
1835
1836 int pipe_reserve[2];
1837
1/2
✓ Branch 1 taken 154 times.
✗ Branch 2 not taken.
 154 MakeReturnPipe(pipe_reserve);
1838
1839 154 LruCommand cmd;
1840 154 cmd.command_type = kReserve;
1841 154 cmd.SetSize(size);
1842
1/2
✓ Branch 1 taken 154 times.
✗ Branch 2 not taken.
 154 cmd.StoreHash(hash);
1843 154 cmd.return_pipe = pipe_reserve[1];
1844
1/2
✓ Branch 1 taken 154 times.
✗ Branch 2 not taken.
 154 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
1845 bool result;
1846
1/2
✓ Branch 1 taken 154 times.
✗ Branch 2 not taken.
 154 ManagedReadHalfPipe(pipe_reserve[0], &result, sizeof(result));
1847
1/2
✓ Branch 1 taken 154 times.
✗ Branch 2 not taken.
 154 CloseReturnPipe(pipe_reserve);
1848
1849
2/2
✓ Branch 0 taken 11 times.
✓ Branch 1 taken 143 times.
 154 if (!result)
1850 11 return false;
1851
3/4
✓ Branch 0 taken 22 times.
✓ Branch 1 taken 121 times.
✓ Branch 3 taken 143 times.
✗ Branch 4 not taken.
 143 DoInsert(hash, size, description, is_catalog ? kPin : kPinRegular);
1852
1853 143 return true;
1854 907 }
1855
1856
1857 1830 PosixQuotaManager::PosixQuotaManager(const uint64_t limit,
1858 const uint64_t cleanup_threshold,
1859 1830 const string &cache_workspace)
1860 1830 : shared_(false)
1861 1830 , spawned_(false)
1862 1830 , limit_(limit)
1863 1830 , cleanup_threshold_(cleanup_threshold)
1864 1830 , gauge_(0)
1865 1830 , pinned_(0)
1866 1830 , seq_(0)
1867 1830 , cache_dir_() // initialized in body
1868 1830 , workspace_dir_() // initialized in body
1869 1830 , fd_lock_cachedb_(-1)
1870 1830 , async_delete_(true)
1871 1830 , cachemgr_pid_(0)
1872 1830 , database_(NULL)
1873 1830 , stmt_touch_(NULL)
1874 1830 , stmt_unpin_(NULL)
1875 1830 , stmt_block_(NULL)
1876 1830 , stmt_unblock_(NULL)
1877 1830 , stmt_new_(NULL)
1878 1830 , stmt_lru_(NULL)
1879 1830 , stmt_size_(NULL)
1880 1830 , stmt_rm_(NULL)
1881 1830 , stmt_rm_batch_(NULL)
1882 1830 , stmt_list_(NULL)
1883 1830 , stmt_list_pinned_(NULL)
1884 1830 , stmt_list_catalogs_(NULL)
1885 1830 , stmt_list_volatile_(NULL)
1886 1830 , initialized_(false)
1887 3660 , cleanup_unused_first_(false) {
1888
2/4
✓ Branch 1 taken 1830 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1830 times.
✗ Branch 5 not taken.
 1830 ParseDirectories(cache_workspace, &cache_dir_, &workspace_dir_);
1889 1830 pipe_lru_[0] = pipe_lru_[1] = -1;
1890
1/2
✓ Branch 1 taken 1830 times.
✗ Branch 2 not taken.
 1830 cleanup_recorder_.AddRecorder(1, 90); // last 1.5 min with second resolution
1891 // last 1.5 h with minute resolution
1892
1/2
✓ Branch 1 taken 1830 times.
✗ Branch 2 not taken.
 1830 cleanup_recorder_.AddRecorder(60, 90 * 60);
1893 // last 18 hours with 20 min resolution
1894
1/2
✓ Branch 1 taken 1830 times.
✗ Branch 2 not taken.
 1830 cleanup_recorder_.AddRecorder(20 * 60, 60 * 60 * 18);
1895 // last 4 days with hour resolution
1896
1/2
✓ Branch 1 taken 1830 times.
✗ Branch 2 not taken.
 1830 cleanup_recorder_.AddRecorder(60 * 60, 60 * 60 * 24 * 4);
1897
1898 1830 lock_open_files_ = reinterpret_cast<pthread_mutex_t *>(
1899 1830 smalloc(sizeof(pthread_mutex_t)));
1900 1830 }
1901
1902
1903 7316 PosixQuotaManager::~PosixQuotaManager() {
1904 3658 free(lock_open_files_);
1905
1906
2/2
✓ Branch 0 taken 33 times.
✓ Branch 1 taken 1796 times.
 3658 if (!initialized_)
1907 66 return;
1908
1909
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1796 times.
 3592 if (shared_) {
1910 // Most of cleanup is done elsewhen by shared cache manager
1911 ✗ close(pipe_lru_[1]);
1912 ✗ return;
1913 }
1914
1915
2/2
✓ Branch 0 taken 341 times.
✓ Branch 1 taken 1455 times.
 3592 if (spawned_) {
1916 682 char fin = 0;
1917 682 WritePipe(pipe_lru_[1], &fin, 1);
1918 682 close(pipe_lru_[1]);
1919 682 pthread_join(thread_lru_, NULL);
1920 } else {
1921 2910 ClosePipe(pipe_lru_);
1922 }
1923
1924 3592 CloseDatabase();
1925
14/14
✓ Branch 1 taken 1796 times.
✓ Branch 2 taken 33 times.
✓ Branch 4 taken 1796 times.
✓ Branch 5 taken 33 times.
✓ Branch 7 taken 1796 times.
✓ Branch 8 taken 33 times.
✓ Branch 10 taken 1796 times.
✓ Branch 11 taken 33 times.
✓ Branch 13 taken 1796 times.
✓ Branch 14 taken 33 times.
✓ Branch 16 taken 1796 times.
✓ Branch 17 taken 33 times.
✓ Branch 19 taken 1796 times.
✓ Branch 20 taken 33 times.
 7712 }
1926
1927
1928 52723 void PosixQuotaManager::ProcessCommandBunch(const unsigned num,
1929 const LruCommand *commands,
1930 const char *descriptions) {
1931 52723 int retval = sqlite3_exec(database_, "BEGIN", NULL, NULL, NULL);
1932
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 52723 times.
 52723 assert(retval == SQLITE_OK);
1933
1934
2/2
✓ Branch 0 taken 1650539 times.
✓ Branch 1 taken 52723 times.
 1703262 for (unsigned i = 0; i < num; ++i) {
1935
1/2
✓ Branch 1 taken 1650539 times.
✗ Branch 2 not taken.
 1650539 const shash::Any hash = commands[i].RetrieveHash();
1936
1/2
✓ Branch 1 taken 1650539 times.
✗ Branch 2 not taken.
 1650539 const string hash_str = hash.ToString();
1937 1650539 const unsigned size = commands[i].GetSize();
1938
1/2
✓ Branch 1 taken 1650539 times.
✗ Branch 2 not taken.
 1650539 LogCvmfs(kLogQuota, kLogDebug, "processing %s (%d)", hash_str.c_str(),
1939 1650539 commands[i].command_type);
1940
1941 bool exists;
1942
3/4
✓ Branch 0 taken 550143 times.
✓ Branch 1 taken 22 times.
✓ Branch 2 taken 1100374 times.
✗ Branch 3 not taken.
 1650539 switch (commands[i].command_type) {
1943 550143 case kTouch:
1944
1/2
✓ Branch 1 taken 550143 times.
✗ Branch 2 not taken.
 550143 sqlite3_bind_int64(stmt_touch_, 1, seq_++);
1945
1/2
✓ Branch 3 taken 550143 times.
✗ Branch 4 not taken.
 550143 sqlite3_bind_text(stmt_touch_, 2, &hash_str[0], hash_str.length(),
1946 SQLITE_STATIC);
1947
1/2
✓ Branch 1 taken 550143 times.
✗ Branch 2 not taken.
 550143 retval = sqlite3_step(stmt_touch_);
1948
1/2
✓ Branch 1 taken 550143 times.
✗ Branch 2 not taken.
 550143 LogCvmfs(kLogQuota, kLogDebug, "touching %s (%ld): %d",
1949 550143 hash_str.c_str(), seq_ - 1, retval);
1950
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 550143 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 550143 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
1951 ✗ PANIC(kLogSyslogErr, "failed to update %s in cachedb, error %d",
1952 hash_str.c_str(), retval);
1953 }
1954
1/2
✓ Branch 1 taken 550143 times.
✗ Branch 2 not taken.
 550143 sqlite3_reset(stmt_touch_);
1955 550143 break;
1956 22 case kUnpin:
1957
1/2
✓ Branch 3 taken 22 times.
✗ Branch 4 not taken.
 22 sqlite3_bind_text(stmt_unpin_, 1, &hash_str[0], hash_str.length(),
1958 SQLITE_STATIC);
1959
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 retval = sqlite3_step(stmt_unpin_);
1960
1/2
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
 22 LogCvmfs(kLogQuota, kLogDebug, "unpinning %s: %d", hash_str.c_str(),
1961 retval);
1962
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 22 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
1963 ✗ PANIC(kLogSyslogErr, "failed to unpin %s in cachedb, error %d",
1964 hash_str.c_str(), retval);
1965 }
1966
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 sqlite3_reset(stmt_unpin_);
1967 22 break;
1968 1100374 case kPin:
1969 case kPinRegular:
1970 case kInsert:
1971 case kInsertVolatile:
1972 // It could already be in, check
1973
1/2
✓ Branch 1 taken 1100374 times.
✗ Branch 2 not taken.
 1100374 exists = Contains(hash_str);
1974
1975 // Cleanup, move to trash and unlink
1976
3/4
✓ Branch 0 taken 1100319 times.
✓ Branch 1 taken 55 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1100319 times.
 1100374 if (!exists && (gauge_ + size > limit_)) {
1977 ✗ LogCvmfs(kLogQuota, kLogDebug, "over limit, gauge %lu, file size %u",
1978 gauge_, size);
1979 ✗ retval = DoCleanup(cleanup_threshold_);
1980 ✗ assert(retval != 0);
1981 }
1982
1983 // Insert or replace
1984
1/2
✓ Branch 3 taken 1100374 times.
✗ Branch 4 not taken.
 1100374 sqlite3_bind_text(stmt_new_, 1, &hash_str[0], hash_str.length(),
1985 SQLITE_STATIC);
1986
1/2
✓ Branch 1 taken 1100374 times.
✗ Branch 2 not taken.
 1100374 sqlite3_bind_int64(stmt_new_, 2, size);
1987
2/2
✓ Branch 0 taken 44 times.
✓ Branch 1 taken 1100330 times.
 1100374 if (commands[i].command_type == kInsertVolatile) {
1988
1/2
✓ Branch 1 taken 44 times.
✗ Branch 2 not taken.
 44 sqlite3_bind_int64(stmt_new_, 3, (seq_++) | kVolatileFlag);
1989 } else {
1990
1/2
✓ Branch 1 taken 1100330 times.
✗ Branch 2 not taken.
 1100330 sqlite3_bind_int64(stmt_new_, 3, seq_++);
1991 }
1992 1100374 sqlite3_bind_text(stmt_new_, 4, &descriptions[i * kMaxDescription],
1993
1/2
✓ Branch 1 taken 1100374 times.
✗ Branch 2 not taken.
 1100374 commands[i].desc_length, SQLITE_STATIC);
1994
1/2
✓ Branch 1 taken 1100374 times.
✗ Branch 2 not taken.
 1100374 sqlite3_bind_int64(
1995 stmt_new_, 5,
1996
2/2
✓ Branch 0 taken 22 times.
✓ Branch 1 taken 1100352 times.
 1100374 (commands[i].command_type == kPin) ? kFileCatalog : kFileRegular);
1997
1/2
✓ Branch 1 taken 1100374 times.
✗ Branch 2 not taken.
 1100374 sqlite3_bind_int64(stmt_new_, 6,
1998
2/2
✓ Branch 0 taken 1100352 times.
✓ Branch 1 taken 22 times.
 1100374 ((commands[i].command_type == kPin)
1999
2/2
✓ Branch 0 taken 121 times.
✓ Branch 1 taken 1100231 times.
 1100352 || (commands[i].command_type == kPinRegular))
2000 ? 1
2001 : 0);
2002
1/2
✓ Branch 1 taken 1100374 times.
✗ Branch 2 not taken.
 1100374 retval = sqlite3_step(stmt_new_);
2003
1/2
✓ Branch 1 taken 1100374 times.
✗ Branch 2 not taken.
 1100374 LogCvmfs(kLogQuota, kLogDebug, "insert or replace %s, method %d: %d",
2004 1100374 hash_str.c_str(), commands[i].command_type, retval);
2005
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1100374 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 1100374 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
2006 ✗ PANIC(kLogSyslogErr, "failed to insert %s in cachedb, error %d",
2007 hash_str.c_str(), retval);
2008 }
2009
1/2
✓ Branch 1 taken 1100374 times.
✗ Branch 2 not taken.
 1100374 sqlite3_reset(stmt_new_);
2010
2011
2/2
✓ Branch 0 taken 1100319 times.
✓ Branch 1 taken 55 times.
 1100374 if (!exists)
2012 1100319 gauge_ += size;
2013 1100374 break;
2014 ✗ default:
2015 // other types should have been taken care of by event loop
2016 ✗ PANIC(NULL);
2017 }
2018 1650539 }
2019
2020 52723 retval = sqlite3_exec(database_, "COMMIT", NULL, NULL, NULL);
2021
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 52723 times.
 52723 if (retval != SQLITE_OK) {
2022 ✗ PANIC(kLogSyslogErr, "failed to commit to cachedb, error %d", retval);
2023 }
2024 52723 }
2025
2026
2027 1808 bool PosixQuotaManager::RebuildDatabase() {
2028 1808 bool result = false;
2029 1808 string sql;
2030 1808 sqlite3_stmt *stmt_select = NULL;
2031 1808 sqlite3_stmt *stmt_insert = NULL;
2032 int sqlerr;
2033 1808 int seq = 0;
2034 char hex[4];
2035 struct stat info;
2036 platform_dirent64 *d;
2037 1808 DIR *dirp = NULL;
2038 1808 string path;
2039
2040
1/2
✓ Branch 1 taken 1808 times.
✗ Branch 2 not taken.
 1808 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug, "re-building cache database");
2041
2042 // Empty cache catalog and fscache
2043
1/2
✓ Branch 1 taken 1808 times.
✗ Branch 2 not taken.
 1808 sql = "DELETE FROM cache_catalog; DELETE FROM fscache;";
2044
1/2
✓ Branch 2 taken 1808 times.
✗ Branch 3 not taken.
 1808 sqlerr = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
2045
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1808 times.
 1808 if (sqlerr != SQLITE_OK) {
2046 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not clear cache database");
2047 ✗ goto build_return;
2048 }
2049
2050 1808 gauge_ = 0;
2051
2052 // Insert files from cache sub-directories 00 - ff
2053 // TODO(jblomer): fs_traversal
2054
1/2
✓ Branch 1 taken 1808 times.
✗ Branch 2 not taken.
 1808 sqlite3_prepare_v2(database_,
2055 "INSERT INTO fscache (sha1, size, actime) "
2056 "VALUES (:sha1, :s, :t);",
2057 -1, &stmt_insert, NULL);
2058
2059
2/2
✓ Branch 0 taken 454433 times.
✓ Branch 1 taken 1775 times.
 456208 for (int i = 0; i <= 0xff; i++) {
2060 454433 snprintf(hex, sizeof(hex), "%02x", i);
2061
3/6
✓ Branch 2 taken 454433 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 454433 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 454433 times.
✗ Branch 9 not taken.
 454433 path = cache_dir_ + "/" + string(hex);
2062
3/4
✓ Branch 2 taken 454433 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 33 times.
✓ Branch 5 taken 454400 times.
 454433 if ((dirp = opendir(path.c_str())) == NULL) {
2063
1/2
✓ Branch 2 taken 33 times.
✗ Branch 3 not taken.
 33 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
2064 "failed to open directory %s (tmpwatch interfering?)",
2065 path.c_str());
2066 33 goto build_return;
2067 }
2068
3/4
✓ Branch 1 taken 1363222 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 908822 times.
✓ Branch 4 taken 454400 times.
 1363222 while ((d = platform_readdir(dirp)) != NULL) {
2069
3/6
✓ Branch 2 taken 908822 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 908822 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 908822 times.
✗ Branch 9 not taken.
 1817644 const string file_path = path + "/" + string(d->d_name);
2070
1/2
✓ Branch 2 taken 908822 times.
✗ Branch 3 not taken.
 908822 if (stat(file_path.c_str(), &info) == 0) {
2071
2/2
✓ Branch 0 taken 908800 times.
✓ Branch 1 taken 22 times.
 908822 if (!S_ISREG(info.st_mode))
2072 908811 continue;
2073
2/2
✓ Branch 0 taken 11 times.
✓ Branch 1 taken 11 times.
 22 if (info.st_size == 0) {
2074
1/2
✓ Branch 2 taken 11 times.
✗ Branch 3 not taken.
 11 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug,
2075 "removing empty file %s during automatic cache db rebuild",
2076 file_path.c_str());
2077 11 unlink(file_path.c_str());
2078 11 continue;
2079 }
2080
2081
3/6
✓ Branch 2 taken 11 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 11 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 11 times.
✗ Branch 10 not taken.
 22 string hash = string(hex) + string(d->d_name);
2082
1/2
✓ Branch 3 taken 11 times.
✗ Branch 4 not taken.
 11 sqlite3_bind_text(stmt_insert, 1, hash.data(), hash.length(),
2083 SQLITE_STATIC);
2084
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 sqlite3_bind_int64(stmt_insert, 2, info.st_size);
2085
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 sqlite3_bind_int64(stmt_insert, 3, info.st_atime);
2086
2/4
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 11 times.
 11 if (sqlite3_step(stmt_insert) != SQLITE_DONE) {
2087 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not insert into temp table");
2088 ✗ goto build_return;
2089 }
2090
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 sqlite3_reset(stmt_insert);
2091
2092 11 gauge_ += info.st_size;
2093
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 } else {
2094 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not stat %s", file_path.c_str());
2095 }
2096
2/3
✓ Branch 1 taken 11 times.
✓ Branch 2 taken 908811 times.
✗ Branch 3 not taken.
 908822 }
2097
1/2
✓ Branch 1 taken 454400 times.
✗ Branch 2 not taken.
 454400 closedir(dirp);
2098 454400 dirp = NULL;
2099 }
2100
1/2
✓ Branch 1 taken 1775 times.
✗ Branch 2 not taken.
 1775 sqlite3_finalize(stmt_insert);
2101 1775 stmt_insert = NULL;
2102
2103 // Transfer from temp table in cache catalog
2104
1/2
✓ Branch 1 taken 1775 times.
✗ Branch 2 not taken.
 1775 sqlite3_prepare_v2(database_,
2105 "SELECT sha1, size FROM fscache ORDER BY actime;", -1,
2106 &stmt_select, NULL);
2107
1/2
✓ Branch 1 taken 1775 times.
✗ Branch 2 not taken.
 1775 sqlite3_prepare_v2(
2108 database_,
2109 "INSERT INTO cache_catalog (sha1, size, acseq, path, type, pinned) "
2110 "VALUES (:sha1, :s, :seq, 'unknown (automatic rebuild)', :t, 0);",
2111 -1, &stmt_insert, NULL);
2112
3/4
✓ Branch 1 taken 1786 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 11 times.
✓ Branch 4 taken 1775 times.
 1786 while (sqlite3_step(stmt_select) == SQLITE_ROW) {
2113 const string hash = string(
2114
2/4
✓ Branch 2 taken 11 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 11 times.
✗ Branch 6 not taken.
 11 reinterpret_cast<const char *>(sqlite3_column_text(stmt_select, 0)));
2115
1/2
✓ Branch 3 taken 11 times.
✗ Branch 4 not taken.
 11 sqlite3_bind_text(stmt_insert, 1, &hash[0], hash.length(), SQLITE_STATIC);
2116
2/4
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 11 times.
✗ Branch 5 not taken.
 11 sqlite3_bind_int64(stmt_insert, 2, sqlite3_column_int64(stmt_select, 1));
2117
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 sqlite3_bind_int64(stmt_insert, 3, seq++);
2118 // Might also be a catalog (information is lost)
2119
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 sqlite3_bind_int64(stmt_insert, 4, kFileRegular);
2120
2121
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 const int retval = sqlite3_step(stmt_insert);
2122
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11 times.
 11 if (retval != SQLITE_DONE) {
2123 // If the file system hosting the cache is full, we'll likely notice here
2124 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
2125 "could not insert into cache catalog (%d - %s)", retval,
2126 sqlite3_errstr(retval));
2127 ✗ goto build_return;
2128 }
2129
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 sqlite3_reset(stmt_insert);
2130
1/2
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
 11 }
2131
2132 // Delete temporary table
2133
1/2
✓ Branch 1 taken 1775 times.
✗ Branch 2 not taken.
 1775 sql = "DELETE FROM fscache;";
2134
1/2
✓ Branch 2 taken 1775 times.
✗ Branch 3 not taken.
 1775 sqlerr = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
2135
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1775 times.
 1775 if (sqlerr != SQLITE_OK) {
2136 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not clear temporary table (%d)",
2137 sqlerr);
2138 ✗ goto build_return;
2139 }
2140
2141 1775 seq_ = seq;
2142 1775 result = true;
2143
1/2
✓ Branch 1 taken 1775 times.
✗ Branch 2 not taken.
 1775 LogCvmfs(kLogQuota, kLogDebug,
2144 "rebuilding finished, sequence %" PRIu64 ", gauge %" PRIu64, seq_,
2145 gauge_);
2146
2147 1808 build_return:
2148
1/2
✓ Branch 0 taken 1808 times.
✗ Branch 1 not taken.
 1808 if (stmt_insert)
2149
1/2
✓ Branch 1 taken 1808 times.
✗ Branch 2 not taken.
 1808 sqlite3_finalize(stmt_insert);
2150
2/2
✓ Branch 0 taken 1775 times.
✓ Branch 1 taken 33 times.
 1808 if (stmt_select)
2151
1/2
✓ Branch 1 taken 1775 times.
✗ Branch 2 not taken.
 1775 sqlite3_finalize(stmt_select);
2152
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1808 times.
 1808 if (dirp)
2153 ✗ closedir(dirp);
2154 1808 return result;
2155 1808 }
2156
2157
2158 /**
2159 * Register a channel that allows the cache manager to trigger action to its
2160 * clients. Currently used for releasing pinned catalogs.
2161 */
2162 44 void PosixQuotaManager::RegisterBackChannel(int back_channel[2],
2163 const string &channel_id) {
2164
1/2
✓ Branch 0 taken 44 times.
✗ Branch 1 not taken.
 44 if (protocol_revision_ >= 1) {
2165
1/2
✓ Branch 2 taken 44 times.
✗ Branch 3 not taken.
 44 shash::Md5 hash = shash::Md5(shash::AsciiPtr(channel_id));
2166
1/2
✓ Branch 1 taken 44 times.
✗ Branch 2 not taken.
 44 MakeReturnPipe(back_channel);
2167
2168 44 LruCommand cmd;
2169 44 cmd.command_type = kRegisterBackChannel;
2170 44 cmd.return_pipe = back_channel[1];
2171 // Not StoreHash(). This is an MD5 hash.
2172 44 memcpy(cmd.digest, hash.digest, hash.GetDigestSize());
2173
1/2
✓ Branch 1 taken 44 times.
✗ Branch 2 not taken.
 44 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2174
2175 char success;
2176
1/2
✓ Branch 1 taken 44 times.
✗ Branch 2 not taken.
 44 ManagedReadHalfPipe(back_channel[0], &success, sizeof(success));
2177 // At this point, the named FIFO is unlinked, so don't use CloseReturnPipe
2178
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 44 times.
 44 if (success != 'S') {
2179 ✗ PANIC(kLogDebug | kLogSyslogErr,
2180 "failed to register quota back channel (%c)", success);
2181 }
2182 } else {
2183 // Dummy pipe to return valid file descriptors
2184 ✗ MakePipe(back_channel);
2185 }
2186 44 }
2187
2188
2189 /**
2190 * Removes a chunk from cache, if it exists.
2191 */
2192 33 void PosixQuotaManager::Remove(const shash::Any &hash) {
2193
1/2
✓ Branch 1 taken 33 times.
✗ Branch 2 not taken.
 33 const string hash_str = hash.ToString();
2194
2195 int pipe_remove[2];
2196
1/2
✓ Branch 1 taken 33 times.
✗ Branch 2 not taken.
 33 MakeReturnPipe(pipe_remove);
2197
2198 33 LruCommand cmd;
2199 33 cmd.command_type = kRemove;
2200 33 cmd.return_pipe = pipe_remove[1];
2201
1/2
✓ Branch 1 taken 33 times.
✗ Branch 2 not taken.
 33 cmd.StoreHash(hash);
2202
1/2
✓ Branch 1 taken 33 times.
✗ Branch 2 not taken.
 33 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2203
2204 bool success;
2205
1/2
✓ Branch 1 taken 33 times.
✗ Branch 2 not taken.
 33 ManagedReadHalfPipe(pipe_remove[0], &success, sizeof(success));
2206
1/2
✓ Branch 1 taken 33 times.
✗ Branch 2 not taken.
 33 CloseReturnPipe(pipe_remove);
2207
2208
3/6
✓ Branch 1 taken 33 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 33 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 33 times.
✗ Branch 8 not taken.
 33 unlink((cache_dir_ + "/" + hash.MakePathWithoutSuffix()).c_str());
2209 33 }
2210
2211
2212 363 void PosixQuotaManager::Spawn() {
2213
2/2
✓ Branch 0 taken 22 times.
✓ Branch 1 taken 341 times.
 363 if (spawned_)
2214 22 return;
2215
2216 341 if (pthread_create(&thread_lru_, NULL, MainCommandServer,
2217 static_cast<void *>(this))
2218
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 341 times.
 341 != 0) {
2219 ✗ PANIC(kLogDebug, "could not create lru thread");
2220 }
2221
2222 341 spawned_ = true;
2223 }
2224
2225
2226 /**
2227 * Updates the sequence number of the file specified by the hash.
2228 */
2229 550476 void PosixQuotaManager::Touch(const shash::Any &hash) {
2230 550476 LruCommand cmd;
2231 550476 cmd.command_type = kTouch;
2232
1/2
✓ Branch 1 taken 550476 times.
✗ Branch 2 not taken.
 550476 cmd.StoreHash(hash);
2233
1/2
✓ Branch 1 taken 550476 times.
✗ Branch 2 not taken.
 550476 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2234 550476 }
2235
2236
2237 946 void PosixQuotaManager::UnbindReturnPipe(int pipe_wronly) {
2238
2/2
✓ Branch 0 taken 11 times.
✓ Branch 1 taken 935 times.
 946 if (shared_)
2239 11 close(pipe_wronly);
2240 946 }
2241
2242
2243 77 void PosixQuotaManager::UnlinkReturnPipe(int pipe_wronly) {
2244
2/2
✓ Branch 0 taken 33 times.
✓ Branch 1 taken 44 times.
 77 if (shared_)
2245
2/4
✓ Branch 2 taken 33 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 33 times.
✗ Branch 6 not taken.
 33 unlink((workspace_dir_ + "/pipe" + StringifyInt(pipe_wronly)).c_str());
2246 77 }
2247
2248
2249 692 void PosixQuotaManager::Unpin(const shash::Any &hash) {
2250
2/4
✓ Branch 1 taken 692 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 692 times.
✗ Branch 6 not taken.
 692 LogCvmfs(kLogQuota, kLogDebug, "Unpin %s", hash.ToString().c_str());
2251
2252 692 LruCommand cmd;
2253 692 cmd.command_type = kUnpin;
2254
1/2
✓ Branch 1 taken 692 times.
✗ Branch 2 not taken.
 692 cmd.StoreHash(hash);
2255
1/2
✓ Branch 1 taken 692 times.
✗ Branch 2 not taken.
 692 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2256 692 }
2257
2258
2259 22 void PosixQuotaManager::UnregisterBackChannel(int back_channel[2],
2260 const string &channel_id) {
2261
1/2
✓ Branch 0 taken 22 times.
✗ Branch 1 not taken.
 22 if (protocol_revision_ >= 1) {
2262
1/2
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
 22 shash::Md5 hash = shash::Md5(shash::AsciiPtr(channel_id));
2263
2264 22 LruCommand cmd;
2265 22 cmd.command_type = kUnregisterBackChannel;
2266 // Not StoreHash(). This is an MD5 hash.
2267 22 memcpy(cmd.digest, hash.digest, hash.GetDigestSize());
2268
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2269
2270 // Writer's end will be closed by cache manager, FIFO is already unlinked
2271
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 close(back_channel[0]);
2272 } else {
2273 ✗ ClosePipe(back_channel);
2274 }
2275 22 }
2276
2277 1101397 void PosixQuotaManager::ManagedReadHalfPipe(int fd, void *buf, size_t nbyte) {
2278
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1101397 times.
 1101397 const unsigned timeout_ms = cachemgr_pid_ ? 1000 : 0;
2279 1101397 bool result = false;
2280 do {
2281 1101397 result = ReadHalfPipe(fd, buf, nbyte, timeout_ms);
2282 // try only as long as the cachemgr is still alive
2283
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 1101397 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 1101397 times.
 1101397 } while (!result && getpgid(cachemgr_pid_) >= 0);
2284
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1101397 times.
 1101397 if (!result) {
2285 ✗ PANIC(kLogStderr,
2286 "Error: quota manager could not read from cachemanager pipe");
2287 }
2288 1101397 }
2289
2290 ✗ void *PosixQuotaManager::CollectMountpointsHashes(void *data) {
2291 #ifndef __APPLE__
2292 ✗ pthread_setname_np(pthread_self(), "hash_collector");
2293 ✗ auto *handler = static_cast<CollectorHandler *>(data);
2294
2295 ✗ const std::string mountpoint = handler->mp[handler->i];
2296 ✗ ssize_t n = getxattr(mountpoint.c_str(), "user.list_open_hashes", nullptr, 0);
2297 ✗ if (n < 0) {
2298 ✗ pthread_exit(nullptr);
2299 }
2300 ✗ std::vector<char> buf((size_t)n);
2301 ✗ n = getxattr(mountpoint.c_str(), "user.list_open_hashes", buf.data(),
2302 buf.size());
2303 ✗ if (n < 0) {
2304 ✗ pthread_exit(nullptr);
2305 }
2306
2307 ✗ std::vector<std::string> hash_strs;
2308 ✗ std::string hash_str;
2309 ✗ for (const char c : buf) {
2310 ✗ if (c == '\n') {
2311 ✗ hash_strs.push_back(hash_str);
2312 ✗ hash_str.clear();
2313 } else {
2314 ✗ hash_str += c;
2315 }
2316 }
2317 ✗ const MutexLockGuard lock_guard(handler->l);
2318 ✗ for (auto hash_str : hash_strs) {
2319 ✗ handler->of.push_back(shash::Short( shash::MkFromHexPtr(shash::HexPtr(hash_str)) ));
2320 }
2321 #endif
2322 ✗ pthread_exit(nullptr);
2323 }
2324
2325 ✗ std::vector<shash::Short> PosixQuotaManager::CollectAllOpenHashes() {
2326 ✗ std::vector<CollectorHandler *> handlers;
2327 ✗ std::vector<pthread_t *> threads;
2328 ✗ open_files_.clear();
2329 #ifndef __APPLE__
2330 ✗ auto &&a_after_b = [](const struct timespec a, const struct timespec b) {
2331 ✗ return (a.tv_sec > b.tv_sec) ? true : false;
2332 };
2333
2334 ✗ for (size_t i = 0; i < mountpoints_.size(); ++i) {
2335 ✗ handlers.push_back(
2336 ✗ new CollectorHandler{open_files_, mountpoints_, lock_open_files_, i});
2337 ✗ threads.push_back(new pthread_t);
2338 }
2339
2340 ✗ const int retval = pthread_mutex_init(lock_open_files_, NULL);
2341 ✗ assert(retval == 0);
2342
2343 ✗ for (size_t i = 0; i < mountpoints_.size(); ++i) {
2344 ✗ pthread_create(threads[i], nullptr, CollectMountpointsHashes, handlers[i]);
2345 }
2346
2347 ✗ std::vector<bool> joined(handlers.size(), false);
2348 struct timespec reference, current;
2349 ✗ clock_gettime(CLOCK_REALTIME, &reference);
2350 ✗ clock_gettime(CLOCK_REALTIME, &current);
2351 ✗ reference.tv_sec += 10; // Give 10sec for hash collection
2352 ✗ size_t i = 0;
2353 ✗ while (
2354 ✗ (not std::all_of(joined.begin(), joined.end(), [](bool b) { return b; }))
2355 ✗ and a_after_b(reference, current)) {
2356 // as long as there are still threads that haven't joined yet
2357 // and for 10 seconds
2358 ✗ if (not joined[i]) {
2359 ✗ const int s = pthread_tryjoin_np(*threads[i], NULL);
2360 ✗ if (s == 0) {
2361 ✗ joined[i] = true;
2362 }
2363 }
2364 ✗ ++i;
2365 ✗ i = i % handlers.size();
2366 ✗ clock_gettime(CLOCK_REALTIME, &current);
2367 }
2368
2369 ✗ for (size_t i = 0; i < handlers.size(); ++i) {
2370 ✗ delete handlers[i];
2371 }
2372
2373 ✗ pthread_mutex_destroy(lock_open_files_);
2374 #endif
2375 ✗ return open_files_;
2376 }
2377
2378