GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/quota_posix.cc
Date: 2026-03-15 02:35:27
Exec Total Coverage
Lines: 954 1396 68.3%
Branches: 749 1942 38.6%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * This module implements a "managed local cache".
5 * This way, we are able to track access times of files in the cache
6 * and remove files based on least recently used strategy.
7 *
8 * We setup another SQLite catalog, a "cache catalog", that helps us
9 * in the bookkeeping of files, file sizes and access times.
10 *
11 * We might choose to not manage the local cache. This is indicated
12 * by limit == 0 and everything succeeds in that case.
13 */
14
15 #define __STDC_LIMIT_MACROS
16 #define __STDC_FORMAT_MACROS
17
18
19 #include "quota_posix.h"
20
21 #include <dirent.h>
22 #include <errno.h>
23 #include <fcntl.h>
24 #include <inttypes.h>
25 #include <pthread.h>
26 #include <signal.h>
27 #include <stdint.h>
28 #include <sys/dir.h>
29 #include <sys/stat.h>
30 #include <sys/xattr.h>
31
32 #ifndef __APPLE__
33 #include <sys/statfs.h>
34 #endif
35 #include <sys/statvfs.h>
36 #include <sys/types.h>
37 #include <sys/wait.h>
38 #include <unistd.h>
39
40 #include <algorithm>
41 #include <cassert>
42 #include <cstdio>
43 #include <cstdlib>
44 #include <cstring>
45 #include <limits>
46 #include <map>
47 #include <set>
48 #include <string>
49 #include <vector>
50
51 #include "crypto/hash.h"
52 #include "duplex_sqlite3.h"
53 #include "monitor.h"
54 #include "statistics.h"
55 #include "util/capabilities.h"
56 #include "util/concurrency.h"
57 #include "util/exception.h"
58 #include "util/logging.h"
59 #include "util/platform.h"
60 #include "util/pointer.h"
61 #include "util/posix.h"
62 #include "util/smalloc.h"
63 #include "util/string.h"
64
65 using namespace std; // NOLINT
66
67
68 3772 int PosixQuotaManager::BindReturnPipe(int pipe_wronly) {
69
2/2
✓ Branch 0 taken 3690 times.
✓ Branch 1 taken 82 times.
 3772 if (!shared_)
70 3690 return pipe_wronly;
71
72 // Connect writer's end
73
1/2
✓ Branch 2 taken 82 times.
✗ Branch 3 not taken.
 82 const int result = open(
74
2/4
✓ Branch 2 taken 82 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 82 times.
✗ Branch 6 not taken.
 164 (workspace_dir_ + "/pipe" + StringifyInt(pipe_wronly)).c_str(),
75 O_WRONLY | O_NONBLOCK);
76
2/2
✓ Branch 0 taken 41 times.
✓ Branch 1 taken 41 times.
 82 if (result >= 0) {
77 41 Nonblock2Block(result);
78 } else {
79 41 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
80 41 "failed to bind return pipe (%d)", errno);
81 }
82 82 return result;
83 }
84
85
86 1084 void PosixQuotaManager::CheckHighPinWatermark() {
87 1084 const uint64_t watermark = kHighPinWatermark * cleanup_threshold_ / 100;
88
3/4
✓ Branch 0 taken 1084 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 123 times.
✓ Branch 3 taken 961 times.
 1084 if ((cleanup_threshold_ > 0) && (pinned_ > watermark)) {
89 123 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
90 "high watermark of pinned files (%" PRIu64 "M > %" PRIu64 "M)",
91 123 pinned_ / (1024 * 1024), watermark / (1024 * 1024));
92
2/4
✓ Branch 2 taken 123 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 123 times.
✗ Branch 6 not taken.
 123 BroadcastBackchannels("R"); // clients: please release pinned catalogs
93 }
94 1084 }
95
96
97 ✗ void PosixQuotaManager::CleanupPipes() {
98 ✗ DIR *dirp = opendir(workspace_dir_.c_str());
99 ✗ assert(dirp != NULL);
100
101 platform_dirent64 *dent;
102 ✗ bool found_leftovers = false;
103 ✗ while ((dent = platform_readdir(dirp)) != NULL) {
104 ✗ const string name = dent->d_name;
105 ✗ const string path = workspace_dir_ + "/" + name;
106 platform_stat64 info;
107 ✗ const int retval = platform_stat(path.c_str(), &info);
108 ✗ if (retval != 0)
109 ✗ continue;
110 ✗ if (S_ISFIFO(info.st_mode) && (name.substr(0, 4) == "pipe")) {
111 ✗ if (!found_leftovers) {
112 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogWarn,
113 "removing left-over FIFOs from cache directory");
114 }
115 ✗ found_leftovers = true;
116 ✗ unlink(path.c_str());
117 }
118 }
119 ✗ closedir(dirp);
120 }
121
122
123 /**
124 * Cleans up in data cache, until cache size is below leave_size.
125 * The actual unlinking is done in a separate process (fork).
126 *
127 * \return True on success, false otherwise
128 */
129 369 bool PosixQuotaManager::Cleanup(const uint64_t leave_size) {
130
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 369 times.
 369 if (!spawned_)
131 ✗ return DoCleanup(leave_size);
132
133 bool result;
134 int pipe_cleanup[2];
135
1/2
✓ Branch 1 taken 369 times.
✗ Branch 2 not taken.
 369 MakeReturnPipe(pipe_cleanup);
136
137 369 LruCommand cmd;
138 369 cmd.command_type = kCleanup;
139 369 cmd.size = leave_size;
140 369 cmd.return_pipe = pipe_cleanup[1];
141
142
1/2
✓ Branch 1 taken 369 times.
✗ Branch 2 not taken.
 369 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
143
1/2
✓ Branch 1 taken 369 times.
✗ Branch 2 not taken.
 369 ManagedReadHalfPipe(pipe_cleanup[0], &result, sizeof(result));
144
1/2
✓ Branch 1 taken 369 times.
✗ Branch 2 not taken.
 369 CloseReturnPipe(pipe_cleanup);
145
146 369 return result;
147 }
148
149
150 3669 void PosixQuotaManager::CloseDatabase() {
151
1/2
✓ Branch 0 taken 3669 times.
✗ Branch 1 not taken.
 3669 if (stmt_list_catalogs_)
152 3669 sqlite3_finalize(stmt_list_catalogs_);
153
1/2
✓ Branch 0 taken 3669 times.
✗ Branch 1 not taken.
 3669 if (stmt_list_pinned_)
154 3669 sqlite3_finalize(stmt_list_pinned_);
155
1/2
✓ Branch 0 taken 3669 times.
✗ Branch 1 not taken.
 3669 if (stmt_list_volatile_)
156 3669 sqlite3_finalize(stmt_list_volatile_);
157
1/2
✓ Branch 0 taken 3669 times.
✗ Branch 1 not taken.
 3669 if (stmt_list_)
158 3669 sqlite3_finalize(stmt_list_);
159
1/2
✓ Branch 0 taken 3669 times.
✗ Branch 1 not taken.
 3669 if (stmt_lru_)
160 3669 sqlite3_finalize(stmt_lru_);
161
1/2
✓ Branch 0 taken 3669 times.
✗ Branch 1 not taken.
 3669 if (stmt_rm_)
162 3669 sqlite3_finalize(stmt_rm_);
163
1/2
✓ Branch 0 taken 3669 times.
✗ Branch 1 not taken.
 3669 if (stmt_rm_batch_)
164 3669 sqlite3_finalize(stmt_rm_batch_);
165
1/2
✓ Branch 0 taken 3669 times.
✗ Branch 1 not taken.
 3669 if (stmt_size_)
166 3669 sqlite3_finalize(stmt_size_);
167
1/2
✓ Branch 0 taken 3669 times.
✗ Branch 1 not taken.
 3669 if (stmt_touch_)
168 3669 sqlite3_finalize(stmt_touch_);
169
1/2
✓ Branch 0 taken 3669 times.
✗ Branch 1 not taken.
 3669 if (stmt_unpin_)
170 3669 sqlite3_finalize(stmt_unpin_);
171
1/2
✓ Branch 0 taken 3669 times.
✗ Branch 1 not taken.
 3669 if (stmt_block_)
172 3669 sqlite3_finalize(stmt_block_);
173
1/2
✓ Branch 0 taken 3669 times.
✗ Branch 1 not taken.
 3669 if (stmt_unblock_)
174 3669 sqlite3_finalize(stmt_unblock_);
175
1/2
✓ Branch 0 taken 3669 times.
✗ Branch 1 not taken.
 3669 if (stmt_new_)
176 3669 sqlite3_finalize(stmt_new_);
177
1/2
✓ Branch 0 taken 3669 times.
✗ Branch 1 not taken.
 3669 if (database_)
178 3669 sqlite3_close(database_);
179 3669 UnlockFile(fd_lock_cachedb_);
180
181 3669 stmt_list_catalogs_ = NULL;
182 3669 stmt_list_pinned_ = NULL;
183 3669 stmt_list_volatile_ = NULL;
184 3669 stmt_list_ = NULL;
185 3669 stmt_rm_ = NULL;
186 3669 stmt_rm_batch_ = NULL;
187 3669 stmt_size_ = NULL;
188 3669 stmt_touch_ = NULL;
189 3669 stmt_unpin_ = NULL;
190 3669 stmt_block_ = NULL;
191 3669 stmt_unblock_ = NULL;
192 3669 stmt_new_ = NULL;
193 3669 database_ = NULL;
194
195 3669 pinned_chunks_.clear();
196 3669 }
197
198
199 3567 void PosixQuotaManager::CloseReturnPipe(int pipe[2]) {
200
2/2
✓ Branch 0 taken 82 times.
✓ Branch 1 taken 3485 times.
 3567 if (shared_) {
201 82 close(pipe[0]);
202 82 UnlinkReturnPipe(pipe[1]);
203 } else {
204 3485 ClosePipe(pipe);
205 }
206 3567 }
207
208
209 4102648 bool PosixQuotaManager::Contains(const string &hash_str) {
210 4102648 bool result = false;
211
212 4102648 sqlite3_bind_text(stmt_size_, 1, &hash_str[0], hash_str.length(),
213 SQLITE_STATIC);
214
2/2
✓ Branch 1 taken 785 times.
✓ Branch 2 taken 4101863 times.
 4102648 if (sqlite3_step(stmt_size_) == SQLITE_ROW)
215 785 result = true;
216 4102648 sqlite3_reset(stmt_size_);
217 4102648 LogCvmfs(kLogQuota, kLogDebug, "contains %s returns %d", hash_str.c_str(),
218 result);
219
220 4102648 return result;
221 }
222
223
224 3588 void PosixQuotaManager::CheckFreeSpace() {
225
3/4
✓ Branch 0 taken 3588 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 41 times.
✓ Branch 3 taken 3547 times.
 3588 if ((limit_ == 0) || (gauge_ >= limit_))
226 41 return;
227
228 struct statvfs vfs_info;
229
1/2
✓ Branch 1 taken 3547 times.
✗ Branch 2 not taken.
 3547 const int retval = statvfs((cache_dir_ + "/cachedb").c_str(), &vfs_info);
230
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3547 times.
 3547 if (retval != 0) {
231 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
232 "failed to query %s for free space (%d)", cache_dir_.c_str(),
233 ✗ errno);
234 ✗ return;
235 }
236 3547 const int64_t free_space_byte = vfs_info.f_bavail * vfs_info.f_bsize;
237
1/2
✓ Branch 1 taken 3547 times.
✗ Branch 2 not taken.
 3547 LogCvmfs(kLogQuota, kLogDebug, "free space: %" PRId64 " MB",
238 free_space_byte / (1024 * 1024));
239
240 3547 const int64_t required_byte = limit_ - gauge_;
241
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3547 times.
 3547 if (free_space_byte < required_byte) {
242 ✗ LogCvmfs(kLogQuota, kLogSyslogWarn,
243 "too little free space on the file system hosting the cache,"
244 " %" PRId64 " MB available",
245 free_space_byte / (1024 * 1024));
246 }
247 }
248
249
250 3711 PosixQuotaManager *PosixQuotaManager::Create(const string &cache_workspace,
251 const uint64_t limit,
252 const uint64_t cleanup_threshold,
253 const bool rebuild_database) {
254
2/2
✓ Branch 0 taken 82 times.
✓ Branch 1 taken 3629 times.
 3711 if (cleanup_threshold >= limit) {
255 82 LogCvmfs(kLogQuota, kLogDebug,
256 "invalid parameters: limit %" PRIu64 ", "
257 "cleanup_threshold %" PRIu64,
258 limit, cleanup_threshold);
259 82 return NULL;
260 }
261
262 PosixQuotaManager *quota_manager = new PosixQuotaManager(
263
1/2
✓ Branch 2 taken 3629 times.
✗ Branch 3 not taken.
 3629 limit, cleanup_threshold, cache_workspace);
264
265 // Initialize cache catalog
266
2/2
✓ Branch 1 taken 41 times.
✓ Branch 2 taken 3588 times.
 3629 if (!quota_manager->InitDatabase(rebuild_database)) {
267
1/2
✓ Branch 0 taken 41 times.
✗ Branch 1 not taken.
 41 delete quota_manager;
268 41 return NULL;
269 }
270 3588 quota_manager->CheckFreeSpace();
271 3588 MakePipe(quota_manager->pipe_lru_);
272
273 3588 quota_manager->protocol_revision_ = kProtocolRevision;
274 3588 quota_manager->initialized_ = true;
275 3588 return quota_manager;
276 }
277
278
279 /**
280 * Connects to a running shared local quota manager. Creates one if necessary.
281 */
282 82 PosixQuotaManager *PosixQuotaManager::CreateShared(
283 const std::string &exe_path,
284 const std::string &cache_workspace,
285 const uint64_t limit,
286 const uint64_t cleanup_threshold,
287 bool foreground) {
288 82 string cache_dir;
289 82 string workspace_dir;
290
2/4
✓ Branch 1 taken 82 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 82 times.
✗ Branch 5 not taken.
 82 ParseDirectories(cache_workspace, &cache_dir, &workspace_dir);
291
292 pid_t new_cachemgr_pid;
293
294 // Create lock file: only one fuse client at a time
295
2/4
✓ Branch 1 taken 82 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 82 times.
✗ Branch 5 not taken.
 82 const int fd_lockfile = LockFile(workspace_dir + "/lock_cachemgr");
296
2/2
✓ Branch 0 taken 41 times.
✓ Branch 1 taken 41 times.
 82 if (fd_lockfile < 0) {
297
1/2
✓ Branch 2 taken 41 times.
✗ Branch 3 not taken.
 41 LogCvmfs(kLogQuota, kLogDebug, "could not open lock file %s (%d)",
298
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 82 (workspace_dir + "/lock_cachemgr").c_str(), errno);
299 41 return NULL;
300 }
301
302 PosixQuotaManager *quota_mgr = new PosixQuotaManager(limit, cleanup_threshold,
303
2/4
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 41 times.
✗ Branch 5 not taken.
 41 cache_workspace);
304 41 quota_mgr->shared_ = true;
305 41 quota_mgr->spawned_ = true;
306
307 // Try to connect to pipe
308
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 const string fifo_path = workspace_dir + "/cachemgr";
309
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 LogCvmfs(kLogQuota, kLogDebug, "trying to connect to existing pipe");
310
1/2
✓ Branch 2 taken 41 times.
✗ Branch 3 not taken.
 41 quota_mgr->pipe_lru_[1] = open(fifo_path.c_str(), O_WRONLY | O_NONBLOCK);
311
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 41 times.
 41 if (quota_mgr->pipe_lru_[1] >= 0) {
312 ✗ const int fd_lockfile_rw = open((workspace_dir + "/lock_cachemgr").c_str(),
313 O_RDWR, 0600);
314 ✗ unsigned lockfile_magicnumber = 0;
315 ✗ const ssize_t result_mn = SafeRead(fd_lockfile_rw, &lockfile_magicnumber,
316 sizeof(lockfile_magicnumber));
317 ✗ const ssize_t result = SafeRead(fd_lockfile_rw, &new_cachemgr_pid,
318 sizeof(new_cachemgr_pid));
319 ✗ close(fd_lockfile_rw);
320
321 ✗ if ((lockfile_magicnumber != kLockFileMagicNumber) || (result < 0)
322 ✗ || (result_mn < 0)
323 ✗ || (static_cast<size_t>(result) < sizeof(new_cachemgr_pid))) {
324 ✗ if (result != 0) {
325 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
326 "could not read cache manager pid from lockfile");
327 ✗ UnlockFile(fd_lockfile);
328 ✗ delete quota_mgr;
329 ✗ return NULL;
330 } else {
331 // support reload from old versions of the cache manager
332 // lock file is empty in this case, try a plain ReadHalfPipe to get pid
333 ✗ quota_mgr->SetCacheMgrPid(quota_mgr->GetPid());
334 }
335 } else {
336 ✗ quota_mgr->SetCacheMgrPid(new_cachemgr_pid);
337 }
338
339
340 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to existing cache manager pipe");
341 ✗ quota_mgr->initialized_ = true;
342 ✗ Nonblock2Block(quota_mgr->pipe_lru_[1]);
343 ✗ UnlockFile(fd_lockfile);
344 ✗ quota_mgr->GetLimits(&quota_mgr->limit_, &quota_mgr->cleanup_threshold_);
345 ✗ LogCvmfs(kLogQuota, kLogDebug,
346 "received limit %" PRIu64 ", threshold %" PRIu64,
347 quota_mgr->limit_, quota_mgr->cleanup_threshold_);
348 ✗ if (FileExists(workspace_dir + "/cachemgr.protocol")) {
349 ✗ quota_mgr->protocol_revision_ = quota_mgr->GetProtocolRevision();
350 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected protocol revision %u",
351 quota_mgr->protocol_revision_);
352 } else {
353 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to ancient cache manager");
354 }
355 ✗ return quota_mgr;
356 }
357 41 const int connect_error = errno;
358
359 // Lock file: let existing cache manager finish first
360
2/4
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 41 times.
✗ Branch 5 not taken.
 41 const int fd_lockfile_fifo = LockFile(workspace_dir + "/lock_cachemgr.fifo");
361
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 41 times.
 41 if (fd_lockfile_fifo < 0) {
362 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not open lock file %s (%d)",
363 ✗ (workspace_dir + "/lock_cachemgr.fifo").c_str(), errno);
364 ✗ UnlockFile(fd_lockfile);
365 ✗ delete quota_mgr;
366 ✗ return NULL;
367 }
368
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 UnlockFile(fd_lockfile_fifo);
369
370
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 41 times.
 41 if (connect_error == ENXIO) {
371 ✗ LogCvmfs(kLogQuota, kLogDebug, "left-over FIFO found, unlinking");
372 ✗ unlink(fifo_path.c_str());
373 }
374
375 // Creating a new FIFO for the cache manager (to be bound later)
376 41 int retval = mkfifo(fifo_path.c_str(), 0600);
377
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 41 times.
 41 if (retval != 0) {
378 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to create cache manager FIFO (%d)",
379 ✗ errno);
380 ✗ UnlockFile(fd_lockfile);
381 ✗ delete quota_mgr;
382 ✗ return NULL;
383 }
384
385 // Create new cache manager
386 int pipe_boot[2];
387 int pipe_handshake[2];
388
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 MakePipe(pipe_boot);
389
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 MakePipe(pipe_handshake);
390
391 41 vector<string> command_line;
392
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 command_line.push_back(exe_path);
393
2/4
✓ Branch 2 taken 41 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 41 times.
✗ Branch 6 not taken.
 41 command_line.push_back("__cachemgr__");
394
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 command_line.push_back(cache_workspace);
395
2/4
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 41 times.
✗ Branch 5 not taken.
 41 command_line.push_back(StringifyInt(pipe_boot[1]));
396
2/4
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 41 times.
✗ Branch 5 not taken.
 41 command_line.push_back(StringifyInt(pipe_handshake[0]));
397
2/4
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 41 times.
✗ Branch 5 not taken.
 41 command_line.push_back(StringifyInt(limit));
398
2/4
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 41 times.
✗ Branch 5 not taken.
 41 command_line.push_back(StringifyInt(cleanup_threshold));
399 // do not propagate foreground in order to reliably get pid from exec
400 // instead, daemonize right here
401
2/4
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 41 times.
✗ Branch 5 not taken.
 41 command_line.push_back(StringifyInt(true)); // foreground
402
3/6
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 41 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 41 times.
✗ Branch 8 not taken.
 41 command_line.push_back(StringifyInt(GetLogSyslogLevel()));
403
3/6
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 41 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 41 times.
✗ Branch 8 not taken.
 41 command_line.push_back(StringifyInt(GetLogSyslogFacility()));
404
5/14
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 41 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 41 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 41 times.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 41 times.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
 41 command_line.push_back(GetLogDebugFile() + ":" + GetLogMicroSyslog());
405
406 41 set<int> preserve_filedes;
407
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 preserve_filedes.insert(0);
408
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 preserve_filedes.insert(1);
409
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 preserve_filedes.insert(2);
410
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 preserve_filedes.insert(pipe_boot[1]);
411
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 preserve_filedes.insert(pipe_handshake[0]);
412
413
1/2
✓ Branch 0 taken 41 times.
✗ Branch 1 not taken.
 41 if (foreground) {
414
1/2
✓ Branch 2 taken 41 times.
✗ Branch 3 not taken.
 41 retval = ManagedExec(command_line, preserve_filedes, map<int, int>(),
415 /*drop_credentials*/ false,
416 /*clear_env*/ false,
417 /*double_fork*/ true, &new_cachemgr_pid);
418 } else {
419 ✗ retval = ExecAsDaemon(command_line, &new_cachemgr_pid);
420 }
421
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 41 times.
 41 if (!retval) {
422 ✗ UnlockFile(fd_lockfile);
423 ✗ ClosePipe(pipe_boot);
424 ✗ ClosePipe(pipe_handshake);
425 ✗ delete quota_mgr;
426 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to start cache manager");
427 ✗ return NULL;
428 }
429
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 LogCvmfs(kLogQuota, kLogDebug, "new cache manager pid: %d", new_cachemgr_pid);
430 41 quota_mgr->SetCacheMgrPid(new_cachemgr_pid);
431
2/4
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 41 times.
✗ Branch 6 not taken.
 41 const int fd_lockfile_rw = open((workspace_dir + "/lock_cachemgr").c_str(),
432 O_RDWR | O_TRUNC, 0600);
433 41 const unsigned magic_number = PosixQuotaManager::kLockFileMagicNumber;
434
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 const bool result_mn = SafeWrite(fd_lockfile_rw, &magic_number,
435 sizeof(magic_number));
436
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 const bool result = SafeWrite(fd_lockfile_rw, &new_cachemgr_pid,
437 sizeof(new_cachemgr_pid));
438
2/4
✓ Branch 0 taken 41 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 41 times.
 41 if (!result || !result_mn) {
439 ✗ PANIC(kLogSyslogErr, "could not write cache manager pid to lockfile");
440 }
441
442
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 close(fd_lockfile_rw);
443 // Wait for cache manager to be ready
444
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 close(pipe_boot[1]);
445
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 close(pipe_handshake[0]);
446 char buf;
447
2/4
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 41 times.
✗ Branch 4 not taken.
 41 if (read(pipe_boot[0], &buf, 1) != 1) {
448
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 UnlockFile(fd_lockfile);
449
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 close(pipe_boot[0]);
450
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 close(pipe_handshake[1]);
451
1/2
✓ Branch 0 taken 41 times.
✗ Branch 1 not taken.
 41 delete quota_mgr;
452
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
453 "cache manager did not start");
454 41 return NULL;
455 }
456 ✗ close(pipe_boot[0]);
457
458 // Connect write end
459 ✗ quota_mgr->pipe_lru_[1] = open(fifo_path.c_str(), O_WRONLY | O_NONBLOCK);
460 ✗ if (quota_mgr->pipe_lru_[1] < 0) {
461 ✗ LogCvmfs(kLogQuota, kLogDebug,
462 ✗ "failed to connect to newly created FIFO (%d)", errno);
463 ✗ close(pipe_handshake[1]);
464 ✗ UnlockFile(fd_lockfile);
465 ✗ delete quota_mgr;
466 ✗ return NULL;
467 }
468
469 // Finalize handshake
470 ✗ buf = 'C';
471 ✗ if (write(pipe_handshake[1], &buf, 1) != 1) {
472 ✗ UnlockFile(fd_lockfile);
473 ✗ close(pipe_handshake[1]);
474 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not finalize handshake");
475 ✗ delete quota_mgr;
476 ✗ return NULL;
477 }
478 ✗ close(pipe_handshake[1]);
479
480 ✗ Nonblock2Block(quota_mgr->pipe_lru_[1]);
481 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to a new cache manager");
482 ✗ quota_mgr->protocol_revision_ = kProtocolRevision;
483
484 ✗ UnlockFile(fd_lockfile);
485
486 ✗ quota_mgr->initialized_ = true;
487 ✗ quota_mgr->GetLimits(&quota_mgr->limit_, &quota_mgr->cleanup_threshold_);
488 ✗ LogCvmfs(kLogQuota, kLogDebug,
489 "received limit %" PRIu64 ", "
490 "threshold %" PRIu64,
491 quota_mgr->limit_, quota_mgr->cleanup_threshold_);
492 ✗ return quota_mgr;
493 82 }
494
495
496 410 bool PosixQuotaManager::DoCleanup(const uint64_t leave_size) {
497
2/2
✓ Branch 0 taken 82 times.
✓ Branch 1 taken 328 times.
 410 if (gauge_ <= leave_size)
498 82 return true;
499
500 // TODO(jblomer) transaction
501
1/2
✓ Branch 1 taken 328 times.
✗ Branch 2 not taken.
 328 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug,
502 "clean up cache until at most %lu KB is used", leave_size / 1024);
503
1/2
✓ Branch 1 taken 328 times.
✗ Branch 2 not taken.
 328 LogCvmfs(kLogQuota, kLogDebug, "gauge %" PRIu64, gauge_);
504
1/2
✓ Branch 1 taken 328 times.
✗ Branch 2 not taken.
 328 cleanup_recorder_.Tick();
505
506 bool result;
507 328 vector<string> trash;
508
509 // Note that volatile files start counting from the smallest int64 number:
510 // the absolute sequence number with the first bit set in two's complement.
511 // So -1 can be a marker that will never appear in the database.
512 328 int64_t max_acseq = -1;
513 328 std::vector<EvictCandidate> lru_ordered_open;
514
515 do {
516
1/2
✓ Branch 1 taken 2378 times.
✗ Branch 2 not taken.
 2378 sqlite3_reset(stmt_lru_);
517
3/4
✓ Branch 0 taken 328 times.
✓ Branch 1 taken 2050 times.
✓ Branch 3 taken 2378 times.
✗ Branch 4 not taken.
 2706 sqlite3_bind_int64(stmt_lru_, 1,
518 328 (max_acseq == -1) ? std::numeric_limits<int64_t>::min()
519 : (max_acseq + 1));
520
521 2378 std::vector<EvictCandidate> candidates;
522
1/2
✓ Branch 1 taken 2378 times.
✗ Branch 2 not taken.
 2378 candidates.reserve(kEvictBatchSize);
523 2378 string hash_str;
524 2378 unsigned i = 0;
525
3/4
✓ Branch 1 taken 2134788 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2132410 times.
✓ Branch 4 taken 2378 times.
 2134788 while (sqlite3_step(stmt_lru_) == SQLITE_ROW) {
526 hash_str = reinterpret_cast<const char *>(
527
2/4
✓ Branch 1 taken 2132410 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2132410 times.
✗ Branch 5 not taken.
 2132410 sqlite3_column_text(stmt_lru_, 0));
528
1/2
✓ Branch 2 taken 2132410 times.
✗ Branch 3 not taken.
 2132410 LogCvmfs(kLogQuota, kLogDebug, "add %s to candidates for eviction",
529 hash_str.c_str());
530
1/2
✓ Branch 1 taken 2132410 times.
✗ Branch 2 not taken.
 2132410 candidates.push_back(
531
1/2
✓ Branch 1 taken 2132410 times.
✗ Branch 2 not taken.
 2132410 EvictCandidate(shash::MkFromHexPtr(shash::HexPtr(hash_str)),
532 2132410 sqlite3_column_int64(stmt_lru_, 1),
533
2/4
✓ Branch 1 taken 2132410 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2132410 times.
✗ Branch 5 not taken.
 2132410 sqlite3_column_int64(stmt_lru_, 2)));
534 2132410 i++;
535 }
536
2/2
✓ Branch 1 taken 41 times.
✓ Branch 2 taken 2337 times.
 2378 if (candidates.empty()) {
537
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 LogCvmfs(kLogQuota, kLogDebug, "no more entries to evict");
538 41 break;
539 }
540
541 2337 const unsigned N = candidates.size();
542
543 2337 open_files_.clear();
544
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2337 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 4674 open_files_ = (cleanup_unused_first_) ? CollectAllOpenHashes()
545 2337 : std::vector<shash::Short>();
546
547
2/2
✓ Branch 0 taken 2050369 times.
✓ Branch 1 taken 2050 times.
 2052419 for (i = 0; i < N; ++i) {
548 // That's a critical condition. We must not delete a not yet inserted
549 // pinned file as it is already reserved (but will be inserted later).
550 // Instead, set the pin bit in the db to not run into an endless loop
551
1/2
✓ Branch 2 taken 2050369 times.
✗ Branch 3 not taken.
 2050369 const bool is_pinned = pinned_chunks_.find(candidates[i].hash)
552 4100738 != pinned_chunks_.end();
553
554 // Avoid evicting open files hopping there are enough more recently used
555 // files to satisfy the cleanup request
556 /*
557 const bool is_open = std::find_if(
558 open_files_.begin(), open_files_.end(),
559 [&candidates, &i](const auto &elem) -> bool {
560 return elem.Collide(candidates[i].hash);
561 })
562 != open_files_.end();
563 */
564 2050369 bool is_open=false;
565
1/2
✗ Branch 4 not taken.
✓ Branch 5 taken 2050369 times.
 2050369 for(auto it=open_files_.begin();it!=open_files_.end();++it){
566 ✗ if (it->Collide(candidates[i].hash)){is_open=true; break;}
567 }
568
569
2/2
✓ Branch 0 taken 41 times.
✓ Branch 1 taken 2050328 times.
 2050369 if (is_pinned) {
570
1/2
✓ Branch 2 taken 41 times.
✗ Branch 3 not taken.
 41 SkipEviction(candidates[i]);
571 41 continue;
572 }
573
574
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2050328 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 2050328 if (cleanup_unused_first_ and is_open) {
575 ✗ SkipEviction(candidates[i]);
576 ✗ lru_ordered_open.push_back(candidates[i]);
577 ✗ continue;
578 }
579
580
2/4
✓ Branch 1 taken 2050328 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2050328 times.
✗ Branch 5 not taken.
 4100656 trash.push_back(cache_dir_ + "/"
581
2/4
✓ Branch 2 taken 2050328 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2050328 times.
✗ Branch 6 not taken.
 6150984 + candidates[i].hash.MakePathWithoutSuffix());
582 2050328 gauge_ -= candidates[i].size;
583 2050328 max_acseq = candidates[i].acseq;
584
1/2
✓ Branch 2 taken 2050328 times.
✗ Branch 3 not taken.
 2050328 LogCvmfs(kLogQuota, kLogDebug, "lru cleanup %s, new gauge %" PRIu64,
585
1/2
✓ Branch 2 taken 2050328 times.
✗ Branch 3 not taken.
 4100656 candidates[i].hash.ToString().c_str(), gauge_);
586
587
2/2
✓ Branch 0 taken 287 times.
✓ Branch 1 taken 2050041 times.
 2050328 if (gauge_ <= leave_size)
588 287 break;
589 }
590
6/6
✓ Branch 1 taken 2337 times.
✓ Branch 2 taken 41 times.
✓ Branch 4 taken 2337 times.
✓ Branch 5 taken 41 times.
✓ Branch 6 taken 2050 times.
✓ Branch 7 taken 287 times.
 4756 } while (gauge_ > leave_size);
591
592
1/2
✓ Branch 0 taken 328 times.
✗ Branch 1 not taken.
 328 if (max_acseq != -1) {
593
1/2
✓ Branch 1 taken 328 times.
✗ Branch 2 not taken.
 328 sqlite3_bind_int64(stmt_rm_batch_, 1, max_acseq);
594
1/2
✓ Branch 1 taken 328 times.
✗ Branch 2 not taken.
 328 result = (sqlite3_step(stmt_rm_batch_) == SQLITE_DONE);
595
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 328 times.
 328 assert(result);
596
1/2
✓ Branch 1 taken 328 times.
✗ Branch 2 not taken.
 328 sqlite3_reset(stmt_rm_batch_);
597
598
1/2
✓ Branch 1 taken 328 times.
✗ Branch 2 not taken.
 328 result = (sqlite3_step(stmt_unblock_) == SQLITE_DONE);
599
1/2
✓ Branch 1 taken 328 times.
✗ Branch 2 not taken.
 328 sqlite3_reset(stmt_unblock_);
600
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 328 times.
 328 assert(result);
601 }
602
603
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 328 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 328 times.
 328 while (!lru_ordered_open.empty() and gauge_ > leave_size) {
604 // cleanup files in use
605 ✗ auto &candidate = lru_ordered_open[0];
606 ✗ trash.push_back(cache_dir_ + "/" + candidate.hash.MakePathWithoutSuffix());
607 ✗ gauge_ -= candidate.size;
608 ✗ max_acseq = candidate.acseq;
609 ✗ LogCvmfs(kLogQuota, kLogDebug, "lru cleanup %s, new gauge %" PRIu64,
610 ✗ candidate.hash.ToString().c_str(), gauge_);
611 ✗ lru_ordered_open.erase(lru_ordered_open.begin());
612 }
613
614
2/4
✓ Branch 1 taken 328 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 328 times.
 328 if (!EmptyTrash(trash))
615 ✗ return false;
616
617
2/2
✓ Branch 0 taken 41 times.
✓ Branch 1 taken 287 times.
 328 if (gauge_ > leave_size) {
618
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
619 "request to clean until %" PRIu64 ", "
620 "but effective gauge is %" PRIu64,
621 leave_size, gauge_);
622 41 return false;
623 }
624 287 return true;
625 328 }
626
627 328 bool PosixQuotaManager::EmptyTrash(const std::vector<std::string> &trash) {
628
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 328 times.
 328 if (trash.empty())
629 ✗ return true;
630
631
2/2
✓ Branch 0 taken 246 times.
✓ Branch 1 taken 82 times.
 328 if (async_delete_) {
632 // Double fork avoids zombie, forked removal process must not flush file
633 // buffers
634 pid_t pid;
635 int statloc;
636
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 246 times.
 246 if ((pid = fork()) == 0) {
637 // TODO(jblomer): eviciting files in the cache should perhaps become a
638 // thread. This would also allow to block the chunks and prevent the
639 // race with re-insertion. Then again, a thread can block umount.
640 #ifndef DEBUGMSG
641 ✗ CloseAllFildes(std::set<int>());
642 #endif
643 ✗ if (fork() == 0) {
644 ✗ for (unsigned i = 0, iEnd = trash.size(); i < iEnd; ++i) {
645 ✗ LogCvmfs(kLogQuota, kLogDebug, "unlink %s", trash[i].c_str());
646 ✗ unlink(trash[i].c_str());
647 }
648 ✗ _exit(0);
649 }
650 ✗ _exit(0);
651 } else {
652
1/2
✓ Branch 0 taken 246 times.
✗ Branch 1 not taken.
 246 if (pid > 0)
653
1/2
✓ Branch 1 taken 246 times.
✗ Branch 2 not taken.
 246 waitpid(pid, &statloc, 0);
654 else
655 ✗ return false;
656 }
657 } else { // !async_delete_
658
2/2
✓ Branch 1 taken 123 times.
✓ Branch 2 taken 82 times.
 205 for (unsigned i = 0, iEnd = trash.size(); i < iEnd; ++i) {
659 123 LogCvmfs(kLogQuota, kLogDebug, "unlink %s", trash[i].c_str());
660 123 unlink(trash[i].c_str());
661 }
662 }
663 328 return true;
664 }
665
666
667 4101734 void PosixQuotaManager::DoInsert(const shash::Any &hash,
668 const uint64_t size,
669 const string &description,
670 const CommandType command_type) {
671
1/2
✓ Branch 1 taken 4101734 times.
✗ Branch 2 not taken.
 4101734 const string hash_str = hash.ToString();
672
1/2
✓ Branch 3 taken 4101734 times.
✗ Branch 4 not taken.
 4101734 LogCvmfs(kLogQuota, kLogDebug, "insert into lru %s, path %s, method %d",
673 hash_str.c_str(), description.c_str(), command_type);
674 4101734 const unsigned desc_length = (description.length() > kMaxDescription)
675 ? kMaxDescription
676
1/2
✓ Branch 0 taken 4101734 times.
✗ Branch 1 not taken.
 4101734 : description.length();
677
678 LruCommand *cmd = reinterpret_cast<LruCommand *>(
679 4101734 alloca(sizeof(LruCommand) + desc_length));
680 4101734 new (cmd) LruCommand;
681 4101734 cmd->command_type = command_type;
682 4101734 cmd->SetSize(size);
683
1/2
✓ Branch 1 taken 4101734 times.
✗ Branch 2 not taken.
 4101734 cmd->StoreHash(hash);
684 4101734 cmd->desc_length = desc_length;
685 4101734 memcpy(reinterpret_cast<char *>(cmd) + sizeof(LruCommand), &description[0],
686 desc_length);
687
1/2
✓ Branch 1 taken 4101734 times.
✗ Branch 2 not taken.
 4101734 WritePipe(pipe_lru_[1], cmd, sizeof(LruCommand) + desc_length);
688 4101734 }
689
690
691 1435 vector<string> PosixQuotaManager::DoList(const CommandType list_command) {
692 1435 vector<string> result;
693
694 int pipe_list[2];
695
1/2
✓ Branch 1 taken 1435 times.
✗ Branch 2 not taken.
 1435 MakeReturnPipe(pipe_list);
696 char description_buffer[kMaxDescription];
697
698 1435 LruCommand cmd;
699 1435 cmd.command_type = list_command;
700 1435 cmd.return_pipe = pipe_list[1];
701
1/2
✓ Branch 1 taken 1435 times.
✗ Branch 2 not taken.
 1435 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
702
703 int length;
704 do {
705
1/2
✓ Branch 1 taken 4103034 times.
✗ Branch 2 not taken.
 4103034 ManagedReadHalfPipe(pipe_list[0], &length, sizeof(length));
706
2/2
✓ Branch 0 taken 4101599 times.
✓ Branch 1 taken 1435 times.
 4103034 if (length > 0) {
707
1/2
✓ Branch 1 taken 4101599 times.
✗ Branch 2 not taken.
 4101599 ReadPipe(pipe_list[0], description_buffer, length);
708
2/4
✓ Branch 2 taken 4101599 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 4101599 times.
✗ Branch 6 not taken.
 4101599 result.push_back(string(description_buffer, length));
709 }
710
2/2
✓ Branch 0 taken 4101599 times.
✓ Branch 1 taken 1435 times.
 4103034 } while (length >= 0);
711
712
1/2
✓ Branch 1 taken 1435 times.
✗ Branch 2 not taken.
 1435 CloseReturnPipe(pipe_list);
713 2870 return result;
714 }
715
716
717 1456 uint64_t PosixQuotaManager::GetCapacity() {
718
1/2
✓ Branch 0 taken 1456 times.
✗ Branch 1 not taken.
 1456 if (limit_ != (uint64_t)(-1))
719 1456 return limit_;
720
721 // Unrestricted cache, look at free space on cache dir fs
722 struct statfs info;
723 ✗ if (statfs(".", &info) == 0) {
724 ✗ return info.f_bavail * info.f_bsize;
725 } else {
726 ✗ LogCvmfs(kLogQuota, kLogSyslogErr | kLogDebug,
727 ✗ "failed to query file system info of cache (%d)", errno);
728 ✗ return limit_;
729 }
730 }
731
732
733 ✗ void PosixQuotaManager::GetLimits(uint64_t *limit,
734 uint64_t *cleanup_threshold) {
735 int pipe_limits[2];
736 ✗ MakeReturnPipe(pipe_limits);
737
738 ✗ LruCommand cmd;
739 ✗ cmd.command_type = kLimits;
740 ✗ cmd.return_pipe = pipe_limits[1];
741 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
742 ✗ ManagedReadHalfPipe(pipe_limits[0], limit, sizeof(*limit));
743 ✗ ReadPipe(pipe_limits[0], cleanup_threshold, sizeof(*cleanup_threshold));
744 ✗ CloseReturnPipe(pipe_limits);
745 }
746
747
748 /**
749 * Since we only cleanup until cleanup_threshold, we can only add
750 * files smaller than limit-cleanup_threshold.
751 */
752 293 uint64_t PosixQuotaManager::GetMaxFileSize() {
753 293 return limit_ - cleanup_threshold_;
754 }
755
756
757 41 pid_t PosixQuotaManager::GetPid() {
758
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 41 if (!shared_ || !spawned_) {
759 41 return getpid();
760 }
761 ✗ if (cachemgr_pid_) {
762 ✗ return cachemgr_pid_;
763 }
764
765 pid_t result;
766 int pipe_pid[2];
767 ✗ MakeReturnPipe(pipe_pid);
768
769 ✗ LruCommand cmd;
770 ✗ cmd.command_type = kPid;
771 ✗ cmd.return_pipe = pipe_pid[1];
772 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
773 ✗ ReadHalfPipe(pipe_pid[0], &result, sizeof(result));
774 ✗ CloseReturnPipe(pipe_pid);
775 ✗ return result;
776 }
777
778
779 41 uint32_t PosixQuotaManager::GetProtocolRevision() {
780 int pipe_revision[2];
781
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 MakeReturnPipe(pipe_revision);
782
783 41 LruCommand cmd;
784 41 cmd.command_type = kGetProtocolRevision;
785 41 cmd.return_pipe = pipe_revision[1];
786
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
787
788 uint32_t revision;
789
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 ManagedReadHalfPipe(pipe_revision[0], &revision, sizeof(revision));
790
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 CloseReturnPipe(pipe_revision);
791 41 return revision;
792 }
793
794 1415 void PosixQuotaManager::SetCleanupPolicy(bool cleanup_unused_first) {
795 1415 LruCommand cmd;
796 1415 cmd.command_type = kSetCleanupPolicy;
797
1/2
✓ Branch 1 taken 1415 times.
✗ Branch 2 not taken.
 1415 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
798
799
1/2
✓ Branch 1 taken 1415 times.
✗ Branch 2 not taken.
 1415 WritePipe(pipe_lru_[1], &cleanup_unused_first, sizeof(bool));
800 1415 }
801
802 ✗ void PosixQuotaManager::RegisterMountpoint(const std::string &mountpoint) {
803 ✗ LruCommand cmd;
804 ✗ cmd.command_type = kRegisterMountpoint;
805 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
806
807 ✗ size_t mp_size = mountpoint.size();
808 ✗ WritePipe(pipe_lru_[1], &mp_size, sizeof(size_t));
809 ✗ WritePipe(pipe_lru_[1], mountpoint.data(), mp_size);
810 }
811
812 ✗ std::string PosixQuotaManager::GetMountpoints() {
813 int pipe_mp[2];
814 ✗ MakeReturnPipe(pipe_mp);
815
816 ✗ LruCommand cmd;
817 ✗ cmd.command_type = kGetMountpoints;
818 ✗ cmd.return_pipe = pipe_mp[1];
819 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
820 ✗ size_t mp_str_size = 0;
821 ✗ ManagedReadHalfPipe(pipe_mp[0], &mp_str_size, sizeof(size_t));
822 ✗ char *buf = (char *)malloc(mp_str_size * sizeof(char));
823 ✗ ManagedReadHalfPipe(pipe_mp[0], buf, mp_str_size);
824 ✗ return std::string(buf);
825 }
826
827 ✗ std::string PosixQuotaManager::GetGroupHashes() {
828 int pipe_gh[2];
829 ✗ MakeReturnPipe(pipe_gh);
830
831 ✗ LruCommand cmd;
832 ✗ cmd.command_type = kGetGroupHashes;
833 ✗ cmd.return_pipe = pipe_gh[1];
834 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
835 ✗ size_t mp_str_size = 0;
836 ✗ ManagedReadHalfPipe(pipe_gh[0], &mp_str_size, sizeof(size_t));
837 ✗ char *buf = (char *)malloc(mp_str_size * sizeof(char));
838 ✗ ManagedReadHalfPipe(pipe_gh[0], buf, mp_str_size);
839 ✗ return std::string(buf);
840 }
841
842 /**
843 * Queries the shared local hard disk quota manager.
844 */
845 738 void PosixQuotaManager::GetSharedStatus(uint64_t *gauge, uint64_t *pinned) {
846 int pipe_status[2];
847
1/2
✓ Branch 1 taken 738 times.
✗ Branch 2 not taken.
 738 MakeReturnPipe(pipe_status);
848
849 738 LruCommand cmd;
850 738 cmd.command_type = kStatus;
851 738 cmd.return_pipe = pipe_status[1];
852
1/2
✓ Branch 1 taken 738 times.
✗ Branch 2 not taken.
 738 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
853
1/2
✓ Branch 1 taken 738 times.
✗ Branch 2 not taken.
 738 ManagedReadHalfPipe(pipe_status[0], gauge, sizeof(*gauge));
854
1/2
✓ Branch 1 taken 738 times.
✗ Branch 2 not taken.
 738 ReadPipe(pipe_status[0], pinned, sizeof(*pinned));
855
1/2
✓ Branch 1 taken 738 times.
✗ Branch 2 not taken.
 738 CloseReturnPipe(pipe_status);
856 738 }
857
858 41 bool PosixQuotaManager::SetSharedLimit(uint64_t limit) {
859 int pipe_set_limit[2];
860 bool result;
861
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 MakeReturnPipe(pipe_set_limit);
862
863 41 LruCommand cmd;
864 41 cmd.command_type = kSetLimit;
865 41 cmd.size = limit;
866 41 cmd.return_pipe = pipe_set_limit[1];
867
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
868
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 ReadHalfPipe(pipe_set_limit[0], &result, sizeof(result));
869
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 CloseReturnPipe(pipe_set_limit);
870 41 return result;
871 }
872
873
874 41 bool PosixQuotaManager::SetLimit(uint64_t size) {
875
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 41 times.
 41 if (!spawned_) {
876 ✗ limit_ = size;
877 ✗ cleanup_threshold_ = size / 2;
878 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
879 "Quota limit set to %lu / threshold %lu", limit_,
880 cleanup_threshold_);
881 ✗ return true;
882 }
883 41 return SetSharedLimit(size);
884 }
885
886 3486 uint64_t PosixQuotaManager::GetSize() {
887
2/2
✓ Branch 0 taken 2830 times.
✓ Branch 1 taken 656 times.
 3486 if (!spawned_)
888 2830 return gauge_;
889 uint64_t gauge, size_pinned;
890
1/2
✓ Branch 1 taken 656 times.
✗ Branch 2 not taken.
 656 GetSharedStatus(&gauge, &size_pinned);
891 656 return gauge;
892 }
893
894
895 82 uint64_t PosixQuotaManager::GetSizePinned() {
896
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 82 times.
 82 if (!spawned_)
897 ✗ return pinned_;
898 uint64_t gauge, size_pinned;
899
1/2
✓ Branch 1 taken 82 times.
✗ Branch 2 not taken.
 82 GetSharedStatus(&gauge, &size_pinned);
900 82 return size_pinned;
901 }
902
903
904 164 uint64_t PosixQuotaManager::GetCleanupRate(uint64_t period_s) {
905
2/4
✓ Branch 0 taken 164 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 164 times.
 164 if (!spawned_ || (protocol_revision_ < 2))
906 ✗ return 0;
907 uint64_t cleanup_rate;
908
909 int pipe_cleanup_rate[2];
910
1/2
✓ Branch 1 taken 164 times.
✗ Branch 2 not taken.
 164 MakeReturnPipe(pipe_cleanup_rate);
911 164 LruCommand cmd;
912 164 cmd.command_type = kCleanupRate;
913 164 cmd.size = period_s;
914 164 cmd.return_pipe = pipe_cleanup_rate[1];
915
1/2
✓ Branch 1 taken 164 times.
✗ Branch 2 not taken.
 164 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
916
1/2
✓ Branch 1 taken 164 times.
✗ Branch 2 not taken.
 164 ManagedReadHalfPipe(pipe_cleanup_rate[0], &cleanup_rate,
917 sizeof(cleanup_rate));
918
1/2
✓ Branch 1 taken 164 times.
✗ Branch 2 not taken.
 164 CloseReturnPipe(pipe_cleanup_rate);
919
920 164 return cleanup_rate;
921 }
922
923
924 3834 bool PosixQuotaManager::InitDatabase(const bool rebuild_database) {
925 3834 string sql;
926 sqlite3_stmt *stmt;
927
928
2/4
✓ Branch 1 taken 3834 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3834 times.
✗ Branch 5 not taken.
 3834 fd_lock_cachedb_ = LockFile(workspace_dir_ + "/lock_cachedb");
929
2/2
✓ Branch 0 taken 41 times.
✓ Branch 1 taken 3793 times.
 3834 if (fd_lock_cachedb_ < 0) {
930
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 LogCvmfs(kLogQuota, kLogDebug, "failed to create cachedb lock");
931 41 return false;
932 }
933
934 3793 bool retry = false;
935
1/2
✓ Branch 1 taken 3793 times.
✗ Branch 2 not taken.
 3793 const string db_file = cache_dir_ + "/cachedb";
936
2/2
✓ Branch 0 taken 3582 times.
✓ Branch 1 taken 211 times.
 3793 if (rebuild_database) {
937
1/2
✓ Branch 2 taken 211 times.
✗ Branch 3 not taken.
 211 LogCvmfs(kLogQuota, kLogDebug, "rebuild database, unlinking existing (%s)",
938 db_file.c_str());
939 211 unlink(db_file.c_str());
940
1/2
✓ Branch 1 taken 211 times.
✗ Branch 2 not taken.
 211 unlink((db_file + "-journal").c_str());
941 }
942
943 3582 init_recover:
944
1/2
✓ Branch 2 taken 3793 times.
✗ Branch 3 not taken.
 3793 int err = sqlite3_open(db_file.c_str(), &database_);
945
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3793 times.
 3793 if (err != SQLITE_OK) {
946 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not open cache database (%d)", err);
947 ✗ goto init_database_fail;
948 }
949 // TODO(reneme): make this a `QuotaDatabase : public sqlite::Database`
950 sql = "PRAGMA synchronous=0; PRAGMA locking_mode=EXCLUSIVE; "
951 "PRAGMA auto_vacuum=1; "
952 "CREATE TABLE IF NOT EXISTS cache_catalog (sha1 TEXT, size INTEGER, "
953 " acseq INTEGER, path TEXT, type INTEGER, pinned INTEGER, "
954 "CONSTRAINT pk_cache_catalog PRIMARY KEY (sha1)); "
955 "CREATE UNIQUE INDEX IF NOT EXISTS idx_cache_catalog_acseq "
956 " ON cache_catalog (acseq); "
957 "CREATE TEMP TABLE fscache (sha1 TEXT, size INTEGER, actime INTEGER, "
958 "CONSTRAINT pk_fscache PRIMARY KEY (sha1)); "
959 "CREATE INDEX idx_fscache_actime ON fscache (actime); "
960 "CREATE TABLE IF NOT EXISTS properties (key TEXT, value TEXT, "
961
1/2
✓ Branch 1 taken 3793 times.
✗ Branch 2 not taken.
 3793 " CONSTRAINT pk_properties PRIMARY KEY(key));";
962
1/2
✓ Branch 2 taken 3793 times.
✗ Branch 3 not taken.
 3793 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
963
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3793 times.
 3793 if (err != SQLITE_OK) {
964 ✗ if (!retry) {
965 ✗ retry = true;
966 ✗ sqlite3_close(database_);
967 ✗ unlink(db_file.c_str());
968 ✗ unlink((db_file + "-journal").c_str());
969 ✗ LogCvmfs(kLogQuota, kLogSyslogWarn,
970 "LRU database corrupted, re-building");
971 ✗ goto init_recover;
972 }
973 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
974 sql.c_str());
975 ✗ goto init_database_fail;
976 }
977
978 // If this an old cache catalog,
979 // add and initialize new columns to cache_catalog
980 sql = "ALTER TABLE cache_catalog ADD type INTEGER; "
981
1/2
✓ Branch 1 taken 3793 times.
✗ Branch 2 not taken.
 3793 "ALTER TABLE cache_catalog ADD pinned INTEGER";
982
1/2
✓ Branch 2 taken 3793 times.
✗ Branch 3 not taken.
 3793 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
983
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3793 times.
 3793 if (err == SQLITE_OK) {
984 ✗ sql = "UPDATE cache_catalog SET type=" + StringifyInt(kFileRegular) + ";";
985 ✗ err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
986 ✗ if (err != SQLITE_OK) {
987 ✗ LogCvmfs(kLogQuota, kLogDebug,
988 "could not init cache database (failed: %s)", sql.c_str());
989 ✗ goto init_database_fail;
990 }
991 }
992
993 // Set pinned back
994
1/2
✓ Branch 1 taken 3793 times.
✗ Branch 2 not taken.
 3793 sql = "UPDATE cache_catalog SET pinned=0;";
995
1/2
✓ Branch 2 taken 3793 times.
✗ Branch 3 not taken.
 3793 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
996
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3793 times.
 3793 if (err != SQLITE_OK) {
997 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
998 sql.c_str());
999 ✗ goto init_database_fail;
1000 }
1001
1002 // Set schema version
1003 sql = "INSERT OR REPLACE INTO properties (key, value) "
1004
1/2
✓ Branch 1 taken 3793 times.
✗ Branch 2 not taken.
 3793 "VALUES ('schema', '1.0')";
1005
1/2
✓ Branch 2 taken 3793 times.
✗ Branch 3 not taken.
 3793 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1006
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3793 times.
 3793 if (err != SQLITE_OK) {
1007 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
1008 sql.c_str());
1009 ✗ goto init_database_fail;
1010 }
1011
1012 // If cache catalog is empty, recreate from file system
1013
1/2
✓ Branch 1 taken 3793 times.
✗ Branch 2 not taken.
 3793 sql = "SELECT count(*) FROM cache_catalog;";
1014
1/2
✓ Branch 2 taken 3793 times.
✗ Branch 3 not taken.
 3793 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1015
2/4
✓ Branch 1 taken 3793 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 3793 times.
✗ Branch 4 not taken.
 3793 if (sqlite3_step(stmt) == SQLITE_ROW) {
1016
6/8
✓ Branch 1 taken 3793 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 164 times.
✓ Branch 4 taken 3629 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 164 times.
✓ Branch 7 taken 3629 times.
✓ Branch 8 taken 164 times.
 3793 if ((sqlite3_column_int64(stmt, 0)) == 0 || rebuild_database) {
1017
1/2
✓ Branch 1 taken 3629 times.
✗ Branch 2 not taken.
 3629 LogCvmfs(kLogCvmfs, kLogDebug,
1018 "CernVM-FS: building lru cache database...");
1019
3/4
✓ Branch 1 taken 3629 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 123 times.
✓ Branch 4 taken 3506 times.
 3629 if (!RebuildDatabase()) {
1020
1/2
✓ Branch 1 taken 123 times.
✗ Branch 2 not taken.
 123 LogCvmfs(kLogQuota, kLogDebug,
1021 "could not build cache database from file system");
1022
1/2
✓ Branch 1 taken 123 times.
✗ Branch 2 not taken.
 123 sqlite3_finalize(stmt);
1023 123 goto init_database_fail;
1024 }
1025 }
1026
1/2
✓ Branch 1 taken 3670 times.
✗ Branch 2 not taken.
 3670 sqlite3_finalize(stmt);
1027 } else {
1028 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not select on cache catalog");
1029 ✗ sqlite3_finalize(stmt);
1030 ✗ goto init_database_fail;
1031 }
1032
1033 // How many bytes do we already have in cache?
1034
1/2
✓ Branch 1 taken 3670 times.
✗ Branch 2 not taken.
 3670 sql = "SELECT sum(size) FROM cache_catalog;";
1035
1/2
✓ Branch 2 taken 3670 times.
✗ Branch 3 not taken.
 3670 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1036
2/4
✓ Branch 1 taken 3670 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 3670 times.
✗ Branch 4 not taken.
 3670 if (sqlite3_step(stmt) == SQLITE_ROW) {
1037
1/2
✓ Branch 1 taken 3670 times.
✗ Branch 2 not taken.
 3670 gauge_ = sqlite3_column_int64(stmt, 0);
1038 } else {
1039 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not determine cache size");
1040 ✗ sqlite3_finalize(stmt);
1041 ✗ goto init_database_fail;
1042 }
1043
1/2
✓ Branch 1 taken 3670 times.
✗ Branch 2 not taken.
 3670 sqlite3_finalize(stmt);
1044
1045 // Highest seq-no?
1046
1/2
✓ Branch 1 taken 3670 times.
✗ Branch 2 not taken.
 3670 sql = "SELECT coalesce(max(acseq & (~(1<<63))), 0) FROM cache_catalog;";
1047
1/2
✓ Branch 2 taken 3670 times.
✗ Branch 3 not taken.
 3670 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1048
2/4
✓ Branch 1 taken 3670 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 3670 times.
✗ Branch 4 not taken.
 3670 if (sqlite3_step(stmt) == SQLITE_ROW) {
1049
1/2
✓ Branch 1 taken 3670 times.
✗ Branch 2 not taken.
 3670 seq_ = sqlite3_column_int64(stmt, 0) + 1;
1050 } else {
1051 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not determine highest seq-no");
1052 ✗ sqlite3_finalize(stmt);
1053 ✗ goto init_database_fail;
1054 }
1055
1/2
✓ Branch 1 taken 3670 times.
✗ Branch 2 not taken.
 3670 sqlite3_finalize(stmt);
1056
1057 // Prepare touch, new, remove statements
1058
1/2
✓ Branch 1 taken 3670 times.
✗ Branch 2 not taken.
 3670 sqlite3_prepare_v2(database_,
1059 "UPDATE cache_catalog SET acseq=:seq | (acseq&(1<<63)) "
1060 "WHERE sha1=:sha1;",
1061 -1, &stmt_touch_, NULL);
1062
1/2
✓ Branch 1 taken 3670 times.
✗ Branch 2 not taken.
 3670 sqlite3_prepare_v2(database_,
1063 "UPDATE cache_catalog SET pinned=0 "
1064 "WHERE sha1=:sha1;",
1065 -1, &stmt_unpin_, NULL);
1066
1/2
✓ Branch 1 taken 3670 times.
✗ Branch 2 not taken.
 3670 sqlite3_prepare_v2(database_,
1067 "UPDATE cache_catalog SET pinned=2 "
1068 "WHERE sha1=:sha1;",
1069 -1, &stmt_block_, NULL);
1070
1/2
✓ Branch 1 taken 3670 times.
✗ Branch 2 not taken.
 3670 sqlite3_prepare_v2(database_,
1071 "UPDATE cache_catalog SET pinned=1 "
1072 "WHERE pinned=2;",
1073 -1, &stmt_unblock_, NULL);
1074
1/2
✓ Branch 1 taken 3670 times.
✗ Branch 2 not taken.
 3670 sqlite3_prepare_v2(database_,
1075 "INSERT OR REPLACE INTO cache_catalog "
1076 "(sha1, size, acseq, path, type, pinned) "
1077 "VALUES (:sha1, :s, :seq, :p, :t, :pin);",
1078 -1, &stmt_new_, NULL);
1079
1/2
✓ Branch 1 taken 3670 times.
✗ Branch 2 not taken.
 3670 sqlite3_prepare_v2(database_,
1080 "SELECT size, pinned FROM cache_catalog WHERE sha1=:sha1;",
1081 -1, &stmt_size_, NULL);
1082
1/2
✓ Branch 1 taken 3670 times.
✗ Branch 2 not taken.
 3670 sqlite3_prepare_v2(database_, "DELETE FROM cache_catalog WHERE sha1=:sha1;",
1083 -1, &stmt_rm_, NULL);
1084
1/2
✓ Branch 1 taken 3670 times.
✗ Branch 2 not taken.
 3670 sqlite3_prepare_v2(database_,
1085 "DELETE FROM cache_catalog WHERE acseq<=:a AND pinned<>2;",
1086 -1, &stmt_rm_batch_, NULL);
1087
1/2
✓ Branch 2 taken 3670 times.
✗ Branch 3 not taken.
 3670 sqlite3_prepare_v2(database_,
1088
1/2
✓ Branch 2 taken 3670 times.
✗ Branch 3 not taken.
 7340 (std::string("SELECT sha1, size, acseq FROM cache_catalog "
1089 "WHERE pinned<>2 AND acseq>=:a "
1090 "ORDER BY acseq ASC "
1091 "LIMIT ")
1092
3/6
✓ Branch 1 taken 3670 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3670 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 3670 times.
✗ Branch 8 not taken.
 14680 + StringifyInt(kEvictBatchSize) + ";")
1093 .c_str(),
1094 -1, &stmt_lru_, NULL);
1095
1/2
✓ Branch 2 taken 3670 times.
✗ Branch 3 not taken.
 3670 sqlite3_prepare_v2(database_,
1096 ("SELECT path FROM cache_catalog WHERE type="
1097
3/6
✓ Branch 1 taken 3670 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3670 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 3670 times.
✗ Branch 8 not taken.
 7340 + StringifyInt(kFileRegular) + ";")
1098 .c_str(),
1099 -1, &stmt_list_, NULL);
1100
1/2
✓ Branch 1 taken 3670 times.
✗ Branch 2 not taken.
 3670 sqlite3_prepare_v2(database_,
1101 "SELECT path FROM cache_catalog WHERE pinned<>0;", -1,
1102 &stmt_list_pinned_, NULL);
1103
1/2
✓ Branch 1 taken 3670 times.
✗ Branch 2 not taken.
 3670 sqlite3_prepare_v2(database_,
1104 "SELECT path FROM cache_catalog WHERE acseq < 0;", -1,
1105 &stmt_list_volatile_, NULL);
1106
1/2
✓ Branch 2 taken 3670 times.
✗ Branch 3 not taken.
 3670 sqlite3_prepare_v2(database_,
1107 ("SELECT path FROM cache_catalog WHERE type="
1108
3/6
✓ Branch 1 taken 3670 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3670 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 3670 times.
✗ Branch 8 not taken.
 7340 + StringifyInt(kFileCatalog) + ";")
1109 .c_str(),
1110 -1, &stmt_list_catalogs_, NULL);
1111 3670 return true;
1112
1113 123 init_database_fail:
1114
1/2
✓ Branch 1 taken 123 times.
✗ Branch 2 not taken.
 123 sqlite3_close(database_);
1115 123 database_ = NULL;
1116
1/2
✓ Branch 1 taken 123 times.
✗ Branch 2 not taken.
 123 UnlockFile(fd_lock_cachedb_);
1117 123 return false;
1118 3834 }
1119
1120
1121 /**
1122 * Inserts a new file into cache catalog. This file gets a new,
1123 * highest sequence number. Does cache cleanup if necessary.
1124 */
1125 4101037 void PosixQuotaManager::Insert(const shash::Any &any_hash,
1126 const uint64_t size,
1127 const string &description) {
1128 4101037 DoInsert(any_hash, size, description, kInsert);
1129 4101037 }
1130
1131
1132 /**
1133 * Inserts a new file into cache catalog. This file is marked as volatile
1134 * and gets a new highest sequence number with the first bit set. Cache cleanup
1135 * treats these files with priority.
1136 */
1137 164 void PosixQuotaManager::InsertVolatile(const shash::Any &any_hash,
1138 const uint64_t size,
1139 const string &description) {
1140 164 DoInsert(any_hash, size, description, kInsertVolatile);
1141 164 }
1142
1143
1144 /**
1145 * Lists all path names from the cache db.
1146 */
1147 861 vector<string> PosixQuotaManager::List() { return DoList(kList); }
1148
1149
1150 /**
1151 * Lists all pinned files from the cache db.
1152 */
1153 328 vector<string> PosixQuotaManager::ListPinned() { return DoList(kListPinned); }
1154
1155
1156 /**
1157 * Lists all sqlite catalog files from the cache db.
1158 */
1159 123 vector<string> PosixQuotaManager::ListCatalogs() {
1160 123 return DoList(kListCatalogs);
1161 }
1162
1163
1164 /**
1165 * Lists only files flagged as volatile (priority removal)
1166 */
1167 123 vector<string> PosixQuotaManager::ListVolatile() {
1168 123 return DoList(kListVolatile);
1169 }
1170
1171
1172 /**
1173 * Entry point for the shared cache manager process
1174 */
1175 ✗ int PosixQuotaManager::MainCacheManager(int argc, char **argv) {
1176 ✗ LogCvmfs(kLogQuota, kLogDebug, "starting quota manager");
1177 int retval;
1178
1179 ✗ PosixQuotaManager shared_manager(0, 0, "");
1180 ✗ shared_manager.shared_ = true;
1181 ✗ shared_manager.spawned_ = true;
1182 ✗ shared_manager.pinned_ = 0;
1183
1184 // Process command line arguments
1185 ✗ ParseDirectories(string(argv[2]),
1186 &shared_manager.cache_dir_,
1187 &shared_manager.workspace_dir_);
1188 ✗ const int pipe_boot = String2Int64(argv[3]);
1189 ✗ const int pipe_handshake = String2Int64(argv[4]);
1190 ✗ shared_manager.limit_ = String2Int64(argv[5]);
1191 ✗ shared_manager.cleanup_threshold_ = String2Int64(argv[6]);
1192 ✗ const int foreground = String2Int64(argv[7]);
1193 ✗ const int syslog_level = String2Int64(argv[8]);
1194 ✗ const int syslog_facility = String2Int64(argv[9]);
1195 ✗ vector<string> logfiles = SplitString(argv[10], ':');
1196
1197 ✗ SetLogSyslogLevel(syslog_level);
1198 ✗ SetLogSyslogFacility(syslog_facility);
1199 ✗ if ((logfiles.size() > 0) && (logfiles[0] != ""))
1200 ✗ SetLogDebugFile(logfiles[0] + ".cachemgr");
1201 ✗ if (logfiles.size() > 1)
1202 ✗ SetLogMicroSyslog(logfiles[1]);
1203
1204 ✗ if (!foreground)
1205 ✗ Daemonize();
1206
1207 ✗ if ((geteuid() != 0) && SetuidCapabilityPermitted()) {
1208 // Permanently drop credentials
1209 ✗ const std::vector<cap_value_t> nocaps;
1210 ✗ assert(ClearPermittedCapabilities(nocaps, nocaps));
1211 // Leave this process ptraceable
1212 ✗ assert(platform_set_dumpable());
1213 // but without core dumps
1214 ✗ assert(SetLimitCore(0));
1215 }
1216
1217 ✗ const UniquePtr<Watchdog> watchdog(Watchdog::Create(NULL));
1218 ✗ assert(watchdog.IsValid());
1219 ✗ watchdog->Spawn("./stacktrace.cachemgr");
1220
1221 // Initialize pipe, open non-blocking as cvmfs is not yet connected
1222 ✗ const int fd_lockfile_fifo = LockFile(shared_manager.workspace_dir_
1223 ✗ + "/lock_cachemgr.fifo");
1224 ✗ if (fd_lockfile_fifo < 0) {
1225 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1226 "could not open lock file "
1227 "%s (%d)",
1228 ✗ (shared_manager.workspace_dir_ + "/lock_cachemgr.fifo").c_str(),
1229 ✗ errno);
1230 ✗ return 1;
1231 }
1232 ✗ const string crash_guard = shared_manager.cache_dir_ + "/cachemgr.running";
1233 ✗ const bool rebuild = FileExists(crash_guard);
1234 ✗ retval = open(crash_guard.c_str(), O_RDONLY | O_CREAT, 0600);
1235 ✗ if (retval < 0) {
1236 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1237 "failed to create shared cache manager crash guard");
1238 ✗ UnlockFile(fd_lockfile_fifo);
1239 ✗ return 1;
1240 }
1241 ✗ close(retval);
1242
1243 // Redirect SQlite temp directory to cache (global variable)
1244 ✗ const string tmp_dir = shared_manager.workspace_dir_;
1245 ✗ sqlite3_temp_directory = static_cast<char *>(
1246 ✗ sqlite3_malloc(tmp_dir.length() + 1));
1247 ✗ snprintf(sqlite3_temp_directory, tmp_dir.length() + 1, "%s", tmp_dir.c_str());
1248
1249 // Cleanup leftover named pipes
1250 ✗ shared_manager.CleanupPipes();
1251
1252 ✗ if (!shared_manager.InitDatabase(rebuild)) {
1253 ✗ UnlockFile(fd_lockfile_fifo);
1254 ✗ return 1;
1255 }
1256 ✗ shared_manager.CheckFreeSpace();
1257
1258 // Save protocol revision to file. If the file is not found, it indicates
1259 // to the client that the cache manager is from times before the protocol
1260 // was versioned.
1261 const string protocol_revision_path = shared_manager.workspace_dir_
1262 ✗ + "/cachemgr.protocol";
1263 ✗ retval = open(protocol_revision_path.c_str(), O_WRONLY | O_CREAT, 0600);
1264 ✗ if (retval < 0) {
1265 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1266 ✗ "failed to open protocol revision file (%d)", errno);
1267 ✗ UnlockFile(fd_lockfile_fifo);
1268 ✗ return 1;
1269 }
1270 ✗ const string revision = StringifyInt(kProtocolRevision);
1271 ✗ const int written = write(retval, revision.data(), revision.length());
1272 ✗ close(retval);
1273 ✗ if ((written < 0) || static_cast<unsigned>(written) != revision.length()) {
1274 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1275 ✗ "failed to write protocol revision (%d)", errno);
1276 ✗ UnlockFile(fd_lockfile_fifo);
1277 ✗ return 1;
1278 }
1279
1280 ✗ const string fifo_path = shared_manager.workspace_dir_ + "/cachemgr";
1281 ✗ shared_manager.pipe_lru_[0] = open(fifo_path.c_str(), O_RDONLY | O_NONBLOCK);
1282 ✗ if (shared_manager.pipe_lru_[0] < 0) {
1283 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to listen on FIFO %s (%d)",
1284 ✗ fifo_path.c_str(), errno);
1285 ✗ UnlockFile(fd_lockfile_fifo);
1286 ✗ return 1;
1287 }
1288 ✗ Nonblock2Block(shared_manager.pipe_lru_[0]);
1289 ✗ LogCvmfs(kLogQuota, kLogDebug, "shared cache manager listening");
1290
1291 ✗ char buf = 'C';
1292 ✗ WritePipe(pipe_boot, &buf, 1);
1293 ✗ close(pipe_boot);
1294
1295 ✗ ReadPipe(pipe_handshake, &buf, 1);
1296 ✗ close(pipe_handshake);
1297 ✗ LogCvmfs(kLogQuota, kLogDebug, "shared cache manager handshake done");
1298
1299 // Ensure that broken pipes from clients do not kill the cache manager
1300 ✗ signal(SIGPIPE, SIG_IGN);
1301 // Don't let Ctrl-C ungracefully kill interactive session
1302 ✗ signal(SIGINT, SIG_IGN);
1303
1304 ✗ shared_manager.MainCommandServer(&shared_manager);
1305 ✗ unlink(fifo_path.c_str());
1306 ✗ unlink(protocol_revision_path.c_str());
1307 ✗ shared_manager.CloseDatabase();
1308 ✗ unlink(crash_guard.c_str());
1309 ✗ UnlockFile(fd_lockfile_fifo);
1310
1311 ✗ if (sqlite3_temp_directory) {
1312 ✗ sqlite3_free(sqlite3_temp_directory);
1313 ✗ sqlite3_temp_directory = NULL;
1314 }
1315
1316 ✗ return 0;
1317 }
1318
1319
1320 1271 void *PosixQuotaManager::MainCommandServer(void *data) {
1321 1271 PosixQuotaManager *quota_mgr = static_cast<PosixQuotaManager *>(data);
1322
1323
1/2
✓ Branch 1 taken 1271 times.
✗ Branch 2 not taken.
 1271 LogCvmfs(kLogQuota, kLogDebug, "starting quota manager");
1324
1/2
✓ Branch 1 taken 1271 times.
✗ Branch 2 not taken.
 1271 sqlite3_soft_heap_limit(quota_mgr->kSqliteMemPerThread);
1325
1326
2/2
✓ Branch 1 taken 40672 times.
✓ Branch 2 taken 1271 times.
 41943 LruCommand command_buffer[kCommandBufferSize];
1327 char description_buffer[kCommandBufferSize * kMaxDescription];
1328 1271 unsigned num_commands = 0;
1329
1330
1/2
✓ Branch 1 taken 6156601 times.
✗ Branch 2 not taken.
 6156601 while (read(quota_mgr->pipe_lru_[0], &command_buffer[num_commands],
1331 sizeof(command_buffer[0]))
1332
2/2
✓ Branch 0 taken 6155330 times.
✓ Branch 1 taken 1271 times.
 6156601 == sizeof(command_buffer[0])) {
1333 6155330 const CommandType command_type = command_buffer[num_commands].command_type;
1334
1/2
✓ Branch 1 taken 6155330 times.
✗ Branch 2 not taken.
 6155330 LogCvmfs(kLogQuota, kLogDebug, "received command %d", command_type);
1335 6155330 const uint64_t size = command_buffer[num_commands].GetSize();
1336
1337 // Inserts and pins come with a description (usually a path)
1338
4/4
✓ Branch 0 taken 2054633 times.
✓ Branch 1 taken 4100697 times.
✓ Branch 2 taken 2054469 times.
✓ Branch 3 taken 164 times.
 6155330 if ((command_type == kInsert) || (command_type == kInsertVolatile)
1339
4/4
✓ Branch 0 taken 2054387 times.
✓ Branch 1 taken 82 times.
✓ Branch 2 taken 451 times.
✓ Branch 3 taken 2053936 times.
 2054469 || (command_type == kPin) || (command_type == kPinRegular)) {
1340 4101394 const int desc_length = command_buffer[num_commands].desc_length;
1341 4101394 ReadPipe(quota_mgr->pipe_lru_[0],
1342
1/2
✓ Branch 1 taken 4101394 times.
✗ Branch 2 not taken.
 4101394 &description_buffer[kMaxDescription * num_commands],
1343 desc_length);
1344 }
1345
1346 // The protocol revision is returned immediately
1347
2/2
✓ Branch 0 taken 41 times.
✓ Branch 1 taken 6155289 times.
 6155330 if (command_type == kGetProtocolRevision) {
1348
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 const int return_pipe = quota_mgr->BindReturnPipe(
1349 command_buffer[num_commands].return_pipe);
1350
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 41 times.
 41 if (return_pipe < 0)
1351 ✗ continue;
1352
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 WritePipe(return_pipe, &quota_mgr->kProtocolRevision,
1353 sizeof(quota_mgr->kProtocolRevision));
1354
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 quota_mgr->UnbindReturnPipe(return_pipe);
1355 41 continue;
1356 41 }
1357
1358 // Register a new mountpoint
1359
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6155289 times.
 6155289 if (command_type == kRegisterMountpoint) {
1360 ✗ size_t mp_size = 0;
1361 ✗ ReadPipe(quota_mgr->pipe_lru_[0], &mp_size, sizeof(size_t));
1362 ✗ std::string mountpoint(mp_size, '\0');
1363 ✗ ReadPipe(quota_mgr->pipe_lru_[0], (void *)mountpoint.data(), mp_size);
1364 ✗ quota_mgr->mountpoints_.push_back(mountpoint);
1365 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
1366 "Mountpoint %s registered in the group", mountpoint.c_str());
1367 ✗ continue;
1368 }
1369
1370 // Set Cleanup Policy
1371
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6155289 times.
 6155289 if (command_type == kSetCleanupPolicy) {
1372 ✗ bool policy = false;
1373 ✗ ReadPipe(quota_mgr->pipe_lru_[0], &policy, sizeof(bool));
1374 ✗ quota_mgr->cleanup_unused_first_ = policy;
1375 ✗ continue;
1376 }
1377 // Mountpoints are returned immediately
1378
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6155289 times.
 6155289 if (command_type == kGetMountpoints) {
1379 ✗ const int return_pipe = quota_mgr->BindReturnPipe(
1380 command_buffer[num_commands].return_pipe);
1381 ✗ if (return_pipe < 0)
1382 ✗ continue;
1383
1384 ✗ std::string mps;
1385 ✗ for (auto it = quota_mgr->mountpoints_.begin();
1386 ✗ it != quota_mgr->mountpoints_.end();
1387 ✗ ++it) {
1388 ✗ mps += *it + "\n";
1389 }
1390 ✗ size_t mp_size = mps.size();
1391 ✗ WritePipe(return_pipe, &mp_size, sizeof(size_t));
1392 ✗ WritePipe(return_pipe, mps.c_str(), mp_size);
1393 ✗ quota_mgr->UnbindReturnPipe(return_pipe);
1394 ✗ continue;
1395 }
1396
1397 // Group hashes are returned immediately
1398
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6155289 times.
 6155289 if (command_type == kGetGroupHashes) {
1399 ✗ const int return_pipe = quota_mgr->BindReturnPipe(
1400 command_buffer[num_commands].return_pipe);
1401 ✗ if (return_pipe < 0)
1402 ✗ continue;
1403
1404 ✗ std::vector<shash::Short> gh = quota_mgr->CollectAllOpenHashes();
1405 ✗ std::string result;
1406 ✗ for (auto it = gh.begin(); it != gh.end(); ++it) {
1407 ✗ result += (*it).ToString() + "\n";
1408 }
1409 ✗ size_t result_size = result.size();
1410 ✗ WritePipe(return_pipe, &result_size, sizeof(size_t));
1411 ✗ WritePipe(return_pipe, result.c_str(), result_size);
1412 ✗ quota_mgr->UnbindReturnPipe(return_pipe);
1413 ✗ continue;
1414 }
1415
1416 // The cleanup rate is returned immediately
1417
2/2
✓ Branch 0 taken 164 times.
✓ Branch 1 taken 6155125 times.
 6155289 if (command_type == kCleanupRate) {
1418
1/2
✓ Branch 1 taken 164 times.
✗ Branch 2 not taken.
 164 const int return_pipe = quota_mgr->BindReturnPipe(
1419 command_buffer[num_commands].return_pipe);
1420
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 164 times.
 164 if (return_pipe < 0)
1421 ✗ continue;
1422 const uint64_t
1423 164 period_s = size; // use the size field to transmit the period
1424
1/2
✓ Branch 1 taken 164 times.
✗ Branch 2 not taken.
 164 uint64_t rate = quota_mgr->cleanup_recorder_.GetNoTicks(period_s);
1425
1/2
✓ Branch 1 taken 164 times.
✗ Branch 2 not taken.
 164 WritePipe(return_pipe, &rate, sizeof(rate));
1426
1/2
✓ Branch 1 taken 164 times.
✗ Branch 2 not taken.
 164 quota_mgr->UnbindReturnPipe(return_pipe);
1427 164 continue;
1428 164 }
1429
1430
2/2
✓ Branch 0 taken 41 times.
✓ Branch 1 taken 6155084 times.
 6155125 if (command_type == kSetLimit) {
1431
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 const int return_pipe = quota_mgr->BindReturnPipe(
1432 command_buffer[num_commands].return_pipe);
1433
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 41 times.
 41 if (return_pipe < 0)
1434 ✗ continue;
1435 41 quota_mgr->limit_ = size; // use the size field to transmit the size
1436 41 quota_mgr->cleanup_threshold_ = size / 2;
1437
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
1438 "Quota limit set to %lu / threshold %lu", quota_mgr->limit_,
1439 quota_mgr->cleanup_threshold_);
1440 41 bool ret = true;
1441
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 WritePipe(return_pipe, &ret, sizeof(ret));
1442
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 quota_mgr->UnbindReturnPipe(return_pipe);
1443 41 continue;
1444 41 }
1445
1446 // Reservations are handled immediately and "out of band"
1447
2/2
✓ Branch 0 taken 574 times.
✓ Branch 1 taken 6154510 times.
 6155084 if (command_type == kReserve) {
1448 574 bool success = true;
1449
1/2
✓ Branch 1 taken 574 times.
✗ Branch 2 not taken.
 574 const int return_pipe = quota_mgr->BindReturnPipe(
1450 command_buffer[num_commands].return_pipe);
1451
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 574 times.
 574 if (return_pipe < 0)
1452 ✗ continue;
1453
1454
1/2
✓ Branch 1 taken 574 times.
✗ Branch 2 not taken.
 574 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1455
1/2
✓ Branch 1 taken 574 times.
✗ Branch 2 not taken.
 574 const string hash_str(hash.ToString());
1456
1/2
✓ Branch 2 taken 574 times.
✗ Branch 3 not taken.
 574 LogCvmfs(kLogQuota, kLogDebug, "reserve %lu bytes for %s", size,
1457 hash_str.c_str());
1458
1459
1/2
✓ Branch 1 taken 574 times.
✗ Branch 2 not taken.
 574 if (quota_mgr->pinned_chunks_.find(hash)
1460
2/2
✓ Branch 2 taken 492 times.
✓ Branch 3 taken 82 times.
 1148 == quota_mgr->pinned_chunks_.end()) {
1461
2/2
✓ Branch 0 taken 41 times.
✓ Branch 1 taken 451 times.
 492 if ((quota_mgr->pinned_ + size) > quota_mgr->cleanup_threshold_) {
1462
1/2
✓ Branch 2 taken 41 times.
✗ Branch 3 not taken.
 41 LogCvmfs(kLogQuota, kLogDebug,
1463 "failed to insert %s (pinned), no space", hash_str.c_str());
1464 41 success = false;
1465 } else {
1466
1/2
✓ Branch 1 taken 451 times.
✗ Branch 2 not taken.
 451 quota_mgr->pinned_chunks_[hash] = size;
1467 451 quota_mgr->pinned_ += size;
1468
1/2
✓ Branch 1 taken 451 times.
✗ Branch 2 not taken.
 451 quota_mgr->CheckHighPinWatermark();
1469 }
1470 }
1471
1472
1/2
✓ Branch 1 taken 574 times.
✗ Branch 2 not taken.
 574 WritePipe(return_pipe, &success, sizeof(success));
1473
1/2
✓ Branch 1 taken 574 times.
✗ Branch 2 not taken.
 574 quota_mgr->UnbindReturnPipe(return_pipe);
1474 574 continue;
1475 574 }
1476
1477 // Back channels are also handled out of band
1478
2/2
✓ Branch 0 taken 164 times.
✓ Branch 1 taken 6154346 times.
 6154510 if (command_type == kRegisterBackChannel) {
1479
1/2
✓ Branch 1 taken 164 times.
✗ Branch 2 not taken.
 164 const int return_pipe = quota_mgr->BindReturnPipe(
1480 command_buffer[num_commands].return_pipe);
1481
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 164 times.
 164 if (return_pipe < 0)
1482 ✗ continue;
1483
1484
1/2
✓ Branch 1 taken 164 times.
✗ Branch 2 not taken.
 164 quota_mgr->UnlinkReturnPipe(command_buffer[num_commands].return_pipe);
1485
1/2
✓ Branch 1 taken 164 times.
✗ Branch 2 not taken.
 164 Block2Nonblock(return_pipe); // back channels are opportunistic
1486
1/2
✓ Branch 1 taken 164 times.
✗ Branch 2 not taken.
 164 shash::Md5 hash;
1487 164 memcpy(hash.digest, command_buffer[num_commands].digest,
1488 164 shash::kDigestSizes[shash::kMd5]);
1489
1490 164 quota_mgr->LockBackChannels();
1491 const map<shash::Md5, int>::const_iterator
1492
1/2
✓ Branch 1 taken 164 times.
✗ Branch 2 not taken.
 164 iter = quota_mgr->back_channels_.find(hash);
1493
1/2
✗ Branch 3 not taken.
✓ Branch 4 taken 164 times.
 164 if (iter != quota_mgr->back_channels_.end()) {
1494 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
1495 ✗ "closing left-over back channel %s", hash.ToString().c_str());
1496 ✗ close(iter->second);
1497 }
1498
1/2
✓ Branch 1 taken 164 times.
✗ Branch 2 not taken.
 164 quota_mgr->back_channels_[hash] = return_pipe;
1499 164 quota_mgr->UnlockBackChannels();
1500
1501 164 char success = 'S';
1502
1/2
✓ Branch 1 taken 164 times.
✗ Branch 2 not taken.
 164 WritePipe(return_pipe, &success, sizeof(success));
1503
1/2
✓ Branch 2 taken 164 times.
✗ Branch 3 not taken.
 164 LogCvmfs(kLogQuota, kLogDebug, "register back channel %s on fd %d",
1504
1/2
✓ Branch 1 taken 164 times.
✗ Branch 2 not taken.
 328 hash.ToString().c_str(), return_pipe);
1505
1506 164 continue;
1507 164 }
1508
1509
2/2
✓ Branch 0 taken 82 times.
✓ Branch 1 taken 6154264 times.
 6154346 if (command_type == kUnregisterBackChannel) {
1510
1/2
✓ Branch 1 taken 82 times.
✗ Branch 2 not taken.
 82 shash::Md5 hash;
1511 82 memcpy(hash.digest, command_buffer[num_commands].digest,
1512 82 shash::kDigestSizes[shash::kMd5]);
1513
1514 82 quota_mgr->LockBackChannels();
1515 const map<shash::Md5, int>::iterator iter = quota_mgr->back_channels_
1516
1/2
✓ Branch 1 taken 82 times.
✗ Branch 2 not taken.
 82 .find(hash);
1517
1/2
✓ Branch 2 taken 82 times.
✗ Branch 3 not taken.
 82 if (iter != quota_mgr->back_channels_.end()) {
1518
1/2
✓ Branch 2 taken 82 times.
✗ Branch 3 not taken.
 82 LogCvmfs(kLogQuota, kLogDebug, "closing back channel %s",
1519
1/2
✓ Branch 1 taken 82 times.
✗ Branch 2 not taken.
 164 hash.ToString().c_str());
1520
1/2
✓ Branch 2 taken 82 times.
✗ Branch 3 not taken.
 82 close(iter->second);
1521
1/2
✓ Branch 1 taken 82 times.
✗ Branch 2 not taken.
 82 quota_mgr->back_channels_.erase(iter);
1522 } else {
1523 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
1524 ✗ "did not find back channel %s", hash.ToString().c_str());
1525 }
1526 82 quota_mgr->UnlockBackChannels();
1527
1528 82 continue;
1529 82 }
1530
1531 // Unpinnings are also handled immediately with respect to the pinned gauge
1532
2/2
✓ Branch 0 taken 82 times.
✓ Branch 1 taken 6154182 times.
 6154264 if (command_type == kUnpin) {
1533
1/2
✓ Branch 1 taken 82 times.
✗ Branch 2 not taken.
 82 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1534
1/2
✓ Branch 1 taken 82 times.
✗ Branch 2 not taken.
 82 const string hash_str(hash.ToString());
1535
1536 const map<shash::Any, uint64_t>::iterator iter = quota_mgr->pinned_chunks_
1537
1/2
✓ Branch 1 taken 82 times.
✗ Branch 2 not taken.
 82 .find(hash);
1538
1/2
✓ Branch 2 taken 82 times.
✗ Branch 3 not taken.
 82 if (iter != quota_mgr->pinned_chunks_.end()) {
1539 82 quota_mgr->pinned_ -= iter->second;
1540
1/2
✓ Branch 1 taken 82 times.
✗ Branch 2 not taken.
 82 quota_mgr->pinned_chunks_.erase(iter);
1541 // It can happen that files get pinned that were removed from the cache
1542 // (see cache.cc). We fix this at this point, where we remove such
1543 // entries from the cache database.
1544
2/4
✓ Branch 1 taken 82 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 82 times.
✗ Branch 5 not taken.
 164 if (!FileExists(quota_mgr->cache_dir_ + "/"
1545
4/6
✓ Branch 1 taken 82 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 82 times.
✗ Branch 5 not taken.
✓ Branch 9 taken 41 times.
✓ Branch 10 taken 41 times.
 246 + hash.MakePathWithoutSuffix())) {
1546
1/2
✓ Branch 2 taken 41 times.
✗ Branch 3 not taken.
 41 LogCvmfs(kLogQuota, kLogDebug,
1547 "remove orphaned pinned hash %s from cache database",
1548 hash_str.c_str());
1549
1/2
✓ Branch 2 taken 41 times.
✗ Branch 3 not taken.
 41 sqlite3_bind_text(quota_mgr->stmt_size_, 1, &hash_str[0],
1550 41 hash_str.length(), SQLITE_STATIC);
1551 int retval;
1552
2/4
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 41 times.
✗ Branch 4 not taken.
 41 if ((retval = sqlite3_step(quota_mgr->stmt_size_)) == SQLITE_ROW) {
1553
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 const uint64_t size = sqlite3_column_int64(quota_mgr->stmt_size_,
1554 41 0);
1555
1/2
✓ Branch 2 taken 41 times.
✗ Branch 3 not taken.
 41 sqlite3_bind_text(quota_mgr->stmt_rm_, 1, &(hash_str[0]),
1556 41 hash_str.length(), SQLITE_STATIC);
1557
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 retval = sqlite3_step(quota_mgr->stmt_rm_);
1558
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 41 if ((retval == SQLITE_DONE) || (retval == SQLITE_OK)) {
1559 41 quota_mgr->gauge_ -= size;
1560 } else {
1561 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1562 "failed to delete %s (%d)", hash_str.c_str(), retval);
1563 }
1564
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 sqlite3_reset(quota_mgr->stmt_rm_);
1565 }
1566
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 sqlite3_reset(quota_mgr->stmt_size_);
1567 }
1568 } else {
1569 ✗ LogCvmfs(kLogQuota, kLogDebug, "this chunk was not pinned");
1570 }
1571 82 }
1572
1573 // Immediate commands trigger flushing of the buffer
1574 6154264 const bool immediate_command = (command_type == kCleanup)
1575
2/2
✓ Branch 0 taken 6153034 times.
✓ Branch 1 taken 861 times.
 6153895 || (command_type == kList)
1576
2/2
✓ Branch 0 taken 6152706 times.
✓ Branch 1 taken 328 times.
 6153034 || (command_type == kListPinned)
1577
2/2
✓ Branch 0 taken 6152583 times.
✓ Branch 1 taken 123 times.
 6152706 || (command_type == kListCatalogs)
1578
2/2
✓ Branch 0 taken 6152460 times.
✓ Branch 1 taken 123 times.
 6152583 || (command_type == kListVolatile)
1579
2/2
✓ Branch 0 taken 6152337 times.
✓ Branch 1 taken 123 times.
 6152460 || (command_type == kRemove)
1580
2/2
✓ Branch 0 taken 6151599 times.
✓ Branch 1 taken 738 times.
 6152337 || (command_type == kStatus)
1581
1/2
✓ Branch 0 taken 6151599 times.
✗ Branch 1 not taken.
 6151599 || (command_type == kLimits)
1582
3/4
✓ Branch 0 taken 6153895 times.
✓ Branch 1 taken 369 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 6151599 times.
 12308159 || (command_type == kPid);
1583
2/2
✓ Branch 0 taken 6151599 times.
✓ Branch 1 taken 2665 times.
 6154264 if (!immediate_command)
1584 6151599 num_commands++;
1585
1586
4/4
✓ Branch 0 taken 5962097 times.
✓ Branch 1 taken 192167 times.
✓ Branch 2 taken 2665 times.
✓ Branch 3 taken 5959432 times.
 6154264 if ((num_commands == kCommandBufferSize) || immediate_command) {
1587
1/2
✓ Branch 1 taken 194832 times.
✗ Branch 2 not taken.
 194832 quota_mgr->ProcessCommandBunch(num_commands, command_buffer,
1588 description_buffer);
1589
2/2
✓ Branch 0 taken 192167 times.
✓ Branch 1 taken 2665 times.
 194832 if (!immediate_command)
1590 192167 num_commands = 0;
1591 }
1592
1593
2/2
✓ Branch 0 taken 2665 times.
✓ Branch 1 taken 6151599 times.
 6154264 if (immediate_command) {
1594 // Process cleanup, listings
1595
1/2
✓ Branch 1 taken 2665 times.
✗ Branch 2 not taken.
 2665 const int return_pipe = quota_mgr->BindReturnPipe(
1596 command_buffer[num_commands].return_pipe);
1597
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2665 times.
 2665 if (return_pipe < 0) {
1598 ✗ num_commands = 0;
1599 ✗ continue;
1600 }
1601
1602 int retval;
1603 2665 sqlite3_stmt *this_stmt_list = NULL;
1604
7/10
✓ Branch 0 taken 123 times.
✓ Branch 1 taken 369 times.
✓ Branch 2 taken 861 times.
✓ Branch 3 taken 328 times.
✓ Branch 4 taken 123 times.
✓ Branch 5 taken 123 times.
✓ Branch 6 taken 738 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
 2665 switch (command_type) {
1605 123 case kRemove: {
1606
1/2
✓ Branch 1 taken 123 times.
✗ Branch 2 not taken.
 123 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1607
1/2
✓ Branch 1 taken 123 times.
✗ Branch 2 not taken.
 123 const string hash_str = hash.ToString();
1608
1/2
✓ Branch 2 taken 123 times.
✗ Branch 3 not taken.
 123 LogCvmfs(kLogQuota, kLogDebug, "manually removing %s",
1609 hash_str.c_str());
1610 123 bool success = false;
1611
1612
1/2
✓ Branch 2 taken 123 times.
✗ Branch 3 not taken.
 123 sqlite3_bind_text(quota_mgr->stmt_size_, 1, &hash_str[0],
1613 123 hash_str.length(), SQLITE_STATIC);
1614 int retval;
1615
3/4
✓ Branch 1 taken 123 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 82 times.
✓ Branch 4 taken 41 times.
 123 if ((retval = sqlite3_step(quota_mgr->stmt_size_)) == SQLITE_ROW) {
1616
1/2
✓ Branch 1 taken 82 times.
✗ Branch 2 not taken.
 82 const uint64_t size = sqlite3_column_int64(quota_mgr->stmt_size_,
1617 82 0);
1618
1/2
✓ Branch 1 taken 82 times.
✗ Branch 2 not taken.
 82 const uint64_t is_pinned = sqlite3_column_int64(
1619 82 quota_mgr->stmt_size_, 1);
1620
1621
1/2
✓ Branch 2 taken 82 times.
✗ Branch 3 not taken.
 82 sqlite3_bind_text(quota_mgr->stmt_rm_, 1, &(hash_str[0]),
1622 82 hash_str.length(), SQLITE_STATIC);
1623
1/2
✓ Branch 1 taken 82 times.
✗ Branch 2 not taken.
 82 retval = sqlite3_step(quota_mgr->stmt_rm_);
1624
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 82 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 82 if ((retval == SQLITE_DONE) || (retval == SQLITE_OK)) {
1625 82 success = true;
1626 82 quota_mgr->gauge_ -= size;
1627
2/2
✓ Branch 0 taken 41 times.
✓ Branch 1 taken 41 times.
 82 if (is_pinned) {
1628
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 quota_mgr->pinned_chunks_.erase(hash);
1629 41 quota_mgr->pinned_ -= size;
1630 }
1631 } else {
1632 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1633 "failed to delete %s (%d)", hash_str.c_str(), retval);
1634 }
1635
1/2
✓ Branch 1 taken 82 times.
✗ Branch 2 not taken.
 82 sqlite3_reset(quota_mgr->stmt_rm_);
1636 } else {
1637 // File does not exist
1638 41 success = true;
1639 }
1640
1/2
✓ Branch 1 taken 123 times.
✗ Branch 2 not taken.
 123 sqlite3_reset(quota_mgr->stmt_size_);
1641
1642
1/2
✓ Branch 1 taken 123 times.
✗ Branch 2 not taken.
 123 WritePipe(return_pipe, &success, sizeof(success));
1643 123 break;
1644 123 }
1645 369 case kCleanup:
1646
1/2
✓ Branch 1 taken 369 times.
✗ Branch 2 not taken.
 369 retval = quota_mgr->DoCleanup(size);
1647
1/2
✓ Branch 1 taken 369 times.
✗ Branch 2 not taken.
 369 WritePipe(return_pipe, &retval, sizeof(retval));
1648 369 break;
1649 861 case kList:
1650
1/2
✓ Branch 0 taken 861 times.
✗ Branch 1 not taken.
 861 if (!this_stmt_list)
1651 861 this_stmt_list = quota_mgr->stmt_list_;
1652 case kListPinned:
1653
2/2
✓ Branch 0 taken 328 times.
✓ Branch 1 taken 861 times.
 1189 if (!this_stmt_list)
1654 328 this_stmt_list = quota_mgr->stmt_list_pinned_;
1655 case kListCatalogs:
1656
2/2
✓ Branch 0 taken 123 times.
✓ Branch 1 taken 1189 times.
 1312 if (!this_stmt_list)
1657 123 this_stmt_list = quota_mgr->stmt_list_catalogs_;
1658 case kListVolatile:
1659
2/2
✓ Branch 0 taken 123 times.
✓ Branch 1 taken 1312 times.
 1435 if (!this_stmt_list)
1660 123 this_stmt_list = quota_mgr->stmt_list_volatile_;
1661
1662 // Pipe back the list, one by one
1663 int length;
1664
3/4
✓ Branch 1 taken 4103034 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 4101599 times.
✓ Branch 4 taken 1435 times.
 4103034 while (sqlite3_step(this_stmt_list) == SQLITE_ROW) {
1665
1/2
✓ Branch 2 taken 4101599 times.
✗ Branch 3 not taken.
 4101599 string path = "(NULL)";
1666
2/4
✓ Branch 1 taken 4101599 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 4101599 times.
✗ Branch 4 not taken.
 4101599 if (sqlite3_column_type(this_stmt_list, 0) != SQLITE_NULL) {
1667 8203198 path = string(reinterpret_cast<const char *>(
1668
2/4
✓ Branch 1 taken 4101599 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 4101599 times.
✗ Branch 5 not taken.
 4101599 sqlite3_column_text(this_stmt_list, 0)));
1669 }
1670 4101599 length = path.length();
1671
1/2
✓ Branch 1 taken 4101599 times.
✗ Branch 2 not taken.
 4101599 WritePipe(return_pipe, &length, sizeof(length));
1672
1/2
✓ Branch 0 taken 4101599 times.
✗ Branch 1 not taken.
 4101599 if (length > 0)
1673
2/4
✓ Branch 1 taken 4101599 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 4101599 times.
✗ Branch 5 not taken.
 4101599 WritePipe(return_pipe, &path[0], length);
1674 4101599 }
1675 1435 length = -1;
1676
1/2
✓ Branch 1 taken 1435 times.
✗ Branch 2 not taken.
 1435 WritePipe(return_pipe, &length, sizeof(length));
1677
1/2
✓ Branch 1 taken 1435 times.
✗ Branch 2 not taken.
 1435 sqlite3_reset(this_stmt_list);
1678 1435 break;
1679 738 case kStatus:
1680
1/2
✓ Branch 1 taken 738 times.
✗ Branch 2 not taken.
 738 WritePipe(return_pipe, &quota_mgr->gauge_, sizeof(quota_mgr->gauge_));
1681
1/2
✓ Branch 1 taken 738 times.
✗ Branch 2 not taken.
 738 WritePipe(return_pipe, &quota_mgr->pinned_,
1682 sizeof(quota_mgr->pinned_));
1683 738 break;
1684 ✗ case kLimits:
1685 ✗ WritePipe(return_pipe, &quota_mgr->limit_, sizeof(quota_mgr->limit_));
1686 ✗ WritePipe(return_pipe, &quota_mgr->cleanup_threshold_,
1687 sizeof(quota_mgr->cleanup_threshold_));
1688 ✗ break;
1689 ✗ case kPid: {
1690 ✗ pid_t pid = getpid();
1691 ✗ WritePipe(return_pipe, &pid, sizeof(pid));
1692 ✗ break;
1693 }
1694 ✗ default:
1695 ✗ PANIC(NULL); // other types are handled by the bunch processor
1696 }
1697
1/2
✓ Branch 1 taken 2665 times.
✗ Branch 2 not taken.
 2665 quota_mgr->UnbindReturnPipe(return_pipe);
1698 2665 num_commands = 0;
1699 }
1700 }
1701
1702
1/2
✓ Branch 1 taken 1271 times.
✗ Branch 2 not taken.
 1271 LogCvmfs(kLogQuota, kLogDebug, "stopping cache manager (%d)", errno);
1703
1/2
✓ Branch 1 taken 1271 times.
✗ Branch 2 not taken.
 1271 close(quota_mgr->pipe_lru_[0]);
1704
1/2
✓ Branch 1 taken 1271 times.
✗ Branch 2 not taken.
 1271 quota_mgr->ProcessCommandBunch(num_commands, command_buffer,
1705 description_buffer);
1706
1707 // Unpin
1708 1271 command_buffer[0].command_type = kTouch;
1709 1271 for (map<shash::Any, uint64_t>::const_iterator
1710 1271 i = quota_mgr->pinned_chunks_.begin(),
1711 1271 iEnd = quota_mgr->pinned_chunks_.end();
1712
2/2
✓ Branch 1 taken 410 times.
✓ Branch 2 taken 1271 times.
 1681 i != iEnd;
1713 410 ++i) {
1714
1/2
✓ Branch 2 taken 410 times.
✗ Branch 3 not taken.
 410 command_buffer[0].StoreHash(i->first);
1715
1/2
✓ Branch 1 taken 410 times.
✗ Branch 2 not taken.
 410 quota_mgr->ProcessCommandBunch(1, command_buffer, description_buffer);
1716 }
1717
1718 1271 return NULL;
1719 }
1720
1721
1722 3772 void PosixQuotaManager::MakeReturnPipe(int pipe[2]) {
1723
2/2
✓ Branch 0 taken 3649 times.
✓ Branch 1 taken 123 times.
 3772 if (!shared_) {
1724 3649 MakePipe(pipe);
1725 3649 return;
1726 }
1727
1728 // Create FIFO in cache directory, store path name (number) in pipe write end
1729 123 int i = 0;
1730 int retval;
1731 do {
1732
2/4
✓ Branch 2 taken 164 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 164 times.
✗ Branch 6 not taken.
 164 retval = mkfifo((workspace_dir_ + "/pipe" + StringifyInt(i)).c_str(), 0600);
1733 164 pipe[1] = i;
1734 164 i++;
1735
3/4
✓ Branch 0 taken 41 times.
✓ Branch 1 taken 123 times.
✓ Branch 2 taken 41 times.
✗ Branch 3 not taken.
 164 } while ((retval == -1) && (errno == EEXIST));
1736
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 123 times.
 123 assert(retval == 0);
1737
1738 // Connect reader's end
1739
3/6
✓ Branch 2 taken 123 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 123 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 123 times.
✗ Branch 10 not taken.
 123 pipe[0] = open((workspace_dir_ + "/pipe" + StringifyInt(pipe[1])).c_str(),
1740 O_RDONLY | O_NONBLOCK);
1741
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 123 times.
 123 assert(pipe[0] >= 0);
1742 123 Nonblock2Block(pipe[0]);
1743 }
1744
1745
1746 3793 void PosixQuotaManager::ParseDirectories(const std::string cache_workspace,
1747 std::string *cache_dir,
1748 std::string *workspace_dir) {
1749
1/2
✓ Branch 1 taken 3793 times.
✗ Branch 2 not taken.
 3793 vector<string> dir_tokens(SplitString(cache_workspace, ':'));
1750
2/3
✓ Branch 1 taken 3711 times.
✓ Branch 2 taken 82 times.
✗ Branch 3 not taken.
 3793 switch (dir_tokens.size()) {
1751 3711 case 1:
1752
2/4
✓ Branch 2 taken 3711 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 3711 times.
✗ Branch 6 not taken.
 3711 *cache_dir = *workspace_dir = dir_tokens[0];
1753 3711 break;
1754 82 case 2:
1755
1/2
✓ Branch 2 taken 82 times.
✗ Branch 3 not taken.
 82 *cache_dir = dir_tokens[0];
1756
1/2
✓ Branch 2 taken 82 times.
✗ Branch 3 not taken.
 82 *workspace_dir = dir_tokens[1];
1757 82 break;
1758 ✗ default:
1759 ✗ PANIC(NULL);
1760 }
1761 3793 }
1762
1763 41 void PosixQuotaManager::SkipEviction(const EvictCandidate &candidate) {
1764 41 bool res = true;
1765
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 std::string hash_str = candidate.hash.ToString();
1766
1/2
✓ Branch 2 taken 41 times.
✗ Branch 3 not taken.
 41 LogCvmfs(kLogQuota, kLogDebug, "Exclude %s from eviction", hash_str.c_str());
1767
2/4
✓ Branch 2 taken 41 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 41 times.
✗ Branch 6 not taken.
 41 sqlite3_bind_text(stmt_block_, 1, &hash_str[0], hash_str.length(),
1768 SQLITE_STATIC);
1769
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 res = (sqlite3_step(stmt_block_) == SQLITE_DONE);
1770
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 sqlite3_reset(stmt_block_);
1771
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 41 times.
 41 assert(res);
1772 41 }
1773
1774 /**
1775 * Immediately inserts a new pinned catalog. Does cache cleanup if necessary.
1776 *
1777 * \return True on success, false otherwise
1778 */
1779 1787 bool PosixQuotaManager::Pin(const shash::Any &hash,
1780 const uint64_t size,
1781 const string &description,
1782 const bool is_catalog) {
1783
3/4
✓ Branch 0 taken 82 times.
✓ Branch 1 taken 1705 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 82 times.
 1787 assert((size > 0) || !is_catalog);
1784
1785
1/2
✓ Branch 1 taken 1787 times.
✗ Branch 2 not taken.
 1787 const string hash_str = hash.ToString();
1786
1/2
✓ Branch 3 taken 1787 times.
✗ Branch 4 not taken.
 1787 LogCvmfs(kLogQuota, kLogDebug, "pin into lru %s, path %s", hash_str.c_str(),
1787 description.c_str());
1788
1789 // Has to run when not yet spawned (cvmfs initialization)
1790
2/2
✓ Branch 0 taken 1213 times.
✓ Branch 1 taken 574 times.
 1787 if (!spawned_) {
1791 // Code duplication here
1792
3/4
✓ Branch 2 taken 1213 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 715 times.
✓ Branch 6 taken 498 times.
 1213 if (pinned_chunks_.find(hash) == pinned_chunks_.end()) {
1793
2/2
✓ Branch 0 taken 82 times.
✓ Branch 1 taken 633 times.
 715 if (pinned_ + size > cleanup_threshold_) {
1794
1/2
✓ Branch 2 taken 82 times.
✗ Branch 3 not taken.
 82 LogCvmfs(kLogQuota, kLogDebug, "failed to insert %s (pinned), no space",
1795 hash_str.c_str());
1796 82 return false;
1797 } else {
1798
1/2
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
 633 pinned_chunks_[hash] = size;
1799 633 pinned_ += size;
1800
1/2
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
 633 CheckHighPinWatermark();
1801 }
1802 }
1803
1/2
✓ Branch 1 taken 1131 times.
✗ Branch 2 not taken.
 1131 const bool exists = Contains(hash_str);
1804
4/4
✓ Branch 0 taken 633 times.
✓ Branch 1 taken 498 times.
✓ Branch 2 taken 41 times.
✓ Branch 3 taken 592 times.
 1131 if (!exists && (gauge_ + size > limit_)) {
1805
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 LogCvmfs(kLogQuota, kLogDebug, "over limit, gauge %lu, file size %lu",
1806 gauge_, size);
1807
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 const int retval = DoCleanup(cleanup_threshold_);
1808
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 41 times.
 41 assert(retval != 0);
1809 }
1810
1/2
✓ Branch 3 taken 1131 times.
✗ Branch 4 not taken.
 1131 sqlite3_bind_text(stmt_new_, 1, &hash_str[0], hash_str.length(),
1811 SQLITE_STATIC);
1812
1/2
✓ Branch 1 taken 1131 times.
✗ Branch 2 not taken.
 1131 sqlite3_bind_int64(stmt_new_, 2, size);
1813
1/2
✓ Branch 1 taken 1131 times.
✗ Branch 2 not taken.
 1131 sqlite3_bind_int64(stmt_new_, 3, seq_++);
1814
1/2
✓ Branch 3 taken 1131 times.
✗ Branch 4 not taken.
 1131 sqlite3_bind_text(stmt_new_, 4, &description[0], description.length(),
1815 SQLITE_STATIC);
1816
3/4
✓ Branch 0 taken 1008 times.
✓ Branch 1 taken 123 times.
✓ Branch 3 taken 1131 times.
✗ Branch 4 not taken.
 1131 sqlite3_bind_int64(stmt_new_, 5, is_catalog ? kFileCatalog : kFileRegular);
1817
1/2
✓ Branch 1 taken 1131 times.
✗ Branch 2 not taken.
 1131 sqlite3_bind_int64(stmt_new_, 6, 1);
1818
1/2
✓ Branch 1 taken 1131 times.
✗ Branch 2 not taken.
 1131 const int retval = sqlite3_step(stmt_new_);
1819
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1131 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 1131 assert((retval == SQLITE_DONE) || (retval == SQLITE_OK));
1820
1/2
✓ Branch 1 taken 1131 times.
✗ Branch 2 not taken.
 1131 sqlite3_reset(stmt_new_);
1821
2/2
✓ Branch 0 taken 633 times.
✓ Branch 1 taken 498 times.
 1131 if (!exists)
1822 633 gauge_ += size;
1823 1131 return true;
1824 }
1825
1826 int pipe_reserve[2];
1827
1/2
✓ Branch 1 taken 574 times.
✗ Branch 2 not taken.
 574 MakeReturnPipe(pipe_reserve);
1828
1829 574 LruCommand cmd;
1830 574 cmd.command_type = kReserve;
1831 574 cmd.SetSize(size);
1832
1/2
✓ Branch 1 taken 574 times.
✗ Branch 2 not taken.
 574 cmd.StoreHash(hash);
1833 574 cmd.return_pipe = pipe_reserve[1];
1834
1/2
✓ Branch 1 taken 574 times.
✗ Branch 2 not taken.
 574 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
1835 bool result;
1836
1/2
✓ Branch 1 taken 574 times.
✗ Branch 2 not taken.
 574 ManagedReadHalfPipe(pipe_reserve[0], &result, sizeof(result));
1837
1/2
✓ Branch 1 taken 574 times.
✗ Branch 2 not taken.
 574 CloseReturnPipe(pipe_reserve);
1838
1839
2/2
✓ Branch 0 taken 41 times.
✓ Branch 1 taken 533 times.
 574 if (!result)
1840 41 return false;
1841
3/4
✓ Branch 0 taken 82 times.
✓ Branch 1 taken 451 times.
✓ Branch 3 taken 533 times.
✗ Branch 4 not taken.
 533 DoInsert(hash, size, description, is_catalog ? kPin : kPinRegular);
1842
1843 533 return true;
1844 1787 }
1845
1846
1847 3711 PosixQuotaManager::PosixQuotaManager(const uint64_t limit,
1848 const uint64_t cleanup_threshold,
1849 3711 const string &cache_workspace)
1850 3711 : shared_(false)
1851 3711 , spawned_(false)
1852 3711 , limit_(limit)
1853 3711 , cleanup_threshold_(cleanup_threshold)
1854 3711 , gauge_(0)
1855 3711 , pinned_(0)
1856 3711 , seq_(0)
1857 3711 , cache_dir_() // initialized in body
1858 3711 , workspace_dir_() // initialized in body
1859 3711 , fd_lock_cachedb_(-1)
1860 3711 , async_delete_(true)
1861 3711 , cachemgr_pid_(0)
1862 3711 , database_(NULL)
1863 3711 , stmt_touch_(NULL)
1864 3711 , stmt_unpin_(NULL)
1865 3711 , stmt_block_(NULL)
1866 3711 , stmt_unblock_(NULL)
1867 3711 , stmt_new_(NULL)
1868 3711 , stmt_lru_(NULL)
1869 3711 , stmt_size_(NULL)
1870 3711 , stmt_rm_(NULL)
1871 3711 , stmt_rm_batch_(NULL)
1872 3711 , stmt_list_(NULL)
1873 3711 , stmt_list_pinned_(NULL)
1874 3711 , stmt_list_catalogs_(NULL)
1875 3711 , stmt_list_volatile_(NULL)
1876 3711 , initialized_(false)
1877 7422 , cleanup_unused_first_(false) {
1878
2/4
✓ Branch 1 taken 3711 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3711 times.
✗ Branch 5 not taken.
 3711 ParseDirectories(cache_workspace, &cache_dir_, &workspace_dir_);
1879 3711 pipe_lru_[0] = pipe_lru_[1] = -1;
1880
1/2
✓ Branch 1 taken 3711 times.
✗ Branch 2 not taken.
 3711 cleanup_recorder_.AddRecorder(1, 90); // last 1.5 min with second resolution
1881 // last 1.5 h with minute resolution
1882
1/2
✓ Branch 1 taken 3711 times.
✗ Branch 2 not taken.
 3711 cleanup_recorder_.AddRecorder(60, 90 * 60);
1883 // last 18 hours with 20 min resolution
1884
1/2
✓ Branch 1 taken 3711 times.
✗ Branch 2 not taken.
 3711 cleanup_recorder_.AddRecorder(20 * 60, 60 * 60 * 18);
1885 // last 4 days with hour resolution
1886
1/2
✓ Branch 1 taken 3711 times.
✗ Branch 2 not taken.
 3711 cleanup_recorder_.AddRecorder(60 * 60, 60 * 60 * 24 * 4);
1887
1888 3711 lock_open_files_ = reinterpret_cast<pthread_mutex_t *>(
1889 3711 smalloc(sizeof(pthread_mutex_t)));
1890 3711 }
1891
1892
1893 14840 PosixQuotaManager::~PosixQuotaManager() {
1894 7420 free(lock_open_files_);
1895
1896
2/2
✓ Branch 0 taken 123 times.
✓ Branch 1 taken 3587 times.
 7420 if (!initialized_)
1897 246 return;
1898
1899
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3587 times.
 7174 if (shared_) {
1900 // Most of cleanup is done elsewhen by shared cache manager
1901 ✗ close(pipe_lru_[1]);
1902 ✗ return;
1903 }
1904
1905
2/2
✓ Branch 0 taken 1271 times.
✓ Branch 1 taken 2316 times.
 7174 if (spawned_) {
1906 2542 char fin = 0;
1907 2542 WritePipe(pipe_lru_[1], &fin, 1);
1908 2542 close(pipe_lru_[1]);
1909 2542 pthread_join(thread_lru_, NULL);
1910 } else {
1911 4632 ClosePipe(pipe_lru_);
1912 }
1913
1914 7174 CloseDatabase();
1915
14/14
✓ Branch 1 taken 3587 times.
✓ Branch 2 taken 123 times.
✓ Branch 4 taken 3587 times.
✓ Branch 5 taken 123 times.
✓ Branch 7 taken 3587 times.
✓ Branch 8 taken 123 times.
✓ Branch 10 taken 3587 times.
✓ Branch 11 taken 123 times.
✓ Branch 13 taken 3587 times.
✓ Branch 14 taken 123 times.
✓ Branch 16 taken 3587 times.
✓ Branch 17 taken 123 times.
✓ Branch 19 taken 3587 times.
✓ Branch 20 taken 123 times.
 16316 }
1916
1917
1918 196513 void PosixQuotaManager::ProcessCommandBunch(const unsigned num,
1919 const LruCommand *commands,
1920 const char *descriptions) {
1921 196513 int retval = sqlite3_exec(database_, "BEGIN", NULL, NULL, NULL);
1922
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 196513 times.
 196513 assert(retval == SQLITE_OK);
1923
1924
2/2
✓ Branch 0 taken 6152009 times.
✓ Branch 1 taken 196513 times.
 6348522 for (unsigned i = 0; i < num; ++i) {
1925
1/2
✓ Branch 1 taken 6152009 times.
✗ Branch 2 not taken.
 6152009 const shash::Any hash = commands[i].RetrieveHash();
1926
1/2
✓ Branch 1 taken 6152009 times.
✗ Branch 2 not taken.
 6152009 const string hash_str = hash.ToString();
1927 6152009 const unsigned size = commands[i].GetSize();
1928
1/2
✓ Branch 1 taken 6152009 times.
✗ Branch 2 not taken.
 6152009 LogCvmfs(kLogQuota, kLogDebug, "processing %s (%d)", hash_str.c_str(),
1929 6152009 commands[i].command_type);
1930
1931 bool exists;
1932
3/4
✓ Branch 0 taken 2050533 times.
✓ Branch 1 taken 82 times.
✓ Branch 2 taken 4101394 times.
✗ Branch 3 not taken.
 6152009 switch (commands[i].command_type) {
1933 2050533 case kTouch:
1934
1/2
✓ Branch 1 taken 2050533 times.
✗ Branch 2 not taken.
 2050533 sqlite3_bind_int64(stmt_touch_, 1, seq_++);
1935
1/2
✓ Branch 3 taken 2050533 times.
✗ Branch 4 not taken.
 2050533 sqlite3_bind_text(stmt_touch_, 2, &hash_str[0], hash_str.length(),
1936 SQLITE_STATIC);
1937
1/2
✓ Branch 1 taken 2050533 times.
✗ Branch 2 not taken.
 2050533 retval = sqlite3_step(stmt_touch_);
1938
1/2
✓ Branch 1 taken 2050533 times.
✗ Branch 2 not taken.
 2050533 LogCvmfs(kLogQuota, kLogDebug, "touching %s (%ld): %d",
1939 2050533 hash_str.c_str(), seq_ - 1, retval);
1940
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2050533 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 2050533 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
1941 ✗ PANIC(kLogSyslogErr, "failed to update %s in cachedb, error %d",
1942 hash_str.c_str(), retval);
1943 }
1944
1/2
✓ Branch 1 taken 2050533 times.
✗ Branch 2 not taken.
 2050533 sqlite3_reset(stmt_touch_);
1945 2050533 break;
1946 82 case kUnpin:
1947
1/2
✓ Branch 3 taken 82 times.
✗ Branch 4 not taken.
 82 sqlite3_bind_text(stmt_unpin_, 1, &hash_str[0], hash_str.length(),
1948 SQLITE_STATIC);
1949
1/2
✓ Branch 1 taken 82 times.
✗ Branch 2 not taken.
 82 retval = sqlite3_step(stmt_unpin_);
1950
1/2
✓ Branch 2 taken 82 times.
✗ Branch 3 not taken.
 82 LogCvmfs(kLogQuota, kLogDebug, "unpinning %s: %d", hash_str.c_str(),
1951 retval);
1952
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 82 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 82 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
1953 ✗ PANIC(kLogSyslogErr, "failed to unpin %s in cachedb, error %d",
1954 hash_str.c_str(), retval);
1955 }
1956
1/2
✓ Branch 1 taken 82 times.
✗ Branch 2 not taken.
 82 sqlite3_reset(stmt_unpin_);
1957 82 break;
1958 4101394 case kPin:
1959 case kPinRegular:
1960 case kInsert:
1961 case kInsertVolatile:
1962 // It could already be in, check
1963
1/2
✓ Branch 1 taken 4101394 times.
✗ Branch 2 not taken.
 4101394 exists = Contains(hash_str);
1964
1965 // Cleanup, move to trash and unlink
1966
3/4
✓ Branch 0 taken 4101189 times.
✓ Branch 1 taken 205 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 4101189 times.
 4101394 if (!exists && (gauge_ + size > limit_)) {
1967 ✗ LogCvmfs(kLogQuota, kLogDebug, "over limit, gauge %lu, file size %u",
1968 gauge_, size);
1969 ✗ retval = DoCleanup(cleanup_threshold_);
1970 ✗ assert(retval != 0);
1971 }
1972
1973 // Insert or replace
1974
1/2
✓ Branch 3 taken 4101394 times.
✗ Branch 4 not taken.
 4101394 sqlite3_bind_text(stmt_new_, 1, &hash_str[0], hash_str.length(),
1975 SQLITE_STATIC);
1976
1/2
✓ Branch 1 taken 4101394 times.
✗ Branch 2 not taken.
 4101394 sqlite3_bind_int64(stmt_new_, 2, size);
1977
2/2
✓ Branch 0 taken 164 times.
✓ Branch 1 taken 4101230 times.
 4101394 if (commands[i].command_type == kInsertVolatile) {
1978
1/2
✓ Branch 1 taken 164 times.
✗ Branch 2 not taken.
 164 sqlite3_bind_int64(stmt_new_, 3, (seq_++) | kVolatileFlag);
1979 } else {
1980
1/2
✓ Branch 1 taken 4101230 times.
✗ Branch 2 not taken.
 4101230 sqlite3_bind_int64(stmt_new_, 3, seq_++);
1981 }
1982 4101394 sqlite3_bind_text(stmt_new_, 4, &descriptions[i * kMaxDescription],
1983
1/2
✓ Branch 1 taken 4101394 times.
✗ Branch 2 not taken.
 4101394 commands[i].desc_length, SQLITE_STATIC);
1984
1/2
✓ Branch 1 taken 4101394 times.
✗ Branch 2 not taken.
 4101394 sqlite3_bind_int64(
1985 stmt_new_, 5,
1986
2/2
✓ Branch 0 taken 82 times.
✓ Branch 1 taken 4101312 times.
 4101394 (commands[i].command_type == kPin) ? kFileCatalog : kFileRegular);
1987
1/2
✓ Branch 1 taken 4101394 times.
✗ Branch 2 not taken.
 4101394 sqlite3_bind_int64(stmt_new_, 6,
1988
2/2
✓ Branch 0 taken 4101312 times.
✓ Branch 1 taken 82 times.
 4101394 ((commands[i].command_type == kPin)
1989
2/2
✓ Branch 0 taken 451 times.
✓ Branch 1 taken 4100861 times.
 4101312 || (commands[i].command_type == kPinRegular))
1990 ? 1
1991 : 0);
1992
1/2
✓ Branch 1 taken 4101394 times.
✗ Branch 2 not taken.
 4101394 retval = sqlite3_step(stmt_new_);
1993
1/2
✓ Branch 1 taken 4101394 times.
✗ Branch 2 not taken.
 4101394 LogCvmfs(kLogQuota, kLogDebug, "insert or replace %s, method %d: %d",
1994 4101394 hash_str.c_str(), commands[i].command_type, retval);
1995
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 4101394 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 4101394 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
1996 ✗ PANIC(kLogSyslogErr, "failed to insert %s in cachedb, error %d",
1997 hash_str.c_str(), retval);
1998 }
1999
1/2
✓ Branch 1 taken 4101394 times.
✗ Branch 2 not taken.
 4101394 sqlite3_reset(stmt_new_);
2000
2001
2/2
✓ Branch 0 taken 4101189 times.
✓ Branch 1 taken 205 times.
 4101394 if (!exists)
2002 4101189 gauge_ += size;
2003 4101394 break;
2004 ✗ default:
2005 // other types should have been taken care of by event loop
2006 ✗ PANIC(NULL);
2007 }
2008 6152009 }
2009
2010 196513 retval = sqlite3_exec(database_, "COMMIT", NULL, NULL, NULL);
2011
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 196513 times.
 196513 if (retval != SQLITE_OK) {
2012 ✗ PANIC(kLogSyslogErr, "failed to commit to cachedb, error %d", retval);
2013 }
2014 196513 }
2015
2016
2017 3629 bool PosixQuotaManager::RebuildDatabase() {
2018 3629 bool result = false;
2019 3629 string sql;
2020 3629 sqlite3_stmt *stmt_select = NULL;
2021 3629 sqlite3_stmt *stmt_insert = NULL;
2022 int sqlerr;
2023 3629 int seq = 0;
2024 char hex[4];
2025 struct stat info;
2026 platform_dirent64 *d;
2027 3629 DIR *dirp = NULL;
2028 3629 string path;
2029
2030
1/2
✓ Branch 1 taken 3629 times.
✗ Branch 2 not taken.
 3629 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug, "re-building cache database");
2031
2032 // Empty cache catalog and fscache
2033
1/2
✓ Branch 1 taken 3629 times.
✗ Branch 2 not taken.
 3629 sql = "DELETE FROM cache_catalog; DELETE FROM fscache;";
2034
1/2
✓ Branch 2 taken 3629 times.
✗ Branch 3 not taken.
 3629 sqlerr = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
2035
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3629 times.
 3629 if (sqlerr != SQLITE_OK) {
2036 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not clear cache database");
2037 ✗ goto build_return;
2038 }
2039
2040 3629 gauge_ = 0;
2041
2042 // Insert files from cache sub-directories 00 - ff
2043 // TODO(jblomer): fs_traversal
2044
1/2
✓ Branch 1 taken 3629 times.
✗ Branch 2 not taken.
 3629 sqlite3_prepare_v2(database_,
2045 "INSERT INTO fscache (sha1, size, actime) "
2046 "VALUES (:sha1, :s, :t);",
2047 -1, &stmt_insert, NULL);
2048
2049
2/2
✓ Branch 0 taken 897659 times.
✓ Branch 1 taken 3506 times.
 901165 for (int i = 0; i <= 0xff; i++) {
2050 897659 snprintf(hex, sizeof(hex), "%02x", i);
2051
3/6
✓ Branch 2 taken 897659 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 897659 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 897659 times.
✗ Branch 9 not taken.
 897659 path = cache_dir_ + "/" + string(hex);
2052
3/4
✓ Branch 2 taken 897659 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 123 times.
✓ Branch 5 taken 897536 times.
 897659 if ((dirp = opendir(path.c_str())) == NULL) {
2053
1/2
✓ Branch 2 taken 123 times.
✗ Branch 3 not taken.
 123 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
2054 "failed to open directory %s (tmpwatch interfering?)",
2055 path.c_str());
2056 123 goto build_return;
2057 }
2058
3/4
✓ Branch 1 taken 2692690 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1795154 times.
✓ Branch 4 taken 897536 times.
 2692690 while ((d = platform_readdir(dirp)) != NULL) {
2059
3/6
✓ Branch 2 taken 1795154 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1795154 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1795154 times.
✗ Branch 9 not taken.
 3590308 const string file_path = path + "/" + string(d->d_name);
2060
1/2
✓ Branch 2 taken 1795154 times.
✗ Branch 3 not taken.
 1795154 if (stat(file_path.c_str(), &info) == 0) {
2061
2/2
✓ Branch 0 taken 1795072 times.
✓ Branch 1 taken 82 times.
 1795154 if (!S_ISREG(info.st_mode))
2062 1795113 continue;
2063
2/2
✓ Branch 0 taken 41 times.
✓ Branch 1 taken 41 times.
 82 if (info.st_size == 0) {
2064
1/2
✓ Branch 2 taken 41 times.
✗ Branch 3 not taken.
 41 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug,
2065 "removing empty file %s during automatic cache db rebuild",
2066 file_path.c_str());
2067 41 unlink(file_path.c_str());
2068 41 continue;
2069 }
2070
2071
3/6
✓ Branch 2 taken 41 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 41 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 41 times.
✗ Branch 10 not taken.
 82 string hash = string(hex) + string(d->d_name);
2072
1/2
✓ Branch 3 taken 41 times.
✗ Branch 4 not taken.
 41 sqlite3_bind_text(stmt_insert, 1, hash.data(), hash.length(),
2073 SQLITE_STATIC);
2074
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 sqlite3_bind_int64(stmt_insert, 2, info.st_size);
2075
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 sqlite3_bind_int64(stmt_insert, 3, info.st_atime);
2076
2/4
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 41 times.
 41 if (sqlite3_step(stmt_insert) != SQLITE_DONE) {
2077 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not insert into temp table");
2078 ✗ goto build_return;
2079 }
2080
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 sqlite3_reset(stmt_insert);
2081
2082 41 gauge_ += info.st_size;
2083
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 } else {
2084 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not stat %s", file_path.c_str());
2085 }
2086
2/3
✓ Branch 1 taken 41 times.
✓ Branch 2 taken 1795113 times.
✗ Branch 3 not taken.
 1795154 }
2087
1/2
✓ Branch 1 taken 897536 times.
✗ Branch 2 not taken.
 897536 closedir(dirp);
2088 897536 dirp = NULL;
2089 }
2090
1/2
✓ Branch 1 taken 3506 times.
✗ Branch 2 not taken.
 3506 sqlite3_finalize(stmt_insert);
2091 3506 stmt_insert = NULL;
2092
2093 // Transfer from temp table in cache catalog
2094
1/2
✓ Branch 1 taken 3506 times.
✗ Branch 2 not taken.
 3506 sqlite3_prepare_v2(database_,
2095 "SELECT sha1, size FROM fscache ORDER BY actime;", -1,
2096 &stmt_select, NULL);
2097
1/2
✓ Branch 1 taken 3506 times.
✗ Branch 2 not taken.
 3506 sqlite3_prepare_v2(
2098 database_,
2099 "INSERT INTO cache_catalog (sha1, size, acseq, path, type, pinned) "
2100 "VALUES (:sha1, :s, :seq, 'unknown (automatic rebuild)', :t, 0);",
2101 -1, &stmt_insert, NULL);
2102
3/4
✓ Branch 1 taken 3547 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 41 times.
✓ Branch 4 taken 3506 times.
 3547 while (sqlite3_step(stmt_select) == SQLITE_ROW) {
2103 const string hash = string(
2104
2/4
✓ Branch 2 taken 41 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 41 times.
✗ Branch 6 not taken.
 41 reinterpret_cast<const char *>(sqlite3_column_text(stmt_select, 0)));
2105
1/2
✓ Branch 3 taken 41 times.
✗ Branch 4 not taken.
 41 sqlite3_bind_text(stmt_insert, 1, &hash[0], hash.length(), SQLITE_STATIC);
2106
2/4
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 41 times.
✗ Branch 5 not taken.
 41 sqlite3_bind_int64(stmt_insert, 2, sqlite3_column_int64(stmt_select, 1));
2107
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 sqlite3_bind_int64(stmt_insert, 3, seq++);
2108 // Might also be a catalog (information is lost)
2109
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 sqlite3_bind_int64(stmt_insert, 4, kFileRegular);
2110
2111
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 const int retval = sqlite3_step(stmt_insert);
2112
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 41 times.
 41 if (retval != SQLITE_DONE) {
2113 // If the file system hosting the cache is full, we'll likely notice here
2114 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
2115 "could not insert into cache catalog (%d - %s)", retval,
2116 sqlite3_errstr(retval));
2117 ✗ goto build_return;
2118 }
2119
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 sqlite3_reset(stmt_insert);
2120
1/2
✓ Branch 1 taken 41 times.
✗ Branch 2 not taken.
 41 }
2121
2122 // Delete temporary table
2123
1/2
✓ Branch 1 taken 3506 times.
✗ Branch 2 not taken.
 3506 sql = "DELETE FROM fscache;";
2124
1/2
✓ Branch 2 taken 3506 times.
✗ Branch 3 not taken.
 3506 sqlerr = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
2125
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3506 times.
 3506 if (sqlerr != SQLITE_OK) {
2126 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not clear temporary table (%d)",
2127 sqlerr);
2128 ✗ goto build_return;
2129 }
2130
2131 3506 seq_ = seq;
2132 3506 result = true;
2133
1/2
✓ Branch 1 taken 3506 times.
✗ Branch 2 not taken.
 3506 LogCvmfs(kLogQuota, kLogDebug,
2134 "rebuilding finished, sequence %" PRIu64 ", gauge %" PRIu64, seq_,
2135 gauge_);
2136
2137 3629 build_return:
2138
1/2
✓ Branch 0 taken 3629 times.
✗ Branch 1 not taken.
 3629 if (stmt_insert)
2139
1/2
✓ Branch 1 taken 3629 times.
✗ Branch 2 not taken.
 3629 sqlite3_finalize(stmt_insert);
2140
2/2
✓ Branch 0 taken 3506 times.
✓ Branch 1 taken 123 times.
 3629 if (stmt_select)
2141
1/2
✓ Branch 1 taken 3506 times.
✗ Branch 2 not taken.
 3506 sqlite3_finalize(stmt_select);
2142
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3629 times.
 3629 if (dirp)
2143 ✗ closedir(dirp);
2144 3629 return result;
2145 3629 }
2146
2147
2148 /**
2149 * Register a channel that allows the cache manager to trigger action to its
2150 * clients. Currently used for releasing pinned catalogs.
2151 */
2152 164 void PosixQuotaManager::RegisterBackChannel(int back_channel[2],
2153 const string &channel_id) {
2154
1/2
✓ Branch 0 taken 164 times.
✗ Branch 1 not taken.
 164 if (protocol_revision_ >= 1) {
2155
1/2
✓ Branch 2 taken 164 times.
✗ Branch 3 not taken.
 164 shash::Md5 hash = shash::Md5(shash::AsciiPtr(channel_id));
2156
1/2
✓ Branch 1 taken 164 times.
✗ Branch 2 not taken.
 164 MakeReturnPipe(back_channel);
2157
2158 164 LruCommand cmd;
2159 164 cmd.command_type = kRegisterBackChannel;
2160 164 cmd.return_pipe = back_channel[1];
2161 // Not StoreHash(). This is an MD5 hash.
2162 164 memcpy(cmd.digest, hash.digest, hash.GetDigestSize());
2163
1/2
✓ Branch 1 taken 164 times.
✗ Branch 2 not taken.
 164 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2164
2165 char success;
2166
1/2
✓ Branch 1 taken 164 times.
✗ Branch 2 not taken.
 164 ManagedReadHalfPipe(back_channel[0], &success, sizeof(success));
2167 // At this point, the named FIFO is unlinked, so don't use CloseReturnPipe
2168
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 164 times.
 164 if (success != 'S') {
2169 ✗ PANIC(kLogDebug | kLogSyslogErr,
2170 "failed to register quota back channel (%c)", success);
2171 }
2172 } else {
2173 // Dummy pipe to return valid file descriptors
2174 ✗ MakePipe(back_channel);
2175 }
2176 164 }
2177
2178
2179 /**
2180 * Removes a chunk from cache, if it exists.
2181 */
2182 123 void PosixQuotaManager::Remove(const shash::Any &hash) {
2183
1/2
✓ Branch 1 taken 123 times.
✗ Branch 2 not taken.
 123 const string hash_str = hash.ToString();
2184
2185 int pipe_remove[2];
2186
1/2
✓ Branch 1 taken 123 times.
✗ Branch 2 not taken.
 123 MakeReturnPipe(pipe_remove);
2187
2188 123 LruCommand cmd;
2189 123 cmd.command_type = kRemove;
2190 123 cmd.return_pipe = pipe_remove[1];
2191
1/2
✓ Branch 1 taken 123 times.
✗ Branch 2 not taken.
 123 cmd.StoreHash(hash);
2192
1/2
✓ Branch 1 taken 123 times.
✗ Branch 2 not taken.
 123 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2193
2194 bool success;
2195
1/2
✓ Branch 1 taken 123 times.
✗ Branch 2 not taken.
 123 ManagedReadHalfPipe(pipe_remove[0], &success, sizeof(success));
2196
1/2
✓ Branch 1 taken 123 times.
✗ Branch 2 not taken.
 123 CloseReturnPipe(pipe_remove);
2197
2198
3/6
✓ Branch 1 taken 123 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 123 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 123 times.
✗ Branch 8 not taken.
 123 unlink((cache_dir_ + "/" + hash.MakePathWithoutSuffix()).c_str());
2199 123 }
2200
2201
2202 1353 void PosixQuotaManager::Spawn() {
2203
2/2
✓ Branch 0 taken 82 times.
✓ Branch 1 taken 1271 times.
 1353 if (spawned_)
2204 82 return;
2205
2206 1271 if (pthread_create(&thread_lru_, NULL, MainCommandServer,
2207 static_cast<void *>(this))
2208
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1271 times.
 1271 != 0) {
2209 ✗ PANIC(kLogDebug, "could not create lru thread");
2210 }
2211
2212 1271 spawned_ = true;
2213 }
2214
2215
2216 /**
2217 * Updates the sequence number of the file specified by the hash.
2218 */
2219 2050721 void PosixQuotaManager::Touch(const shash::Any &hash) {
2220 2050721 LruCommand cmd;
2221 2050721 cmd.command_type = kTouch;
2222
1/2
✓ Branch 1 taken 2050721 times.
✗ Branch 2 not taken.
 2050721 cmd.StoreHash(hash);
2223
1/2
✓ Branch 1 taken 2050721 times.
✗ Branch 2 not taken.
 2050721 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2224 2050721 }
2225
2226
2227 3526 void PosixQuotaManager::UnbindReturnPipe(int pipe_wronly) {
2228
2/2
✓ Branch 0 taken 41 times.
✓ Branch 1 taken 3485 times.
 3526 if (shared_)
2229 41 close(pipe_wronly);
2230 3526 }
2231
2232
2233 287 void PosixQuotaManager::UnlinkReturnPipe(int pipe_wronly) {
2234
2/2
✓ Branch 0 taken 123 times.
✓ Branch 1 taken 164 times.
 287 if (shared_)
2235
2/4
✓ Branch 2 taken 123 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 123 times.
✗ Branch 6 not taken.
 123 unlink((workspace_dir_ + "/pipe" + StringifyInt(pipe_wronly)).c_str());
2236 287 }
2237
2238
2239 845 void PosixQuotaManager::Unpin(const shash::Any &hash) {
2240
2/4
✓ Branch 1 taken 845 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 845 times.
✗ Branch 6 not taken.
 845 LogCvmfs(kLogQuota, kLogDebug, "Unpin %s", hash.ToString().c_str());
2241
2242 845 LruCommand cmd;
2243 845 cmd.command_type = kUnpin;
2244
1/2
✓ Branch 1 taken 845 times.
✗ Branch 2 not taken.
 845 cmd.StoreHash(hash);
2245
1/2
✓ Branch 1 taken 845 times.
✗ Branch 2 not taken.
 845 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2246 845 }
2247
2248
2249 82 void PosixQuotaManager::UnregisterBackChannel(int back_channel[2],
2250 const string &channel_id) {
2251
1/2
✓ Branch 0 taken 82 times.
✗ Branch 1 not taken.
 82 if (protocol_revision_ >= 1) {
2252
1/2
✓ Branch 2 taken 82 times.
✗ Branch 3 not taken.
 82 shash::Md5 hash = shash::Md5(shash::AsciiPtr(channel_id));
2253
2254 82 LruCommand cmd;
2255 82 cmd.command_type = kUnregisterBackChannel;
2256 // Not StoreHash(). This is an MD5 hash.
2257 82 memcpy(cmd.digest, hash.digest, hash.GetDigestSize());
2258
1/2
✓ Branch 1 taken 82 times.
✗ Branch 2 not taken.
 82 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2259
2260 // Writer's end will be closed by cache manager, FIFO is already unlinked
2261
1/2
✓ Branch 1 taken 82 times.
✗ Branch 2 not taken.
 82 close(back_channel[0]);
2262 } else {
2263 ✗ ClosePipe(back_channel);
2264 }
2265 82 }
2266
2267 4105207 void PosixQuotaManager::ManagedReadHalfPipe(int fd, void *buf, size_t nbyte) {
2268
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4105207 times.
 4105207 const unsigned timeout_ms = cachemgr_pid_ ? 1000 : 0;
2269 4105207 bool result = false;
2270 do {
2271 4105207 result = ReadHalfPipe(fd, buf, nbyte, timeout_ms);
2272 // try only as long as the cachemgr is still alive
2273
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 4105207 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 4105207 times.
 4105207 } while (!result && getpgid(cachemgr_pid_) >= 0);
2274
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4105207 times.
 4105207 if (!result) {
2275 ✗ PANIC(kLogStderr,
2276 "Error: quota manager could not read from cachemanager pipe");
2277 }
2278 4105207 }
2279
2280 ✗ void *PosixQuotaManager::CollectMountpointsHashes(void *data) {
2281 #ifndef __APPLE__
2282 ✗ pthread_setname_np(pthread_self(), "hash_collector");
2283 ✗ auto *handler = static_cast<CollectorHandler *>(data);
2284
2285 ✗ const std::string mountpoint = handler->mp[handler->i];
2286 ✗ ssize_t n = getxattr(mountpoint.c_str(), "user.list_open_hashes", nullptr, 0);
2287 ✗ if (n < 0) {
2288 ✗ pthread_exit(nullptr);
2289 }
2290 ✗ std::vector<char> buf((size_t)n);
2291 ✗ n = getxattr(mountpoint.c_str(), "user.list_open_hashes", buf.data(),
2292 buf.size());
2293 ✗ if (n < 0) {
2294 ✗ pthread_exit(nullptr);
2295 }
2296
2297 ✗ std::vector<std::string> hash_strs;
2298 ✗ std::string hash_str;
2299 ✗ for (const char c : buf) {
2300 ✗ if (c == '\n') {
2301 ✗ hash_strs.push_back(hash_str);
2302 ✗ hash_str.clear();
2303 } else {
2304 ✗ hash_str += c;
2305 }
2306 }
2307 ✗ const MutexLockGuard lock_guard(handler->l);
2308 ✗ for (auto hash_str : hash_strs) {
2309 ✗ handler->of.push_back(shash::Short( shash::MkFromHexPtr(shash::HexPtr(hash_str)) ));
2310 }
2311 #endif
2312 ✗ pthread_exit(nullptr);
2313 }
2314
2315 ✗ std::vector<shash::Short> PosixQuotaManager::CollectAllOpenHashes() {
2316 ✗ std::vector<CollectorHandler *> handlers;
2317 ✗ std::vector<pthread_t *> threads;
2318 ✗ open_files_.clear();
2319 #ifndef __APPLE__
2320 ✗ auto &&a_after_b = [](const struct timespec a, const struct timespec b) {
2321 ✗ return (a.tv_sec > b.tv_sec) ? true : false;
2322 };
2323
2324 ✗ for (size_t i = 0; i < mountpoints_.size(); ++i) {
2325 ✗ handlers.push_back(
2326 ✗ new CollectorHandler{open_files_, mountpoints_, lock_open_files_, i});
2327 ✗ threads.push_back(new pthread_t);
2328 }
2329
2330 ✗ const int retval = pthread_mutex_init(lock_open_files_, NULL);
2331 ✗ assert(retval == 0);
2332
2333 ✗ for (size_t i = 0; i < mountpoints_.size(); ++i) {
2334 ✗ pthread_create(threads[i], nullptr, CollectMountpointsHashes, handlers[i]);
2335 }
2336
2337 ✗ std::vector<bool> joined(handlers.size(), false);
2338 struct timespec reference, current;
2339 ✗ clock_gettime(CLOCK_REALTIME, &reference);
2340 ✗ clock_gettime(CLOCK_REALTIME, &current);
2341 ✗ reference.tv_sec += 10; // Give 10sec for hash collection
2342 ✗ size_t i = 0;
2343 ✗ while (
2344 ✗ (not std::all_of(joined.begin(), joined.end(), [](bool b) { return b; }))
2345 ✗ and a_after_b(reference, current)) {
2346 // as long as there are still threads that haven't joined yet
2347 // and for 10 seconds
2348 ✗ if (not joined[i]) {
2349 ✗ const int s = pthread_tryjoin_np(*threads[i], NULL);
2350 ✗ if (s == 0) {
2351 ✗ joined[i] = true;
2352 }
2353 }
2354 ✗ ++i;
2355 ✗ i = i % handlers.size();
2356 ✗ clock_gettime(CLOCK_REALTIME, &current);
2357 }
2358
2359 ✗ for (size_t i = 0; i < handlers.size(); ++i) {
2360 ✗ delete handlers[i];
2361 }
2362
2363 ✗ pthread_mutex_destroy(lock_open_files_);
2364 #endif
2365 ✗ return open_files_;
2366 }
2367
2368