GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/quota_posix.cc
Date: 2026-03-22 02:40:38
Exec Total Coverage
Lines: 954 1399 68.2%
Branches: 750 1954 38.4%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * This module implements a "managed local cache".
5 * This way, we are able to track access times of files in the cache
6 * and remove files based on least recently used strategy.
7 *
8 * We setup another SQLite catalog, a "cache catalog", that helps us
9 * in the bookkeeping of files, file sizes and access times.
10 *
11 * We might choose to not manage the local cache. This is indicated
12 * by limit == 0 and everything succeeds in that case.
13 */
14
15 #define __STDC_LIMIT_MACROS
16 #define __STDC_FORMAT_MACROS
17
18
19 #include "quota_posix.h"
20
21 #include <dirent.h>
22 #include <errno.h>
23 #include <fcntl.h>
24 #include <inttypes.h>
25 #include <pthread.h>
26 #include <signal.h>
27 #include <stdint.h>
28 #include <sys/dir.h>
29 #include <sys/stat.h>
30 #include <sys/xattr.h>
31
32 #ifndef __APPLE__
33 #include <sys/statfs.h>
34 #endif
35 #include <sys/statvfs.h>
36 #include <sys/types.h>
37 #include <sys/wait.h>
38 #include <unistd.h>
39
40 #include <algorithm>
41 #include <cassert>
42 #include <cstdio>
43 #include <cstdlib>
44 #include <cstring>
45 #include <limits>
46 #include <map>
47 #include <set>
48 #include <string>
49 #include <vector>
50
51 #include "crypto/hash.h"
52 #include "duplex_sqlite3.h"
53 #include "monitor.h"
54 #include "statistics.h"
55 #include "util/capabilities.h"
56 #include "util/concurrency.h"
57 #include "util/exception.h"
58 #include "util/logging.h"
59 #include "util/platform.h"
60 #include "util/pointer.h"
61 #include "util/posix.h"
62 #include "util/smalloc.h"
63 #include "util/string.h"
64
65 using namespace std; // NOLINT
66
67
68 4232 int PosixQuotaManager::BindReturnPipe(int pipe_wronly) {
69
2/2
✓ Branch 0 taken 4140 times.
✓ Branch 1 taken 92 times.
 4232 if (!shared_)
70 4140 return pipe_wronly;
71
72 // Connect writer's end
73
1/2
✓ Branch 2 taken 92 times.
✗ Branch 3 not taken.
 92 const int result = open(
74
2/4
✓ Branch 2 taken 92 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 92 times.
✗ Branch 6 not taken.
 184 (workspace_dir_ + "/pipe" + StringifyInt(pipe_wronly)).c_str(),
75 O_WRONLY | O_NONBLOCK);
76
2/2
✓ Branch 0 taken 46 times.
✓ Branch 1 taken 46 times.
 92 if (result >= 0) {
77 46 Nonblock2Block(result);
78 } else {
79 46 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
80 46 "failed to bind return pipe (%d)", errno);
81 }
82 92 return result;
83 }
84
85
86 924 void PosixQuotaManager::CheckHighPinWatermark() {
87 924 const uint64_t watermark = kHighPinWatermark * cleanup_threshold_ / 100;
88
3/4
✓ Branch 0 taken 924 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 138 times.
✓ Branch 3 taken 786 times.
 924 if ((cleanup_threshold_ > 0) && (pinned_ > watermark)) {
89 138 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
90 "high watermark of pinned files (%" PRIu64 "M > %" PRIu64 "M)",
91 138 pinned_ / (1024 * 1024), watermark / (1024 * 1024));
92
2/4
✓ Branch 2 taken 138 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 138 times.
✗ Branch 6 not taken.
 138 BroadcastBackchannels("R"); // clients: please release pinned catalogs
93 }
94 924 }
95
96
97 ✗ void PosixQuotaManager::CleanupPipes() {
98 ✗ DIR *dirp = opendir(workspace_dir_.c_str());
99 ✗ assert(dirp != NULL);
100
101 platform_dirent64 *dent;
102 ✗ bool found_leftovers = false;
103 ✗ while ((dent = platform_readdir(dirp)) != NULL) {
104 ✗ const string name = dent->d_name;
105 ✗ const string path = workspace_dir_ + "/" + name;
106 platform_stat64 info;
107 ✗ const int retval = platform_stat(path.c_str(), &info);
108 ✗ if (retval != 0)
109 ✗ continue;
110 ✗ if (S_ISFIFO(info.st_mode) && (name.substr(0, 4) == "pipe")) {
111 ✗ if (!found_leftovers) {
112 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogWarn,
113 "removing left-over FIFOs from cache directory");
114 }
115 ✗ found_leftovers = true;
116 ✗ unlink(path.c_str());
117 }
118 }
119 ✗ closedir(dirp);
120 }
121
122
123 /**
124 * Cleans up in data cache, until cache size is below leave_size.
125 * The actual unlinking is done in a separate process (fork).
126 *
127 * \return True on success, false otherwise
128 */
129 414 bool PosixQuotaManager::Cleanup(const uint64_t leave_size) {
130
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 414 times.
 414 if (!spawned_)
131 ✗ return DoCleanup(leave_size);
132
133 bool result;
134 int pipe_cleanup[2];
135
1/2
✓ Branch 1 taken 414 times.
✗ Branch 2 not taken.
 414 MakeReturnPipe(pipe_cleanup);
136
137 414 LruCommand cmd;
138 414 cmd.command_type = kCleanup;
139 414 cmd.size = leave_size;
140 414 cmd.return_pipe = pipe_cleanup[1];
141
142
1/2
✓ Branch 1 taken 414 times.
✗ Branch 2 not taken.
 414 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
143
1/2
✓ Branch 1 taken 414 times.
✗ Branch 2 not taken.
 414 ManagedReadHalfPipe(pipe_cleanup[0], &result, sizeof(result));
144
1/2
✓ Branch 1 taken 414 times.
✗ Branch 2 not taken.
 414 CloseReturnPipe(pipe_cleanup);
145
146 414 return result;
147 }
148
149
150 3041 void PosixQuotaManager::CloseDatabase() {
151
1/2
✓ Branch 0 taken 3041 times.
✗ Branch 1 not taken.
 3041 if (stmt_list_catalogs_)
152 3041 sqlite3_finalize(stmt_list_catalogs_);
153
1/2
✓ Branch 0 taken 3041 times.
✗ Branch 1 not taken.
 3041 if (stmt_list_pinned_)
154 3041 sqlite3_finalize(stmt_list_pinned_);
155
1/2
✓ Branch 0 taken 3041 times.
✗ Branch 1 not taken.
 3041 if (stmt_list_volatile_)
156 3041 sqlite3_finalize(stmt_list_volatile_);
157
1/2
✓ Branch 0 taken 3041 times.
✗ Branch 1 not taken.
 3041 if (stmt_list_)
158 3041 sqlite3_finalize(stmt_list_);
159
1/2
✓ Branch 0 taken 3041 times.
✗ Branch 1 not taken.
 3041 if (stmt_lru_)
160 3041 sqlite3_finalize(stmt_lru_);
161
1/2
✓ Branch 0 taken 3041 times.
✗ Branch 1 not taken.
 3041 if (stmt_rm_)
162 3041 sqlite3_finalize(stmt_rm_);
163
1/2
✓ Branch 0 taken 3041 times.
✗ Branch 1 not taken.
 3041 if (stmt_rm_batch_)
164 3041 sqlite3_finalize(stmt_rm_batch_);
165
1/2
✓ Branch 0 taken 3041 times.
✗ Branch 1 not taken.
 3041 if (stmt_size_)
166 3041 sqlite3_finalize(stmt_size_);
167
1/2
✓ Branch 0 taken 3041 times.
✗ Branch 1 not taken.
 3041 if (stmt_touch_)
168 3041 sqlite3_finalize(stmt_touch_);
169
1/2
✓ Branch 0 taken 3041 times.
✗ Branch 1 not taken.
 3041 if (stmt_unpin_)
170 3041 sqlite3_finalize(stmt_unpin_);
171
1/2
✓ Branch 0 taken 3041 times.
✗ Branch 1 not taken.
 3041 if (stmt_block_)
172 3041 sqlite3_finalize(stmt_block_);
173
1/2
✓ Branch 0 taken 3041 times.
✗ Branch 1 not taken.
 3041 if (stmt_unblock_)
174 3041 sqlite3_finalize(stmt_unblock_);
175
1/2
✓ Branch 0 taken 3041 times.
✗ Branch 1 not taken.
 3041 if (stmt_new_)
176 3041 sqlite3_finalize(stmt_new_);
177
1/2
✓ Branch 0 taken 3041 times.
✗ Branch 1 not taken.
 3041 if (database_)
178 3041 sqlite3_close(database_);
179 3041 UnlockFile(fd_lock_cachedb_);
180
181 3041 stmt_list_catalogs_ = NULL;
182 3041 stmt_list_pinned_ = NULL;
183 3041 stmt_list_volatile_ = NULL;
184 3041 stmt_list_ = NULL;
185 3041 stmt_rm_ = NULL;
186 3041 stmt_rm_batch_ = NULL;
187 3041 stmt_size_ = NULL;
188 3041 stmt_touch_ = NULL;
189 3041 stmt_unpin_ = NULL;
190 3041 stmt_block_ = NULL;
191 3041 stmt_unblock_ = NULL;
192 3041 stmt_new_ = NULL;
193 3041 database_ = NULL;
194
195 3041 pinned_chunks_.clear();
196 3041 }
197
198
199 4002 void PosixQuotaManager::CloseReturnPipe(int pipe[2]) {
200
2/2
✓ Branch 0 taken 92 times.
✓ Branch 1 taken 3910 times.
 4002 if (shared_) {
201 92 close(pipe[0]);
202 92 UnlinkReturnPipe(pipe[1]);
203 } else {
204 3910 ClosePipe(pipe);
205 }
206 4002 }
207
208
209 4602458 bool PosixQuotaManager::Contains(const string &hash_str) {
210 4602458 bool result = false;
211
212 4602458 sqlite3_bind_text(stmt_size_, 1, &hash_str[0], hash_str.length(),
213 SQLITE_STATIC);
214
2/2
✓ Branch 1 taken 660 times.
✓ Branch 2 taken 4601798 times.
 4602458 if (sqlite3_step(stmt_size_) == SQLITE_ROW)
215 660 result = true;
216 4602458 sqlite3_reset(stmt_size_);
217 4602458 LogCvmfs(kLogQuota, kLogDebug, "contains %s returns %d", hash_str.c_str(),
218 result);
219
220 4602458 return result;
221 }
222
223
224 2950 void PosixQuotaManager::CheckFreeSpace() {
225
3/4
✓ Branch 0 taken 2950 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 46 times.
✓ Branch 3 taken 2904 times.
 2950 if ((limit_ == 0) || (gauge_ >= limit_))
226 46 return;
227
228 struct statvfs vfs_info;
229
1/2
✓ Branch 1 taken 2904 times.
✗ Branch 2 not taken.
 2904 const int retval = statvfs((cache_dir_ + "/cachedb").c_str(), &vfs_info);
230
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2904 times.
 2904 if (retval != 0) {
231 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
232 "failed to query %s for free space (%d)", cache_dir_.c_str(),
233 ✗ errno);
234 ✗ return;
235 }
236 2904 const int64_t free_space_byte = vfs_info.f_bavail * vfs_info.f_bsize;
237
1/2
✓ Branch 1 taken 2904 times.
✗ Branch 2 not taken.
 2904 LogCvmfs(kLogQuota, kLogDebug, "free space: %" PRId64 " MB",
238 free_space_byte / (1024 * 1024));
239
240 2904 const int64_t required_byte = limit_ - gauge_;
241
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2904 times.
 2904 if (free_space_byte < required_byte) {
242 ✗ LogCvmfs(kLogQuota, kLogSyslogWarn,
243 "too little free space on the file system hosting the cache,"
244 " %" PRId64 " MB available",
245 free_space_byte / (1024 * 1024));
246 }
247 }
248
249
250 3088 PosixQuotaManager *PosixQuotaManager::Create(const string &cache_workspace,
251 const uint64_t limit,
252 const uint64_t cleanup_threshold,
253 const bool rebuild_database) {
254
2/2
✓ Branch 0 taken 92 times.
✓ Branch 1 taken 2996 times.
 3088 if (cleanup_threshold >= limit) {
255 92 LogCvmfs(kLogQuota, kLogDebug,
256 "invalid parameters: limit %" PRIu64 ", "
257 "cleanup_threshold %" PRIu64,
258 limit, cleanup_threshold);
259 92 return NULL;
260 }
261
262 PosixQuotaManager *quota_manager = new PosixQuotaManager(
263
1/2
✓ Branch 2 taken 2996 times.
✗ Branch 3 not taken.
 2996 limit, cleanup_threshold, cache_workspace);
264
265 // Initialize cache catalog
266
2/2
✓ Branch 1 taken 46 times.
✓ Branch 2 taken 2950 times.
 2996 if (!quota_manager->InitDatabase(rebuild_database)) {
267
1/2
✓ Branch 0 taken 46 times.
✗ Branch 1 not taken.
 46 delete quota_manager;
268 46 return NULL;
269 }
270 2950 quota_manager->CheckFreeSpace();
271 2950 MakePipe(quota_manager->pipe_lru_);
272
273 2950 quota_manager->protocol_revision_ = kProtocolRevision;
274 2950 quota_manager->initialized_ = true;
275 2950 return quota_manager;
276 }
277
278
279 /**
280 * Connects to a running shared local quota manager. Creates one if necessary.
281 */
282 92 PosixQuotaManager *PosixQuotaManager::CreateShared(
283 const std::string &exe_path,
284 const std::string &cache_workspace,
285 const uint64_t limit,
286 const uint64_t cleanup_threshold,
287 bool foreground) {
288 92 string cache_dir;
289 92 string workspace_dir;
290
2/4
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 92 times.
✗ Branch 5 not taken.
 92 ParseDirectories(cache_workspace, &cache_dir, &workspace_dir);
291
292 pid_t new_cachemgr_pid;
293
294 // Create lock file: only one fuse client at a time
295
2/4
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 92 times.
✗ Branch 5 not taken.
 92 const int fd_lockfile = LockFile(workspace_dir + "/lock_cachemgr");
296
2/2
✓ Branch 0 taken 46 times.
✓ Branch 1 taken 46 times.
 92 if (fd_lockfile < 0) {
297
1/2
✓ Branch 2 taken 46 times.
✗ Branch 3 not taken.
 46 LogCvmfs(kLogQuota, kLogDebug, "could not open lock file %s (%d)",
298
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 92 (workspace_dir + "/lock_cachemgr").c_str(), errno);
299 46 return NULL;
300 }
301
302 PosixQuotaManager *quota_mgr = new PosixQuotaManager(limit, cleanup_threshold,
303
2/4
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 46 times.
✗ Branch 5 not taken.
 46 cache_workspace);
304 46 quota_mgr->shared_ = true;
305 46 quota_mgr->spawned_ = true;
306
307 // Try to connect to pipe
308
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 const string fifo_path = workspace_dir + "/cachemgr";
309
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 LogCvmfs(kLogQuota, kLogDebug, "trying to connect to existing pipe");
310
1/2
✓ Branch 2 taken 46 times.
✗ Branch 3 not taken.
 46 quota_mgr->pipe_lru_[1] = open(fifo_path.c_str(), O_WRONLY | O_NONBLOCK);
311
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 46 times.
 46 if (quota_mgr->pipe_lru_[1] >= 0) {
312 ✗ const int fd_lockfile_rw = open((workspace_dir + "/lock_cachemgr").c_str(),
313 O_RDWR, 0600);
314 ✗ unsigned lockfile_magicnumber = 0;
315 ✗ const ssize_t result_mn = SafeRead(fd_lockfile_rw, &lockfile_magicnumber,
316 sizeof(lockfile_magicnumber));
317 ✗ const ssize_t result = SafeRead(fd_lockfile_rw, &new_cachemgr_pid,
318 sizeof(new_cachemgr_pid));
319 ✗ close(fd_lockfile_rw);
320
321 ✗ if ((lockfile_magicnumber != kLockFileMagicNumber) || (result < 0)
322 ✗ || (result_mn < 0)
323 ✗ || (static_cast<size_t>(result) < sizeof(new_cachemgr_pid))) {
324 ✗ if (result != 0) {
325 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
326 "could not read cache manager pid from lockfile");
327 ✗ UnlockFile(fd_lockfile);
328 ✗ delete quota_mgr;
329 ✗ return NULL;
330 } else {
331 // support reload from old versions of the cache manager
332 // lock file is empty in this case, try a plain ReadHalfPipe to get pid
333 ✗ quota_mgr->SetCacheMgrPid(quota_mgr->GetPid());
334 }
335 } else {
336 ✗ quota_mgr->SetCacheMgrPid(new_cachemgr_pid);
337 }
338
339
340 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to existing cache manager pipe");
341 ✗ quota_mgr->initialized_ = true;
342 ✗ Nonblock2Block(quota_mgr->pipe_lru_[1]);
343 ✗ UnlockFile(fd_lockfile);
344 ✗ quota_mgr->GetLimits(&quota_mgr->limit_, &quota_mgr->cleanup_threshold_);
345 ✗ LogCvmfs(kLogQuota, kLogDebug,
346 "received limit %" PRIu64 ", threshold %" PRIu64,
347 quota_mgr->limit_, quota_mgr->cleanup_threshold_);
348 ✗ if (FileExists(workspace_dir + "/cachemgr.protocol")) {
349 ✗ quota_mgr->protocol_revision_ = quota_mgr->GetProtocolRevision();
350 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected protocol revision %u",
351 quota_mgr->protocol_revision_);
352 } else {
353 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to ancient cache manager");
354 }
355 ✗ return quota_mgr;
356 }
357 46 const int connect_error = errno;
358
359 // Lock file: let existing cache manager finish first
360
2/4
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 46 times.
✗ Branch 5 not taken.
 46 const int fd_lockfile_fifo = LockFile(workspace_dir + "/lock_cachemgr.fifo");
361
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 46 times.
 46 if (fd_lockfile_fifo < 0) {
362 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not open lock file %s (%d)",
363 ✗ (workspace_dir + "/lock_cachemgr.fifo").c_str(), errno);
364 ✗ UnlockFile(fd_lockfile);
365 ✗ delete quota_mgr;
366 ✗ return NULL;
367 }
368
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 UnlockFile(fd_lockfile_fifo);
369
370
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 46 times.
 46 if (connect_error == ENXIO) {
371 ✗ LogCvmfs(kLogQuota, kLogDebug, "left-over FIFO found, unlinking");
372 ✗ unlink(fifo_path.c_str());
373 }
374
375 // Creating a new FIFO for the cache manager (to be bound later)
376 46 int retval = mkfifo(fifo_path.c_str(), 0600);
377
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 46 times.
 46 if (retval != 0) {
378 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to create cache manager FIFO (%d)",
379 ✗ errno);
380 ✗ UnlockFile(fd_lockfile);
381 ✗ delete quota_mgr;
382 ✗ return NULL;
383 }
384
385 // Create new cache manager
386 int pipe_boot[2];
387 int pipe_handshake[2];
388
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 MakePipe(pipe_boot);
389
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 MakePipe(pipe_handshake);
390
391 46 vector<string> command_line;
392
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 command_line.push_back(exe_path);
393
2/4
✓ Branch 2 taken 46 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 46 times.
✗ Branch 6 not taken.
 46 command_line.push_back("__cachemgr__");
394
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 command_line.push_back(cache_workspace);
395
2/4
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 46 times.
✗ Branch 5 not taken.
 46 command_line.push_back(StringifyInt(pipe_boot[1]));
396
2/4
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 46 times.
✗ Branch 5 not taken.
 46 command_line.push_back(StringifyInt(pipe_handshake[0]));
397
2/4
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 46 times.
✗ Branch 5 not taken.
 46 command_line.push_back(StringifyInt(limit));
398
2/4
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 46 times.
✗ Branch 5 not taken.
 46 command_line.push_back(StringifyInt(cleanup_threshold));
399 // do not propagate foreground in order to reliably get pid from exec
400 // instead, daemonize right here
401
2/4
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 46 times.
✗ Branch 5 not taken.
 46 command_line.push_back(StringifyInt(true)); // foreground
402
3/6
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 46 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 46 times.
✗ Branch 8 not taken.
 46 command_line.push_back(StringifyInt(GetLogSyslogLevel()));
403
3/6
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 46 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 46 times.
✗ Branch 8 not taken.
 46 command_line.push_back(StringifyInt(GetLogSyslogFacility()));
404
5/14
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 46 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 46 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 46 times.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 46 times.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
 46 command_line.push_back(GetLogDebugFile() + ":" + GetLogMicroSyslog());
405
406 46 set<int> preserve_filedes;
407
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 preserve_filedes.insert(0);
408
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 preserve_filedes.insert(1);
409
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 preserve_filedes.insert(2);
410
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 preserve_filedes.insert(pipe_boot[1]);
411
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 preserve_filedes.insert(pipe_handshake[0]);
412
413
1/2
✓ Branch 0 taken 46 times.
✗ Branch 1 not taken.
 46 if (foreground) {
414
1/2
✓ Branch 2 taken 46 times.
✗ Branch 3 not taken.
 46 retval = ManagedExec(command_line, preserve_filedes, map<int, int>(),
415 /*drop_credentials*/ false,
416 /*clear_env*/ false,
417 /*double_fork*/ true, &new_cachemgr_pid);
418 } else {
419 ✗ retval = ExecAsDaemon(command_line, &new_cachemgr_pid);
420 }
421
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 46 times.
 46 if (!retval) {
422 ✗ UnlockFile(fd_lockfile);
423 ✗ ClosePipe(pipe_boot);
424 ✗ ClosePipe(pipe_handshake);
425 ✗ delete quota_mgr;
426 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to start cache manager");
427 ✗ return NULL;
428 }
429
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 LogCvmfs(kLogQuota, kLogDebug,
430 "new cache manager pid: %d protocol revision %d",
431 new_cachemgr_pid, QuotaManager::kProtocolRevision);
432 46 quota_mgr->SetCacheMgrPid(new_cachemgr_pid);
433
2/4
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 46 times.
✗ Branch 6 not taken.
 46 const int fd_lockfile_rw = open((workspace_dir + "/lock_cachemgr").c_str(),
434 O_RDWR | O_TRUNC, 0600);
435 46 const unsigned magic_number = PosixQuotaManager::kLockFileMagicNumber;
436
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 const bool result_mn = SafeWrite(fd_lockfile_rw, &magic_number,
437 sizeof(magic_number));
438
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 const bool result = SafeWrite(fd_lockfile_rw, &new_cachemgr_pid,
439 sizeof(new_cachemgr_pid));
440
2/4
✓ Branch 0 taken 46 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 46 times.
 46 if (!result || !result_mn) {
441 ✗ PANIC(kLogSyslogErr, "could not write cache manager pid to lockfile");
442 }
443
444
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 close(fd_lockfile_rw);
445 // Wait for cache manager to be ready
446
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 close(pipe_boot[1]);
447
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 close(pipe_handshake[0]);
448 char buf;
449
2/4
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 46 times.
✗ Branch 4 not taken.
 46 if (read(pipe_boot[0], &buf, 1) != 1) {
450
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 UnlockFile(fd_lockfile);
451
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 close(pipe_boot[0]);
452
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 close(pipe_handshake[1]);
453
1/2
✓ Branch 0 taken 46 times.
✗ Branch 1 not taken.
 46 delete quota_mgr;
454
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
455 "cache manager did not start");
456 46 return NULL;
457 }
458 ✗ close(pipe_boot[0]);
459
460 // Connect write end
461 ✗ quota_mgr->pipe_lru_[1] = open(fifo_path.c_str(), O_WRONLY | O_NONBLOCK);
462 ✗ if (quota_mgr->pipe_lru_[1] < 0) {
463 ✗ LogCvmfs(kLogQuota, kLogDebug,
464 ✗ "failed to connect to newly created FIFO (%d)", errno);
465 ✗ close(pipe_handshake[1]);
466 ✗ UnlockFile(fd_lockfile);
467 ✗ delete quota_mgr;
468 ✗ return NULL;
469 }
470
471 // Finalize handshake
472 ✗ buf = 'C';
473 ✗ if (write(pipe_handshake[1], &buf, 1) != 1) {
474 ✗ UnlockFile(fd_lockfile);
475 ✗ close(pipe_handshake[1]);
476 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not finalize handshake");
477 ✗ delete quota_mgr;
478 ✗ return NULL;
479 }
480 ✗ close(pipe_handshake[1]);
481
482 ✗ Nonblock2Block(quota_mgr->pipe_lru_[1]);
483 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to a new cache manager");
484 ✗ quota_mgr->protocol_revision_ = kProtocolRevision;
485
486 ✗ UnlockFile(fd_lockfile);
487
488 ✗ quota_mgr->initialized_ = true;
489 ✗ quota_mgr->GetLimits(&quota_mgr->limit_, &quota_mgr->cleanup_threshold_);
490 ✗ LogCvmfs(kLogQuota, kLogDebug,
491 "received limit %" PRIu64 ", "
492 "threshold %" PRIu64,
493 quota_mgr->limit_, quota_mgr->cleanup_threshold_);
494 ✗ return quota_mgr;
495 92 }
496
497
498 460 bool PosixQuotaManager::DoCleanup(const uint64_t leave_size) {
499
2/2
✓ Branch 0 taken 92 times.
✓ Branch 1 taken 368 times.
 460 if (gauge_ <= leave_size)
500 92 return true;
501
502 // TODO(jblomer) transaction
503
1/2
✓ Branch 1 taken 368 times.
✗ Branch 2 not taken.
 368 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug,
504 "clean up cache until at most %lu KB is used", leave_size / 1024);
505
1/2
✓ Branch 1 taken 368 times.
✗ Branch 2 not taken.
 368 LogCvmfs(kLogQuota, kLogDebug, "gauge %" PRIu64, gauge_);
506
1/2
✓ Branch 1 taken 368 times.
✗ Branch 2 not taken.
 368 cleanup_recorder_.Tick();
507
508 bool result;
509 368 vector<string> trash;
510
511 // Note that volatile files start counting from the smallest int64 number:
512 // the absolute sequence number with the first bit set in two's complement.
513 // So -1 can be a marker that will never appear in the database.
514 368 int64_t max_acseq = -1;
515 368 std::vector<EvictCandidate> lru_ordered_open;
516
517 do {
518
1/2
✓ Branch 1 taken 2668 times.
✗ Branch 2 not taken.
 2668 sqlite3_reset(stmt_lru_);
519
3/4
✓ Branch 0 taken 368 times.
✓ Branch 1 taken 2300 times.
✓ Branch 3 taken 2668 times.
✗ Branch 4 not taken.
 3036 sqlite3_bind_int64(stmt_lru_, 1,
520 368 (max_acseq == -1) ? std::numeric_limits<int64_t>::min()
521 : (max_acseq + 1));
522
523 2668 std::vector<EvictCandidate> candidates;
524
1/2
✓ Branch 1 taken 2668 times.
✗ Branch 2 not taken.
 2668 candidates.reserve(kEvictBatchSize);
525 2668 string hash_str;
526 2668 unsigned i = 0;
527
3/4
✓ Branch 1 taken 2395128 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2392460 times.
✓ Branch 4 taken 2668 times.
 2395128 while (sqlite3_step(stmt_lru_) == SQLITE_ROW) {
528 hash_str = reinterpret_cast<const char *>(
529
2/4
✓ Branch 1 taken 2392460 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2392460 times.
✗ Branch 5 not taken.
 2392460 sqlite3_column_text(stmt_lru_, 0));
530
1/2
✓ Branch 2 taken 2392460 times.
✗ Branch 3 not taken.
 2392460 LogCvmfs(kLogQuota, kLogDebug, "add %s to candidates for eviction",
531 hash_str.c_str());
532
1/2
✓ Branch 1 taken 2392460 times.
✗ Branch 2 not taken.
 2392460 candidates.push_back(
533
1/2
✓ Branch 1 taken 2392460 times.
✗ Branch 2 not taken.
 2392460 EvictCandidate(shash::MkFromHexPtr(shash::HexPtr(hash_str)),
534 2392460 sqlite3_column_int64(stmt_lru_, 1),
535
2/4
✓ Branch 1 taken 2392460 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2392460 times.
✗ Branch 5 not taken.
 2392460 sqlite3_column_int64(stmt_lru_, 2)));
536 2392460 i++;
537 }
538
2/2
✓ Branch 1 taken 46 times.
✓ Branch 2 taken 2622 times.
 2668 if (candidates.empty()) {
539
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 LogCvmfs(kLogQuota, kLogDebug, "no more entries to evict");
540 46 break;
541 }
542
543 2622 const unsigned N = candidates.size();
544
545 2622 open_files_.clear();
546
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2622 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 5244 open_files_ = (cleanup_unused_first_) ? CollectAllOpenHashes()
547 2622 : std::vector<shash::Short>();
548
549
2/2
✓ Branch 0 taken 2300414 times.
✓ Branch 1 taken 2300 times.
 2302714 for (i = 0; i < N; ++i) {
550 // That's a critical condition. We must not delete a not yet inserted
551 // pinned file as it is already reserved (but will be inserted later).
552 // Instead, set the pin bit in the db to not run into an endless loop
553
1/2
✓ Branch 2 taken 2300414 times.
✗ Branch 3 not taken.
 2300414 const bool is_pinned = pinned_chunks_.find(candidates[i].hash)
554 4600828 != pinned_chunks_.end();
555
556 // Avoid evicting open files hopping there are enough more recently used
557 // files to satisfy the cleanup request
558 /*
559 const bool is_open = std::find_if(
560 open_files_.begin(), open_files_.end(),
561 [&candidates, &i](const auto &elem) -> bool {
562 return elem.Collide(candidates[i].hash);
563 })
564 != open_files_.end();
565 */
566 2300414 bool is_open=false;
567
1/2
✗ Branch 4 not taken.
✓ Branch 5 taken 2300414 times.
 2300414 for(auto it=open_files_.begin();it!=open_files_.end();++it){
568 ✗ if (it->Collide(candidates[i].hash)){is_open=true; break;}
569 }
570
571
2/2
✓ Branch 0 taken 46 times.
✓ Branch 1 taken 2300368 times.
 2300414 if (is_pinned) {
572
1/2
✓ Branch 2 taken 46 times.
✗ Branch 3 not taken.
 46 SkipEviction(candidates[i]);
573 46 continue;
574 }
575
576
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2300368 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 2300368 if (cleanup_unused_first_ and is_open) {
577 ✗ SkipEviction(candidates[i]);
578 ✗ lru_ordered_open.push_back(candidates[i]);
579 ✗ continue;
580 }
581
582
2/4
✓ Branch 1 taken 2300368 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2300368 times.
✗ Branch 5 not taken.
 4600736 trash.push_back(cache_dir_ + "/"
583
2/4
✓ Branch 2 taken 2300368 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2300368 times.
✗ Branch 6 not taken.
 6901104 + candidates[i].hash.MakePathWithoutSuffix());
584 2300368 gauge_ -= candidates[i].size;
585 2300368 max_acseq = candidates[i].acseq;
586
1/2
✓ Branch 2 taken 2300368 times.
✗ Branch 3 not taken.
 2300368 LogCvmfs(kLogQuota, kLogDebug, "lru cleanup %s, new gauge %" PRIu64,
587
1/2
✓ Branch 2 taken 2300368 times.
✗ Branch 3 not taken.
 4600736 candidates[i].hash.ToString().c_str(), gauge_);
588
589
2/2
✓ Branch 0 taken 322 times.
✓ Branch 1 taken 2300046 times.
 2300368 if (gauge_ <= leave_size)
590 322 break;
591 }
592
6/6
✓ Branch 1 taken 2622 times.
✓ Branch 2 taken 46 times.
✓ Branch 4 taken 2622 times.
✓ Branch 5 taken 46 times.
✓ Branch 6 taken 2300 times.
✓ Branch 7 taken 322 times.
 5336 } while (gauge_ > leave_size);
593
594
1/2
✓ Branch 0 taken 368 times.
✗ Branch 1 not taken.
 368 if (max_acseq != -1) {
595
1/2
✓ Branch 1 taken 368 times.
✗ Branch 2 not taken.
 368 sqlite3_bind_int64(stmt_rm_batch_, 1, max_acseq);
596
1/2
✓ Branch 1 taken 368 times.
✗ Branch 2 not taken.
 368 result = (sqlite3_step(stmt_rm_batch_) == SQLITE_DONE);
597
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 368 times.
 368 assert(result);
598
1/2
✓ Branch 1 taken 368 times.
✗ Branch 2 not taken.
 368 sqlite3_reset(stmt_rm_batch_);
599
600
1/2
✓ Branch 1 taken 368 times.
✗ Branch 2 not taken.
 368 result = (sqlite3_step(stmt_unblock_) == SQLITE_DONE);
601
1/2
✓ Branch 1 taken 368 times.
✗ Branch 2 not taken.
 368 sqlite3_reset(stmt_unblock_);
602
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 368 times.
 368 assert(result);
603 }
604
605
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 368 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 368 times.
 368 while (!lru_ordered_open.empty() and gauge_ > leave_size) {
606 // cleanup files in use
607 ✗ auto &candidate = lru_ordered_open[0];
608 ✗ trash.push_back(cache_dir_ + "/" + candidate.hash.MakePathWithoutSuffix());
609 ✗ gauge_ -= candidate.size;
610 ✗ max_acseq = candidate.acseq;
611 ✗ LogCvmfs(kLogQuota, kLogDebug, "lru cleanup %s, new gauge %" PRIu64,
612 ✗ candidate.hash.ToString().c_str(), gauge_);
613 ✗ lru_ordered_open.erase(lru_ordered_open.begin());
614 }
615
616
2/4
✓ Branch 1 taken 368 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 368 times.
 368 if (!EmptyTrash(trash))
617 ✗ return false;
618
619
2/2
✓ Branch 0 taken 46 times.
✓ Branch 1 taken 322 times.
 368 if (gauge_ > leave_size) {
620
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
621 "request to clean until %" PRIu64 ", "
622 "but effective gauge is %" PRIu64,
623 leave_size, gauge_);
624 46 return false;
625 }
626 322 return true;
627 368 }
628
629 368 bool PosixQuotaManager::EmptyTrash(const std::vector<std::string> &trash) {
630
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 368 times.
 368 if (trash.empty())
631 ✗ return true;
632
633
2/2
✓ Branch 0 taken 276 times.
✓ Branch 1 taken 92 times.
 368 if (async_delete_) {
634 // Double fork avoids zombie, forked removal process must not flush file
635 // buffers
636 pid_t pid;
637 int statloc;
638
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 276 times.
 276 if ((pid = fork()) == 0) {
639 // TODO(jblomer): eviciting files in the cache should perhaps become a
640 // thread. This would also allow to block the chunks and prevent the
641 // race with re-insertion. Then again, a thread can block umount.
642 #ifndef DEBUGMSG
643 ✗ CloseAllFildes(std::set<int>());
644 #endif
645 ✗ if (fork() == 0) {
646 ✗ for (unsigned i = 0, iEnd = trash.size(); i < iEnd; ++i) {
647 ✗ LogCvmfs(kLogQuota, kLogDebug, "unlink %s", trash[i].c_str());
648 ✗ unlink(trash[i].c_str());
649 }
650 ✗ _exit(0);
651 }
652 ✗ _exit(0);
653 } else {
654
1/2
✓ Branch 0 taken 276 times.
✗ Branch 1 not taken.
 276 if (pid > 0)
655
1/2
✓ Branch 1 taken 276 times.
✗ Branch 2 not taken.
 276 waitpid(pid, &statloc, 0);
656 else
657 ✗ return false;
658 }
659 } else { // !async_delete_
660
2/2
✓ Branch 1 taken 138 times.
✓ Branch 2 taken 92 times.
 230 for (unsigned i = 0, iEnd = trash.size(); i < iEnd; ++i) {
661 138 LogCvmfs(kLogQuota, kLogDebug, "unlink %s", trash[i].c_str());
662 138 unlink(trash[i].c_str());
663 }
664 }
665 368 return true;
666 }
667
668
669 4601763 void PosixQuotaManager::DoInsert(const shash::Any &hash,
670 const uint64_t size,
671 const string &description,
672 const CommandType command_type) {
673
1/2
✓ Branch 1 taken 4601763 times.
✗ Branch 2 not taken.
 4601763 const string hash_str = hash.ToString();
674
1/2
✓ Branch 3 taken 4601763 times.
✗ Branch 4 not taken.
 4601763 LogCvmfs(kLogQuota, kLogDebug, "insert into lru %s, path %s, method %d",
675 hash_str.c_str(), description.c_str(), command_type);
676 4601763 const unsigned desc_length = (description.length() > kMaxDescription)
677 ? kMaxDescription
678
1/2
✓ Branch 0 taken 4601763 times.
✗ Branch 1 not taken.
 4601763 : description.length();
679
680 LruCommand *cmd = reinterpret_cast<LruCommand *>(
681 4601763 alloca(sizeof(LruCommand) + desc_length));
682 4601763 new (cmd) LruCommand;
683 4601763 cmd->command_type = command_type;
684 4601763 cmd->SetSize(size);
685
1/2
✓ Branch 1 taken 4601763 times.
✗ Branch 2 not taken.
 4601763 cmd->StoreHash(hash);
686 4601763 cmd->desc_length = desc_length;
687 4601763 memcpy(reinterpret_cast<char *>(cmd) + sizeof(LruCommand), &description[0],
688 desc_length);
689
1/2
✓ Branch 1 taken 4601763 times.
✗ Branch 2 not taken.
 4601763 WritePipe(pipe_lru_[1], cmd, sizeof(LruCommand) + desc_length);
690 4601763 }
691
692
693 1610 vector<string> PosixQuotaManager::DoList(const CommandType list_command) {
694 1610 vector<string> result;
695
696 int pipe_list[2];
697
1/2
✓ Branch 1 taken 1610 times.
✗ Branch 2 not taken.
 1610 MakeReturnPipe(pipe_list);
698 char description_buffer[kMaxDescription];
699
700 1610 LruCommand cmd;
701 1610 cmd.command_type = list_command;
702 1610 cmd.return_pipe = pipe_list[1];
703
1/2
✓ Branch 1 taken 1610 times.
✗ Branch 2 not taken.
 1610 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
704
705 int length;
706 do {
707
1/2
✓ Branch 1 taken 4603404 times.
✗ Branch 2 not taken.
 4603404 ManagedReadHalfPipe(pipe_list[0], &length, sizeof(length));
708
2/2
✓ Branch 0 taken 4601794 times.
✓ Branch 1 taken 1610 times.
 4603404 if (length > 0) {
709
1/2
✓ Branch 1 taken 4601794 times.
✗ Branch 2 not taken.
 4601794 ReadPipe(pipe_list[0], description_buffer, length);
710
2/4
✓ Branch 2 taken 4601794 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 4601794 times.
✗ Branch 6 not taken.
 4601794 result.push_back(string(description_buffer, length));
711 }
712
2/2
✓ Branch 0 taken 4601794 times.
✓ Branch 1 taken 1610 times.
 4603404 } while (length >= 0);
713
714
1/2
✓ Branch 1 taken 1610 times.
✗ Branch 2 not taken.
 1610 CloseReturnPipe(pipe_list);
715 3220 return result;
716 }
717
718
719 558 uint64_t PosixQuotaManager::GetCapacity() {
720
1/2
✓ Branch 0 taken 558 times.
✗ Branch 1 not taken.
 558 if (limit_ != (uint64_t)(-1))
721 558 return limit_;
722
723 // Unrestricted cache, look at free space on cache dir fs
724 struct statfs info;
725 ✗ if (statfs(".", &info) == 0) {
726 ✗ return info.f_bavail * info.f_bsize;
727 } else {
728 ✗ LogCvmfs(kLogQuota, kLogSyslogErr | kLogDebug,
729 ✗ "failed to query file system info of cache (%d)", errno);
730 ✗ return limit_;
731 }
732 }
733
734
735 ✗ void PosixQuotaManager::GetLimits(uint64_t *limit,
736 uint64_t *cleanup_threshold) {
737 int pipe_limits[2];
738 ✗ MakeReturnPipe(pipe_limits);
739
740 ✗ LruCommand cmd;
741 ✗ cmd.command_type = kLimits;
742 ✗ cmd.return_pipe = pipe_limits[1];
743 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
744 ✗ ManagedReadHalfPipe(pipe_limits[0], limit, sizeof(*limit));
745 ✗ ReadPipe(pipe_limits[0], cleanup_threshold, sizeof(*cleanup_threshold));
746 ✗ CloseReturnPipe(pipe_limits);
747 }
748
749
750 /**
751 * Since we only cleanup until cleanup_threshold, we can only add
752 * files smaller than limit-cleanup_threshold.
753 */
754 182 uint64_t PosixQuotaManager::GetMaxFileSize() {
755 182 return limit_ - cleanup_threshold_;
756 }
757
758
759 46 pid_t PosixQuotaManager::GetPid() {
760
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 46 if (!shared_ || !spawned_) {
761 46 return getpid();
762 }
763 ✗ if (cachemgr_pid_) {
764 ✗ return cachemgr_pid_;
765 }
766
767 pid_t result;
768 int pipe_pid[2];
769 ✗ MakeReturnPipe(pipe_pid);
770
771 ✗ LruCommand cmd;
772 ✗ cmd.command_type = kPid;
773 ✗ cmd.return_pipe = pipe_pid[1];
774 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
775 ✗ ReadHalfPipe(pipe_pid[0], &result, sizeof(result));
776 ✗ CloseReturnPipe(pipe_pid);
777 ✗ return result;
778 }
779
780
781 46 uint32_t PosixQuotaManager::GetProtocolRevision() {
782 int pipe_revision[2];
783
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 MakeReturnPipe(pipe_revision);
784
785 46 LruCommand cmd;
786 46 cmd.command_type = kGetProtocolRevision;
787 46 cmd.return_pipe = pipe_revision[1];
788
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
789
790 uint32_t revision;
791
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 ManagedReadHalfPipe(pipe_revision[0], &revision, sizeof(revision));
792
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 CloseReturnPipe(pipe_revision);
793 46 return revision;
794 }
795
796 512 void PosixQuotaManager::SetCleanupPolicy(bool cleanup_unused_first) {
797
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 512 times.
 512 if (protocol_revision_ < 3) return;
798
799 512 LruCommand cmd;
800 512 cmd.command_type = kSetCleanupPolicy;
801
1/2
✓ Branch 1 taken 512 times.
✗ Branch 2 not taken.
 512 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
802
803
1/2
✓ Branch 1 taken 512 times.
✗ Branch 2 not taken.
 512 WritePipe(pipe_lru_[1], &cleanup_unused_first, sizeof(bool));
804 }
805
806 ✗ void PosixQuotaManager::RegisterMountpoint(const std::string &mountpoint) {
807 ✗ if (protocol_revision_ < 3) return;
808
809 ✗ LruCommand cmd;
810 ✗ cmd.command_type = kRegisterMountpoint;
811 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
812
813 ✗ size_t mp_size = mountpoint.size();
814 ✗ WritePipe(pipe_lru_[1], &mp_size, sizeof(size_t));
815 ✗ WritePipe(pipe_lru_[1], mountpoint.data(), mp_size);
816 }
817
818 ✗ std::string PosixQuotaManager::GetMountpoints() {
819 ✗ if (protocol_revision_ < 3) return "";
820
821 int pipe_mp[2];
822 ✗ MakeReturnPipe(pipe_mp);
823
824 ✗ LruCommand cmd;
825 ✗ cmd.command_type = kGetMountpoints;
826 ✗ cmd.return_pipe = pipe_mp[1];
827 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
828 ✗ size_t mp_str_size = 0;
829 ✗ ManagedReadHalfPipe(pipe_mp[0], &mp_str_size, sizeof(size_t));
830 ✗ char *buf = (char *)malloc(mp_str_size * sizeof(char));
831 ✗ ManagedReadHalfPipe(pipe_mp[0], buf, mp_str_size);
832 ✗ return std::string(buf);
833 }
834
835 ✗ std::string PosixQuotaManager::GetGroupHashes() {
836 ✗ if (protocol_revision_ < 3) return "";
837
838 int pipe_gh[2];
839 ✗ MakeReturnPipe(pipe_gh);
840
841 ✗ LruCommand cmd;
842 ✗ cmd.command_type = kGetGroupHashes;
843 ✗ cmd.return_pipe = pipe_gh[1];
844 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
845 ✗ size_t mp_str_size = 0;
846 ✗ ManagedReadHalfPipe(pipe_gh[0], &mp_str_size, sizeof(size_t));
847 ✗ char *buf = (char *)malloc(mp_str_size * sizeof(char));
848 ✗ ManagedReadHalfPipe(pipe_gh[0], buf, mp_str_size);
849 ✗ return std::string(buf);
850 }
851
852 /**
853 * Queries the shared local hard disk quota manager.
854 */
855 828 void PosixQuotaManager::GetSharedStatus(uint64_t *gauge, uint64_t *pinned) {
856 int pipe_status[2];
857
1/2
✓ Branch 1 taken 828 times.
✗ Branch 2 not taken.
 828 MakeReturnPipe(pipe_status);
858
859 828 LruCommand cmd;
860 828 cmd.command_type = kStatus;
861 828 cmd.return_pipe = pipe_status[1];
862
1/2
✓ Branch 1 taken 828 times.
✗ Branch 2 not taken.
 828 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
863
1/2
✓ Branch 1 taken 828 times.
✗ Branch 2 not taken.
 828 ManagedReadHalfPipe(pipe_status[0], gauge, sizeof(*gauge));
864
1/2
✓ Branch 1 taken 828 times.
✗ Branch 2 not taken.
 828 ReadPipe(pipe_status[0], pinned, sizeof(*pinned));
865
1/2
✓ Branch 1 taken 828 times.
✗ Branch 2 not taken.
 828 CloseReturnPipe(pipe_status);
866 828 }
867
868 46 bool PosixQuotaManager::SetSharedLimit(uint64_t limit) {
869 int pipe_set_limit[2];
870 bool result;
871
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 MakeReturnPipe(pipe_set_limit);
872
873 46 LruCommand cmd;
874 46 cmd.command_type = kSetLimit;
875 46 cmd.size = limit;
876 46 cmd.return_pipe = pipe_set_limit[1];
877
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
878
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 ReadHalfPipe(pipe_set_limit[0], &result, sizeof(result));
879
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 CloseReturnPipe(pipe_set_limit);
880 46 return result;
881 }
882
883
884 46 bool PosixQuotaManager::SetLimit(uint64_t size) {
885
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 46 times.
 46 if (!spawned_) {
886 ✗ limit_ = size;
887 ✗ cleanup_threshold_ = size / 2;
888 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
889 "Quota limit set to %lu / threshold %lu", limit_,
890 cleanup_threshold_);
891 ✗ return true;
892 }
893 46 return SetSharedLimit(size);
894 }
895
896 1760 uint64_t PosixQuotaManager::GetSize() {
897
2/2
✓ Branch 0 taken 1024 times.
✓ Branch 1 taken 736 times.
 1760 if (!spawned_)
898 1024 return gauge_;
899 uint64_t gauge, size_pinned;
900
1/2
✓ Branch 1 taken 736 times.
✗ Branch 2 not taken.
 736 GetSharedStatus(&gauge, &size_pinned);
901 736 return gauge;
902 }
903
904
905 92 uint64_t PosixQuotaManager::GetSizePinned() {
906
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 92 times.
 92 if (!spawned_)
907 ✗ return pinned_;
908 uint64_t gauge, size_pinned;
909
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 GetSharedStatus(&gauge, &size_pinned);
910 92 return size_pinned;
911 }
912
913
914 184 uint64_t PosixQuotaManager::GetCleanupRate(uint64_t period_s) {
915
2/4
✓ Branch 0 taken 184 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 184 times.
 184 if (!spawned_ || (protocol_revision_ < 2))
916 ✗ return 0;
917 uint64_t cleanup_rate;
918
919 int pipe_cleanup_rate[2];
920
1/2
✓ Branch 1 taken 184 times.
✗ Branch 2 not taken.
 184 MakeReturnPipe(pipe_cleanup_rate);
921 184 LruCommand cmd;
922 184 cmd.command_type = kCleanupRate;
923 184 cmd.size = period_s;
924 184 cmd.return_pipe = pipe_cleanup_rate[1];
925
1/2
✓ Branch 1 taken 184 times.
✗ Branch 2 not taken.
 184 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
926
1/2
✓ Branch 1 taken 184 times.
✗ Branch 2 not taken.
 184 ManagedReadHalfPipe(pipe_cleanup_rate[0], &cleanup_rate,
927 sizeof(cleanup_rate));
928
1/2
✓ Branch 1 taken 184 times.
✗ Branch 2 not taken.
 184 CloseReturnPipe(pipe_cleanup_rate);
929
930 184 return cleanup_rate;
931 }
932
933
934 3226 bool PosixQuotaManager::InitDatabase(const bool rebuild_database) {
935 3226 string sql;
936 sqlite3_stmt *stmt;
937
938
2/4
✓ Branch 1 taken 3226 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3226 times.
✗ Branch 5 not taken.
 3226 fd_lock_cachedb_ = LockFile(workspace_dir_ + "/lock_cachedb");
939
2/2
✓ Branch 0 taken 46 times.
✓ Branch 1 taken 3180 times.
 3226 if (fd_lock_cachedb_ < 0) {
940
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 LogCvmfs(kLogQuota, kLogDebug, "failed to create cachedb lock");
941 46 return false;
942 }
943
944 3180 bool retry = false;
945
1/2
✓ Branch 1 taken 3180 times.
✗ Branch 2 not taken.
 3180 const string db_file = cache_dir_ + "/cachedb";
946
2/2
✓ Branch 0 taken 2979 times.
✓ Branch 1 taken 201 times.
 3180 if (rebuild_database) {
947
1/2
✓ Branch 2 taken 201 times.
✗ Branch 3 not taken.
 201 LogCvmfs(kLogQuota, kLogDebug, "rebuild database, unlinking existing (%s)",
948 db_file.c_str());
949 201 unlink(db_file.c_str());
950
1/2
✓ Branch 1 taken 201 times.
✗ Branch 2 not taken.
 201 unlink((db_file + "-journal").c_str());
951 }
952
953 2979 init_recover:
954
1/2
✓ Branch 2 taken 3180 times.
✗ Branch 3 not taken.
 3180 int err = sqlite3_open(db_file.c_str(), &database_);
955
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3180 times.
 3180 if (err != SQLITE_OK) {
956 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not open cache database (%d)", err);
957 ✗ goto init_database_fail;
958 }
959 // TODO(reneme): make this a `QuotaDatabase : public sqlite::Database`
960 sql = "PRAGMA synchronous=0; PRAGMA locking_mode=EXCLUSIVE; "
961 "PRAGMA auto_vacuum=1; "
962 "CREATE TABLE IF NOT EXISTS cache_catalog (sha1 TEXT, size INTEGER, "
963 " acseq INTEGER, path TEXT, type INTEGER, pinned INTEGER, "
964 "CONSTRAINT pk_cache_catalog PRIMARY KEY (sha1)); "
965 "CREATE UNIQUE INDEX IF NOT EXISTS idx_cache_catalog_acseq "
966 " ON cache_catalog (acseq); "
967 "CREATE TEMP TABLE fscache (sha1 TEXT, size INTEGER, actime INTEGER, "
968 "CONSTRAINT pk_fscache PRIMARY KEY (sha1)); "
969 "CREATE INDEX idx_fscache_actime ON fscache (actime); "
970 "CREATE TABLE IF NOT EXISTS properties (key TEXT, value TEXT, "
971
1/2
✓ Branch 1 taken 3180 times.
✗ Branch 2 not taken.
 3180 " CONSTRAINT pk_properties PRIMARY KEY(key));";
972
1/2
✓ Branch 2 taken 3180 times.
✗ Branch 3 not taken.
 3180 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
973
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3180 times.
 3180 if (err != SQLITE_OK) {
974 ✗ if (!retry) {
975 ✗ retry = true;
976 ✗ sqlite3_close(database_);
977 ✗ unlink(db_file.c_str());
978 ✗ unlink((db_file + "-journal").c_str());
979 ✗ LogCvmfs(kLogQuota, kLogSyslogWarn,
980 "LRU database corrupted, re-building");
981 ✗ goto init_recover;
982 }
983 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
984 sql.c_str());
985 ✗ goto init_database_fail;
986 }
987
988 // If this an old cache catalog,
989 // add and initialize new columns to cache_catalog
990 sql = "ALTER TABLE cache_catalog ADD type INTEGER; "
991
1/2
✓ Branch 1 taken 3180 times.
✗ Branch 2 not taken.
 3180 "ALTER TABLE cache_catalog ADD pinned INTEGER";
992
1/2
✓ Branch 2 taken 3180 times.
✗ Branch 3 not taken.
 3180 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
993
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3180 times.
 3180 if (err == SQLITE_OK) {
994 ✗ sql = "UPDATE cache_catalog SET type=" + StringifyInt(kFileRegular) + ";";
995 ✗ err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
996 ✗ if (err != SQLITE_OK) {
997 ✗ LogCvmfs(kLogQuota, kLogDebug,
998 "could not init cache database (failed: %s)", sql.c_str());
999 ✗ goto init_database_fail;
1000 }
1001 }
1002
1003 // Set pinned back
1004
1/2
✓ Branch 1 taken 3180 times.
✗ Branch 2 not taken.
 3180 sql = "UPDATE cache_catalog SET pinned=0;";
1005
1/2
✓ Branch 2 taken 3180 times.
✗ Branch 3 not taken.
 3180 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1006
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3180 times.
 3180 if (err != SQLITE_OK) {
1007 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
1008 sql.c_str());
1009 ✗ goto init_database_fail;
1010 }
1011
1012 // Set schema version
1013 sql = "INSERT OR REPLACE INTO properties (key, value) "
1014
1/2
✓ Branch 1 taken 3180 times.
✗ Branch 2 not taken.
 3180 "VALUES ('schema', '1.0')";
1015
1/2
✓ Branch 2 taken 3180 times.
✗ Branch 3 not taken.
 3180 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1016
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3180 times.
 3180 if (err != SQLITE_OK) {
1017 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
1018 sql.c_str());
1019 ✗ goto init_database_fail;
1020 }
1021
1022 // If cache catalog is empty, recreate from file system
1023
1/2
✓ Branch 1 taken 3180 times.
✗ Branch 2 not taken.
 3180 sql = "SELECT count(*) FROM cache_catalog;";
1024
1/2
✓ Branch 2 taken 3180 times.
✗ Branch 3 not taken.
 3180 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1025
2/4
✓ Branch 1 taken 3180 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 3180 times.
✗ Branch 4 not taken.
 3180 if (sqlite3_step(stmt) == SQLITE_ROW) {
1026
6/8
✓ Branch 1 taken 3180 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 184 times.
✓ Branch 4 taken 2996 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 184 times.
✓ Branch 7 taken 2996 times.
✓ Branch 8 taken 184 times.
 3180 if ((sqlite3_column_int64(stmt, 0)) == 0 || rebuild_database) {
1027
1/2
✓ Branch 1 taken 2996 times.
✗ Branch 2 not taken.
 2996 LogCvmfs(kLogCvmfs, kLogDebug,
1028 "CernVM-FS: building lru cache database...");
1029
3/4
✓ Branch 1 taken 2996 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 138 times.
✓ Branch 4 taken 2858 times.
 2996 if (!RebuildDatabase()) {
1030
1/2
✓ Branch 1 taken 138 times.
✗ Branch 2 not taken.
 138 LogCvmfs(kLogQuota, kLogDebug,
1031 "could not build cache database from file system");
1032
1/2
✓ Branch 1 taken 138 times.
✗ Branch 2 not taken.
 138 sqlite3_finalize(stmt);
1033 138 goto init_database_fail;
1034 }
1035 }
1036
1/2
✓ Branch 1 taken 3042 times.
✗ Branch 2 not taken.
 3042 sqlite3_finalize(stmt);
1037 } else {
1038 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not select on cache catalog");
1039 ✗ sqlite3_finalize(stmt);
1040 ✗ goto init_database_fail;
1041 }
1042
1043 // How many bytes do we already have in cache?
1044
1/2
✓ Branch 1 taken 3042 times.
✗ Branch 2 not taken.
 3042 sql = "SELECT sum(size) FROM cache_catalog;";
1045
1/2
✓ Branch 2 taken 3042 times.
✗ Branch 3 not taken.
 3042 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1046
2/4
✓ Branch 1 taken 3042 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 3042 times.
✗ Branch 4 not taken.
 3042 if (sqlite3_step(stmt) == SQLITE_ROW) {
1047
1/2
✓ Branch 1 taken 3042 times.
✗ Branch 2 not taken.
 3042 gauge_ = sqlite3_column_int64(stmt, 0);
1048 } else {
1049 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not determine cache size");
1050 ✗ sqlite3_finalize(stmt);
1051 ✗ goto init_database_fail;
1052 }
1053
1/2
✓ Branch 1 taken 3042 times.
✗ Branch 2 not taken.
 3042 sqlite3_finalize(stmt);
1054
1055 // Highest seq-no?
1056
1/2
✓ Branch 1 taken 3042 times.
✗ Branch 2 not taken.
 3042 sql = "SELECT coalesce(max(acseq & (~(1<<63))), 0) FROM cache_catalog;";
1057
1/2
✓ Branch 2 taken 3042 times.
✗ Branch 3 not taken.
 3042 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1058
2/4
✓ Branch 1 taken 3042 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 3042 times.
✗ Branch 4 not taken.
 3042 if (sqlite3_step(stmt) == SQLITE_ROW) {
1059
1/2
✓ Branch 1 taken 3042 times.
✗ Branch 2 not taken.
 3042 seq_ = sqlite3_column_int64(stmt, 0) + 1;
1060 } else {
1061 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not determine highest seq-no");
1062 ✗ sqlite3_finalize(stmt);
1063 ✗ goto init_database_fail;
1064 }
1065
1/2
✓ Branch 1 taken 3042 times.
✗ Branch 2 not taken.
 3042 sqlite3_finalize(stmt);
1066
1067 // Prepare touch, new, remove statements
1068
1/2
✓ Branch 1 taken 3042 times.
✗ Branch 2 not taken.
 3042 sqlite3_prepare_v2(database_,
1069 "UPDATE cache_catalog SET acseq=:seq | (acseq&(1<<63)) "
1070 "WHERE sha1=:sha1;",
1071 -1, &stmt_touch_, NULL);
1072
1/2
✓ Branch 1 taken 3042 times.
✗ Branch 2 not taken.
 3042 sqlite3_prepare_v2(database_,
1073 "UPDATE cache_catalog SET pinned=0 "
1074 "WHERE sha1=:sha1;",
1075 -1, &stmt_unpin_, NULL);
1076
1/2
✓ Branch 1 taken 3042 times.
✗ Branch 2 not taken.
 3042 sqlite3_prepare_v2(database_,
1077 "UPDATE cache_catalog SET pinned=2 "
1078 "WHERE sha1=:sha1;",
1079 -1, &stmt_block_, NULL);
1080
1/2
✓ Branch 1 taken 3042 times.
✗ Branch 2 not taken.
 3042 sqlite3_prepare_v2(database_,
1081 "UPDATE cache_catalog SET pinned=1 "
1082 "WHERE pinned=2;",
1083 -1, &stmt_unblock_, NULL);
1084
1/2
✓ Branch 1 taken 3042 times.
✗ Branch 2 not taken.
 3042 sqlite3_prepare_v2(database_,
1085 "INSERT OR REPLACE INTO cache_catalog "
1086 "(sha1, size, acseq, path, type, pinned) "
1087 "VALUES (:sha1, :s, :seq, :p, :t, :pin);",
1088 -1, &stmt_new_, NULL);
1089
1/2
✓ Branch 1 taken 3042 times.
✗ Branch 2 not taken.
 3042 sqlite3_prepare_v2(database_,
1090 "SELECT size, pinned FROM cache_catalog WHERE sha1=:sha1;",
1091 -1, &stmt_size_, NULL);
1092
1/2
✓ Branch 1 taken 3042 times.
✗ Branch 2 not taken.
 3042 sqlite3_prepare_v2(database_, "DELETE FROM cache_catalog WHERE sha1=:sha1;",
1093 -1, &stmt_rm_, NULL);
1094
1/2
✓ Branch 1 taken 3042 times.
✗ Branch 2 not taken.
 3042 sqlite3_prepare_v2(database_,
1095 "DELETE FROM cache_catalog WHERE acseq<=:a AND pinned<>2;",
1096 -1, &stmt_rm_batch_, NULL);
1097
1/2
✓ Branch 2 taken 3042 times.
✗ Branch 3 not taken.
 3042 sqlite3_prepare_v2(database_,
1098
1/2
✓ Branch 2 taken 3042 times.
✗ Branch 3 not taken.
 6084 (std::string("SELECT sha1, size, acseq FROM cache_catalog "
1099 "WHERE pinned<>2 AND acseq>=:a "
1100 "ORDER BY acseq ASC "
1101 "LIMIT ")
1102
3/6
✓ Branch 1 taken 3042 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3042 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 3042 times.
✗ Branch 8 not taken.
 12168 + StringifyInt(kEvictBatchSize) + ";")
1103 .c_str(),
1104 -1, &stmt_lru_, NULL);
1105
1/2
✓ Branch 2 taken 3042 times.
✗ Branch 3 not taken.
 3042 sqlite3_prepare_v2(database_,
1106 ("SELECT path FROM cache_catalog WHERE type="
1107
3/6
✓ Branch 1 taken 3042 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3042 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 3042 times.
✗ Branch 8 not taken.
 6084 + StringifyInt(kFileRegular) + ";")
1108 .c_str(),
1109 -1, &stmt_list_, NULL);
1110
1/2
✓ Branch 1 taken 3042 times.
✗ Branch 2 not taken.
 3042 sqlite3_prepare_v2(database_,
1111 "SELECT path FROM cache_catalog WHERE pinned<>0;", -1,
1112 &stmt_list_pinned_, NULL);
1113
1/2
✓ Branch 1 taken 3042 times.
✗ Branch 2 not taken.
 3042 sqlite3_prepare_v2(database_,
1114 "SELECT path FROM cache_catalog WHERE acseq < 0;", -1,
1115 &stmt_list_volatile_, NULL);
1116
1/2
✓ Branch 2 taken 3042 times.
✗ Branch 3 not taken.
 3042 sqlite3_prepare_v2(database_,
1117 ("SELECT path FROM cache_catalog WHERE type="
1118
3/6
✓ Branch 1 taken 3042 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3042 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 3042 times.
✗ Branch 8 not taken.
 6084 + StringifyInt(kFileCatalog) + ";")
1119 .c_str(),
1120 -1, &stmt_list_catalogs_, NULL);
1121 3042 return true;
1122
1123 138 init_database_fail:
1124
1/2
✓ Branch 1 taken 138 times.
✗ Branch 2 not taken.
 138 sqlite3_close(database_);
1125 138 database_ = NULL;
1126
1/2
✓ Branch 1 taken 138 times.
✗ Branch 2 not taken.
 138 UnlockFile(fd_lock_cachedb_);
1127 138 return false;
1128 3226 }
1129
1130
1131 /**
1132 * Inserts a new file into cache catalog. This file gets a new,
1133 * highest sequence number. Does cache cleanup if necessary.
1134 */
1135 4600981 void PosixQuotaManager::Insert(const shash::Any &any_hash,
1136 const uint64_t size,
1137 const string &description) {
1138 4600981 DoInsert(any_hash, size, description, kInsert);
1139 4600981 }
1140
1141
1142 /**
1143 * Inserts a new file into cache catalog. This file is marked as volatile
1144 * and gets a new highest sequence number with the first bit set. Cache cleanup
1145 * treats these files with priority.
1146 */
1147 184 void PosixQuotaManager::InsertVolatile(const shash::Any &any_hash,
1148 const uint64_t size,
1149 const string &description) {
1150 184 DoInsert(any_hash, size, description, kInsertVolatile);
1151 184 }
1152
1153
1154 /**
1155 * Lists all path names from the cache db.
1156 */
1157 966 vector<string> PosixQuotaManager::List() { return DoList(kList); }
1158
1159
1160 /**
1161 * Lists all pinned files from the cache db.
1162 */
1163 368 vector<string> PosixQuotaManager::ListPinned() { return DoList(kListPinned); }
1164
1165
1166 /**
1167 * Lists all sqlite catalog files from the cache db.
1168 */
1169 138 vector<string> PosixQuotaManager::ListCatalogs() {
1170 138 return DoList(kListCatalogs);
1171 }
1172
1173
1174 /**
1175 * Lists only files flagged as volatile (priority removal)
1176 */
1177 138 vector<string> PosixQuotaManager::ListVolatile() {
1178 138 return DoList(kListVolatile);
1179 }
1180
1181
1182 /**
1183 * Entry point for the shared cache manager process
1184 */
1185 ✗ int PosixQuotaManager::MainCacheManager(int argc, char **argv) {
1186 ✗ LogCvmfs(kLogQuota, kLogDebug, "starting quota manager");
1187 int retval;
1188
1189 ✗ PosixQuotaManager shared_manager(0, 0, "");
1190 ✗ shared_manager.shared_ = true;
1191 ✗ shared_manager.spawned_ = true;
1192 ✗ shared_manager.pinned_ = 0;
1193
1194 // Process command line arguments
1195 ✗ ParseDirectories(string(argv[2]),
1196 &shared_manager.cache_dir_,
1197 &shared_manager.workspace_dir_);
1198 ✗ const int pipe_boot = String2Int64(argv[3]);
1199 ✗ const int pipe_handshake = String2Int64(argv[4]);
1200 ✗ shared_manager.limit_ = String2Int64(argv[5]);
1201 ✗ shared_manager.cleanup_threshold_ = String2Int64(argv[6]);
1202 ✗ const int foreground = String2Int64(argv[7]);
1203 ✗ const int syslog_level = String2Int64(argv[8]);
1204 ✗ const int syslog_facility = String2Int64(argv[9]);
1205 ✗ vector<string> logfiles = SplitString(argv[10], ':');
1206
1207 ✗ SetLogSyslogLevel(syslog_level);
1208 ✗ SetLogSyslogFacility(syslog_facility);
1209 ✗ if ((logfiles.size() > 0) && (logfiles[0] != ""))
1210 ✗ SetLogDebugFile(logfiles[0] + ".cachemgr");
1211 ✗ if (logfiles.size() > 1)
1212 ✗ SetLogMicroSyslog(logfiles[1]);
1213
1214 ✗ if (!foreground)
1215 ✗ Daemonize();
1216
1217 ✗ if ((geteuid() != 0) && SetuidCapabilityPermitted()) {
1218 // Permanently drop credentials
1219 ✗ const std::vector<cap_value_t> nocaps;
1220 ✗ assert(ClearPermittedCapabilities(nocaps, nocaps));
1221 // Leave this process ptraceable
1222 ✗ assert(platform_set_dumpable());
1223 // but without core dumps
1224 ✗ assert(SetLimitCore(0));
1225 }
1226
1227 ✗ const UniquePtr<Watchdog> watchdog(Watchdog::Create(NULL, false /* needs_read_environ */));
1228 ✗ assert(watchdog.IsValid());
1229 ✗ watchdog->Spawn("./stacktrace.cachemgr");
1230
1231 // Initialize pipe, open non-blocking as cvmfs is not yet connected
1232 ✗ const int fd_lockfile_fifo = LockFile(shared_manager.workspace_dir_
1233 ✗ + "/lock_cachemgr.fifo");
1234 ✗ if (fd_lockfile_fifo < 0) {
1235 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1236 "could not open lock file "
1237 "%s (%d)",
1238 ✗ (shared_manager.workspace_dir_ + "/lock_cachemgr.fifo").c_str(),
1239 ✗ errno);
1240 ✗ return 1;
1241 }
1242 ✗ const string crash_guard = shared_manager.cache_dir_ + "/cachemgr.running";
1243 ✗ const bool rebuild = FileExists(crash_guard);
1244 ✗ retval = open(crash_guard.c_str(), O_RDONLY | O_CREAT, 0600);
1245 ✗ if (retval < 0) {
1246 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1247 "failed to create shared cache manager crash guard");
1248 ✗ UnlockFile(fd_lockfile_fifo);
1249 ✗ return 1;
1250 }
1251 ✗ close(retval);
1252
1253 // Redirect SQlite temp directory to cache (global variable)
1254 ✗ const string tmp_dir = shared_manager.workspace_dir_;
1255 ✗ sqlite3_temp_directory = static_cast<char *>(
1256 ✗ sqlite3_malloc(tmp_dir.length() + 1));
1257 ✗ snprintf(sqlite3_temp_directory, tmp_dir.length() + 1, "%s", tmp_dir.c_str());
1258
1259 // Cleanup leftover named pipes
1260 ✗ shared_manager.CleanupPipes();
1261
1262 ✗ if (!shared_manager.InitDatabase(rebuild)) {
1263 ✗ UnlockFile(fd_lockfile_fifo);
1264 ✗ return 1;
1265 }
1266 ✗ shared_manager.CheckFreeSpace();
1267
1268 // Save protocol revision to file. If the file is not found, it indicates
1269 // to the client that the cache manager is from times before the protocol
1270 // was versioned.
1271 const string protocol_revision_path = shared_manager.workspace_dir_
1272 ✗ + "/cachemgr.protocol";
1273 ✗ retval = open(protocol_revision_path.c_str(), O_WRONLY | O_CREAT, 0600);
1274 ✗ if (retval < 0) {
1275 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1276 ✗ "failed to open protocol revision file (%d)", errno);
1277 ✗ UnlockFile(fd_lockfile_fifo);
1278 ✗ return 1;
1279 }
1280 ✗ const string revision = StringifyInt(kProtocolRevision);
1281 ✗ const int written = write(retval, revision.data(), revision.length());
1282 ✗ close(retval);
1283 ✗ if ((written < 0) || static_cast<unsigned>(written) != revision.length()) {
1284 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1285 ✗ "failed to write protocol revision (%d)", errno);
1286 ✗ UnlockFile(fd_lockfile_fifo);
1287 ✗ return 1;
1288 }
1289
1290 ✗ const string fifo_path = shared_manager.workspace_dir_ + "/cachemgr";
1291 ✗ shared_manager.pipe_lru_[0] = open(fifo_path.c_str(), O_RDONLY | O_NONBLOCK);
1292 ✗ if (shared_manager.pipe_lru_[0] < 0) {
1293 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to listen on FIFO %s (%d)",
1294 ✗ fifo_path.c_str(), errno);
1295 ✗ UnlockFile(fd_lockfile_fifo);
1296 ✗ return 1;
1297 }
1298 ✗ Nonblock2Block(shared_manager.pipe_lru_[0]);
1299 ✗ LogCvmfs(kLogQuota, kLogDebug, "shared cache manager listening");
1300
1301 ✗ char buf = 'C';
1302 ✗ WritePipe(pipe_boot, &buf, 1);
1303 ✗ close(pipe_boot);
1304
1305 ✗ ReadPipe(pipe_handshake, &buf, 1);
1306 ✗ close(pipe_handshake);
1307 ✗ LogCvmfs(kLogQuota, kLogDebug, "shared cache manager handshake done");
1308
1309 // Ensure that broken pipes from clients do not kill the cache manager
1310 ✗ signal(SIGPIPE, SIG_IGN);
1311 // Don't let Ctrl-C ungracefully kill interactive session
1312 ✗ signal(SIGINT, SIG_IGN);
1313
1314 ✗ shared_manager.MainCommandServer(&shared_manager);
1315 ✗ unlink(fifo_path.c_str());
1316 ✗ unlink(protocol_revision_path.c_str());
1317 ✗ shared_manager.CloseDatabase();
1318 ✗ unlink(crash_guard.c_str());
1319 ✗ UnlockFile(fd_lockfile_fifo);
1320
1321 ✗ if (sqlite3_temp_directory) {
1322 ✗ sqlite3_free(sqlite3_temp_directory);
1323 ✗ sqlite3_temp_directory = NULL;
1324 }
1325
1326 ✗ return 0;
1327 }
1328
1329
1330 1426 void *PosixQuotaManager::MainCommandServer(void *data) {
1331 1426 PosixQuotaManager *quota_mgr = static_cast<PosixQuotaManager *>(data);
1332
1333
1/2
✓ Branch 1 taken 1426 times.
✗ Branch 2 not taken.
 1426 LogCvmfs(kLogQuota, kLogDebug, "starting quota manager");
1334
1/2
✓ Branch 1 taken 1426 times.
✗ Branch 2 not taken.
 1426 sqlite3_soft_heap_limit(quota_mgr->kSqliteMemPerThread);
1335
1336
2/2
✓ Branch 1 taken 45632 times.
✓ Branch 2 taken 1426 times.
 47058 LruCommand command_buffer[kCommandBufferSize];
1337 char description_buffer[kCommandBufferSize * kMaxDescription];
1338 1426 unsigned num_commands = 0;
1339
1340
1/2
✓ Branch 1 taken 6907406 times.
✗ Branch 2 not taken.
 6907406 while (read(quota_mgr->pipe_lru_[0], &command_buffer[num_commands],
1341 sizeof(command_buffer[0]))
1342
2/2
✓ Branch 0 taken 6905980 times.
✓ Branch 1 taken 1426 times.
 6907406 == sizeof(command_buffer[0])) {
1343 6905980 const CommandType command_type = command_buffer[num_commands].command_type;
1344
1/2
✓ Branch 1 taken 6905980 times.
✗ Branch 2 not taken.
 6905980 LogCvmfs(kLogQuota, kLogDebug, "received command %d", command_type);
1345 6905980 const uint64_t size = command_buffer[num_commands].GetSize();
1346
1347 // Inserts and pins come with a description (usually a path)
1348
4/4
✓ Branch 0 taken 2305198 times.
✓ Branch 1 taken 4600782 times.
✓ Branch 2 taken 2305014 times.
✓ Branch 3 taken 184 times.
 6905980 if ((command_type == kInsert) || (command_type == kInsertVolatile)
1349
4/4
✓ Branch 0 taken 2304922 times.
✓ Branch 1 taken 92 times.
✓ Branch 2 taken 506 times.
✓ Branch 3 taken 2304416 times.
 2305014 || (command_type == kPin) || (command_type == kPinRegular)) {
1350 4601564 const int desc_length = command_buffer[num_commands].desc_length;
1351 4601564 ReadPipe(quota_mgr->pipe_lru_[0],
1352
1/2
✓ Branch 1 taken 4601564 times.
✗ Branch 2 not taken.
 4601564 &description_buffer[kMaxDescription * num_commands],
1353 desc_length);
1354 }
1355
1356 // The protocol revision is returned immediately
1357
2/2
✓ Branch 0 taken 46 times.
✓ Branch 1 taken 6905934 times.
 6905980 if (command_type == kGetProtocolRevision) {
1358
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 const int return_pipe = quota_mgr->BindReturnPipe(
1359 command_buffer[num_commands].return_pipe);
1360
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 46 times.
 46 if (return_pipe < 0)
1361 ✗ continue;
1362
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 WritePipe(return_pipe, &quota_mgr->kProtocolRevision,
1363 sizeof(quota_mgr->kProtocolRevision));
1364
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 quota_mgr->UnbindReturnPipe(return_pipe);
1365 46 continue;
1366 46 }
1367
1368 // Register a new mountpoint
1369
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6905934 times.
 6905934 if (command_type == kRegisterMountpoint) {
1370 ✗ size_t mp_size = 0;
1371 ✗ ReadPipe(quota_mgr->pipe_lru_[0], &mp_size, sizeof(size_t));
1372 ✗ std::string mountpoint(mp_size, '\0');
1373 ✗ ReadPipe(quota_mgr->pipe_lru_[0], (void *)mountpoint.data(), mp_size);
1374 ✗ quota_mgr->mountpoints_.push_back(mountpoint);
1375 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
1376 "Mountpoint %s registered in the group", mountpoint.c_str());
1377 ✗ continue;
1378 }
1379
1380 // Set Cleanup Policy
1381
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6905934 times.
 6905934 if (command_type == kSetCleanupPolicy) {
1382 ✗ bool policy = false;
1383 ✗ ReadPipe(quota_mgr->pipe_lru_[0], &policy, sizeof(bool));
1384 ✗ quota_mgr->cleanup_unused_first_ = policy;
1385 ✗ continue;
1386 }
1387 // Mountpoints are returned immediately
1388
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6905934 times.
 6905934 if (command_type == kGetMountpoints) {
1389 ✗ const int return_pipe = quota_mgr->BindReturnPipe(
1390 command_buffer[num_commands].return_pipe);
1391 ✗ if (return_pipe < 0)
1392 ✗ continue;
1393
1394 ✗ std::string mps;
1395 ✗ for (auto it = quota_mgr->mountpoints_.begin();
1396 ✗ it != quota_mgr->mountpoints_.end();
1397 ✗ ++it) {
1398 ✗ mps += *it + "\n";
1399 }
1400 ✗ size_t mp_size = mps.size();
1401 ✗ WritePipe(return_pipe, &mp_size, sizeof(size_t));
1402 ✗ WritePipe(return_pipe, mps.c_str(), mp_size);
1403 ✗ quota_mgr->UnbindReturnPipe(return_pipe);
1404 ✗ continue;
1405 }
1406
1407 // Group hashes are returned immediately
1408
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6905934 times.
 6905934 if (command_type == kGetGroupHashes) {
1409 ✗ const int return_pipe = quota_mgr->BindReturnPipe(
1410 command_buffer[num_commands].return_pipe);
1411 ✗ if (return_pipe < 0)
1412 ✗ continue;
1413
1414 ✗ std::vector<shash::Short> gh = quota_mgr->CollectAllOpenHashes();
1415 ✗ std::string result;
1416 ✗ for (auto it = gh.begin(); it != gh.end(); ++it) {
1417 ✗ result += (*it).ToString() + "\n";
1418 }
1419 ✗ size_t result_size = result.size();
1420 ✗ WritePipe(return_pipe, &result_size, sizeof(size_t));
1421 ✗ WritePipe(return_pipe, result.c_str(), result_size);
1422 ✗ quota_mgr->UnbindReturnPipe(return_pipe);
1423 ✗ continue;
1424 }
1425
1426 // The cleanup rate is returned immediately
1427
2/2
✓ Branch 0 taken 184 times.
✓ Branch 1 taken 6905750 times.
 6905934 if (command_type == kCleanupRate) {
1428
1/2
✓ Branch 1 taken 184 times.
✗ Branch 2 not taken.
 184 const int return_pipe = quota_mgr->BindReturnPipe(
1429 command_buffer[num_commands].return_pipe);
1430
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 184 times.
 184 if (return_pipe < 0)
1431 ✗ continue;
1432 const uint64_t
1433 184 period_s = size; // use the size field to transmit the period
1434
1/2
✓ Branch 1 taken 184 times.
✗ Branch 2 not taken.
 184 uint64_t rate = quota_mgr->cleanup_recorder_.GetNoTicks(period_s);
1435
1/2
✓ Branch 1 taken 184 times.
✗ Branch 2 not taken.
 184 WritePipe(return_pipe, &rate, sizeof(rate));
1436
1/2
✓ Branch 1 taken 184 times.
✗ Branch 2 not taken.
 184 quota_mgr->UnbindReturnPipe(return_pipe);
1437 184 continue;
1438 184 }
1439
1440
2/2
✓ Branch 0 taken 46 times.
✓ Branch 1 taken 6905704 times.
 6905750 if (command_type == kSetLimit) {
1441
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 const int return_pipe = quota_mgr->BindReturnPipe(
1442 command_buffer[num_commands].return_pipe);
1443
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 46 times.
 46 if (return_pipe < 0)
1444 ✗ continue;
1445 46 quota_mgr->limit_ = size; // use the size field to transmit the size
1446 46 quota_mgr->cleanup_threshold_ = size / 2;
1447
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
1448 "Quota limit set to %lu / threshold %lu", quota_mgr->limit_,
1449 quota_mgr->cleanup_threshold_);
1450 46 bool ret = true;
1451
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 WritePipe(return_pipe, &ret, sizeof(ret));
1452
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 quota_mgr->UnbindReturnPipe(return_pipe);
1453 46 continue;
1454 46 }
1455
1456 // Reservations are handled immediately and "out of band"
1457
2/2
✓ Branch 0 taken 644 times.
✓ Branch 1 taken 6905060 times.
 6905704 if (command_type == kReserve) {
1458 644 bool success = true;
1459
1/2
✓ Branch 1 taken 644 times.
✗ Branch 2 not taken.
 644 const int return_pipe = quota_mgr->BindReturnPipe(
1460 command_buffer[num_commands].return_pipe);
1461
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 644 times.
 644 if (return_pipe < 0)
1462 ✗ continue;
1463
1464
1/2
✓ Branch 1 taken 644 times.
✗ Branch 2 not taken.
 644 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1465
1/2
✓ Branch 1 taken 644 times.
✗ Branch 2 not taken.
 644 const string hash_str(hash.ToString());
1466
1/2
✓ Branch 2 taken 644 times.
✗ Branch 3 not taken.
 644 LogCvmfs(kLogQuota, kLogDebug, "reserve %lu bytes for %s", size,
1467 hash_str.c_str());
1468
1469
1/2
✓ Branch 1 taken 644 times.
✗ Branch 2 not taken.
 644 if (quota_mgr->pinned_chunks_.find(hash)
1470
2/2
✓ Branch 2 taken 552 times.
✓ Branch 3 taken 92 times.
 1288 == quota_mgr->pinned_chunks_.end()) {
1471
2/2
✓ Branch 0 taken 46 times.
✓ Branch 1 taken 506 times.
 552 if ((quota_mgr->pinned_ + size) > quota_mgr->cleanup_threshold_) {
1472
1/2
✓ Branch 2 taken 46 times.
✗ Branch 3 not taken.
 46 LogCvmfs(kLogQuota, kLogDebug,
1473 "failed to insert %s (pinned), no space", hash_str.c_str());
1474 46 success = false;
1475 } else {
1476
1/2
✓ Branch 1 taken 506 times.
✗ Branch 2 not taken.
 506 quota_mgr->pinned_chunks_[hash] = size;
1477 506 quota_mgr->pinned_ += size;
1478
1/2
✓ Branch 1 taken 506 times.
✗ Branch 2 not taken.
 506 quota_mgr->CheckHighPinWatermark();
1479 }
1480 }
1481
1482
1/2
✓ Branch 1 taken 644 times.
✗ Branch 2 not taken.
 644 WritePipe(return_pipe, &success, sizeof(success));
1483
1/2
✓ Branch 1 taken 644 times.
✗ Branch 2 not taken.
 644 quota_mgr->UnbindReturnPipe(return_pipe);
1484 644 continue;
1485 644 }
1486
1487 // Back channels are also handled out of band
1488
2/2
✓ Branch 0 taken 184 times.
✓ Branch 1 taken 6904876 times.
 6905060 if (command_type == kRegisterBackChannel) {
1489
1/2
✓ Branch 1 taken 184 times.
✗ Branch 2 not taken.
 184 const int return_pipe = quota_mgr->BindReturnPipe(
1490 command_buffer[num_commands].return_pipe);
1491
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 184 times.
 184 if (return_pipe < 0)
1492 ✗ continue;
1493
1494
1/2
✓ Branch 1 taken 184 times.
✗ Branch 2 not taken.
 184 quota_mgr->UnlinkReturnPipe(command_buffer[num_commands].return_pipe);
1495
1/2
✓ Branch 1 taken 184 times.
✗ Branch 2 not taken.
 184 Block2Nonblock(return_pipe); // back channels are opportunistic
1496
1/2
✓ Branch 1 taken 184 times.
✗ Branch 2 not taken.
 184 shash::Md5 hash;
1497 184 memcpy(hash.digest, command_buffer[num_commands].digest,
1498 184 shash::kDigestSizes[shash::kMd5]);
1499
1500 184 quota_mgr->LockBackChannels();
1501 const map<shash::Md5, int>::const_iterator
1502
1/2
✓ Branch 1 taken 184 times.
✗ Branch 2 not taken.
 184 iter = quota_mgr->back_channels_.find(hash);
1503
1/2
✗ Branch 3 not taken.
✓ Branch 4 taken 184 times.
 184 if (iter != quota_mgr->back_channels_.end()) {
1504 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
1505 ✗ "closing left-over back channel %s", hash.ToString().c_str());
1506 ✗ close(iter->second);
1507 }
1508
1/2
✓ Branch 1 taken 184 times.
✗ Branch 2 not taken.
 184 quota_mgr->back_channels_[hash] = return_pipe;
1509 184 quota_mgr->UnlockBackChannels();
1510
1511 184 char success = 'S';
1512
1/2
✓ Branch 1 taken 184 times.
✗ Branch 2 not taken.
 184 WritePipe(return_pipe, &success, sizeof(success));
1513
1/2
✓ Branch 2 taken 184 times.
✗ Branch 3 not taken.
 184 LogCvmfs(kLogQuota, kLogDebug, "register back channel %s on fd %d",
1514
1/2
✓ Branch 1 taken 184 times.
✗ Branch 2 not taken.
 368 hash.ToString().c_str(), return_pipe);
1515
1516 184 continue;
1517 184 }
1518
1519
2/2
✓ Branch 0 taken 92 times.
✓ Branch 1 taken 6904784 times.
 6904876 if (command_type == kUnregisterBackChannel) {
1520
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 shash::Md5 hash;
1521 92 memcpy(hash.digest, command_buffer[num_commands].digest,
1522 92 shash::kDigestSizes[shash::kMd5]);
1523
1524 92 quota_mgr->LockBackChannels();
1525 const map<shash::Md5, int>::iterator iter = quota_mgr->back_channels_
1526
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 .find(hash);
1527
1/2
✓ Branch 2 taken 92 times.
✗ Branch 3 not taken.
 92 if (iter != quota_mgr->back_channels_.end()) {
1528
1/2
✓ Branch 2 taken 92 times.
✗ Branch 3 not taken.
 92 LogCvmfs(kLogQuota, kLogDebug, "closing back channel %s",
1529
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 184 hash.ToString().c_str());
1530
1/2
✓ Branch 2 taken 92 times.
✗ Branch 3 not taken.
 92 close(iter->second);
1531
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 quota_mgr->back_channels_.erase(iter);
1532 } else {
1533 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
1534 ✗ "did not find back channel %s", hash.ToString().c_str());
1535 }
1536 92 quota_mgr->UnlockBackChannels();
1537
1538 92 continue;
1539 92 }
1540
1541 // Unpinnings are also handled immediately with respect to the pinned gauge
1542
2/2
✓ Branch 0 taken 92 times.
✓ Branch 1 taken 6904692 times.
 6904784 if (command_type == kUnpin) {
1543
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1544
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 const string hash_str(hash.ToString());
1545
1546 const map<shash::Any, uint64_t>::iterator iter = quota_mgr->pinned_chunks_
1547
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 .find(hash);
1548
1/2
✓ Branch 2 taken 92 times.
✗ Branch 3 not taken.
 92 if (iter != quota_mgr->pinned_chunks_.end()) {
1549 92 quota_mgr->pinned_ -= iter->second;
1550
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 quota_mgr->pinned_chunks_.erase(iter);
1551 // It can happen that files get pinned that were removed from the cache
1552 // (see cache.cc). We fix this at this point, where we remove such
1553 // entries from the cache database.
1554
2/4
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 92 times.
✗ Branch 5 not taken.
 184 if (!FileExists(quota_mgr->cache_dir_ + "/"
1555
4/6
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 92 times.
✗ Branch 5 not taken.
✓ Branch 9 taken 46 times.
✓ Branch 10 taken 46 times.
 276 + hash.MakePathWithoutSuffix())) {
1556
1/2
✓ Branch 2 taken 46 times.
✗ Branch 3 not taken.
 46 LogCvmfs(kLogQuota, kLogDebug,
1557 "remove orphaned pinned hash %s from cache database",
1558 hash_str.c_str());
1559
1/2
✓ Branch 2 taken 46 times.
✗ Branch 3 not taken.
 46 sqlite3_bind_text(quota_mgr->stmt_size_, 1, &hash_str[0],
1560 46 hash_str.length(), SQLITE_STATIC);
1561 int retval;
1562
2/4
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 46 times.
✗ Branch 4 not taken.
 46 if ((retval = sqlite3_step(quota_mgr->stmt_size_)) == SQLITE_ROW) {
1563
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 const uint64_t size = sqlite3_column_int64(quota_mgr->stmt_size_,
1564 46 0);
1565
1/2
✓ Branch 2 taken 46 times.
✗ Branch 3 not taken.
 46 sqlite3_bind_text(quota_mgr->stmt_rm_, 1, &(hash_str[0]),
1566 46 hash_str.length(), SQLITE_STATIC);
1567
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 retval = sqlite3_step(quota_mgr->stmt_rm_);
1568
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 46 if ((retval == SQLITE_DONE) || (retval == SQLITE_OK)) {
1569 46 quota_mgr->gauge_ -= size;
1570 } else {
1571 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1572 "failed to delete %s (%d)", hash_str.c_str(), retval);
1573 }
1574
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 sqlite3_reset(quota_mgr->stmt_rm_);
1575 }
1576
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 sqlite3_reset(quota_mgr->stmt_size_);
1577 }
1578 } else {
1579 ✗ LogCvmfs(kLogQuota, kLogDebug, "this chunk was not pinned");
1580 }
1581 92 }
1582
1583 // Immediate commands trigger flushing of the buffer
1584 6904784 const bool immediate_command = (command_type == kCleanup)
1585
2/2
✓ Branch 0 taken 6903404 times.
✓ Branch 1 taken 966 times.
 6904370 || (command_type == kList)
1586
2/2
✓ Branch 0 taken 6903036 times.
✓ Branch 1 taken 368 times.
 6903404 || (command_type == kListPinned)
1587
2/2
✓ Branch 0 taken 6902898 times.
✓ Branch 1 taken 138 times.
 6903036 || (command_type == kListCatalogs)
1588
2/2
✓ Branch 0 taken 6902760 times.
✓ Branch 1 taken 138 times.
 6902898 || (command_type == kListVolatile)
1589
2/2
✓ Branch 0 taken 6902622 times.
✓ Branch 1 taken 138 times.
 6902760 || (command_type == kRemove)
1590
2/2
✓ Branch 0 taken 6901794 times.
✓ Branch 1 taken 828 times.
 6902622 || (command_type == kStatus)
1591
1/2
✓ Branch 0 taken 6901794 times.
✗ Branch 1 not taken.
 6901794 || (command_type == kLimits)
1592
3/4
✓ Branch 0 taken 6904370 times.
✓ Branch 1 taken 414 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 6901794 times.
 13809154 || (command_type == kPid);
1593
2/2
✓ Branch 0 taken 6901794 times.
✓ Branch 1 taken 2990 times.
 6904784 if (!immediate_command)
1594 6901794 num_commands++;
1595
1596
4/4
✓ Branch 0 taken 6689182 times.
✓ Branch 1 taken 215602 times.
✓ Branch 2 taken 2990 times.
✓ Branch 3 taken 6686192 times.
 6904784 if ((num_commands == kCommandBufferSize) || immediate_command) {
1597
1/2
✓ Branch 1 taken 218592 times.
✗ Branch 2 not taken.
 218592 quota_mgr->ProcessCommandBunch(num_commands, command_buffer,
1598 description_buffer);
1599
2/2
✓ Branch 0 taken 215602 times.
✓ Branch 1 taken 2990 times.
 218592 if (!immediate_command)
1600 215602 num_commands = 0;
1601 }
1602
1603
2/2
✓ Branch 0 taken 2990 times.
✓ Branch 1 taken 6901794 times.
 6904784 if (immediate_command) {
1604 // Process cleanup, listings
1605
1/2
✓ Branch 1 taken 2990 times.
✗ Branch 2 not taken.
 2990 const int return_pipe = quota_mgr->BindReturnPipe(
1606 command_buffer[num_commands].return_pipe);
1607
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2990 times.
 2990 if (return_pipe < 0) {
1608 ✗ num_commands = 0;
1609 ✗ continue;
1610 }
1611
1612 int retval;
1613 2990 sqlite3_stmt *this_stmt_list = NULL;
1614
7/10
✓ Branch 0 taken 138 times.
✓ Branch 1 taken 414 times.
✓ Branch 2 taken 966 times.
✓ Branch 3 taken 368 times.
✓ Branch 4 taken 138 times.
✓ Branch 5 taken 138 times.
✓ Branch 6 taken 828 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
 2990 switch (command_type) {
1615 138 case kRemove: {
1616
1/2
✓ Branch 1 taken 138 times.
✗ Branch 2 not taken.
 138 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1617
1/2
✓ Branch 1 taken 138 times.
✗ Branch 2 not taken.
 138 const string hash_str = hash.ToString();
1618
1/2
✓ Branch 2 taken 138 times.
✗ Branch 3 not taken.
 138 LogCvmfs(kLogQuota, kLogDebug, "manually removing %s",
1619 hash_str.c_str());
1620 138 bool success = false;
1621
1622
1/2
✓ Branch 2 taken 138 times.
✗ Branch 3 not taken.
 138 sqlite3_bind_text(quota_mgr->stmt_size_, 1, &hash_str[0],
1623 138 hash_str.length(), SQLITE_STATIC);
1624 int retval;
1625
3/4
✓ Branch 1 taken 138 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 92 times.
✓ Branch 4 taken 46 times.
 138 if ((retval = sqlite3_step(quota_mgr->stmt_size_)) == SQLITE_ROW) {
1626
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 const uint64_t size = sqlite3_column_int64(quota_mgr->stmt_size_,
1627 92 0);
1628
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 const uint64_t is_pinned = sqlite3_column_int64(
1629 92 quota_mgr->stmt_size_, 1);
1630
1631
1/2
✓ Branch 2 taken 92 times.
✗ Branch 3 not taken.
 92 sqlite3_bind_text(quota_mgr->stmt_rm_, 1, &(hash_str[0]),
1632 92 hash_str.length(), SQLITE_STATIC);
1633
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 retval = sqlite3_step(quota_mgr->stmt_rm_);
1634
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 92 if ((retval == SQLITE_DONE) || (retval == SQLITE_OK)) {
1635 92 success = true;
1636 92 quota_mgr->gauge_ -= size;
1637
2/2
✓ Branch 0 taken 46 times.
✓ Branch 1 taken 46 times.
 92 if (is_pinned) {
1638
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 quota_mgr->pinned_chunks_.erase(hash);
1639 46 quota_mgr->pinned_ -= size;
1640 }
1641 } else {
1642 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1643 "failed to delete %s (%d)", hash_str.c_str(), retval);
1644 }
1645
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 sqlite3_reset(quota_mgr->stmt_rm_);
1646 } else {
1647 // File does not exist
1648 46 success = true;
1649 }
1650
1/2
✓ Branch 1 taken 138 times.
✗ Branch 2 not taken.
 138 sqlite3_reset(quota_mgr->stmt_size_);
1651
1652
1/2
✓ Branch 1 taken 138 times.
✗ Branch 2 not taken.
 138 WritePipe(return_pipe, &success, sizeof(success));
1653 138 break;
1654 138 }
1655 414 case kCleanup:
1656
1/2
✓ Branch 1 taken 414 times.
✗ Branch 2 not taken.
 414 retval = quota_mgr->DoCleanup(size);
1657
1/2
✓ Branch 1 taken 414 times.
✗ Branch 2 not taken.
 414 WritePipe(return_pipe, &retval, sizeof(retval));
1658 414 break;
1659 966 case kList:
1660
1/2
✓ Branch 0 taken 966 times.
✗ Branch 1 not taken.
 966 if (!this_stmt_list)
1661 966 this_stmt_list = quota_mgr->stmt_list_;
1662 case kListPinned:
1663
2/2
✓ Branch 0 taken 368 times.
✓ Branch 1 taken 966 times.
 1334 if (!this_stmt_list)
1664 368 this_stmt_list = quota_mgr->stmt_list_pinned_;
1665 case kListCatalogs:
1666
2/2
✓ Branch 0 taken 138 times.
✓ Branch 1 taken 1334 times.
 1472 if (!this_stmt_list)
1667 138 this_stmt_list = quota_mgr->stmt_list_catalogs_;
1668 case kListVolatile:
1669
2/2
✓ Branch 0 taken 138 times.
✓ Branch 1 taken 1472 times.
 1610 if (!this_stmt_list)
1670 138 this_stmt_list = quota_mgr->stmt_list_volatile_;
1671
1672 // Pipe back the list, one by one
1673 int length;
1674
3/4
✓ Branch 1 taken 4603404 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 4601794 times.
✓ Branch 4 taken 1610 times.
 4603404 while (sqlite3_step(this_stmt_list) == SQLITE_ROW) {
1675
1/2
✓ Branch 2 taken 4601794 times.
✗ Branch 3 not taken.
 4601794 string path = "(NULL)";
1676
2/4
✓ Branch 1 taken 4601794 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 4601794 times.
✗ Branch 4 not taken.
 4601794 if (sqlite3_column_type(this_stmt_list, 0) != SQLITE_NULL) {
1677 9203588 path = string(reinterpret_cast<const char *>(
1678
2/4
✓ Branch 1 taken 4601794 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 4601794 times.
✗ Branch 5 not taken.
 4601794 sqlite3_column_text(this_stmt_list, 0)));
1679 }
1680 4601794 length = path.length();
1681
1/2
✓ Branch 1 taken 4601794 times.
✗ Branch 2 not taken.
 4601794 WritePipe(return_pipe, &length, sizeof(length));
1682
1/2
✓ Branch 0 taken 4601794 times.
✗ Branch 1 not taken.
 4601794 if (length > 0)
1683
2/4
✓ Branch 1 taken 4601794 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 4601794 times.
✗ Branch 5 not taken.
 4601794 WritePipe(return_pipe, &path[0], length);
1684 4601794 }
1685 1610 length = -1;
1686
1/2
✓ Branch 1 taken 1610 times.
✗ Branch 2 not taken.
 1610 WritePipe(return_pipe, &length, sizeof(length));
1687
1/2
✓ Branch 1 taken 1610 times.
✗ Branch 2 not taken.
 1610 sqlite3_reset(this_stmt_list);
1688 1610 break;
1689 828 case kStatus:
1690
1/2
✓ Branch 1 taken 828 times.
✗ Branch 2 not taken.
 828 WritePipe(return_pipe, &quota_mgr->gauge_, sizeof(quota_mgr->gauge_));
1691
1/2
✓ Branch 1 taken 828 times.
✗ Branch 2 not taken.
 828 WritePipe(return_pipe, &quota_mgr->pinned_,
1692 sizeof(quota_mgr->pinned_));
1693 828 break;
1694 ✗ case kLimits:
1695 ✗ WritePipe(return_pipe, &quota_mgr->limit_, sizeof(quota_mgr->limit_));
1696 ✗ WritePipe(return_pipe, &quota_mgr->cleanup_threshold_,
1697 sizeof(quota_mgr->cleanup_threshold_));
1698 ✗ break;
1699 ✗ case kPid: {
1700 ✗ pid_t pid = getpid();
1701 ✗ WritePipe(return_pipe, &pid, sizeof(pid));
1702 ✗ break;
1703 }
1704 ✗ default:
1705 ✗ PANIC(NULL); // other types are handled by the bunch processor
1706 }
1707
1/2
✓ Branch 1 taken 2990 times.
✗ Branch 2 not taken.
 2990 quota_mgr->UnbindReturnPipe(return_pipe);
1708 2990 num_commands = 0;
1709 }
1710 }
1711
1712
1/2
✓ Branch 1 taken 1426 times.
✗ Branch 2 not taken.
 1426 LogCvmfs(kLogQuota, kLogDebug, "stopping cache manager (%d)", errno);
1713
1/2
✓ Branch 1 taken 1426 times.
✗ Branch 2 not taken.
 1426 close(quota_mgr->pipe_lru_[0]);
1714
1/2
✓ Branch 1 taken 1426 times.
✗ Branch 2 not taken.
 1426 quota_mgr->ProcessCommandBunch(num_commands, command_buffer,
1715 description_buffer);
1716
1717 // Unpin
1718 1426 command_buffer[0].command_type = kTouch;
1719 1426 for (map<shash::Any, uint64_t>::const_iterator
1720 1426 i = quota_mgr->pinned_chunks_.begin(),
1721 1426 iEnd = quota_mgr->pinned_chunks_.end();
1722
2/2
✓ Branch 1 taken 460 times.
✓ Branch 2 taken 1426 times.
 1886 i != iEnd;
1723 460 ++i) {
1724
1/2
✓ Branch 2 taken 460 times.
✗ Branch 3 not taken.
 460 command_buffer[0].StoreHash(i->first);
1725
1/2
✓ Branch 1 taken 460 times.
✗ Branch 2 not taken.
 460 quota_mgr->ProcessCommandBunch(1, command_buffer, description_buffer);
1726 }
1727
1728 1426 return NULL;
1729 }
1730
1731
1732 4232 void PosixQuotaManager::MakeReturnPipe(int pipe[2]) {
1733
2/2
✓ Branch 0 taken 4094 times.
✓ Branch 1 taken 138 times.
 4232 if (!shared_) {
1734 4094 MakePipe(pipe);
1735 4094 return;
1736 }
1737
1738 // Create FIFO in cache directory, store path name (number) in pipe write end
1739 138 int i = 0;
1740 int retval;
1741 do {
1742
2/4
✓ Branch 2 taken 184 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 184 times.
✗ Branch 6 not taken.
 184 retval = mkfifo((workspace_dir_ + "/pipe" + StringifyInt(i)).c_str(), 0600);
1743 184 pipe[1] = i;
1744 184 i++;
1745
3/4
✓ Branch 0 taken 46 times.
✓ Branch 1 taken 138 times.
✓ Branch 2 taken 46 times.
✗ Branch 3 not taken.
 184 } while ((retval == -1) && (errno == EEXIST));
1746
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 138 times.
 138 assert(retval == 0);
1747
1748 // Connect reader's end
1749
3/6
✓ Branch 2 taken 138 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 138 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 138 times.
✗ Branch 10 not taken.
 138 pipe[0] = open((workspace_dir_ + "/pipe" + StringifyInt(pipe[1])).c_str(),
1750 O_RDONLY | O_NONBLOCK);
1751
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 138 times.
 138 assert(pipe[0] >= 0);
1752 138 Nonblock2Block(pipe[0]);
1753 }
1754
1755
1756 3180 void PosixQuotaManager::ParseDirectories(const std::string cache_workspace,
1757 std::string *cache_dir,
1758 std::string *workspace_dir) {
1759
1/2
✓ Branch 1 taken 3180 times.
✗ Branch 2 not taken.
 3180 vector<string> dir_tokens(SplitString(cache_workspace, ':'));
1760
2/3
✓ Branch 1 taken 3088 times.
✓ Branch 2 taken 92 times.
✗ Branch 3 not taken.
 3180 switch (dir_tokens.size()) {
1761 3088 case 1:
1762
2/4
✓ Branch 2 taken 3088 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 3088 times.
✗ Branch 6 not taken.
 3088 *cache_dir = *workspace_dir = dir_tokens[0];
1763 3088 break;
1764 92 case 2:
1765
1/2
✓ Branch 2 taken 92 times.
✗ Branch 3 not taken.
 92 *cache_dir = dir_tokens[0];
1766
1/2
✓ Branch 2 taken 92 times.
✗ Branch 3 not taken.
 92 *workspace_dir = dir_tokens[1];
1767 92 break;
1768 ✗ default:
1769 ✗ PANIC(NULL);
1770 }
1771 3180 }
1772
1773 46 void PosixQuotaManager::SkipEviction(const EvictCandidate &candidate) {
1774 46 bool res = true;
1775
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 std::string hash_str = candidate.hash.ToString();
1776
1/2
✓ Branch 2 taken 46 times.
✗ Branch 3 not taken.
 46 LogCvmfs(kLogQuota, kLogDebug, "Exclude %s from eviction", hash_str.c_str());
1777
2/4
✓ Branch 2 taken 46 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 46 times.
✗ Branch 6 not taken.
 46 sqlite3_bind_text(stmt_block_, 1, &hash_str[0], hash_str.length(),
1778 SQLITE_STATIC);
1779
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 res = (sqlite3_step(stmt_block_) == SQLITE_DONE);
1780
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 sqlite3_reset(stmt_block_);
1781
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 46 times.
 46 assert(res);
1782 46 }
1783
1784 /**
1785 * Immediately inserts a new pinned catalog. Does cache cleanup if necessary.
1786 *
1787 * \return True on success, false otherwise
1788 */
1789 1492 bool PosixQuotaManager::Pin(const shash::Any &hash,
1790 const uint64_t size,
1791 const string &description,
1792 const bool is_catalog) {
1793
3/4
✓ Branch 0 taken 92 times.
✓ Branch 1 taken 1400 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 92 times.
 1492 assert((size > 0) || !is_catalog);
1794
1795
1/2
✓ Branch 1 taken 1492 times.
✗ Branch 2 not taken.
 1492 const string hash_str = hash.ToString();
1796
1/2
✓ Branch 3 taken 1492 times.
✗ Branch 4 not taken.
 1492 LogCvmfs(kLogQuota, kLogDebug, "pin into lru %s, path %s", hash_str.c_str(),
1797 description.c_str());
1798
1799 // Has to run when not yet spawned (cvmfs initialization)
1800
2/2
✓ Branch 0 taken 848 times.
✓ Branch 1 taken 644 times.
 1492 if (!spawned_) {
1801 // Code duplication here
1802
3/4
✓ Branch 2 taken 848 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 510 times.
✓ Branch 6 taken 338 times.
 848 if (pinned_chunks_.find(hash) == pinned_chunks_.end()) {
1803
2/2
✓ Branch 0 taken 92 times.
✓ Branch 1 taken 418 times.
 510 if (pinned_ + size > cleanup_threshold_) {
1804
1/2
✓ Branch 2 taken 92 times.
✗ Branch 3 not taken.
 92 LogCvmfs(kLogQuota, kLogDebug, "failed to insert %s (pinned), no space",
1805 hash_str.c_str());
1806 92 return false;
1807 } else {
1808
1/2
✓ Branch 1 taken 418 times.
✗ Branch 2 not taken.
 418 pinned_chunks_[hash] = size;
1809 418 pinned_ += size;
1810
1/2
✓ Branch 1 taken 418 times.
✗ Branch 2 not taken.
 418 CheckHighPinWatermark();
1811 }
1812 }
1813
1/2
✓ Branch 1 taken 756 times.
✗ Branch 2 not taken.
 756 const bool exists = Contains(hash_str);
1814
4/4
✓ Branch 0 taken 418 times.
✓ Branch 1 taken 338 times.
✓ Branch 2 taken 46 times.
✓ Branch 3 taken 372 times.
 756 if (!exists && (gauge_ + size > limit_)) {
1815
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 LogCvmfs(kLogQuota, kLogDebug, "over limit, gauge %lu, file size %lu",
1816 gauge_, size);
1817
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 const int retval = DoCleanup(cleanup_threshold_);
1818
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 46 times.
 46 assert(retval != 0);
1819 }
1820
1/2
✓ Branch 3 taken 756 times.
✗ Branch 4 not taken.
 756 sqlite3_bind_text(stmt_new_, 1, &hash_str[0], hash_str.length(),
1821 SQLITE_STATIC);
1822
1/2
✓ Branch 1 taken 756 times.
✗ Branch 2 not taken.
 756 sqlite3_bind_int64(stmt_new_, 2, size);
1823
1/2
✓ Branch 1 taken 756 times.
✗ Branch 2 not taken.
 756 sqlite3_bind_int64(stmt_new_, 3, seq_++);
1824
1/2
✓ Branch 3 taken 756 times.
✗ Branch 4 not taken.
 756 sqlite3_bind_text(stmt_new_, 4, &description[0], description.length(),
1825 SQLITE_STATIC);
1826
3/4
✓ Branch 0 taken 618 times.
✓ Branch 1 taken 138 times.
✓ Branch 3 taken 756 times.
✗ Branch 4 not taken.
 756 sqlite3_bind_int64(stmt_new_, 5, is_catalog ? kFileCatalog : kFileRegular);
1827
1/2
✓ Branch 1 taken 756 times.
✗ Branch 2 not taken.
 756 sqlite3_bind_int64(stmt_new_, 6, 1);
1828
1/2
✓ Branch 1 taken 756 times.
✗ Branch 2 not taken.
 756 const int retval = sqlite3_step(stmt_new_);
1829
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 756 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 756 assert((retval == SQLITE_DONE) || (retval == SQLITE_OK));
1830
1/2
✓ Branch 1 taken 756 times.
✗ Branch 2 not taken.
 756 sqlite3_reset(stmt_new_);
1831
2/2
✓ Branch 0 taken 418 times.
✓ Branch 1 taken 338 times.
 756 if (!exists)
1832 418 gauge_ += size;
1833 756 return true;
1834 }
1835
1836 int pipe_reserve[2];
1837
1/2
✓ Branch 1 taken 644 times.
✗ Branch 2 not taken.
 644 MakeReturnPipe(pipe_reserve);
1838
1839 644 LruCommand cmd;
1840 644 cmd.command_type = kReserve;
1841 644 cmd.SetSize(size);
1842
1/2
✓ Branch 1 taken 644 times.
✗ Branch 2 not taken.
 644 cmd.StoreHash(hash);
1843 644 cmd.return_pipe = pipe_reserve[1];
1844
1/2
✓ Branch 1 taken 644 times.
✗ Branch 2 not taken.
 644 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
1845 bool result;
1846
1/2
✓ Branch 1 taken 644 times.
✗ Branch 2 not taken.
 644 ManagedReadHalfPipe(pipe_reserve[0], &result, sizeof(result));
1847
1/2
✓ Branch 1 taken 644 times.
✗ Branch 2 not taken.
 644 CloseReturnPipe(pipe_reserve);
1848
1849
2/2
✓ Branch 0 taken 46 times.
✓ Branch 1 taken 598 times.
 644 if (!result)
1850 46 return false;
1851
3/4
✓ Branch 0 taken 92 times.
✓ Branch 1 taken 506 times.
✓ Branch 3 taken 598 times.
✗ Branch 4 not taken.
 598 DoInsert(hash, size, description, is_catalog ? kPin : kPinRegular);
1852
1853 598 return true;
1854 1492 }
1855
1856
1857 3088 PosixQuotaManager::PosixQuotaManager(const uint64_t limit,
1858 const uint64_t cleanup_threshold,
1859 3088 const string &cache_workspace)
1860 3088 : shared_(false)
1861 3088 , spawned_(false)
1862 3088 , limit_(limit)
1863 3088 , cleanup_threshold_(cleanup_threshold)
1864 3088 , gauge_(0)
1865 3088 , pinned_(0)
1866 3088 , seq_(0)
1867 3088 , cache_dir_() // initialized in body
1868 3088 , workspace_dir_() // initialized in body
1869 3088 , fd_lock_cachedb_(-1)
1870 3088 , async_delete_(true)
1871 3088 , cachemgr_pid_(0)
1872 3088 , database_(NULL)
1873 3088 , stmt_touch_(NULL)
1874 3088 , stmt_unpin_(NULL)
1875 3088 , stmt_block_(NULL)
1876 3088 , stmt_unblock_(NULL)
1877 3088 , stmt_new_(NULL)
1878 3088 , stmt_lru_(NULL)
1879 3088 , stmt_size_(NULL)
1880 3088 , stmt_rm_(NULL)
1881 3088 , stmt_rm_batch_(NULL)
1882 3088 , stmt_list_(NULL)
1883 3088 , stmt_list_pinned_(NULL)
1884 3088 , stmt_list_catalogs_(NULL)
1885 3088 , stmt_list_volatile_(NULL)
1886 3088 , initialized_(false)
1887 6176 , cleanup_unused_first_(false) {
1888
2/4
✓ Branch 1 taken 3088 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3088 times.
✗ Branch 5 not taken.
 3088 ParseDirectories(cache_workspace, &cache_dir_, &workspace_dir_);
1889 3088 pipe_lru_[0] = pipe_lru_[1] = -1;
1890
1/2
✓ Branch 1 taken 3088 times.
✗ Branch 2 not taken.
 3088 cleanup_recorder_.AddRecorder(1, 90); // last 1.5 min with second resolution
1891 // last 1.5 h with minute resolution
1892
1/2
✓ Branch 1 taken 3088 times.
✗ Branch 2 not taken.
 3088 cleanup_recorder_.AddRecorder(60, 90 * 60);
1893 // last 18 hours with 20 min resolution
1894
1/2
✓ Branch 1 taken 3088 times.
✗ Branch 2 not taken.
 3088 cleanup_recorder_.AddRecorder(20 * 60, 60 * 60 * 18);
1895 // last 4 days with hour resolution
1896
1/2
✓ Branch 1 taken 3088 times.
✗ Branch 2 not taken.
 3088 cleanup_recorder_.AddRecorder(60 * 60, 60 * 60 * 24 * 4);
1897
1898 3088 lock_open_files_ = reinterpret_cast<pthread_mutex_t *>(
1899 3088 smalloc(sizeof(pthread_mutex_t)));
1900 3088 }
1901
1902
1903 12348 PosixQuotaManager::~PosixQuotaManager() {
1904 6174 free(lock_open_files_);
1905
1906
2/2
✓ Branch 0 taken 138 times.
✓ Branch 1 taken 2949 times.
 6174 if (!initialized_)
1907 276 return;
1908
1909
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2949 times.
 5898 if (shared_) {
1910 // Most of cleanup is done elsewhen by shared cache manager
1911 ✗ close(pipe_lru_[1]);
1912 ✗ return;
1913 }
1914
1915
2/2
✓ Branch 0 taken 1426 times.
✓ Branch 1 taken 1523 times.
 5898 if (spawned_) {
1916 2852 char fin = 0;
1917 2852 WritePipe(pipe_lru_[1], &fin, 1);
1918 2852 close(pipe_lru_[1]);
1919 2852 pthread_join(thread_lru_, NULL);
1920 } else {
1921 3046 ClosePipe(pipe_lru_);
1922 }
1923
1924 5898 CloseDatabase();
1925
14/14
✓ Branch 1 taken 2949 times.
✓ Branch 2 taken 138 times.
✓ Branch 4 taken 2949 times.
✓ Branch 5 taken 138 times.
✓ Branch 7 taken 2949 times.
✓ Branch 8 taken 138 times.
✓ Branch 10 taken 2949 times.
✓ Branch 11 taken 138 times.
✓ Branch 13 taken 2949 times.
✓ Branch 14 taken 138 times.
✓ Branch 16 taken 2949 times.
✓ Branch 17 taken 138 times.
✓ Branch 19 taken 2949 times.
✓ Branch 20 taken 138 times.
 14004 }
1926
1927
1928 220478 void PosixQuotaManager::ProcessCommandBunch(const unsigned num,
1929 const LruCommand *commands,
1930 const char *descriptions) {
1931 220478 int retval = sqlite3_exec(database_, "BEGIN", NULL, NULL, NULL);
1932
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 220478 times.
 220478 assert(retval == SQLITE_OK);
1933
1934
2/2
✓ Branch 0 taken 6902254 times.
✓ Branch 1 taken 220478 times.
 7122732 for (unsigned i = 0; i < num; ++i) {
1935
1/2
✓ Branch 1 taken 6902254 times.
✗ Branch 2 not taken.
 6902254 const shash::Any hash = commands[i].RetrieveHash();
1936
1/2
✓ Branch 1 taken 6902254 times.
✗ Branch 2 not taken.
 6902254 const string hash_str = hash.ToString();
1937 6902254 const unsigned size = commands[i].GetSize();
1938
1/2
✓ Branch 1 taken 6902254 times.
✗ Branch 2 not taken.
 6902254 LogCvmfs(kLogQuota, kLogDebug, "processing %s (%d)", hash_str.c_str(),
1939 6902254 commands[i].command_type);
1940
1941 bool exists;
1942
3/4
✓ Branch 0 taken 2300598 times.
✓ Branch 1 taken 92 times.
✓ Branch 2 taken 4601564 times.
✗ Branch 3 not taken.
 6902254 switch (commands[i].command_type) {
1943 2300598 case kTouch:
1944
1/2
✓ Branch 1 taken 2300598 times.
✗ Branch 2 not taken.
 2300598 sqlite3_bind_int64(stmt_touch_, 1, seq_++);
1945
1/2
✓ Branch 3 taken 2300598 times.
✗ Branch 4 not taken.
 2300598 sqlite3_bind_text(stmt_touch_, 2, &hash_str[0], hash_str.length(),
1946 SQLITE_STATIC);
1947
1/2
✓ Branch 1 taken 2300598 times.
✗ Branch 2 not taken.
 2300598 retval = sqlite3_step(stmt_touch_);
1948
1/2
✓ Branch 1 taken 2300598 times.
✗ Branch 2 not taken.
 2300598 LogCvmfs(kLogQuota, kLogDebug, "touching %s (%ld): %d",
1949 2300598 hash_str.c_str(), seq_ - 1, retval);
1950
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2300598 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 2300598 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
1951 ✗ PANIC(kLogSyslogErr, "failed to update %s in cachedb, error %d",
1952 hash_str.c_str(), retval);
1953 }
1954
1/2
✓ Branch 1 taken 2300598 times.
✗ Branch 2 not taken.
 2300598 sqlite3_reset(stmt_touch_);
1955 2300598 break;
1956 92 case kUnpin:
1957
1/2
✓ Branch 3 taken 92 times.
✗ Branch 4 not taken.
 92 sqlite3_bind_text(stmt_unpin_, 1, &hash_str[0], hash_str.length(),
1958 SQLITE_STATIC);
1959
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 retval = sqlite3_step(stmt_unpin_);
1960
1/2
✓ Branch 2 taken 92 times.
✗ Branch 3 not taken.
 92 LogCvmfs(kLogQuota, kLogDebug, "unpinning %s: %d", hash_str.c_str(),
1961 retval);
1962
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 92 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
1963 ✗ PANIC(kLogSyslogErr, "failed to unpin %s in cachedb, error %d",
1964 hash_str.c_str(), retval);
1965 }
1966
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 sqlite3_reset(stmt_unpin_);
1967 92 break;
1968 4601564 case kPin:
1969 case kPinRegular:
1970 case kInsert:
1971 case kInsertVolatile:
1972 // It could already be in, check
1973
1/2
✓ Branch 1 taken 4601564 times.
✗ Branch 2 not taken.
 4601564 exists = Contains(hash_str);
1974
1975 // Cleanup, move to trash and unlink
1976
3/4
✓ Branch 0 taken 4601334 times.
✓ Branch 1 taken 230 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 4601334 times.
 4601564 if (!exists && (gauge_ + size > limit_)) {
1977 ✗ LogCvmfs(kLogQuota, kLogDebug, "over limit, gauge %lu, file size %u",
1978 gauge_, size);
1979 ✗ retval = DoCleanup(cleanup_threshold_);
1980 ✗ assert(retval != 0);
1981 }
1982
1983 // Insert or replace
1984
1/2
✓ Branch 3 taken 4601564 times.
✗ Branch 4 not taken.
 4601564 sqlite3_bind_text(stmt_new_, 1, &hash_str[0], hash_str.length(),
1985 SQLITE_STATIC);
1986
1/2
✓ Branch 1 taken 4601564 times.
✗ Branch 2 not taken.
 4601564 sqlite3_bind_int64(stmt_new_, 2, size);
1987
2/2
✓ Branch 0 taken 184 times.
✓ Branch 1 taken 4601380 times.
 4601564 if (commands[i].command_type == kInsertVolatile) {
1988
1/2
✓ Branch 1 taken 184 times.
✗ Branch 2 not taken.
 184 sqlite3_bind_int64(stmt_new_, 3, (seq_++) | kVolatileFlag);
1989 } else {
1990
1/2
✓ Branch 1 taken 4601380 times.
✗ Branch 2 not taken.
 4601380 sqlite3_bind_int64(stmt_new_, 3, seq_++);
1991 }
1992 4601564 sqlite3_bind_text(stmt_new_, 4, &descriptions[i * kMaxDescription],
1993
1/2
✓ Branch 1 taken 4601564 times.
✗ Branch 2 not taken.
 4601564 commands[i].desc_length, SQLITE_STATIC);
1994
1/2
✓ Branch 1 taken 4601564 times.
✗ Branch 2 not taken.
 4601564 sqlite3_bind_int64(
1995 stmt_new_, 5,
1996
2/2
✓ Branch 0 taken 92 times.
✓ Branch 1 taken 4601472 times.
 4601564 (commands[i].command_type == kPin) ? kFileCatalog : kFileRegular);
1997
1/2
✓ Branch 1 taken 4601564 times.
✗ Branch 2 not taken.
 4601564 sqlite3_bind_int64(stmt_new_, 6,
1998
2/2
✓ Branch 0 taken 4601472 times.
✓ Branch 1 taken 92 times.
 4601564 ((commands[i].command_type == kPin)
1999
2/2
✓ Branch 0 taken 506 times.
✓ Branch 1 taken 4600966 times.
 4601472 || (commands[i].command_type == kPinRegular))
2000 ? 1
2001 : 0);
2002
1/2
✓ Branch 1 taken 4601564 times.
✗ Branch 2 not taken.
 4601564 retval = sqlite3_step(stmt_new_);
2003
1/2
✓ Branch 1 taken 4601564 times.
✗ Branch 2 not taken.
 4601564 LogCvmfs(kLogQuota, kLogDebug, "insert or replace %s, method %d: %d",
2004 4601564 hash_str.c_str(), commands[i].command_type, retval);
2005
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 4601564 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 4601564 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
2006 ✗ PANIC(kLogSyslogErr, "failed to insert %s in cachedb, error %d",
2007 hash_str.c_str(), retval);
2008 }
2009
1/2
✓ Branch 1 taken 4601564 times.
✗ Branch 2 not taken.
 4601564 sqlite3_reset(stmt_new_);
2010
2011
2/2
✓ Branch 0 taken 4601334 times.
✓ Branch 1 taken 230 times.
 4601564 if (!exists)
2012 4601334 gauge_ += size;
2013 4601564 break;
2014 ✗ default:
2015 // other types should have been taken care of by event loop
2016 ✗ PANIC(NULL);
2017 }
2018 6902254 }
2019
2020 220478 retval = sqlite3_exec(database_, "COMMIT", NULL, NULL, NULL);
2021
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 220478 times.
 220478 if (retval != SQLITE_OK) {
2022 ✗ PANIC(kLogSyslogErr, "failed to commit to cachedb, error %d", retval);
2023 }
2024 220478 }
2025
2026
2027 2996 bool PosixQuotaManager::RebuildDatabase() {
2028 2996 bool result = false;
2029 2996 string sql;
2030 2996 sqlite3_stmt *stmt_select = NULL;
2031 2996 sqlite3_stmt *stmt_insert = NULL;
2032 int sqlerr;
2033 2996 int seq = 0;
2034 char hex[4];
2035 struct stat info;
2036 platform_dirent64 *d;
2037 2996 DIR *dirp = NULL;
2038 2996 string path;
2039
2040
1/2
✓ Branch 1 taken 2996 times.
✗ Branch 2 not taken.
 2996 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug, "re-building cache database");
2041
2042 // Empty cache catalog and fscache
2043
1/2
✓ Branch 1 taken 2996 times.
✗ Branch 2 not taken.
 2996 sql = "DELETE FROM cache_catalog; DELETE FROM fscache;";
2044
1/2
✓ Branch 2 taken 2996 times.
✗ Branch 3 not taken.
 2996 sqlerr = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
2045
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2996 times.
 2996 if (sqlerr != SQLITE_OK) {
2046 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not clear cache database");
2047 ✗ goto build_return;
2048 }
2049
2050 2996 gauge_ = 0;
2051
2052 // Insert files from cache sub-directories 00 - ff
2053 // TODO(jblomer): fs_traversal
2054
1/2
✓ Branch 1 taken 2996 times.
✗ Branch 2 not taken.
 2996 sqlite3_prepare_v2(database_,
2055 "INSERT INTO fscache (sha1, size, actime) "
2056 "VALUES (:sha1, :s, :t);",
2057 -1, &stmt_insert, NULL);
2058
2059
2/2
✓ Branch 0 taken 731786 times.
✓ Branch 1 taken 2858 times.
 734644 for (int i = 0; i <= 0xff; i++) {
2060 731786 snprintf(hex, sizeof(hex), "%02x", i);
2061
3/6
✓ Branch 2 taken 731786 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 731786 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 731786 times.
✗ Branch 9 not taken.
 731786 path = cache_dir_ + "/" + string(hex);
2062
3/4
✓ Branch 2 taken 731786 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 138 times.
✓ Branch 5 taken 731648 times.
 731786 if ((dirp = opendir(path.c_str())) == NULL) {
2063
1/2
✓ Branch 2 taken 138 times.
✗ Branch 3 not taken.
 138 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
2064 "failed to open directory %s (tmpwatch interfering?)",
2065 path.c_str());
2066 138 goto build_return;
2067 }
2068
3/4
✓ Branch 1 taken 2195036 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1463388 times.
✓ Branch 4 taken 731648 times.
 2195036 while ((d = platform_readdir(dirp)) != NULL) {
2069
3/6
✓ Branch 2 taken 1463388 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1463388 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1463388 times.
✗ Branch 9 not taken.
 2926776 const string file_path = path + "/" + string(d->d_name);
2070
1/2
✓ Branch 2 taken 1463388 times.
✗ Branch 3 not taken.
 1463388 if (stat(file_path.c_str(), &info) == 0) {
2071
2/2
✓ Branch 0 taken 1463296 times.
✓ Branch 1 taken 92 times.
 1463388 if (!S_ISREG(info.st_mode))
2072 1463342 continue;
2073
2/2
✓ Branch 0 taken 46 times.
✓ Branch 1 taken 46 times.
 92 if (info.st_size == 0) {
2074
1/2
✓ Branch 2 taken 46 times.
✗ Branch 3 not taken.
 46 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug,
2075 "removing empty file %s during automatic cache db rebuild",
2076 file_path.c_str());
2077 46 unlink(file_path.c_str());
2078 46 continue;
2079 }
2080
2081
3/6
✓ Branch 2 taken 46 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 46 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 46 times.
✗ Branch 10 not taken.
 92 string hash = string(hex) + string(d->d_name);
2082
1/2
✓ Branch 3 taken 46 times.
✗ Branch 4 not taken.
 46 sqlite3_bind_text(stmt_insert, 1, hash.data(), hash.length(),
2083 SQLITE_STATIC);
2084
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 sqlite3_bind_int64(stmt_insert, 2, info.st_size);
2085
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 sqlite3_bind_int64(stmt_insert, 3, info.st_atime);
2086
2/4
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 46 times.
 46 if (sqlite3_step(stmt_insert) != SQLITE_DONE) {
2087 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not insert into temp table");
2088 ✗ goto build_return;
2089 }
2090
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 sqlite3_reset(stmt_insert);
2091
2092 46 gauge_ += info.st_size;
2093
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 } else {
2094 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not stat %s", file_path.c_str());
2095 }
2096
2/3
✓ Branch 1 taken 46 times.
✓ Branch 2 taken 1463342 times.
✗ Branch 3 not taken.
 1463388 }
2097
1/2
✓ Branch 1 taken 731648 times.
✗ Branch 2 not taken.
 731648 closedir(dirp);
2098 731648 dirp = NULL;
2099 }
2100
1/2
✓ Branch 1 taken 2858 times.
✗ Branch 2 not taken.
 2858 sqlite3_finalize(stmt_insert);
2101 2858 stmt_insert = NULL;
2102
2103 // Transfer from temp table in cache catalog
2104
1/2
✓ Branch 1 taken 2858 times.
✗ Branch 2 not taken.
 2858 sqlite3_prepare_v2(database_,
2105 "SELECT sha1, size FROM fscache ORDER BY actime;", -1,
2106 &stmt_select, NULL);
2107
1/2
✓ Branch 1 taken 2858 times.
✗ Branch 2 not taken.
 2858 sqlite3_prepare_v2(
2108 database_,
2109 "INSERT INTO cache_catalog (sha1, size, acseq, path, type, pinned) "
2110 "VALUES (:sha1, :s, :seq, 'unknown (automatic rebuild)', :t, 0);",
2111 -1, &stmt_insert, NULL);
2112
3/4
✓ Branch 1 taken 2904 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 46 times.
✓ Branch 4 taken 2858 times.
 2904 while (sqlite3_step(stmt_select) == SQLITE_ROW) {
2113 const string hash = string(
2114
2/4
✓ Branch 2 taken 46 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 46 times.
✗ Branch 6 not taken.
 46 reinterpret_cast<const char *>(sqlite3_column_text(stmt_select, 0)));
2115
1/2
✓ Branch 3 taken 46 times.
✗ Branch 4 not taken.
 46 sqlite3_bind_text(stmt_insert, 1, &hash[0], hash.length(), SQLITE_STATIC);
2116
2/4
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 46 times.
✗ Branch 5 not taken.
 46 sqlite3_bind_int64(stmt_insert, 2, sqlite3_column_int64(stmt_select, 1));
2117
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 sqlite3_bind_int64(stmt_insert, 3, seq++);
2118 // Might also be a catalog (information is lost)
2119
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 sqlite3_bind_int64(stmt_insert, 4, kFileRegular);
2120
2121
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 const int retval = sqlite3_step(stmt_insert);
2122
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 46 times.
 46 if (retval != SQLITE_DONE) {
2123 // If the file system hosting the cache is full, we'll likely notice here
2124 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
2125 "could not insert into cache catalog (%d - %s)", retval,
2126 sqlite3_errstr(retval));
2127 ✗ goto build_return;
2128 }
2129
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 sqlite3_reset(stmt_insert);
2130
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 }
2131
2132 // Delete temporary table
2133
1/2
✓ Branch 1 taken 2858 times.
✗ Branch 2 not taken.
 2858 sql = "DELETE FROM fscache;";
2134
1/2
✓ Branch 2 taken 2858 times.
✗ Branch 3 not taken.
 2858 sqlerr = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
2135
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2858 times.
 2858 if (sqlerr != SQLITE_OK) {
2136 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not clear temporary table (%d)",
2137 sqlerr);
2138 ✗ goto build_return;
2139 }
2140
2141 2858 seq_ = seq;
2142 2858 result = true;
2143
1/2
✓ Branch 1 taken 2858 times.
✗ Branch 2 not taken.
 2858 LogCvmfs(kLogQuota, kLogDebug,
2144 "rebuilding finished, sequence %" PRIu64 ", gauge %" PRIu64, seq_,
2145 gauge_);
2146
2147 2996 build_return:
2148
1/2
✓ Branch 0 taken 2996 times.
✗ Branch 1 not taken.
 2996 if (stmt_insert)
2149
1/2
✓ Branch 1 taken 2996 times.
✗ Branch 2 not taken.
 2996 sqlite3_finalize(stmt_insert);
2150
2/2
✓ Branch 0 taken 2858 times.
✓ Branch 1 taken 138 times.
 2996 if (stmt_select)
2151
1/2
✓ Branch 1 taken 2858 times.
✗ Branch 2 not taken.
 2858 sqlite3_finalize(stmt_select);
2152
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2996 times.
 2996 if (dirp)
2153 ✗ closedir(dirp);
2154 2996 return result;
2155 2996 }
2156
2157
2158 /**
2159 * Register a channel that allows the cache manager to trigger action to its
2160 * clients. Currently used for releasing pinned catalogs.
2161 */
2162 184 void PosixQuotaManager::RegisterBackChannel(int back_channel[2],
2163 const string &channel_id) {
2164
1/2
✓ Branch 0 taken 184 times.
✗ Branch 1 not taken.
 184 if (protocol_revision_ >= 1) {
2165
1/2
✓ Branch 2 taken 184 times.
✗ Branch 3 not taken.
 184 shash::Md5 hash = shash::Md5(shash::AsciiPtr(channel_id));
2166
1/2
✓ Branch 1 taken 184 times.
✗ Branch 2 not taken.
 184 MakeReturnPipe(back_channel);
2167
2168 184 LruCommand cmd;
2169 184 cmd.command_type = kRegisterBackChannel;
2170 184 cmd.return_pipe = back_channel[1];
2171 // Not StoreHash(). This is an MD5 hash.
2172 184 memcpy(cmd.digest, hash.digest, hash.GetDigestSize());
2173
1/2
✓ Branch 1 taken 184 times.
✗ Branch 2 not taken.
 184 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2174
2175 char success;
2176
1/2
✓ Branch 1 taken 184 times.
✗ Branch 2 not taken.
 184 ManagedReadHalfPipe(back_channel[0], &success, sizeof(success));
2177 // At this point, the named FIFO is unlinked, so don't use CloseReturnPipe
2178
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 184 times.
 184 if (success != 'S') {
2179 ✗ PANIC(kLogDebug | kLogSyslogErr,
2180 "failed to register quota back channel (%c)", success);
2181 }
2182 } else {
2183 // Dummy pipe to return valid file descriptors
2184 ✗ MakePipe(back_channel);
2185 }
2186 184 }
2187
2188
2189 /**
2190 * Removes a chunk from cache, if it exists.
2191 */
2192 138 void PosixQuotaManager::Remove(const shash::Any &hash) {
2193
1/2
✓ Branch 1 taken 138 times.
✗ Branch 2 not taken.
 138 const string hash_str = hash.ToString();
2194
2195 int pipe_remove[2];
2196
1/2
✓ Branch 1 taken 138 times.
✗ Branch 2 not taken.
 138 MakeReturnPipe(pipe_remove);
2197
2198 138 LruCommand cmd;
2199 138 cmd.command_type = kRemove;
2200 138 cmd.return_pipe = pipe_remove[1];
2201
1/2
✓ Branch 1 taken 138 times.
✗ Branch 2 not taken.
 138 cmd.StoreHash(hash);
2202
1/2
✓ Branch 1 taken 138 times.
✗ Branch 2 not taken.
 138 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2203
2204 bool success;
2205
1/2
✓ Branch 1 taken 138 times.
✗ Branch 2 not taken.
 138 ManagedReadHalfPipe(pipe_remove[0], &success, sizeof(success));
2206
1/2
✓ Branch 1 taken 138 times.
✗ Branch 2 not taken.
 138 CloseReturnPipe(pipe_remove);
2207
2208
3/6
✓ Branch 1 taken 138 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 138 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 138 times.
✗ Branch 8 not taken.
 138 unlink((cache_dir_ + "/" + hash.MakePathWithoutSuffix()).c_str());
2209 138 }
2210
2211
2212 1518 void PosixQuotaManager::Spawn() {
2213
2/2
✓ Branch 0 taken 92 times.
✓ Branch 1 taken 1426 times.
 1518 if (spawned_)
2214 92 return;
2215
2216 1426 if (pthread_create(&thread_lru_, NULL, MainCommandServer,
2217 static_cast<void *>(this))
2218
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1426 times.
 1426 != 0) {
2219 ✗ PANIC(kLogDebug, "could not create lru thread");
2220 }
2221
2222 1426 spawned_ = true;
2223 }
2224
2225
2226 /**
2227 * Updates the sequence number of the file specified by the hash.
2228 */
2229 2300481 void PosixQuotaManager::Touch(const shash::Any &hash) {
2230 2300481 LruCommand cmd;
2231 2300481 cmd.command_type = kTouch;
2232
1/2
✓ Branch 1 taken 2300481 times.
✗ Branch 2 not taken.
 2300481 cmd.StoreHash(hash);
2233
1/2
✓ Branch 1 taken 2300481 times.
✗ Branch 2 not taken.
 2300481 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2234 2300481 }
2235
2236
2237 3956 void PosixQuotaManager::UnbindReturnPipe(int pipe_wronly) {
2238
2/2
✓ Branch 0 taken 46 times.
✓ Branch 1 taken 3910 times.
 3956 if (shared_)
2239 46 close(pipe_wronly);
2240 3956 }
2241
2242
2243 322 void PosixQuotaManager::UnlinkReturnPipe(int pipe_wronly) {
2244
2/2
✓ Branch 0 taken 138 times.
✓ Branch 1 taken 184 times.
 322 if (shared_)
2245
2/4
✓ Branch 2 taken 138 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 138 times.
✗ Branch 6 not taken.
 138 unlink((workspace_dir_ + "/pipe" + StringifyInt(pipe_wronly)).c_str());
2246 322 }
2247
2248
2249 444 void PosixQuotaManager::Unpin(const shash::Any &hash) {
2250
2/4
✓ Branch 1 taken 444 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 444 times.
✗ Branch 6 not taken.
 444 LogCvmfs(kLogQuota, kLogDebug, "Unpin %s", hash.ToString().c_str());
2251
2252 444 LruCommand cmd;
2253 444 cmd.command_type = kUnpin;
2254
1/2
✓ Branch 1 taken 444 times.
✗ Branch 2 not taken.
 444 cmd.StoreHash(hash);
2255
1/2
✓ Branch 1 taken 444 times.
✗ Branch 2 not taken.
 444 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2256 444 }
2257
2258
2259 92 void PosixQuotaManager::UnregisterBackChannel(int back_channel[2],
2260 const string &channel_id) {
2261
1/2
✓ Branch 0 taken 92 times.
✗ Branch 1 not taken.
 92 if (protocol_revision_ >= 1) {
2262
1/2
✓ Branch 2 taken 92 times.
✗ Branch 3 not taken.
 92 shash::Md5 hash = shash::Md5(shash::AsciiPtr(channel_id));
2263
2264 92 LruCommand cmd;
2265 92 cmd.command_type = kUnregisterBackChannel;
2266 // Not StoreHash(). This is an MD5 hash.
2267 92 memcpy(cmd.digest, hash.digest, hash.GetDigestSize());
2268
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2269
2270 // Writer's end will be closed by cache manager, FIFO is already unlinked
2271
1/2
✓ Branch 1 taken 92 times.
✗ Branch 2 not taken.
 92 close(back_channel[0]);
2272 } else {
2273 ✗ ClosePipe(back_channel);
2274 }
2275 92 }
2276
2277 4605842 void PosixQuotaManager::ManagedReadHalfPipe(int fd, void *buf, size_t nbyte) {
2278
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4605842 times.
 4605842 const unsigned timeout_ms = cachemgr_pid_ ? 1000 : 0;
2279 4605842 bool result = false;
2280 do {
2281 4605842 result = ReadHalfPipe(fd, buf, nbyte, timeout_ms);
2282 // try only as long as the cachemgr is still alive
2283
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 4605842 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 4605842 times.
 4605842 } while (!result && getpgid(cachemgr_pid_) >= 0);
2284
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4605842 times.
 4605842 if (!result) {
2285 ✗ PANIC(kLogStderr,
2286 "Error: quota manager could not read from cachemanager pipe");
2287 }
2288 4605842 }
2289
2290 ✗ void *PosixQuotaManager::CollectMountpointsHashes(void *data) {
2291 #ifndef __APPLE__
2292 ✗ pthread_setname_np(pthread_self(), "hash_collector");
2293 ✗ auto *handler = static_cast<CollectorHandler *>(data);
2294
2295 ✗ const std::string mountpoint = handler->mp[handler->i];
2296 ✗ ssize_t n = getxattr(mountpoint.c_str(), "user.list_open_hashes", nullptr, 0);
2297 ✗ if (n < 0) {
2298 ✗ pthread_exit(nullptr);
2299 }
2300 ✗ std::vector<char> buf((size_t)n);
2301 ✗ n = getxattr(mountpoint.c_str(), "user.list_open_hashes", buf.data(),
2302 buf.size());
2303 ✗ if (n < 0) {
2304 ✗ pthread_exit(nullptr);
2305 }
2306
2307 ✗ std::vector<std::string> hash_strs;
2308 ✗ std::string hash_str;
2309 ✗ for (const char c : buf) {
2310 ✗ if (c == '\n') {
2311 ✗ hash_strs.push_back(hash_str);
2312 ✗ hash_str.clear();
2313 } else {
2314 ✗ hash_str += c;
2315 }
2316 }
2317 ✗ const MutexLockGuard lock_guard(handler->l);
2318 ✗ for (auto hash_str : hash_strs) {
2319 ✗ handler->of.push_back(shash::Short( shash::MkFromHexPtr(shash::HexPtr(hash_str)) ));
2320 }
2321 #endif
2322 ✗ pthread_exit(nullptr);
2323 }
2324
2325 ✗ std::vector<shash::Short> PosixQuotaManager::CollectAllOpenHashes() {
2326 ✗ std::vector<CollectorHandler *> handlers;
2327 ✗ std::vector<pthread_t *> threads;
2328 ✗ open_files_.clear();
2329 #ifndef __APPLE__
2330 ✗ auto &&a_after_b = [](const struct timespec a, const struct timespec b) {
2331 ✗ return (a.tv_sec > b.tv_sec) ? true : false;
2332 };
2333
2334 ✗ for (size_t i = 0; i < mountpoints_.size(); ++i) {
2335 ✗ handlers.push_back(
2336 ✗ new CollectorHandler{open_files_, mountpoints_, lock_open_files_, i});
2337 ✗ threads.push_back(new pthread_t);
2338 }
2339
2340 ✗ const int retval = pthread_mutex_init(lock_open_files_, NULL);
2341 ✗ assert(retval == 0);
2342
2343 ✗ for (size_t i = 0; i < mountpoints_.size(); ++i) {
2344 ✗ pthread_create(threads[i], nullptr, CollectMountpointsHashes, handlers[i]);
2345 }
2346
2347 ✗ std::vector<bool> joined(handlers.size(), false);
2348 struct timespec reference, current;
2349 ✗ clock_gettime(CLOCK_REALTIME, &reference);
2350 ✗ clock_gettime(CLOCK_REALTIME, &current);
2351 ✗ reference.tv_sec += 10; // Give 10sec for hash collection
2352 ✗ size_t i = 0;
2353 ✗ while (
2354 ✗ (not std::all_of(joined.begin(), joined.end(), [](bool b) { return b; }))
2355 ✗ and a_after_b(reference, current)) {
2356 // as long as there are still threads that haven't joined yet
2357 // and for 10 seconds
2358 ✗ if (not joined[i]) {
2359 ✗ const int s = pthread_tryjoin_np(*threads[i], NULL);
2360 ✗ if (s == 0) {
2361 ✗ joined[i] = true;
2362 }
2363 }
2364 ✗ ++i;
2365 ✗ i = i % handlers.size();
2366 ✗ clock_gettime(CLOCK_REALTIME, &current);
2367 }
2368
2369 ✗ for (size_t i = 0; i < handlers.size(); ++i) {
2370 ✗ delete handlers[i];
2371 }
2372
2373 ✗ pthread_mutex_destroy(lock_open_files_);
2374 #endif
2375 ✗ return open_files_;
2376 }
2377
2378