GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/quota_posix.cc
Date: 2026-04-19 02:41:37
Exec Total Coverage
Lines: 954 1399 68.2%
Branches: 750 1954 38.4%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * This module implements a "managed local cache".
5 * This way, we are able to track access times of files in the cache
6 * and remove files based on least recently used strategy.
7 *
8 * We setup another SQLite catalog, a "cache catalog", that helps us
9 * in the bookkeeping of files, file sizes and access times.
10 *
11 * We might choose to not manage the local cache. This is indicated
12 * by limit == 0 and everything succeeds in that case.
13 */
14
15 #define __STDC_LIMIT_MACROS
16 #define __STDC_FORMAT_MACROS
17
18
19 #include "quota_posix.h"
20
21 #include <dirent.h>
22 #include <errno.h>
23 #include <fcntl.h>
24 #include <inttypes.h>
25 #include <pthread.h>
26 #include <signal.h>
27 #include <stdint.h>
28 #include <sys/dir.h>
29 #include <sys/stat.h>
30 #include <sys/xattr.h>
31
32 #ifndef __APPLE__
33 #include <sys/statfs.h>
34 #endif
35 #include <sys/statvfs.h>
36 #include <sys/types.h>
37 #include <sys/wait.h>
38 #include <unistd.h>
39
40 #include <algorithm>
41 #include <cassert>
42 #include <cstdio>
43 #include <cstdlib>
44 #include <cstring>
45 #include <limits>
46 #include <map>
47 #include <set>
48 #include <string>
49 #include <vector>
50
51 #include "crypto/hash.h"
52 #include "duplex_sqlite3.h"
53 #include "monitor.h"
54 #include "statistics.h"
55 #include "util/capabilities.h"
56 #include "util/concurrency.h"
57 #include "util/exception.h"
58 #include "util/logging.h"
59 #include "util/pointer.h"
60 #include "util/posix.h"
61 #include "util/smalloc.h"
62 #include "util/string.h"
63
64 using namespace std; // NOLINT
65
66
67 3220 int PosixQuotaManager::BindReturnPipe(int pipe_wronly) {
68
2/2
✓ Branch 0 taken 3150 times.
✓ Branch 1 taken 70 times.
 3220 if (!shared_)
69 3150 return pipe_wronly;
70
71 // Connect writer's end
72
1/2
✓ Branch 2 taken 70 times.
✗ Branch 3 not taken.
 70 const int result = open(
73
2/4
✓ Branch 2 taken 70 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 70 times.
✗ Branch 6 not taken.
 140 (workspace_dir_ + "/pipe" + StringifyInt(pipe_wronly)).c_str(),
74 O_WRONLY | O_NONBLOCK);
75
2/2
✓ Branch 0 taken 35 times.
✓ Branch 1 taken 35 times.
 70 if (result >= 0) {
76 35 Nonblock2Block(result);
77 } else {
78 35 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
79 35 "failed to bind return pipe (%d)", errno);
80 }
81 70 return result;
82 }
83
84
85 574 void PosixQuotaManager::CheckHighPinWatermark() {
86 574 const uint64_t watermark = kHighPinWatermark * cleanup_threshold_ / 100;
87
3/4
✓ Branch 0 taken 574 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 105 times.
✓ Branch 3 taken 469 times.
 574 if ((cleanup_threshold_ > 0) && (pinned_ > watermark)) {
88 105 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
89 "high watermark of pinned files (%" PRIu64 "M > %" PRIu64 "M)",
90 105 pinned_ / (1024 * 1024), watermark / (1024 * 1024));
91
2/4
✓ Branch 2 taken 105 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 105 times.
✗ Branch 6 not taken.
 105 BroadcastBackchannels("R"); // clients: please release pinned catalogs
92 }
93 574 }
94
95
96 ✗ void PosixQuotaManager::CleanupPipes() {
97 ✗ DIR *dirp = opendir(workspace_dir_.c_str());
98 ✗ assert(dirp != NULL);
99
100 platform_dirent64 *dent;
101 ✗ bool found_leftovers = false;
102 ✗ while ((dent = platform_readdir(dirp)) != NULL) {
103 ✗ const string name = dent->d_name;
104 ✗ const string path = workspace_dir_ + "/" + name;
105 platform_stat64 info;
106 ✗ const int retval = platform_stat(path.c_str(), &info);
107 ✗ if (retval != 0)
108 ✗ continue;
109 ✗ if (S_ISFIFO(info.st_mode) && (name.substr(0, 4) == "pipe")) {
110 ✗ if (!found_leftovers) {
111 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogWarn,
112 "removing left-over FIFOs from cache directory");
113 }
114 ✗ found_leftovers = true;
115 ✗ unlink(path.c_str());
116 }
117 }
118 ✗ closedir(dirp);
119 }
120
121
122 /**
123 * Cleans up in data cache, until cache size is below leave_size.
124 * The actual unlinking is done in a separate process (fork).
125 *
126 * \return True on success, false otherwise
127 */
128 315 bool PosixQuotaManager::Cleanup(const uint64_t leave_size) {
129
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 315 times.
 315 if (!spawned_)
130 ✗ return DoCleanup(leave_size);
131
132 bool result;
133 int pipe_cleanup[2];
134
1/2
✓ Branch 1 taken 315 times.
✗ Branch 2 not taken.
 315 MakeReturnPipe(pipe_cleanup);
135
136 315 LruCommand cmd;
137 315 cmd.command_type = kCleanup;
138 315 cmd.size = leave_size;
139 315 cmd.return_pipe = pipe_cleanup[1];
140
141
1/2
✓ Branch 1 taken 315 times.
✗ Branch 2 not taken.
 315 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
142
1/2
✓ Branch 1 taken 315 times.
✗ Branch 2 not taken.
 315 ManagedReadHalfPipe(pipe_cleanup[0], &result, sizeof(result));
143
1/2
✓ Branch 1 taken 315 times.
✗ Branch 2 not taken.
 315 CloseReturnPipe(pipe_cleanup);
144
145 315 return result;
146 }
147
148
149 2437 void PosixQuotaManager::CloseDatabase() {
150
1/2
✓ Branch 0 taken 2437 times.
✗ Branch 1 not taken.
 2437 if (stmt_list_catalogs_)
151 2437 sqlite3_finalize(stmt_list_catalogs_);
152
1/2
✓ Branch 0 taken 2437 times.
✗ Branch 1 not taken.
 2437 if (stmt_list_pinned_)
153 2437 sqlite3_finalize(stmt_list_pinned_);
154
1/2
✓ Branch 0 taken 2437 times.
✗ Branch 1 not taken.
 2437 if (stmt_list_volatile_)
155 2437 sqlite3_finalize(stmt_list_volatile_);
156
1/2
✓ Branch 0 taken 2437 times.
✗ Branch 1 not taken.
 2437 if (stmt_list_)
157 2437 sqlite3_finalize(stmt_list_);
158
1/2
✓ Branch 0 taken 2437 times.
✗ Branch 1 not taken.
 2437 if (stmt_lru_)
159 2437 sqlite3_finalize(stmt_lru_);
160
1/2
✓ Branch 0 taken 2437 times.
✗ Branch 1 not taken.
 2437 if (stmt_rm_)
161 2437 sqlite3_finalize(stmt_rm_);
162
1/2
✓ Branch 0 taken 2437 times.
✗ Branch 1 not taken.
 2437 if (stmt_rm_batch_)
163 2437 sqlite3_finalize(stmt_rm_batch_);
164
1/2
✓ Branch 0 taken 2437 times.
✗ Branch 1 not taken.
 2437 if (stmt_size_)
165 2437 sqlite3_finalize(stmt_size_);
166
1/2
✓ Branch 0 taken 2437 times.
✗ Branch 1 not taken.
 2437 if (stmt_touch_)
167 2437 sqlite3_finalize(stmt_touch_);
168
1/2
✓ Branch 0 taken 2437 times.
✗ Branch 1 not taken.
 2437 if (stmt_unpin_)
169 2437 sqlite3_finalize(stmt_unpin_);
170
1/2
✓ Branch 0 taken 2437 times.
✗ Branch 1 not taken.
 2437 if (stmt_block_)
171 2437 sqlite3_finalize(stmt_block_);
172
1/2
✓ Branch 0 taken 2437 times.
✗ Branch 1 not taken.
 2437 if (stmt_unblock_)
173 2437 sqlite3_finalize(stmt_unblock_);
174
1/2
✓ Branch 0 taken 2437 times.
✗ Branch 1 not taken.
 2437 if (stmt_new_)
175 2437 sqlite3_finalize(stmt_new_);
176
1/2
✓ Branch 0 taken 2437 times.
✗ Branch 1 not taken.
 2437 if (database_)
177 2437 sqlite3_close(database_);
178 2437 UnlockFile(fd_lock_cachedb_);
179
180 2437 stmt_list_catalogs_ = NULL;
181 2437 stmt_list_pinned_ = NULL;
182 2437 stmt_list_volatile_ = NULL;
183 2437 stmt_list_ = NULL;
184 2437 stmt_rm_ = NULL;
185 2437 stmt_rm_batch_ = NULL;
186 2437 stmt_size_ = NULL;
187 2437 stmt_touch_ = NULL;
188 2437 stmt_unpin_ = NULL;
189 2437 stmt_block_ = NULL;
190 2437 stmt_unblock_ = NULL;
191 2437 stmt_new_ = NULL;
192 2437 database_ = NULL;
193
194 2437 pinned_chunks_.clear();
195 2437 }
196
197
198 3045 void PosixQuotaManager::CloseReturnPipe(int pipe[2]) {
199
2/2
✓ Branch 0 taken 70 times.
✓ Branch 1 taken 2975 times.
 3045 if (shared_) {
200 70 close(pipe[0]);
201 70 UnlinkReturnPipe(pipe[1]);
202 } else {
203 2975 ClosePipe(pipe);
204 }
205 3045 }
206
207
208 3501612 bool PosixQuotaManager::Contains(const string &hash_str) {
209 3501612 bool result = false;
210
211 3501612 sqlite3_bind_text(stmt_size_, 1, &hash_str[0], hash_str.length(),
212 SQLITE_STATIC);
213
2/2
✓ Branch 1 taken 373 times.
✓ Branch 2 taken 3501239 times.
 3501612 if (sqlite3_step(stmt_size_) == SQLITE_ROW)
214 373 result = true;
215 3501612 sqlite3_reset(stmt_size_);
216 3501612 LogCvmfs(kLogQuota, kLogDebug, "contains %s returns %d", hash_str.c_str(),
217 result);
218
219 3501612 return result;
220 }
221
222
223 2368 void PosixQuotaManager::CheckFreeSpace() {
224
3/4
✓ Branch 0 taken 2368 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 35 times.
✓ Branch 3 taken 2333 times.
 2368 if ((limit_ == 0) || (gauge_ >= limit_))
225 35 return;
226
227 struct statvfs vfs_info;
228
1/2
✓ Branch 1 taken 2333 times.
✗ Branch 2 not taken.
 2333 const int retval = statvfs((cache_dir_ + "/cachedb").c_str(), &vfs_info);
229
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2333 times.
 2333 if (retval != 0) {
230 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
231 "failed to query %s for free space (%d)", cache_dir_.c_str(),
232 ✗ errno);
233 ✗ return;
234 }
235 2333 const int64_t free_space_byte = vfs_info.f_bavail * vfs_info.f_bsize;
236
1/2
✓ Branch 1 taken 2333 times.
✗ Branch 2 not taken.
 2333 LogCvmfs(kLogQuota, kLogDebug, "free space: %" PRId64 " MB",
237 free_space_byte / (1024 * 1024));
238
239 2333 const int64_t required_byte = limit_ - gauge_;
240
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2333 times.
 2333 if (free_space_byte < required_byte) {
241 ✗ LogCvmfs(kLogQuota, kLogSyslogWarn,
242 "too little free space on the file system hosting the cache,"
243 " %" PRId64 " MB available",
244 free_space_byte / (1024 * 1024));
245 }
246 }
247
248
249 2473 PosixQuotaManager *PosixQuotaManager::Create(const string &cache_workspace,
250 const uint64_t limit,
251 const uint64_t cleanup_threshold,
252 const bool rebuild_database) {
253
2/2
✓ Branch 0 taken 70 times.
✓ Branch 1 taken 2403 times.
 2473 if (cleanup_threshold >= limit) {
254 70 LogCvmfs(kLogQuota, kLogDebug,
255 "invalid parameters: limit %" PRIu64 ", "
256 "cleanup_threshold %" PRIu64,
257 limit, cleanup_threshold);
258 70 return NULL;
259 }
260
261 PosixQuotaManager *quota_manager = new PosixQuotaManager(
262
1/2
✓ Branch 2 taken 2403 times.
✗ Branch 3 not taken.
 2403 limit, cleanup_threshold, cache_workspace);
263
264 // Initialize cache catalog
265
2/2
✓ Branch 1 taken 35 times.
✓ Branch 2 taken 2368 times.
 2403 if (!quota_manager->InitDatabase(rebuild_database)) {
266
1/2
✓ Branch 0 taken 35 times.
✗ Branch 1 not taken.
 35 delete quota_manager;
267 35 return NULL;
268 }
269 2368 quota_manager->CheckFreeSpace();
270 2368 MakePipe(quota_manager->pipe_lru_);
271
272 2368 quota_manager->protocol_revision_ = kProtocolRevision;
273 2368 quota_manager->initialized_ = true;
274 2368 return quota_manager;
275 }
276
277
278 /**
279 * Connects to a running shared local quota manager. Creates one if necessary.
280 */
281 70 PosixQuotaManager *PosixQuotaManager::CreateShared(
282 const std::string &exe_path,
283 const std::string &cache_workspace,
284 const uint64_t limit,
285 const uint64_t cleanup_threshold,
286 bool foreground) {
287 70 string cache_dir;
288 70 string workspace_dir;
289
2/4
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 70 times.
✗ Branch 5 not taken.
 70 ParseDirectories(cache_workspace, &cache_dir, &workspace_dir);
290
291 pid_t new_cachemgr_pid;
292
293 // Create lock file: only one fuse client at a time
294
2/4
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 70 times.
✗ Branch 5 not taken.
 70 const int fd_lockfile = LockFile(workspace_dir + "/lock_cachemgr");
295
2/2
✓ Branch 0 taken 35 times.
✓ Branch 1 taken 35 times.
 70 if (fd_lockfile < 0) {
296
1/2
✓ Branch 2 taken 35 times.
✗ Branch 3 not taken.
 35 LogCvmfs(kLogQuota, kLogDebug, "could not open lock file %s (%d)",
297
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 70 (workspace_dir + "/lock_cachemgr").c_str(), errno);
298 35 return NULL;
299 }
300
301 PosixQuotaManager *quota_mgr = new PosixQuotaManager(limit, cleanup_threshold,
302
2/4
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 35 times.
✗ Branch 5 not taken.
 35 cache_workspace);
303 35 quota_mgr->shared_ = true;
304 35 quota_mgr->spawned_ = true;
305
306 // Try to connect to pipe
307
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 const string fifo_path = workspace_dir + "/cachemgr";
308
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 LogCvmfs(kLogQuota, kLogDebug, "trying to connect to existing pipe");
309
1/2
✓ Branch 2 taken 35 times.
✗ Branch 3 not taken.
 35 quota_mgr->pipe_lru_[1] = open(fifo_path.c_str(), O_WRONLY | O_NONBLOCK);
310
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 35 times.
 35 if (quota_mgr->pipe_lru_[1] >= 0) {
311 ✗ const int fd_lockfile_rw = open((workspace_dir + "/lock_cachemgr").c_str(),
312 O_RDWR, 0600);
313 ✗ unsigned lockfile_magicnumber = 0;
314 ✗ const ssize_t result_mn = SafeRead(fd_lockfile_rw, &lockfile_magicnumber,
315 sizeof(lockfile_magicnumber));
316 ✗ const ssize_t result = SafeRead(fd_lockfile_rw, &new_cachemgr_pid,
317 sizeof(new_cachemgr_pid));
318 ✗ close(fd_lockfile_rw);
319
320 ✗ if ((lockfile_magicnumber != kLockFileMagicNumber) || (result < 0)
321 ✗ || (result_mn < 0)
322 ✗ || (static_cast<size_t>(result) < sizeof(new_cachemgr_pid))) {
323 ✗ if (result != 0) {
324 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
325 "could not read cache manager pid from lockfile");
326 ✗ UnlockFile(fd_lockfile);
327 ✗ delete quota_mgr;
328 ✗ return NULL;
329 } else {
330 // support reload from old versions of the cache manager
331 // lock file is empty in this case, try a plain ReadHalfPipe to get pid
332 ✗ quota_mgr->SetCacheMgrPid(quota_mgr->GetPid());
333 }
334 } else {
335 ✗ quota_mgr->SetCacheMgrPid(new_cachemgr_pid);
336 }
337
338
339 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to existing cache manager pipe");
340 ✗ quota_mgr->initialized_ = true;
341 ✗ Nonblock2Block(quota_mgr->pipe_lru_[1]);
342 ✗ UnlockFile(fd_lockfile);
343 ✗ quota_mgr->GetLimits(&quota_mgr->limit_, &quota_mgr->cleanup_threshold_);
344 ✗ LogCvmfs(kLogQuota, kLogDebug,
345 "received limit %" PRIu64 ", threshold %" PRIu64,
346 quota_mgr->limit_, quota_mgr->cleanup_threshold_);
347 ✗ if (FileExists(workspace_dir + "/cachemgr.protocol")) {
348 ✗ quota_mgr->protocol_revision_ = quota_mgr->GetProtocolRevision();
349 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected protocol revision %u",
350 quota_mgr->protocol_revision_);
351 } else {
352 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to ancient cache manager");
353 }
354 ✗ return quota_mgr;
355 }
356 35 const int connect_error = errno;
357
358 // Lock file: let existing cache manager finish first
359
2/4
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 35 times.
✗ Branch 5 not taken.
 35 const int fd_lockfile_fifo = LockFile(workspace_dir + "/lock_cachemgr.fifo");
360
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 35 times.
 35 if (fd_lockfile_fifo < 0) {
361 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not open lock file %s (%d)",
362 ✗ (workspace_dir + "/lock_cachemgr.fifo").c_str(), errno);
363 ✗ UnlockFile(fd_lockfile);
364 ✗ delete quota_mgr;
365 ✗ return NULL;
366 }
367
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 UnlockFile(fd_lockfile_fifo);
368
369
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 35 times.
 35 if (connect_error == ENXIO) {
370 ✗ LogCvmfs(kLogQuota, kLogDebug, "left-over FIFO found, unlinking");
371 ✗ unlink(fifo_path.c_str());
372 }
373
374 // Creating a new FIFO for the cache manager (to be bound later)
375 35 int retval = mkfifo(fifo_path.c_str(), 0600);
376
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 35 times.
 35 if (retval != 0) {
377 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to create cache manager FIFO (%d)",
378 ✗ errno);
379 ✗ UnlockFile(fd_lockfile);
380 ✗ delete quota_mgr;
381 ✗ return NULL;
382 }
383
384 // Create new cache manager
385 int pipe_boot[2];
386 int pipe_handshake[2];
387
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 MakePipe(pipe_boot);
388
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 MakePipe(pipe_handshake);
389
390 35 vector<string> command_line;
391
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 command_line.push_back(exe_path);
392
2/4
✓ Branch 2 taken 35 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 35 times.
✗ Branch 6 not taken.
 35 command_line.push_back("__cachemgr__");
393
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 command_line.push_back(cache_workspace);
394
2/4
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 35 times.
✗ Branch 5 not taken.
 35 command_line.push_back(StringifyInt(pipe_boot[1]));
395
2/4
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 35 times.
✗ Branch 5 not taken.
 35 command_line.push_back(StringifyInt(pipe_handshake[0]));
396
2/4
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 35 times.
✗ Branch 5 not taken.
 35 command_line.push_back(StringifyInt(limit));
397
2/4
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 35 times.
✗ Branch 5 not taken.
 35 command_line.push_back(StringifyInt(cleanup_threshold));
398 // do not propagate foreground in order to reliably get pid from exec
399 // instead, daemonize right here
400
2/4
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 35 times.
✗ Branch 5 not taken.
 35 command_line.push_back(StringifyInt(true)); // foreground
401
3/6
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 35 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 35 times.
✗ Branch 8 not taken.
 35 command_line.push_back(StringifyInt(GetLogSyslogLevel()));
402
3/6
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 35 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 35 times.
✗ Branch 8 not taken.
 35 command_line.push_back(StringifyInt(GetLogSyslogFacility()));
403
5/14
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 35 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 35 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 35 times.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 35 times.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
 35 command_line.push_back(GetLogDebugFile() + ":" + GetLogMicroSyslog());
404
405 35 set<int> preserve_filedes;
406
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 preserve_filedes.insert(0);
407
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 preserve_filedes.insert(1);
408
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 preserve_filedes.insert(2);
409
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 preserve_filedes.insert(pipe_boot[1]);
410
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 preserve_filedes.insert(pipe_handshake[0]);
411
412
1/2
✓ Branch 0 taken 35 times.
✗ Branch 1 not taken.
 35 if (foreground) {
413
1/2
✓ Branch 2 taken 35 times.
✗ Branch 3 not taken.
 35 retval = ManagedExec(command_line, preserve_filedes, map<int, int>(),
414 /*drop_credentials*/ false,
415 /*clear_env*/ false,
416 /*double_fork*/ true, &new_cachemgr_pid);
417 } else {
418 ✗ retval = ExecAsDaemon(command_line, &new_cachemgr_pid);
419 }
420
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 35 times.
 35 if (!retval) {
421 ✗ UnlockFile(fd_lockfile);
422 ✗ ClosePipe(pipe_boot);
423 ✗ ClosePipe(pipe_handshake);
424 ✗ delete quota_mgr;
425 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to start cache manager");
426 ✗ return NULL;
427 }
428
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 LogCvmfs(kLogQuota, kLogDebug,
429 "new cache manager pid: %d protocol revision %d",
430 new_cachemgr_pid, QuotaManager::kProtocolRevision);
431 35 quota_mgr->SetCacheMgrPid(new_cachemgr_pid);
432
2/4
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 35 times.
✗ Branch 6 not taken.
 35 const int fd_lockfile_rw = open((workspace_dir + "/lock_cachemgr").c_str(),
433 O_RDWR | O_TRUNC, 0600);
434 35 const unsigned magic_number = PosixQuotaManager::kLockFileMagicNumber;
435
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 const bool result_mn = SafeWrite(fd_lockfile_rw, &magic_number,
436 sizeof(magic_number));
437
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 const bool result = SafeWrite(fd_lockfile_rw, &new_cachemgr_pid,
438 sizeof(new_cachemgr_pid));
439
2/4
✓ Branch 0 taken 35 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 35 times.
 35 if (!result || !result_mn) {
440 ✗ PANIC(kLogSyslogErr, "could not write cache manager pid to lockfile");
441 }
442
443
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 close(fd_lockfile_rw);
444 // Wait for cache manager to be ready
445
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 close(pipe_boot[1]);
446
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 close(pipe_handshake[0]);
447 char buf;
448
2/4
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 35 times.
✗ Branch 4 not taken.
 35 if (read(pipe_boot[0], &buf, 1) != 1) {
449
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 UnlockFile(fd_lockfile);
450
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 close(pipe_boot[0]);
451
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 close(pipe_handshake[1]);
452
1/2
✓ Branch 0 taken 35 times.
✗ Branch 1 not taken.
 35 delete quota_mgr;
453
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
454 "cache manager did not start");
455 35 return NULL;
456 }
457 ✗ close(pipe_boot[0]);
458
459 // Connect write end
460 ✗ quota_mgr->pipe_lru_[1] = open(fifo_path.c_str(), O_WRONLY | O_NONBLOCK);
461 ✗ if (quota_mgr->pipe_lru_[1] < 0) {
462 ✗ LogCvmfs(kLogQuota, kLogDebug,
463 ✗ "failed to connect to newly created FIFO (%d)", errno);
464 ✗ close(pipe_handshake[1]);
465 ✗ UnlockFile(fd_lockfile);
466 ✗ delete quota_mgr;
467 ✗ return NULL;
468 }
469
470 // Finalize handshake
471 ✗ buf = 'C';
472 ✗ if (write(pipe_handshake[1], &buf, 1) != 1) {
473 ✗ UnlockFile(fd_lockfile);
474 ✗ close(pipe_handshake[1]);
475 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not finalize handshake");
476 ✗ delete quota_mgr;
477 ✗ return NULL;
478 }
479 ✗ close(pipe_handshake[1]);
480
481 ✗ Nonblock2Block(quota_mgr->pipe_lru_[1]);
482 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to a new cache manager");
483 ✗ quota_mgr->protocol_revision_ = kProtocolRevision;
484
485 ✗ UnlockFile(fd_lockfile);
486
487 ✗ quota_mgr->initialized_ = true;
488 ✗ quota_mgr->GetLimits(&quota_mgr->limit_, &quota_mgr->cleanup_threshold_);
489 ✗ LogCvmfs(kLogQuota, kLogDebug,
490 "received limit %" PRIu64 ", "
491 "threshold %" PRIu64,
492 quota_mgr->limit_, quota_mgr->cleanup_threshold_);
493 ✗ return quota_mgr;
494 70 }
495
496
497 350 bool PosixQuotaManager::DoCleanup(const uint64_t leave_size) {
498
2/2
✓ Branch 0 taken 70 times.
✓ Branch 1 taken 280 times.
 350 if (gauge_ <= leave_size)
499 70 return true;
500
501 // TODO(jblomer) transaction
502
1/2
✓ Branch 1 taken 280 times.
✗ Branch 2 not taken.
 280 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug,
503 "clean up cache until at most %lu KB is used", leave_size / 1024);
504
1/2
✓ Branch 1 taken 280 times.
✗ Branch 2 not taken.
 280 LogCvmfs(kLogQuota, kLogDebug, "gauge %" PRIu64, gauge_);
505
1/2
✓ Branch 1 taken 280 times.
✗ Branch 2 not taken.
 280 cleanup_recorder_.Tick();
506
507 bool result;
508 280 vector<string> trash;
509
510 // Note that volatile files start counting from the smallest int64 number:
511 // the absolute sequence number with the first bit set in two's complement.
512 // So -1 can be a marker that will never appear in the database.
513 280 int64_t max_acseq = -1;
514 280 std::vector<EvictCandidate> lru_ordered_open;
515
516 do {
517
1/2
✓ Branch 1 taken 2030 times.
✗ Branch 2 not taken.
 2030 sqlite3_reset(stmt_lru_);
518
3/4
✓ Branch 0 taken 280 times.
✓ Branch 1 taken 1750 times.
✓ Branch 3 taken 2030 times.
✗ Branch 4 not taken.
 2310 sqlite3_bind_int64(stmt_lru_, 1,
519 280 (max_acseq == -1) ? std::numeric_limits<int64_t>::min()
520 : (max_acseq + 1));
521
522 2030 std::vector<EvictCandidate> candidates;
523
1/2
✓ Branch 1 taken 2030 times.
✗ Branch 2 not taken.
 2030 candidates.reserve(kEvictBatchSize);
524 2030 string hash_str;
525 2030 unsigned i = 0;
526
3/4
✓ Branch 1 taken 1822380 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1820350 times.
✓ Branch 4 taken 2030 times.
 1822380 while (sqlite3_step(stmt_lru_) == SQLITE_ROW) {
527 hash_str = reinterpret_cast<const char *>(
528
2/4
✓ Branch 1 taken 1820350 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1820350 times.
✗ Branch 5 not taken.
 1820350 sqlite3_column_text(stmt_lru_, 0));
529
1/2
✓ Branch 2 taken 1820350 times.
✗ Branch 3 not taken.
 1820350 LogCvmfs(kLogQuota, kLogDebug, "add %s to candidates for eviction",
530 hash_str.c_str());
531
1/2
✓ Branch 1 taken 1820350 times.
✗ Branch 2 not taken.
 1820350 candidates.push_back(
532
1/2
✓ Branch 1 taken 1820350 times.
✗ Branch 2 not taken.
 1820350 EvictCandidate(shash::MkFromHexPtr(shash::HexPtr(hash_str)),
533 1820350 sqlite3_column_int64(stmt_lru_, 1),
534
2/4
✓ Branch 1 taken 1820350 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1820350 times.
✗ Branch 5 not taken.
 1820350 sqlite3_column_int64(stmt_lru_, 2)));
535 1820350 i++;
536 }
537
2/2
✓ Branch 1 taken 35 times.
✓ Branch 2 taken 1995 times.
 2030 if (candidates.empty()) {
538
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 LogCvmfs(kLogQuota, kLogDebug, "no more entries to evict");
539 35 break;
540 }
541
542 1995 const unsigned N = candidates.size();
543
544 1995 open_files_.clear();
545
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1995 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 3990 open_files_ = (cleanup_unused_first_) ? CollectAllOpenHashes()
546 1995 : std::vector<shash::Short>();
547
548
2/2
✓ Branch 0 taken 1750315 times.
✓ Branch 1 taken 1750 times.
 1752065 for (i = 0; i < N; ++i) {
549 // That's a critical condition. We must not delete a not yet inserted
550 // pinned file as it is already reserved (but will be inserted later).
551 // Instead, set the pin bit in the db to not run into an endless loop
552
1/2
✓ Branch 2 taken 1750315 times.
✗ Branch 3 not taken.
 1750315 const bool is_pinned = pinned_chunks_.find(candidates[i].hash)
553 3500630 != pinned_chunks_.end();
554
555 // Avoid evicting open files hopping there are enough more recently used
556 // files to satisfy the cleanup request
557 /*
558 const bool is_open = std::find_if(
559 open_files_.begin(), open_files_.end(),
560 [&candidates, &i](const auto &elem) -> bool {
561 return elem.Collide(candidates[i].hash);
562 })
563 != open_files_.end();
564 */
565 1750315 bool is_open=false;
566
1/2
✗ Branch 4 not taken.
✓ Branch 5 taken 1750315 times.
 1750315 for(auto it=open_files_.begin();it!=open_files_.end();++it){
567 ✗ if (it->Collide(candidates[i].hash)){is_open=true; break;}
568 }
569
570
2/2
✓ Branch 0 taken 35 times.
✓ Branch 1 taken 1750280 times.
 1750315 if (is_pinned) {
571
1/2
✓ Branch 2 taken 35 times.
✗ Branch 3 not taken.
 35 SkipEviction(candidates[i]);
572 35 continue;
573 }
574
575
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1750280 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 1750280 if (cleanup_unused_first_ and is_open) {
576 ✗ SkipEviction(candidates[i]);
577 ✗ lru_ordered_open.push_back(candidates[i]);
578 ✗ continue;
579 }
580
581
2/4
✓ Branch 1 taken 1750280 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1750280 times.
✗ Branch 5 not taken.
 3500560 trash.push_back(cache_dir_ + "/"
582
2/4
✓ Branch 2 taken 1750280 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1750280 times.
✗ Branch 6 not taken.
 5250840 + candidates[i].hash.MakePathWithoutSuffix());
583 1750280 gauge_ -= candidates[i].size;
584 1750280 max_acseq = candidates[i].acseq;
585
1/2
✓ Branch 2 taken 1750280 times.
✗ Branch 3 not taken.
 1750280 LogCvmfs(kLogQuota, kLogDebug, "lru cleanup %s, new gauge %" PRIu64,
586
1/2
✓ Branch 2 taken 1750280 times.
✗ Branch 3 not taken.
 3500560 candidates[i].hash.ToString().c_str(), gauge_);
587
588
2/2
✓ Branch 0 taken 245 times.
✓ Branch 1 taken 1750035 times.
 1750280 if (gauge_ <= leave_size)
589 245 break;
590 }
591
6/6
✓ Branch 1 taken 1995 times.
✓ Branch 2 taken 35 times.
✓ Branch 4 taken 1995 times.
✓ Branch 5 taken 35 times.
✓ Branch 6 taken 1750 times.
✓ Branch 7 taken 245 times.
 4060 } while (gauge_ > leave_size);
592
593
1/2
✓ Branch 0 taken 280 times.
✗ Branch 1 not taken.
 280 if (max_acseq != -1) {
594
1/2
✓ Branch 1 taken 280 times.
✗ Branch 2 not taken.
 280 sqlite3_bind_int64(stmt_rm_batch_, 1, max_acseq);
595
1/2
✓ Branch 1 taken 280 times.
✗ Branch 2 not taken.
 280 result = (sqlite3_step(stmt_rm_batch_) == SQLITE_DONE);
596
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 280 times.
 280 assert(result);
597
1/2
✓ Branch 1 taken 280 times.
✗ Branch 2 not taken.
 280 sqlite3_reset(stmt_rm_batch_);
598
599
1/2
✓ Branch 1 taken 280 times.
✗ Branch 2 not taken.
 280 result = (sqlite3_step(stmt_unblock_) == SQLITE_DONE);
600
1/2
✓ Branch 1 taken 280 times.
✗ Branch 2 not taken.
 280 sqlite3_reset(stmt_unblock_);
601
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 280 times.
 280 assert(result);
602 }
603
604
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 280 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 280 times.
 280 while (!lru_ordered_open.empty() and gauge_ > leave_size) {
605 // cleanup files in use
606 ✗ auto &candidate = lru_ordered_open[0];
607 ✗ trash.push_back(cache_dir_ + "/" + candidate.hash.MakePathWithoutSuffix());
608 ✗ gauge_ -= candidate.size;
609 ✗ max_acseq = candidate.acseq;
610 ✗ LogCvmfs(kLogQuota, kLogDebug, "lru cleanup %s, new gauge %" PRIu64,
611 ✗ candidate.hash.ToString().c_str(), gauge_);
612 ✗ lru_ordered_open.erase(lru_ordered_open.begin());
613 }
614
615
2/4
✓ Branch 1 taken 280 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 280 times.
 280 if (!EmptyTrash(trash))
616 ✗ return false;
617
618
2/2
✓ Branch 0 taken 35 times.
✓ Branch 1 taken 245 times.
 280 if (gauge_ > leave_size) {
619
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
620 "request to clean until %" PRIu64 ", "
621 "but effective gauge is %" PRIu64,
622 leave_size, gauge_);
623 35 return false;
624 }
625 245 return true;
626 280 }
627
628 280 bool PosixQuotaManager::EmptyTrash(const std::vector<std::string> &trash) {
629
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 280 times.
 280 if (trash.empty())
630 ✗ return true;
631
632
2/2
✓ Branch 0 taken 210 times.
✓ Branch 1 taken 70 times.
 280 if (async_delete_) {
633 // Double fork avoids zombie, forked removal process must not flush file
634 // buffers
635 pid_t pid;
636 int statloc;
637
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 210 times.
 210 if ((pid = fork()) == 0) {
638 // TODO(jblomer): eviciting files in the cache should perhaps become a
639 // thread. This would also allow to block the chunks and prevent the
640 // race with re-insertion. Then again, a thread can block umount.
641 #ifndef DEBUGMSG
642 ✗ CloseAllFildes(std::set<int>());
643 #endif
644 ✗ if (fork() == 0) {
645 ✗ for (unsigned i = 0, iEnd = trash.size(); i < iEnd; ++i) {
646 ✗ LogCvmfs(kLogQuota, kLogDebug, "unlink %s", trash[i].c_str());
647 ✗ unlink(trash[i].c_str());
648 }
649 ✗ _exit(0);
650 }
651 ✗ _exit(0);
652 } else {
653
1/2
✓ Branch 0 taken 210 times.
✗ Branch 1 not taken.
 210 if (pid > 0)
654
1/2
✓ Branch 1 taken 210 times.
✗ Branch 2 not taken.
 210 waitpid(pid, &statloc, 0);
655 else
656 ✗ return false;
657 }
658 } else { // !async_delete_
659
2/2
✓ Branch 1 taken 105 times.
✓ Branch 2 taken 70 times.
 175 for (unsigned i = 0, iEnd = trash.size(); i < iEnd; ++i) {
660 105 LogCvmfs(kLogQuota, kLogDebug, "unlink %s", trash[i].c_str());
661 105 unlink(trash[i].c_str());
662 }
663 }
664 280 return true;
665 }
666
667
668 3501264 void PosixQuotaManager::DoInsert(const shash::Any &hash,
669 const uint64_t size,
670 const string &description,
671 const CommandType command_type) {
672
1/2
✓ Branch 1 taken 3501264 times.
✗ Branch 2 not taken.
 3501264 const string hash_str = hash.ToString();
673
1/2
✓ Branch 3 taken 3501264 times.
✗ Branch 4 not taken.
 3501264 LogCvmfs(kLogQuota, kLogDebug, "insert into lru %s, path %s, method %d",
674 hash_str.c_str(), description.c_str(), command_type);
675 3501264 const unsigned desc_length = (description.length() > kMaxDescription)
676 ? kMaxDescription
677
1/2
✓ Branch 0 taken 3501264 times.
✗ Branch 1 not taken.
 3501264 : description.length();
678
679 LruCommand *cmd = reinterpret_cast<LruCommand *>(
680 3501264 alloca(sizeof(LruCommand) + desc_length));
681 3501264 new (cmd) LruCommand;
682 3501264 cmd->command_type = command_type;
683 3501264 cmd->SetSize(size);
684
1/2
✓ Branch 1 taken 3501264 times.
✗ Branch 2 not taken.
 3501264 cmd->StoreHash(hash);
685 3501264 cmd->desc_length = desc_length;
686 3501264 memcpy(reinterpret_cast<char *>(cmd) + sizeof(LruCommand), &description[0],
687 desc_length);
688
1/2
✓ Branch 1 taken 3501264 times.
✗ Branch 2 not taken.
 3501264 WritePipe(pipe_lru_[1], cmd, sizeof(LruCommand) + desc_length);
689 3501264 }
690
691
692 1225 vector<string> PosixQuotaManager::DoList(const CommandType list_command) {
693 1225 vector<string> result;
694
695 int pipe_list[2];
696
1/2
✓ Branch 1 taken 1225 times.
✗ Branch 2 not taken.
 1225 MakeReturnPipe(pipe_list);
697 char description_buffer[kMaxDescription];
698
699 1225 LruCommand cmd;
700 1225 cmd.command_type = list_command;
701 1225 cmd.return_pipe = pipe_list[1];
702
1/2
✓ Branch 1 taken 1225 times.
✗ Branch 2 not taken.
 1225 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
703
704 int length;
705 do {
706
1/2
✓ Branch 1 taken 3502590 times.
✗ Branch 2 not taken.
 3502590 ManagedReadHalfPipe(pipe_list[0], &length, sizeof(length));
707
2/2
✓ Branch 0 taken 3501365 times.
✓ Branch 1 taken 1225 times.
 3502590 if (length > 0) {
708
1/2
✓ Branch 1 taken 3501365 times.
✗ Branch 2 not taken.
 3501365 ReadPipe(pipe_list[0], description_buffer, length);
709
2/4
✓ Branch 2 taken 3501365 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 3501365 times.
✗ Branch 6 not taken.
 3501365 result.push_back(string(description_buffer, length));
710 }
711
2/2
✓ Branch 0 taken 3501365 times.
✓ Branch 1 taken 1225 times.
 3502590 } while (length >= 0);
712
713
1/2
✓ Branch 1 taken 1225 times.
✗ Branch 2 not taken.
 1225 CloseReturnPipe(pipe_list);
714 2450 return result;
715 }
716
717
718 548 uint64_t PosixQuotaManager::GetCapacity() {
719
1/2
✓ Branch 0 taken 548 times.
✗ Branch 1 not taken.
 548 if (limit_ != (uint64_t)(-1))
720 548 return limit_;
721
722 // Unrestricted cache, look at free space on cache dir fs
723 struct statfs info;
724 ✗ if (statfs(".", &info) == 0) {
725 ✗ return info.f_bavail * info.f_bsize;
726 } else {
727 ✗ LogCvmfs(kLogQuota, kLogSyslogErr | kLogDebug,
728 ✗ "failed to query file system info of cache (%d)", errno);
729 ✗ return limit_;
730 }
731 }
732
733
734 ✗ void PosixQuotaManager::GetLimits(uint64_t *limit,
735 uint64_t *cleanup_threshold) {
736 int pipe_limits[2];
737 ✗ MakeReturnPipe(pipe_limits);
738
739 ✗ LruCommand cmd;
740 ✗ cmd.command_type = kLimits;
741 ✗ cmd.return_pipe = pipe_limits[1];
742 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
743 ✗ ManagedReadHalfPipe(pipe_limits[0], limit, sizeof(*limit));
744 ✗ ReadPipe(pipe_limits[0], cleanup_threshold, sizeof(*cleanup_threshold));
745 ✗ CloseReturnPipe(pipe_limits);
746 }
747
748
749 /**
750 * Since we only cleanup until cleanup_threshold, we can only add
751 * files smaller than limit-cleanup_threshold.
752 */
753 61 uint64_t PosixQuotaManager::GetMaxFileSize() {
754 61 return limit_ - cleanup_threshold_;
755 }
756
757
758 35 pid_t PosixQuotaManager::GetPid() {
759
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 35 if (!shared_ || !spawned_) {
760 35 return getpid();
761 }
762 ✗ if (cachemgr_pid_) {
763 ✗ return cachemgr_pid_;
764 }
765
766 pid_t result;
767 int pipe_pid[2];
768 ✗ MakeReturnPipe(pipe_pid);
769
770 ✗ LruCommand cmd;
771 ✗ cmd.command_type = kPid;
772 ✗ cmd.return_pipe = pipe_pid[1];
773 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
774 ✗ ReadHalfPipe(pipe_pid[0], &result, sizeof(result));
775 ✗ CloseReturnPipe(pipe_pid);
776 ✗ return result;
777 }
778
779
780 35 uint32_t PosixQuotaManager::GetProtocolRevision() {
781 int pipe_revision[2];
782
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 MakeReturnPipe(pipe_revision);
783
784 35 LruCommand cmd;
785 35 cmd.command_type = kGetProtocolRevision;
786 35 cmd.return_pipe = pipe_revision[1];
787
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
788
789 uint32_t revision;
790
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 ManagedReadHalfPipe(pipe_revision[0], &revision, sizeof(revision));
791
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 CloseReturnPipe(pipe_revision);
792 35 return revision;
793 }
794
795 513 void PosixQuotaManager::SetCleanupPolicy(bool cleanup_unused_first) {
796
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 513 times.
 513 if (protocol_revision_ < 3) return;
797
798 513 LruCommand cmd;
799 513 cmd.command_type = kSetCleanupPolicy;
800
1/2
✓ Branch 1 taken 513 times.
✗ Branch 2 not taken.
 513 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
801
802
1/2
✓ Branch 1 taken 513 times.
✗ Branch 2 not taken.
 513 WritePipe(pipe_lru_[1], &cleanup_unused_first, sizeof(bool));
803 }
804
805 ✗ void PosixQuotaManager::RegisterMountpoint(const std::string &mountpoint) {
806 ✗ if (protocol_revision_ < 3) return;
807
808 ✗ LruCommand cmd;
809 ✗ cmd.command_type = kRegisterMountpoint;
810 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
811
812 ✗ size_t mp_size = mountpoint.size();
813 ✗ WritePipe(pipe_lru_[1], &mp_size, sizeof(size_t));
814 ✗ WritePipe(pipe_lru_[1], mountpoint.data(), mp_size);
815 }
816
817 ✗ std::string PosixQuotaManager::GetMountpoints() {
818 ✗ if (protocol_revision_ < 3) return "";
819
820 int pipe_mp[2];
821 ✗ MakeReturnPipe(pipe_mp);
822
823 ✗ LruCommand cmd;
824 ✗ cmd.command_type = kGetMountpoints;
825 ✗ cmd.return_pipe = pipe_mp[1];
826 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
827 ✗ size_t mp_str_size = 0;
828 ✗ ManagedReadHalfPipe(pipe_mp[0], &mp_str_size, sizeof(size_t));
829 ✗ char *buf = (char *)malloc(mp_str_size * sizeof(char));
830 ✗ ManagedReadHalfPipe(pipe_mp[0], buf, mp_str_size);
831 ✗ return std::string(buf);
832 }
833
834 ✗ std::string PosixQuotaManager::GetGroupHashes() {
835 ✗ if (protocol_revision_ < 3) return "";
836
837 int pipe_gh[2];
838 ✗ MakeReturnPipe(pipe_gh);
839
840 ✗ LruCommand cmd;
841 ✗ cmd.command_type = kGetGroupHashes;
842 ✗ cmd.return_pipe = pipe_gh[1];
843 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
844 ✗ size_t mp_str_size = 0;
845 ✗ ManagedReadHalfPipe(pipe_gh[0], &mp_str_size, sizeof(size_t));
846 ✗ char *buf = (char *)malloc(mp_str_size * sizeof(char));
847 ✗ ManagedReadHalfPipe(pipe_gh[0], buf, mp_str_size);
848 ✗ return std::string(buf);
849 }
850
851 /**
852 * Queries the shared local hard disk quota manager.
853 */
854 630 void PosixQuotaManager::GetSharedStatus(uint64_t *gauge, uint64_t *pinned) {
855 int pipe_status[2];
856
1/2
✓ Branch 1 taken 630 times.
✗ Branch 2 not taken.
 630 MakeReturnPipe(pipe_status);
857
858 630 LruCommand cmd;
859 630 cmd.command_type = kStatus;
860 630 cmd.return_pipe = pipe_status[1];
861
1/2
✓ Branch 1 taken 630 times.
✗ Branch 2 not taken.
 630 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
862
1/2
✓ Branch 1 taken 630 times.
✗ Branch 2 not taken.
 630 ManagedReadHalfPipe(pipe_status[0], gauge, sizeof(*gauge));
863
1/2
✓ Branch 1 taken 630 times.
✗ Branch 2 not taken.
 630 ReadPipe(pipe_status[0], pinned, sizeof(*pinned));
864
1/2
✓ Branch 1 taken 630 times.
✗ Branch 2 not taken.
 630 CloseReturnPipe(pipe_status);
865 630 }
866
867 35 bool PosixQuotaManager::SetSharedLimit(uint64_t limit) {
868 int pipe_set_limit[2];
869 bool result;
870
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 MakeReturnPipe(pipe_set_limit);
871
872 35 LruCommand cmd;
873 35 cmd.command_type = kSetLimit;
874 35 cmd.size = limit;
875 35 cmd.return_pipe = pipe_set_limit[1];
876
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
877
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 ReadHalfPipe(pipe_set_limit[0], &result, sizeof(result));
878
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 CloseReturnPipe(pipe_set_limit);
879 35 return result;
880 }
881
882
883 35 bool PosixQuotaManager::SetLimit(uint64_t size) {
884
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 35 times.
 35 if (!spawned_) {
885 ✗ limit_ = size;
886 ✗ cleanup_threshold_ = size / 2;
887 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
888 "Quota limit set to %lu / threshold %lu", limit_,
889 cleanup_threshold_);
890 ✗ return true;
891 }
892 35 return SetSharedLimit(size);
893 }
894
895 1586 uint64_t PosixQuotaManager::GetSize() {
896
2/2
✓ Branch 0 taken 1026 times.
✓ Branch 1 taken 560 times.
 1586 if (!spawned_)
897 1026 return gauge_;
898 uint64_t gauge, size_pinned;
899
1/2
✓ Branch 1 taken 560 times.
✗ Branch 2 not taken.
 560 GetSharedStatus(&gauge, &size_pinned);
900 560 return gauge;
901 }
902
903
904 70 uint64_t PosixQuotaManager::GetSizePinned() {
905
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 70 times.
 70 if (!spawned_)
906 ✗ return pinned_;
907 uint64_t gauge, size_pinned;
908
1/2
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
 70 GetSharedStatus(&gauge, &size_pinned);
909 70 return size_pinned;
910 }
911
912
913 140 uint64_t PosixQuotaManager::GetCleanupRate(uint64_t period_s) {
914
2/4
✓ Branch 0 taken 140 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 140 times.
 140 if (!spawned_ || (protocol_revision_ < 2))
915 ✗ return 0;
916 uint64_t cleanup_rate;
917
918 int pipe_cleanup_rate[2];
919
1/2
✓ Branch 1 taken 140 times.
✗ Branch 2 not taken.
 140 MakeReturnPipe(pipe_cleanup_rate);
920 140 LruCommand cmd;
921 140 cmd.command_type = kCleanupRate;
922 140 cmd.size = period_s;
923 140 cmd.return_pipe = pipe_cleanup_rate[1];
924
1/2
✓ Branch 1 taken 140 times.
✗ Branch 2 not taken.
 140 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
925
1/2
✓ Branch 1 taken 140 times.
✗ Branch 2 not taken.
 140 ManagedReadHalfPipe(pipe_cleanup_rate[0], &cleanup_rate,
926 sizeof(cleanup_rate));
927
1/2
✓ Branch 1 taken 140 times.
✗ Branch 2 not taken.
 140 CloseReturnPipe(pipe_cleanup_rate);
928
929 140 return cleanup_rate;
930 }
931
932
933 2578 bool PosixQuotaManager::InitDatabase(const bool rebuild_database) {
934 2578 string sql;
935 sqlite3_stmt *stmt;
936
937
2/4
✓ Branch 1 taken 2578 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2578 times.
✗ Branch 5 not taken.
 2578 fd_lock_cachedb_ = LockFile(workspace_dir_ + "/lock_cachedb");
938
2/2
✓ Branch 0 taken 35 times.
✓ Branch 1 taken 2543 times.
 2578 if (fd_lock_cachedb_ < 0) {
939
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 LogCvmfs(kLogQuota, kLogDebug, "failed to create cachedb lock");
940 35 return false;
941 }
942
943 2543 bool retry = false;
944
1/2
✓ Branch 1 taken 2543 times.
✗ Branch 2 not taken.
 2543 const string db_file = cache_dir_ + "/cachedb";
945
2/2
✓ Branch 0 taken 2390 times.
✓ Branch 1 taken 153 times.
 2543 if (rebuild_database) {
946
1/2
✓ Branch 2 taken 153 times.
✗ Branch 3 not taken.
 153 LogCvmfs(kLogQuota, kLogDebug, "rebuild database, unlinking existing (%s)",
947 db_file.c_str());
948 153 unlink(db_file.c_str());
949
1/2
✓ Branch 1 taken 153 times.
✗ Branch 2 not taken.
 153 unlink((db_file + "-journal").c_str());
950 }
951
952 2390 init_recover:
953
1/2
✓ Branch 2 taken 2543 times.
✗ Branch 3 not taken.
 2543 int err = sqlite3_open(db_file.c_str(), &database_);
954
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2543 times.
 2543 if (err != SQLITE_OK) {
955 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not open cache database (%d)", err);
956 ✗ goto init_database_fail;
957 }
958 // TODO(reneme): make this a `QuotaDatabase : public sqlite::Database`
959 sql = "PRAGMA synchronous=0; PRAGMA locking_mode=EXCLUSIVE; "
960 "PRAGMA auto_vacuum=1; "
961 "CREATE TABLE IF NOT EXISTS cache_catalog (sha1 TEXT, size INTEGER, "
962 " acseq INTEGER, path TEXT, type INTEGER, pinned INTEGER, "
963 "CONSTRAINT pk_cache_catalog PRIMARY KEY (sha1)); "
964 "CREATE UNIQUE INDEX IF NOT EXISTS idx_cache_catalog_acseq "
965 " ON cache_catalog (acseq); "
966 "CREATE TEMP TABLE fscache (sha1 TEXT, size INTEGER, actime INTEGER, "
967 "CONSTRAINT pk_fscache PRIMARY KEY (sha1)); "
968 "CREATE INDEX idx_fscache_actime ON fscache (actime); "
969 "CREATE TABLE IF NOT EXISTS properties (key TEXT, value TEXT, "
970
1/2
✓ Branch 1 taken 2543 times.
✗ Branch 2 not taken.
 2543 " CONSTRAINT pk_properties PRIMARY KEY(key));";
971
1/2
✓ Branch 2 taken 2543 times.
✗ Branch 3 not taken.
 2543 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
972
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2543 times.
 2543 if (err != SQLITE_OK) {
973 ✗ if (!retry) {
974 ✗ retry = true;
975 ✗ sqlite3_close(database_);
976 ✗ unlink(db_file.c_str());
977 ✗ unlink((db_file + "-journal").c_str());
978 ✗ LogCvmfs(kLogQuota, kLogSyslogWarn,
979 "LRU database corrupted, re-building");
980 ✗ goto init_recover;
981 }
982 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
983 sql.c_str());
984 ✗ goto init_database_fail;
985 }
986
987 // If this an old cache catalog,
988 // add and initialize new columns to cache_catalog
989 sql = "ALTER TABLE cache_catalog ADD type INTEGER; "
990
1/2
✓ Branch 1 taken 2543 times.
✗ Branch 2 not taken.
 2543 "ALTER TABLE cache_catalog ADD pinned INTEGER";
991
1/2
✓ Branch 2 taken 2543 times.
✗ Branch 3 not taken.
 2543 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
992
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2543 times.
 2543 if (err == SQLITE_OK) {
993 ✗ sql = "UPDATE cache_catalog SET type=" + StringifyInt(kFileRegular) + ";";
994 ✗ err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
995 ✗ if (err != SQLITE_OK) {
996 ✗ LogCvmfs(kLogQuota, kLogDebug,
997 "could not init cache database (failed: %s)", sql.c_str());
998 ✗ goto init_database_fail;
999 }
1000 }
1001
1002 // Set pinned back
1003
1/2
✓ Branch 1 taken 2543 times.
✗ Branch 2 not taken.
 2543 sql = "UPDATE cache_catalog SET pinned=0;";
1004
1/2
✓ Branch 2 taken 2543 times.
✗ Branch 3 not taken.
 2543 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1005
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2543 times.
 2543 if (err != SQLITE_OK) {
1006 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
1007 sql.c_str());
1008 ✗ goto init_database_fail;
1009 }
1010
1011 // Set schema version
1012 sql = "INSERT OR REPLACE INTO properties (key, value) "
1013
1/2
✓ Branch 1 taken 2543 times.
✗ Branch 2 not taken.
 2543 "VALUES ('schema', '1.0')";
1014
1/2
✓ Branch 2 taken 2543 times.
✗ Branch 3 not taken.
 2543 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
1015
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2543 times.
 2543 if (err != SQLITE_OK) {
1016 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
1017 sql.c_str());
1018 ✗ goto init_database_fail;
1019 }
1020
1021 // If cache catalog is empty, recreate from file system
1022
1/2
✓ Branch 1 taken 2543 times.
✗ Branch 2 not taken.
 2543 sql = "SELECT count(*) FROM cache_catalog;";
1023
1/2
✓ Branch 2 taken 2543 times.
✗ Branch 3 not taken.
 2543 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1024
2/4
✓ Branch 1 taken 2543 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2543 times.
✗ Branch 4 not taken.
 2543 if (sqlite3_step(stmt) == SQLITE_ROW) {
1025
6/8
✓ Branch 1 taken 2543 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 140 times.
✓ Branch 4 taken 2403 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 140 times.
✓ Branch 7 taken 2403 times.
✓ Branch 8 taken 140 times.
 2543 if ((sqlite3_column_int64(stmt, 0)) == 0 || rebuild_database) {
1026
1/2
✓ Branch 1 taken 2403 times.
✗ Branch 2 not taken.
 2403 LogCvmfs(kLogCvmfs, kLogDebug,
1027 "CernVM-FS: building lru cache database...");
1028
3/4
✓ Branch 1 taken 2403 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 105 times.
✓ Branch 4 taken 2298 times.
 2403 if (!RebuildDatabase()) {
1029
1/2
✓ Branch 1 taken 105 times.
✗ Branch 2 not taken.
 105 LogCvmfs(kLogQuota, kLogDebug,
1030 "could not build cache database from file system");
1031
1/2
✓ Branch 1 taken 105 times.
✗ Branch 2 not taken.
 105 sqlite3_finalize(stmt);
1032 105 goto init_database_fail;
1033 }
1034 }
1035
1/2
✓ Branch 1 taken 2438 times.
✗ Branch 2 not taken.
 2438 sqlite3_finalize(stmt);
1036 } else {
1037 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not select on cache catalog");
1038 ✗ sqlite3_finalize(stmt);
1039 ✗ goto init_database_fail;
1040 }
1041
1042 // How many bytes do we already have in cache?
1043
1/2
✓ Branch 1 taken 2438 times.
✗ Branch 2 not taken.
 2438 sql = "SELECT sum(size) FROM cache_catalog;";
1044
1/2
✓ Branch 2 taken 2438 times.
✗ Branch 3 not taken.
 2438 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1045
2/4
✓ Branch 1 taken 2438 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2438 times.
✗ Branch 4 not taken.
 2438 if (sqlite3_step(stmt) == SQLITE_ROW) {
1046
1/2
✓ Branch 1 taken 2438 times.
✗ Branch 2 not taken.
 2438 gauge_ = sqlite3_column_int64(stmt, 0);
1047 } else {
1048 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not determine cache size");
1049 ✗ sqlite3_finalize(stmt);
1050 ✗ goto init_database_fail;
1051 }
1052
1/2
✓ Branch 1 taken 2438 times.
✗ Branch 2 not taken.
 2438 sqlite3_finalize(stmt);
1053
1054 // Highest seq-no?
1055
1/2
✓ Branch 1 taken 2438 times.
✗ Branch 2 not taken.
 2438 sql = "SELECT coalesce(max(acseq & (~(1<<63))), 0) FROM cache_catalog;";
1056
1/2
✓ Branch 2 taken 2438 times.
✗ Branch 3 not taken.
 2438 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1057
2/4
✓ Branch 1 taken 2438 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2438 times.
✗ Branch 4 not taken.
 2438 if (sqlite3_step(stmt) == SQLITE_ROW) {
1058
1/2
✓ Branch 1 taken 2438 times.
✗ Branch 2 not taken.
 2438 seq_ = sqlite3_column_int64(stmt, 0) + 1;
1059 } else {
1060 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not determine highest seq-no");
1061 ✗ sqlite3_finalize(stmt);
1062 ✗ goto init_database_fail;
1063 }
1064
1/2
✓ Branch 1 taken 2438 times.
✗ Branch 2 not taken.
 2438 sqlite3_finalize(stmt);
1065
1066 // Prepare touch, new, remove statements
1067
1/2
✓ Branch 1 taken 2438 times.
✗ Branch 2 not taken.
 2438 sqlite3_prepare_v2(database_,
1068 "UPDATE cache_catalog SET acseq=:seq | (acseq&(1<<63)) "
1069 "WHERE sha1=:sha1;",
1070 -1, &stmt_touch_, NULL);
1071
1/2
✓ Branch 1 taken 2438 times.
✗ Branch 2 not taken.
 2438 sqlite3_prepare_v2(database_,
1072 "UPDATE cache_catalog SET pinned=0 "
1073 "WHERE sha1=:sha1;",
1074 -1, &stmt_unpin_, NULL);
1075
1/2
✓ Branch 1 taken 2438 times.
✗ Branch 2 not taken.
 2438 sqlite3_prepare_v2(database_,
1076 "UPDATE cache_catalog SET pinned=2 "
1077 "WHERE sha1=:sha1;",
1078 -1, &stmt_block_, NULL);
1079
1/2
✓ Branch 1 taken 2438 times.
✗ Branch 2 not taken.
 2438 sqlite3_prepare_v2(database_,
1080 "UPDATE cache_catalog SET pinned=1 "
1081 "WHERE pinned=2;",
1082 -1, &stmt_unblock_, NULL);
1083
1/2
✓ Branch 1 taken 2438 times.
✗ Branch 2 not taken.
 2438 sqlite3_prepare_v2(database_,
1084 "INSERT OR REPLACE INTO cache_catalog "
1085 "(sha1, size, acseq, path, type, pinned) "
1086 "VALUES (:sha1, :s, :seq, :p, :t, :pin);",
1087 -1, &stmt_new_, NULL);
1088
1/2
✓ Branch 1 taken 2438 times.
✗ Branch 2 not taken.
 2438 sqlite3_prepare_v2(database_,
1089 "SELECT size, pinned FROM cache_catalog WHERE sha1=:sha1;",
1090 -1, &stmt_size_, NULL);
1091
1/2
✓ Branch 1 taken 2438 times.
✗ Branch 2 not taken.
 2438 sqlite3_prepare_v2(database_, "DELETE FROM cache_catalog WHERE sha1=:sha1;",
1092 -1, &stmt_rm_, NULL);
1093
1/2
✓ Branch 1 taken 2438 times.
✗ Branch 2 not taken.
 2438 sqlite3_prepare_v2(database_,
1094 "DELETE FROM cache_catalog WHERE acseq<=:a AND pinned<>2;",
1095 -1, &stmt_rm_batch_, NULL);
1096
1/2
✓ Branch 2 taken 2438 times.
✗ Branch 3 not taken.
 2438 sqlite3_prepare_v2(database_,
1097
1/2
✓ Branch 2 taken 2438 times.
✗ Branch 3 not taken.
 4876 (std::string("SELECT sha1, size, acseq FROM cache_catalog "
1098 "WHERE pinned<>2 AND acseq>=:a "
1099 "ORDER BY acseq ASC "
1100 "LIMIT ")
1101
3/6
✓ Branch 1 taken 2438 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2438 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 2438 times.
✗ Branch 8 not taken.
 9752 + StringifyInt(kEvictBatchSize) + ";")
1102 .c_str(),
1103 -1, &stmt_lru_, NULL);
1104
1/2
✓ Branch 2 taken 2438 times.
✗ Branch 3 not taken.
 2438 sqlite3_prepare_v2(database_,
1105 ("SELECT path FROM cache_catalog WHERE type="
1106
3/6
✓ Branch 1 taken 2438 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2438 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 2438 times.
✗ Branch 8 not taken.
 4876 + StringifyInt(kFileRegular) + ";")
1107 .c_str(),
1108 -1, &stmt_list_, NULL);
1109
1/2
✓ Branch 1 taken 2438 times.
✗ Branch 2 not taken.
 2438 sqlite3_prepare_v2(database_,
1110 "SELECT path FROM cache_catalog WHERE pinned<>0;", -1,
1111 &stmt_list_pinned_, NULL);
1112
1/2
✓ Branch 1 taken 2438 times.
✗ Branch 2 not taken.
 2438 sqlite3_prepare_v2(database_,
1113 "SELECT path FROM cache_catalog WHERE acseq < 0;", -1,
1114 &stmt_list_volatile_, NULL);
1115
1/2
✓ Branch 2 taken 2438 times.
✗ Branch 3 not taken.
 2438 sqlite3_prepare_v2(database_,
1116 ("SELECT path FROM cache_catalog WHERE type="
1117
3/6
✓ Branch 1 taken 2438 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2438 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 2438 times.
✗ Branch 8 not taken.
 4876 + StringifyInt(kFileCatalog) + ";")
1118 .c_str(),
1119 -1, &stmt_list_catalogs_, NULL);
1120 2438 return true;
1121
1122 105 init_database_fail:
1123
1/2
✓ Branch 1 taken 105 times.
✗ Branch 2 not taken.
 105 sqlite3_close(database_);
1124 105 database_ = NULL;
1125
1/2
✓ Branch 1 taken 105 times.
✗ Branch 2 not taken.
 105 UnlockFile(fd_lock_cachedb_);
1126 105 return false;
1127 2578 }
1128
1129
1130 /**
1131 * Inserts a new file into cache catalog. This file gets a new,
1132 * highest sequence number. Does cache cleanup if necessary.
1133 */
1134 3500669 void PosixQuotaManager::Insert(const shash::Any &any_hash,
1135 const uint64_t size,
1136 const string &description) {
1137 3500669 DoInsert(any_hash, size, description, kInsert);
1138 3500669 }
1139
1140
1141 /**
1142 * Inserts a new file into cache catalog. This file is marked as volatile
1143 * and gets a new highest sequence number with the first bit set. Cache cleanup
1144 * treats these files with priority.
1145 */
1146 140 void PosixQuotaManager::InsertVolatile(const shash::Any &any_hash,
1147 const uint64_t size,
1148 const string &description) {
1149 140 DoInsert(any_hash, size, description, kInsertVolatile);
1150 140 }
1151
1152
1153 /**
1154 * Lists all path names from the cache db.
1155 */
1156 735 vector<string> PosixQuotaManager::List() { return DoList(kList); }
1157
1158
1159 /**
1160 * Lists all pinned files from the cache db.
1161 */
1162 280 vector<string> PosixQuotaManager::ListPinned() { return DoList(kListPinned); }
1163
1164
1165 /**
1166 * Lists all sqlite catalog files from the cache db.
1167 */
1168 105 vector<string> PosixQuotaManager::ListCatalogs() {
1169 105 return DoList(kListCatalogs);
1170 }
1171
1172
1173 /**
1174 * Lists only files flagged as volatile (priority removal)
1175 */
1176 105 vector<string> PosixQuotaManager::ListVolatile() {
1177 105 return DoList(kListVolatile);
1178 }
1179
1180
1181 /**
1182 * Entry point for the shared cache manager process
1183 */
1184 ✗ int PosixQuotaManager::MainCacheManager(int argc, char **argv) {
1185 ✗ LogCvmfs(kLogQuota, kLogDebug, "starting quota manager");
1186 int retval;
1187
1188 ✗ PosixQuotaManager shared_manager(0, 0, "");
1189 ✗ shared_manager.shared_ = true;
1190 ✗ shared_manager.spawned_ = true;
1191 ✗ shared_manager.pinned_ = 0;
1192
1193 // Process command line arguments
1194 ✗ ParseDirectories(string(argv[2]),
1195 &shared_manager.cache_dir_,
1196 &shared_manager.workspace_dir_);
1197 ✗ const int pipe_boot = String2Int64(argv[3]);
1198 ✗ const int pipe_handshake = String2Int64(argv[4]);
1199 ✗ shared_manager.limit_ = String2Int64(argv[5]);
1200 ✗ shared_manager.cleanup_threshold_ = String2Int64(argv[6]);
1201 ✗ const int foreground = String2Int64(argv[7]);
1202 ✗ const int syslog_level = String2Int64(argv[8]);
1203 ✗ const int syslog_facility = String2Int64(argv[9]);
1204 ✗ vector<string> logfiles = SplitString(argv[10], ':');
1205
1206 ✗ SetLogSyslogLevel(syslog_level);
1207 ✗ SetLogSyslogFacility(syslog_facility);
1208 ✗ if ((logfiles.size() > 0) && (logfiles[0] != ""))
1209 ✗ SetLogDebugFile(logfiles[0] + ".cachemgr");
1210 ✗ if (logfiles.size() > 1)
1211 ✗ SetLogMicroSyslog(logfiles[1]);
1212
1213 ✗ if (!foreground)
1214 ✗ Daemonize();
1215
1216 ✗ if ((geteuid() != 0) && SetuidCapabilityPermitted()) {
1217 // Permanently drop credentials
1218 ✗ const std::vector<cap_value_t> nocaps;
1219 ✗ assert(ClearPermittedCapabilities(nocaps, nocaps));
1220 // Leave this process ptraceable
1221 ✗ assert(platform_set_dumpable());
1222 // but without core dumps
1223 ✗ assert(SetLimitCore(0));
1224 }
1225
1226 ✗ const UniquePtr<Watchdog> watchdog(Watchdog::Create(NULL, false /* needs_read_environ */));
1227 ✗ assert(watchdog.IsValid());
1228 ✗ watchdog->Spawn("./stacktrace.cachemgr");
1229
1230 // Initialize pipe, open non-blocking as cvmfs is not yet connected
1231 ✗ const int fd_lockfile_fifo = LockFile(shared_manager.workspace_dir_
1232 ✗ + "/lock_cachemgr.fifo");
1233 ✗ if (fd_lockfile_fifo < 0) {
1234 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1235 "could not open lock file "
1236 "%s (%d)",
1237 ✗ (shared_manager.workspace_dir_ + "/lock_cachemgr.fifo").c_str(),
1238 ✗ errno);
1239 ✗ return 1;
1240 }
1241 ✗ const string crash_guard = shared_manager.cache_dir_ + "/cachemgr.running";
1242 ✗ const bool rebuild = FileExists(crash_guard);
1243 ✗ retval = open(crash_guard.c_str(), O_RDONLY | O_CREAT, 0600);
1244 ✗ if (retval < 0) {
1245 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1246 "failed to create shared cache manager crash guard");
1247 ✗ UnlockFile(fd_lockfile_fifo);
1248 ✗ return 1;
1249 }
1250 ✗ close(retval);
1251
1252 // Redirect SQlite temp directory to cache (global variable)
1253 ✗ const string tmp_dir = shared_manager.workspace_dir_;
1254 ✗ sqlite3_temp_directory = static_cast<char *>(
1255 ✗ sqlite3_malloc(tmp_dir.length() + 1));
1256 ✗ snprintf(sqlite3_temp_directory, tmp_dir.length() + 1, "%s", tmp_dir.c_str());
1257
1258 // Cleanup leftover named pipes
1259 ✗ shared_manager.CleanupPipes();
1260
1261 ✗ if (!shared_manager.InitDatabase(rebuild)) {
1262 ✗ UnlockFile(fd_lockfile_fifo);
1263 ✗ return 1;
1264 }
1265 ✗ shared_manager.CheckFreeSpace();
1266
1267 // Save protocol revision to file. If the file is not found, it indicates
1268 // to the client that the cache manager is from times before the protocol
1269 // was versioned.
1270 const string protocol_revision_path = shared_manager.workspace_dir_
1271 ✗ + "/cachemgr.protocol";
1272 ✗ retval = open(protocol_revision_path.c_str(), O_WRONLY | O_CREAT, 0600);
1273 ✗ if (retval < 0) {
1274 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1275 ✗ "failed to open protocol revision file (%d)", errno);
1276 ✗ UnlockFile(fd_lockfile_fifo);
1277 ✗ return 1;
1278 }
1279 ✗ const string revision = StringifyInt(kProtocolRevision);
1280 ✗ const int written = write(retval, revision.data(), revision.length());
1281 ✗ close(retval);
1282 ✗ if ((written < 0) || static_cast<unsigned>(written) != revision.length()) {
1283 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1284 ✗ "failed to write protocol revision (%d)", errno);
1285 ✗ UnlockFile(fd_lockfile_fifo);
1286 ✗ return 1;
1287 }
1288
1289 ✗ const string fifo_path = shared_manager.workspace_dir_ + "/cachemgr";
1290 ✗ shared_manager.pipe_lru_[0] = open(fifo_path.c_str(), O_RDONLY | O_NONBLOCK);
1291 ✗ if (shared_manager.pipe_lru_[0] < 0) {
1292 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to listen on FIFO %s (%d)",
1293 ✗ fifo_path.c_str(), errno);
1294 ✗ UnlockFile(fd_lockfile_fifo);
1295 ✗ return 1;
1296 }
1297 ✗ Nonblock2Block(shared_manager.pipe_lru_[0]);
1298 ✗ LogCvmfs(kLogQuota, kLogDebug, "shared cache manager listening");
1299
1300 ✗ char buf = 'C';
1301 ✗ WritePipe(pipe_boot, &buf, 1);
1302 ✗ close(pipe_boot);
1303
1304 ✗ ReadPipe(pipe_handshake, &buf, 1);
1305 ✗ close(pipe_handshake);
1306 ✗ LogCvmfs(kLogQuota, kLogDebug, "shared cache manager handshake done");
1307
1308 // Ensure that broken pipes from clients do not kill the cache manager
1309 ✗ signal(SIGPIPE, SIG_IGN);
1310 // Don't let Ctrl-C ungracefully kill interactive session
1311 ✗ signal(SIGINT, SIG_IGN);
1312
1313 ✗ shared_manager.MainCommandServer(&shared_manager);
1314 ✗ unlink(fifo_path.c_str());
1315 ✗ unlink(protocol_revision_path.c_str());
1316 ✗ shared_manager.CloseDatabase();
1317 ✗ unlink(crash_guard.c_str());
1318 ✗ UnlockFile(fd_lockfile_fifo);
1319
1320 ✗ if (sqlite3_temp_directory) {
1321 ✗ sqlite3_free(sqlite3_temp_directory);
1322 ✗ sqlite3_temp_directory = NULL;
1323 }
1324
1325 ✗ return 0;
1326 }
1327
1328
1329 1085 void *PosixQuotaManager::MainCommandServer(void *data) {
1330 1085 PosixQuotaManager *quota_mgr = static_cast<PosixQuotaManager *>(data);
1331
1332
1/2
✓ Branch 1 taken 1085 times.
✗ Branch 2 not taken.
 1085 LogCvmfs(kLogQuota, kLogDebug, "starting quota manager");
1333
1/2
✓ Branch 1 taken 1085 times.
✗ Branch 2 not taken.
 1085 sqlite3_soft_heap_limit(quota_mgr->kSqliteMemPerThread);
1334
1335
2/2
✓ Branch 1 taken 34720 times.
✓ Branch 2 taken 1085 times.
 35805 LruCommand command_buffer[kCommandBufferSize];
1336 char description_buffer[kCommandBufferSize * kMaxDescription];
1337 1085 unsigned num_commands = 0;
1338
1339
1/2
✓ Branch 1 taken 5255635 times.
✗ Branch 2 not taken.
 5255635 while (read(quota_mgr->pipe_lru_[0], &command_buffer[num_commands],
1340 sizeof(command_buffer[0]))
1341
2/2
✓ Branch 0 taken 5254550 times.
✓ Branch 1 taken 1085 times.
 5255635 == sizeof(command_buffer[0])) {
1342 5254550 const CommandType command_type = command_buffer[num_commands].command_type;
1343
1/2
✓ Branch 1 taken 5254550 times.
✗ Branch 2 not taken.
 5254550 LogCvmfs(kLogQuota, kLogDebug, "received command %d", command_type);
1344 5254550 const uint64_t size = command_buffer[num_commands].GetSize();
1345
1346 // Inserts and pins come with a description (usually a path)
1347
4/4
✓ Branch 0 taken 1753955 times.
✓ Branch 1 taken 3500595 times.
✓ Branch 2 taken 1753815 times.
✓ Branch 3 taken 140 times.
 5254550 if ((command_type == kInsert) || (command_type == kInsertVolatile)
1348
4/4
✓ Branch 0 taken 1753745 times.
✓ Branch 1 taken 70 times.
✓ Branch 2 taken 385 times.
✓ Branch 3 taken 1753360 times.
 1753815 || (command_type == kPin) || (command_type == kPinRegular)) {
1349 3501190 const int desc_length = command_buffer[num_commands].desc_length;
1350 3501190 ReadPipe(quota_mgr->pipe_lru_[0],
1351
1/2
✓ Branch 1 taken 3501190 times.
✗ Branch 2 not taken.
 3501190 &description_buffer[kMaxDescription * num_commands],
1352 desc_length);
1353 }
1354
1355 // The protocol revision is returned immediately
1356
2/2
✓ Branch 0 taken 35 times.
✓ Branch 1 taken 5254515 times.
 5254550 if (command_type == kGetProtocolRevision) {
1357
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 const int return_pipe = quota_mgr->BindReturnPipe(
1358 command_buffer[num_commands].return_pipe);
1359
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 35 times.
 35 if (return_pipe < 0)
1360 ✗ continue;
1361
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 WritePipe(return_pipe, &quota_mgr->kProtocolRevision,
1362 sizeof(quota_mgr->kProtocolRevision));
1363
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 quota_mgr->UnbindReturnPipe(return_pipe);
1364 35 continue;
1365 35 }
1366
1367 // Register a new mountpoint
1368
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5254515 times.
 5254515 if (command_type == kRegisterMountpoint) {
1369 ✗ size_t mp_size = 0;
1370 ✗ ReadPipe(quota_mgr->pipe_lru_[0], &mp_size, sizeof(size_t));
1371 ✗ std::string mountpoint(mp_size, '\0');
1372 ✗ ReadPipe(quota_mgr->pipe_lru_[0], (void *)mountpoint.data(), mp_size);
1373 ✗ quota_mgr->mountpoints_.push_back(mountpoint);
1374 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
1375 "Mountpoint %s registered in the group", mountpoint.c_str());
1376 ✗ continue;
1377 }
1378
1379 // Set Cleanup Policy
1380
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5254515 times.
 5254515 if (command_type == kSetCleanupPolicy) {
1381 ✗ bool policy = false;
1382 ✗ ReadPipe(quota_mgr->pipe_lru_[0], &policy, sizeof(bool));
1383 ✗ quota_mgr->cleanup_unused_first_ = policy;
1384 ✗ continue;
1385 }
1386 // Mountpoints are returned immediately
1387
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5254515 times.
 5254515 if (command_type == kGetMountpoints) {
1388 ✗ const int return_pipe = quota_mgr->BindReturnPipe(
1389 command_buffer[num_commands].return_pipe);
1390 ✗ if (return_pipe < 0)
1391 ✗ continue;
1392
1393 ✗ std::string mps;
1394 ✗ for (auto it = quota_mgr->mountpoints_.begin();
1395 ✗ it != quota_mgr->mountpoints_.end();
1396 ✗ ++it) {
1397 ✗ mps += *it + "\n";
1398 }
1399 ✗ size_t mp_size = mps.size();
1400 ✗ WritePipe(return_pipe, &mp_size, sizeof(size_t));
1401 ✗ WritePipe(return_pipe, mps.c_str(), mp_size);
1402 ✗ quota_mgr->UnbindReturnPipe(return_pipe);
1403 ✗ continue;
1404 }
1405
1406 // Group hashes are returned immediately
1407
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5254515 times.
 5254515 if (command_type == kGetGroupHashes) {
1408 ✗ const int return_pipe = quota_mgr->BindReturnPipe(
1409 command_buffer[num_commands].return_pipe);
1410 ✗ if (return_pipe < 0)
1411 ✗ continue;
1412
1413 ✗ std::vector<shash::Short> gh = quota_mgr->CollectAllOpenHashes();
1414 ✗ std::string result;
1415 ✗ for (auto it = gh.begin(); it != gh.end(); ++it) {
1416 ✗ result += (*it).ToString() + "\n";
1417 }
1418 ✗ size_t result_size = result.size();
1419 ✗ WritePipe(return_pipe, &result_size, sizeof(size_t));
1420 ✗ WritePipe(return_pipe, result.c_str(), result_size);
1421 ✗ quota_mgr->UnbindReturnPipe(return_pipe);
1422 ✗ continue;
1423 }
1424
1425 // The cleanup rate is returned immediately
1426
2/2
✓ Branch 0 taken 140 times.
✓ Branch 1 taken 5254375 times.
 5254515 if (command_type == kCleanupRate) {
1427
1/2
✓ Branch 1 taken 140 times.
✗ Branch 2 not taken.
 140 const int return_pipe = quota_mgr->BindReturnPipe(
1428 command_buffer[num_commands].return_pipe);
1429
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 140 times.
 140 if (return_pipe < 0)
1430 ✗ continue;
1431 const uint64_t
1432 140 period_s = size; // use the size field to transmit the period
1433
1/2
✓ Branch 1 taken 140 times.
✗ Branch 2 not taken.
 140 uint64_t rate = quota_mgr->cleanup_recorder_.GetNoTicks(period_s);
1434
1/2
✓ Branch 1 taken 140 times.
✗ Branch 2 not taken.
 140 WritePipe(return_pipe, &rate, sizeof(rate));
1435
1/2
✓ Branch 1 taken 140 times.
✗ Branch 2 not taken.
 140 quota_mgr->UnbindReturnPipe(return_pipe);
1436 140 continue;
1437 140 }
1438
1439
2/2
✓ Branch 0 taken 35 times.
✓ Branch 1 taken 5254340 times.
 5254375 if (command_type == kSetLimit) {
1440
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 const int return_pipe = quota_mgr->BindReturnPipe(
1441 command_buffer[num_commands].return_pipe);
1442
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 35 times.
 35 if (return_pipe < 0)
1443 ✗ continue;
1444 35 quota_mgr->limit_ = size; // use the size field to transmit the size
1445 35 quota_mgr->cleanup_threshold_ = size / 2;
1446
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
1447 "Quota limit set to %lu / threshold %lu", quota_mgr->limit_,
1448 quota_mgr->cleanup_threshold_);
1449 35 bool ret = true;
1450
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 WritePipe(return_pipe, &ret, sizeof(ret));
1451
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 quota_mgr->UnbindReturnPipe(return_pipe);
1452 35 continue;
1453 35 }
1454
1455 // Reservations are handled immediately and "out of band"
1456
2/2
✓ Branch 0 taken 490 times.
✓ Branch 1 taken 5253850 times.
 5254340 if (command_type == kReserve) {
1457 490 bool success = true;
1458
1/2
✓ Branch 1 taken 490 times.
✗ Branch 2 not taken.
 490 const int return_pipe = quota_mgr->BindReturnPipe(
1459 command_buffer[num_commands].return_pipe);
1460
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 490 times.
 490 if (return_pipe < 0)
1461 ✗ continue;
1462
1463
1/2
✓ Branch 1 taken 490 times.
✗ Branch 2 not taken.
 490 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1464
1/2
✓ Branch 1 taken 490 times.
✗ Branch 2 not taken.
 490 const string hash_str(hash.ToString());
1465
1/2
✓ Branch 2 taken 490 times.
✗ Branch 3 not taken.
 490 LogCvmfs(kLogQuota, kLogDebug, "reserve %lu bytes for %s", size,
1466 hash_str.c_str());
1467
1468
1/2
✓ Branch 1 taken 490 times.
✗ Branch 2 not taken.
 490 if (quota_mgr->pinned_chunks_.find(hash)
1469
2/2
✓ Branch 2 taken 420 times.
✓ Branch 3 taken 70 times.
 980 == quota_mgr->pinned_chunks_.end()) {
1470
2/2
✓ Branch 0 taken 35 times.
✓ Branch 1 taken 385 times.
 420 if ((quota_mgr->pinned_ + size) > quota_mgr->cleanup_threshold_) {
1471
1/2
✓ Branch 2 taken 35 times.
✗ Branch 3 not taken.
 35 LogCvmfs(kLogQuota, kLogDebug,
1472 "failed to insert %s (pinned), no space", hash_str.c_str());
1473 35 success = false;
1474 } else {
1475
1/2
✓ Branch 1 taken 385 times.
✗ Branch 2 not taken.
 385 quota_mgr->pinned_chunks_[hash] = size;
1476 385 quota_mgr->pinned_ += size;
1477
1/2
✓ Branch 1 taken 385 times.
✗ Branch 2 not taken.
 385 quota_mgr->CheckHighPinWatermark();
1478 }
1479 }
1480
1481
1/2
✓ Branch 1 taken 490 times.
✗ Branch 2 not taken.
 490 WritePipe(return_pipe, &success, sizeof(success));
1482
1/2
✓ Branch 1 taken 490 times.
✗ Branch 2 not taken.
 490 quota_mgr->UnbindReturnPipe(return_pipe);
1483 490 continue;
1484 490 }
1485
1486 // Back channels are also handled out of band
1487
2/2
✓ Branch 0 taken 140 times.
✓ Branch 1 taken 5253710 times.
 5253850 if (command_type == kRegisterBackChannel) {
1488
1/2
✓ Branch 1 taken 140 times.
✗ Branch 2 not taken.
 140 const int return_pipe = quota_mgr->BindReturnPipe(
1489 command_buffer[num_commands].return_pipe);
1490
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 140 times.
 140 if (return_pipe < 0)
1491 ✗ continue;
1492
1493
1/2
✓ Branch 1 taken 140 times.
✗ Branch 2 not taken.
 140 quota_mgr->UnlinkReturnPipe(command_buffer[num_commands].return_pipe);
1494
1/2
✓ Branch 1 taken 140 times.
✗ Branch 2 not taken.
 140 Block2Nonblock(return_pipe); // back channels are opportunistic
1495
1/2
✓ Branch 1 taken 140 times.
✗ Branch 2 not taken.
 140 shash::Md5 hash;
1496 140 memcpy(hash.digest, command_buffer[num_commands].digest,
1497 140 shash::kDigestSizes[shash::kMd5]);
1498
1499 140 quota_mgr->LockBackChannels();
1500 const map<shash::Md5, int>::const_iterator
1501
1/2
✓ Branch 1 taken 140 times.
✗ Branch 2 not taken.
 140 iter = quota_mgr->back_channels_.find(hash);
1502
1/2
✗ Branch 3 not taken.
✓ Branch 4 taken 140 times.
 140 if (iter != quota_mgr->back_channels_.end()) {
1503 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
1504 ✗ "closing left-over back channel %s", hash.ToString().c_str());
1505 ✗ close(iter->second);
1506 }
1507
1/2
✓ Branch 1 taken 140 times.
✗ Branch 2 not taken.
 140 quota_mgr->back_channels_[hash] = return_pipe;
1508 140 quota_mgr->UnlockBackChannels();
1509
1510 140 char success = 'S';
1511
1/2
✓ Branch 1 taken 140 times.
✗ Branch 2 not taken.
 140 WritePipe(return_pipe, &success, sizeof(success));
1512
1/2
✓ Branch 2 taken 140 times.
✗ Branch 3 not taken.
 140 LogCvmfs(kLogQuota, kLogDebug, "register back channel %s on fd %d",
1513
1/2
✓ Branch 1 taken 140 times.
✗ Branch 2 not taken.
 280 hash.ToString().c_str(), return_pipe);
1514
1515 140 continue;
1516 140 }
1517
1518
2/2
✓ Branch 0 taken 70 times.
✓ Branch 1 taken 5253640 times.
 5253710 if (command_type == kUnregisterBackChannel) {
1519
1/2
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
 70 shash::Md5 hash;
1520 70 memcpy(hash.digest, command_buffer[num_commands].digest,
1521 70 shash::kDigestSizes[shash::kMd5]);
1522
1523 70 quota_mgr->LockBackChannels();
1524 const map<shash::Md5, int>::iterator iter = quota_mgr->back_channels_
1525
1/2
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
 70 .find(hash);
1526
1/2
✓ Branch 2 taken 70 times.
✗ Branch 3 not taken.
 70 if (iter != quota_mgr->back_channels_.end()) {
1527
1/2
✓ Branch 2 taken 70 times.
✗ Branch 3 not taken.
 70 LogCvmfs(kLogQuota, kLogDebug, "closing back channel %s",
1528
1/2
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
 140 hash.ToString().c_str());
1529
1/2
✓ Branch 2 taken 70 times.
✗ Branch 3 not taken.
 70 close(iter->second);
1530
1/2
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
 70 quota_mgr->back_channels_.erase(iter);
1531 } else {
1532 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
1533 ✗ "did not find back channel %s", hash.ToString().c_str());
1534 }
1535 70 quota_mgr->UnlockBackChannels();
1536
1537 70 continue;
1538 70 }
1539
1540 // Unpinnings are also handled immediately with respect to the pinned gauge
1541
2/2
✓ Branch 0 taken 70 times.
✓ Branch 1 taken 5253570 times.
 5253640 if (command_type == kUnpin) {
1542
1/2
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
 70 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1543
1/2
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
 70 const string hash_str(hash.ToString());
1544
1545 const map<shash::Any, uint64_t>::iterator iter = quota_mgr->pinned_chunks_
1546
1/2
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
 70 .find(hash);
1547
1/2
✓ Branch 2 taken 70 times.
✗ Branch 3 not taken.
 70 if (iter != quota_mgr->pinned_chunks_.end()) {
1548 70 quota_mgr->pinned_ -= iter->second;
1549
1/2
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
 70 quota_mgr->pinned_chunks_.erase(iter);
1550 // It can happen that files get pinned that were removed from the cache
1551 // (see cache.cc). We fix this at this point, where we remove such
1552 // entries from the cache database.
1553
2/4
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 70 times.
✗ Branch 5 not taken.
 140 if (!FileExists(quota_mgr->cache_dir_ + "/"
1554
4/6
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 70 times.
✗ Branch 5 not taken.
✓ Branch 9 taken 35 times.
✓ Branch 10 taken 35 times.
 210 + hash.MakePathWithoutSuffix())) {
1555
1/2
✓ Branch 2 taken 35 times.
✗ Branch 3 not taken.
 35 LogCvmfs(kLogQuota, kLogDebug,
1556 "remove orphaned pinned hash %s from cache database",
1557 hash_str.c_str());
1558
1/2
✓ Branch 2 taken 35 times.
✗ Branch 3 not taken.
 35 sqlite3_bind_text(quota_mgr->stmt_size_, 1, &hash_str[0],
1559 35 hash_str.length(), SQLITE_STATIC);
1560 int retval;
1561
2/4
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 35 times.
✗ Branch 4 not taken.
 35 if ((retval = sqlite3_step(quota_mgr->stmt_size_)) == SQLITE_ROW) {
1562
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 const uint64_t size = sqlite3_column_int64(quota_mgr->stmt_size_,
1563 35 0);
1564
1/2
✓ Branch 2 taken 35 times.
✗ Branch 3 not taken.
 35 sqlite3_bind_text(quota_mgr->stmt_rm_, 1, &(hash_str[0]),
1565 35 hash_str.length(), SQLITE_STATIC);
1566
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 retval = sqlite3_step(quota_mgr->stmt_rm_);
1567
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 35 if ((retval == SQLITE_DONE) || (retval == SQLITE_OK)) {
1568 35 quota_mgr->gauge_ -= size;
1569 } else {
1570 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1571 "failed to delete %s (%d)", hash_str.c_str(), retval);
1572 }
1573
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 sqlite3_reset(quota_mgr->stmt_rm_);
1574 }
1575
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 sqlite3_reset(quota_mgr->stmt_size_);
1576 }
1577 } else {
1578 ✗ LogCvmfs(kLogQuota, kLogDebug, "this chunk was not pinned");
1579 }
1580 70 }
1581
1582 // Immediate commands trigger flushing of the buffer
1583 5253640 const bool immediate_command = (command_type == kCleanup)
1584
2/2
✓ Branch 0 taken 5252590 times.
✓ Branch 1 taken 735 times.
 5253325 || (command_type == kList)
1585
2/2
✓ Branch 0 taken 5252310 times.
✓ Branch 1 taken 280 times.
 5252590 || (command_type == kListPinned)
1586
2/2
✓ Branch 0 taken 5252205 times.
✓ Branch 1 taken 105 times.
 5252310 || (command_type == kListCatalogs)
1587
2/2
✓ Branch 0 taken 5252100 times.
✓ Branch 1 taken 105 times.
 5252205 || (command_type == kListVolatile)
1588
2/2
✓ Branch 0 taken 5251995 times.
✓ Branch 1 taken 105 times.
 5252100 || (command_type == kRemove)
1589
2/2
✓ Branch 0 taken 5251365 times.
✓ Branch 1 taken 630 times.
 5251995 || (command_type == kStatus)
1590
1/2
✓ Branch 0 taken 5251365 times.
✗ Branch 1 not taken.
 5251365 || (command_type == kLimits)
1591
3/4
✓ Branch 0 taken 5253325 times.
✓ Branch 1 taken 315 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 5251365 times.
 10506965 || (command_type == kPid);
1592
2/2
✓ Branch 0 taken 5251365 times.
✓ Branch 1 taken 2275 times.
 5253640 if (!immediate_command)
1593 5251365 num_commands++;
1594
1595
4/4
✓ Branch 0 taken 5089595 times.
✓ Branch 1 taken 164045 times.
✓ Branch 2 taken 2275 times.
✓ Branch 3 taken 5087320 times.
 5253640 if ((num_commands == kCommandBufferSize) || immediate_command) {
1596
1/2
✓ Branch 1 taken 166320 times.
✗ Branch 2 not taken.
 166320 quota_mgr->ProcessCommandBunch(num_commands, command_buffer,
1597 description_buffer);
1598
2/2
✓ Branch 0 taken 164045 times.
✓ Branch 1 taken 2275 times.
 166320 if (!immediate_command)
1599 164045 num_commands = 0;
1600 }
1601
1602
2/2
✓ Branch 0 taken 2275 times.
✓ Branch 1 taken 5251365 times.
 5253640 if (immediate_command) {
1603 // Process cleanup, listings
1604
1/2
✓ Branch 1 taken 2275 times.
✗ Branch 2 not taken.
 2275 const int return_pipe = quota_mgr->BindReturnPipe(
1605 command_buffer[num_commands].return_pipe);
1606
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2275 times.
 2275 if (return_pipe < 0) {
1607 ✗ num_commands = 0;
1608 ✗ continue;
1609 }
1610
1611 int retval;
1612 2275 sqlite3_stmt *this_stmt_list = NULL;
1613
7/10
✓ Branch 0 taken 105 times.
✓ Branch 1 taken 315 times.
✓ Branch 2 taken 735 times.
✓ Branch 3 taken 280 times.
✓ Branch 4 taken 105 times.
✓ Branch 5 taken 105 times.
✓ Branch 6 taken 630 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
 2275 switch (command_type) {
1614 105 case kRemove: {
1615
1/2
✓ Branch 1 taken 105 times.
✗ Branch 2 not taken.
 105 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1616
1/2
✓ Branch 1 taken 105 times.
✗ Branch 2 not taken.
 105 const string hash_str = hash.ToString();
1617
1/2
✓ Branch 2 taken 105 times.
✗ Branch 3 not taken.
 105 LogCvmfs(kLogQuota, kLogDebug, "manually removing %s",
1618 hash_str.c_str());
1619 105 bool success = false;
1620
1621
1/2
✓ Branch 2 taken 105 times.
✗ Branch 3 not taken.
 105 sqlite3_bind_text(quota_mgr->stmt_size_, 1, &hash_str[0],
1622 105 hash_str.length(), SQLITE_STATIC);
1623 int retval;
1624
3/4
✓ Branch 1 taken 105 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 70 times.
✓ Branch 4 taken 35 times.
 105 if ((retval = sqlite3_step(quota_mgr->stmt_size_)) == SQLITE_ROW) {
1625
1/2
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
 70 const uint64_t size = sqlite3_column_int64(quota_mgr->stmt_size_,
1626 70 0);
1627
1/2
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
 70 const uint64_t is_pinned = sqlite3_column_int64(
1628 70 quota_mgr->stmt_size_, 1);
1629
1630
1/2
✓ Branch 2 taken 70 times.
✗ Branch 3 not taken.
 70 sqlite3_bind_text(quota_mgr->stmt_rm_, 1, &(hash_str[0]),
1631 70 hash_str.length(), SQLITE_STATIC);
1632
1/2
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
 70 retval = sqlite3_step(quota_mgr->stmt_rm_);
1633
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 70 if ((retval == SQLITE_DONE) || (retval == SQLITE_OK)) {
1634 70 success = true;
1635 70 quota_mgr->gauge_ -= size;
1636
2/2
✓ Branch 0 taken 35 times.
✓ Branch 1 taken 35 times.
 70 if (is_pinned) {
1637
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 quota_mgr->pinned_chunks_.erase(hash);
1638 35 quota_mgr->pinned_ -= size;
1639 }
1640 } else {
1641 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1642 "failed to delete %s (%d)", hash_str.c_str(), retval);
1643 }
1644
1/2
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
 70 sqlite3_reset(quota_mgr->stmt_rm_);
1645 } else {
1646 // File does not exist
1647 35 success = true;
1648 }
1649
1/2
✓ Branch 1 taken 105 times.
✗ Branch 2 not taken.
 105 sqlite3_reset(quota_mgr->stmt_size_);
1650
1651
1/2
✓ Branch 1 taken 105 times.
✗ Branch 2 not taken.
 105 WritePipe(return_pipe, &success, sizeof(success));
1652 105 break;
1653 105 }
1654 315 case kCleanup:
1655
1/2
✓ Branch 1 taken 315 times.
✗ Branch 2 not taken.
 315 retval = quota_mgr->DoCleanup(size);
1656
1/2
✓ Branch 1 taken 315 times.
✗ Branch 2 not taken.
 315 WritePipe(return_pipe, &retval, sizeof(retval));
1657 315 break;
1658 735 case kList:
1659
1/2
✓ Branch 0 taken 735 times.
✗ Branch 1 not taken.
 735 if (!this_stmt_list)
1660 735 this_stmt_list = quota_mgr->stmt_list_;
1661 case kListPinned:
1662
2/2
✓ Branch 0 taken 280 times.
✓ Branch 1 taken 735 times.
 1015 if (!this_stmt_list)
1663 280 this_stmt_list = quota_mgr->stmt_list_pinned_;
1664 case kListCatalogs:
1665
2/2
✓ Branch 0 taken 105 times.
✓ Branch 1 taken 1015 times.
 1120 if (!this_stmt_list)
1666 105 this_stmt_list = quota_mgr->stmt_list_catalogs_;
1667 case kListVolatile:
1668
2/2
✓ Branch 0 taken 105 times.
✓ Branch 1 taken 1120 times.
 1225 if (!this_stmt_list)
1669 105 this_stmt_list = quota_mgr->stmt_list_volatile_;
1670
1671 // Pipe back the list, one by one
1672 int length;
1673
3/4
✓ Branch 1 taken 3502590 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 3501365 times.
✓ Branch 4 taken 1225 times.
 3502590 while (sqlite3_step(this_stmt_list) == SQLITE_ROW) {
1674
1/2
✓ Branch 2 taken 3501365 times.
✗ Branch 3 not taken.
 3501365 string path = "(NULL)";
1675
2/4
✓ Branch 1 taken 3501365 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 3501365 times.
✗ Branch 4 not taken.
 3501365 if (sqlite3_column_type(this_stmt_list, 0) != SQLITE_NULL) {
1676 7002730 path = string(reinterpret_cast<const char *>(
1677
2/4
✓ Branch 1 taken 3501365 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3501365 times.
✗ Branch 5 not taken.
 3501365 sqlite3_column_text(this_stmt_list, 0)));
1678 }
1679 3501365 length = path.length();
1680
1/2
✓ Branch 1 taken 3501365 times.
✗ Branch 2 not taken.
 3501365 WritePipe(return_pipe, &length, sizeof(length));
1681
1/2
✓ Branch 0 taken 3501365 times.
✗ Branch 1 not taken.
 3501365 if (length > 0)
1682
2/4
✓ Branch 1 taken 3501365 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3501365 times.
✗ Branch 5 not taken.
 3501365 WritePipe(return_pipe, &path[0], length);
1683 3501365 }
1684 1225 length = -1;
1685
1/2
✓ Branch 1 taken 1225 times.
✗ Branch 2 not taken.
 1225 WritePipe(return_pipe, &length, sizeof(length));
1686
1/2
✓ Branch 1 taken 1225 times.
✗ Branch 2 not taken.
 1225 sqlite3_reset(this_stmt_list);
1687 1225 break;
1688 630 case kStatus:
1689
1/2
✓ Branch 1 taken 630 times.
✗ Branch 2 not taken.
 630 WritePipe(return_pipe, &quota_mgr->gauge_, sizeof(quota_mgr->gauge_));
1690
1/2
✓ Branch 1 taken 630 times.
✗ Branch 2 not taken.
 630 WritePipe(return_pipe, &quota_mgr->pinned_,
1691 sizeof(quota_mgr->pinned_));
1692 630 break;
1693 ✗ case kLimits:
1694 ✗ WritePipe(return_pipe, &quota_mgr->limit_, sizeof(quota_mgr->limit_));
1695 ✗ WritePipe(return_pipe, &quota_mgr->cleanup_threshold_,
1696 sizeof(quota_mgr->cleanup_threshold_));
1697 ✗ break;
1698 ✗ case kPid: {
1699 ✗ pid_t pid = getpid();
1700 ✗ WritePipe(return_pipe, &pid, sizeof(pid));
1701 ✗ break;
1702 }
1703 ✗ default:
1704 ✗ PANIC(NULL); // other types are handled by the bunch processor
1705 }
1706
1/2
✓ Branch 1 taken 2275 times.
✗ Branch 2 not taken.
 2275 quota_mgr->UnbindReturnPipe(return_pipe);
1707 2275 num_commands = 0;
1708 }
1709 }
1710
1711
1/2
✓ Branch 1 taken 1085 times.
✗ Branch 2 not taken.
 1085 LogCvmfs(kLogQuota, kLogDebug, "stopping cache manager (%d)", errno);
1712
1/2
✓ Branch 1 taken 1085 times.
✗ Branch 2 not taken.
 1085 close(quota_mgr->pipe_lru_[0]);
1713
1/2
✓ Branch 1 taken 1085 times.
✗ Branch 2 not taken.
 1085 quota_mgr->ProcessCommandBunch(num_commands, command_buffer,
1714 description_buffer);
1715
1716 // Unpin
1717 1085 command_buffer[0].command_type = kTouch;
1718 1085 for (map<shash::Any, uint64_t>::const_iterator
1719 1085 i = quota_mgr->pinned_chunks_.begin(),
1720 1085 iEnd = quota_mgr->pinned_chunks_.end();
1721
2/2
✓ Branch 1 taken 350 times.
✓ Branch 2 taken 1085 times.
 1435 i != iEnd;
1722 350 ++i) {
1723
1/2
✓ Branch 2 taken 350 times.
✗ Branch 3 not taken.
 350 command_buffer[0].StoreHash(i->first);
1724
1/2
✓ Branch 1 taken 350 times.
✗ Branch 2 not taken.
 350 quota_mgr->ProcessCommandBunch(1, command_buffer, description_buffer);
1725 }
1726
1727 1085 return NULL;
1728 }
1729
1730
1731 3220 void PosixQuotaManager::MakeReturnPipe(int pipe[2]) {
1732
2/2
✓ Branch 0 taken 3115 times.
✓ Branch 1 taken 105 times.
 3220 if (!shared_) {
1733 3115 MakePipe(pipe);
1734 3115 return;
1735 }
1736
1737 // Create FIFO in cache directory, store path name (number) in pipe write end
1738 105 int i = 0;
1739 int retval;
1740 do {
1741
2/4
✓ Branch 2 taken 140 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 140 times.
✗ Branch 6 not taken.
 140 retval = mkfifo((workspace_dir_ + "/pipe" + StringifyInt(i)).c_str(), 0600);
1742 140 pipe[1] = i;
1743 140 i++;
1744
3/4
✓ Branch 0 taken 35 times.
✓ Branch 1 taken 105 times.
✓ Branch 2 taken 35 times.
✗ Branch 3 not taken.
 140 } while ((retval == -1) && (errno == EEXIST));
1745
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 105 times.
 105 assert(retval == 0);
1746
1747 // Connect reader's end
1748
3/6
✓ Branch 2 taken 105 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 105 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 105 times.
✗ Branch 10 not taken.
 105 pipe[0] = open((workspace_dir_ + "/pipe" + StringifyInt(pipe[1])).c_str(),
1749 O_RDONLY | O_NONBLOCK);
1750
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 105 times.
 105 assert(pipe[0] >= 0);
1751 105 Nonblock2Block(pipe[0]);
1752 }
1753
1754
1755 2543 void PosixQuotaManager::ParseDirectories(const std::string cache_workspace,
1756 std::string *cache_dir,
1757 std::string *workspace_dir) {
1758
1/2
✓ Branch 1 taken 2543 times.
✗ Branch 2 not taken.
 2543 vector<string> dir_tokens(SplitString(cache_workspace, ':'));
1759
2/3
✓ Branch 1 taken 2473 times.
✓ Branch 2 taken 70 times.
✗ Branch 3 not taken.
 2543 switch (dir_tokens.size()) {
1760 2473 case 1:
1761
2/4
✓ Branch 2 taken 2473 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2473 times.
✗ Branch 6 not taken.
 2473 *cache_dir = *workspace_dir = dir_tokens[0];
1762 2473 break;
1763 70 case 2:
1764
1/2
✓ Branch 2 taken 70 times.
✗ Branch 3 not taken.
 70 *cache_dir = dir_tokens[0];
1765
1/2
✓ Branch 2 taken 70 times.
✗ Branch 3 not taken.
 70 *workspace_dir = dir_tokens[1];
1766 70 break;
1767 ✗ default:
1768 ✗ PANIC(NULL);
1769 }
1770 2543 }
1771
1772 35 void PosixQuotaManager::SkipEviction(const EvictCandidate &candidate) {
1773 35 bool res = true;
1774
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 std::string hash_str = candidate.hash.ToString();
1775
1/2
✓ Branch 2 taken 35 times.
✗ Branch 3 not taken.
 35 LogCvmfs(kLogQuota, kLogDebug, "Exclude %s from eviction", hash_str.c_str());
1776
2/4
✓ Branch 2 taken 35 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 35 times.
✗ Branch 6 not taken.
 35 sqlite3_bind_text(stmt_block_, 1, &hash_str[0], hash_str.length(),
1777 SQLITE_STATIC);
1778
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 res = (sqlite3_step(stmt_block_) == SQLITE_DONE);
1779
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 sqlite3_reset(stmt_block_);
1780
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 35 times.
 35 assert(res);
1781 35 }
1782
1783 /**
1784 * Immediately inserts a new pinned catalog. Does cache cleanup if necessary.
1785 *
1786 * \return True on success, false otherwise
1787 */
1788 877 bool PosixQuotaManager::Pin(const shash::Any &hash,
1789 const uint64_t size,
1790 const string &description,
1791 const bool is_catalog) {
1792
3/4
✓ Branch 0 taken 70 times.
✓ Branch 1 taken 807 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 70 times.
 877 assert((size > 0) || !is_catalog);
1793
1794
1/2
✓ Branch 1 taken 877 times.
✗ Branch 2 not taken.
 877 const string hash_str = hash.ToString();
1795
1/2
✓ Branch 3 taken 877 times.
✗ Branch 4 not taken.
 877 LogCvmfs(kLogQuota, kLogDebug, "pin into lru %s, path %s", hash_str.c_str(),
1796 description.c_str());
1797
1798 // Has to run when not yet spawned (cvmfs initialization)
1799
2/2
✓ Branch 0 taken 387 times.
✓ Branch 1 taken 490 times.
 877 if (!spawned_) {
1800 // Code duplication here
1801
3/4
✓ Branch 2 taken 387 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 259 times.
✓ Branch 6 taken 128 times.
 387 if (pinned_chunks_.find(hash) == pinned_chunks_.end()) {
1802
2/2
✓ Branch 0 taken 70 times.
✓ Branch 1 taken 189 times.
 259 if (pinned_ + size > cleanup_threshold_) {
1803
1/2
✓ Branch 2 taken 70 times.
✗ Branch 3 not taken.
 70 LogCvmfs(kLogQuota, kLogDebug, "failed to insert %s (pinned), no space",
1804 hash_str.c_str());
1805 70 return false;
1806 } else {
1807
1/2
✓ Branch 1 taken 189 times.
✗ Branch 2 not taken.
 189 pinned_chunks_[hash] = size;
1808 189 pinned_ += size;
1809
1/2
✓ Branch 1 taken 189 times.
✗ Branch 2 not taken.
 189 CheckHighPinWatermark();
1810 }
1811 }
1812
1/2
✓ Branch 1 taken 317 times.
✗ Branch 2 not taken.
 317 const bool exists = Contains(hash_str);
1813
4/4
✓ Branch 0 taken 189 times.
✓ Branch 1 taken 128 times.
✓ Branch 2 taken 35 times.
✓ Branch 3 taken 154 times.
 317 if (!exists && (gauge_ + size > limit_)) {
1814
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 LogCvmfs(kLogQuota, kLogDebug, "over limit, gauge %lu, file size %lu",
1815 gauge_, size);
1816
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 const int retval = DoCleanup(cleanup_threshold_);
1817
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 35 times.
 35 assert(retval != 0);
1818 }
1819
1/2
✓ Branch 3 taken 317 times.
✗ Branch 4 not taken.
 317 sqlite3_bind_text(stmt_new_, 1, &hash_str[0], hash_str.length(),
1820 SQLITE_STATIC);
1821
1/2
✓ Branch 1 taken 317 times.
✗ Branch 2 not taken.
 317 sqlite3_bind_int64(stmt_new_, 2, size);
1822
1/2
✓ Branch 1 taken 317 times.
✗ Branch 2 not taken.
 317 sqlite3_bind_int64(stmt_new_, 3, seq_++);
1823
1/2
✓ Branch 3 taken 317 times.
✗ Branch 4 not taken.
 317 sqlite3_bind_text(stmt_new_, 4, &description[0], description.length(),
1824 SQLITE_STATIC);
1825
3/4
✓ Branch 0 taken 212 times.
✓ Branch 1 taken 105 times.
✓ Branch 3 taken 317 times.
✗ Branch 4 not taken.
 317 sqlite3_bind_int64(stmt_new_, 5, is_catalog ? kFileCatalog : kFileRegular);
1826
1/2
✓ Branch 1 taken 317 times.
✗ Branch 2 not taken.
 317 sqlite3_bind_int64(stmt_new_, 6, 1);
1827
1/2
✓ Branch 1 taken 317 times.
✗ Branch 2 not taken.
 317 const int retval = sqlite3_step(stmt_new_);
1828
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 317 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 317 assert((retval == SQLITE_DONE) || (retval == SQLITE_OK));
1829
1/2
✓ Branch 1 taken 317 times.
✗ Branch 2 not taken.
 317 sqlite3_reset(stmt_new_);
1830
2/2
✓ Branch 0 taken 189 times.
✓ Branch 1 taken 128 times.
 317 if (!exists)
1831 189 gauge_ += size;
1832 317 return true;
1833 }
1834
1835 int pipe_reserve[2];
1836
1/2
✓ Branch 1 taken 490 times.
✗ Branch 2 not taken.
 490 MakeReturnPipe(pipe_reserve);
1837
1838 490 LruCommand cmd;
1839 490 cmd.command_type = kReserve;
1840 490 cmd.SetSize(size);
1841
1/2
✓ Branch 1 taken 490 times.
✗ Branch 2 not taken.
 490 cmd.StoreHash(hash);
1842 490 cmd.return_pipe = pipe_reserve[1];
1843
1/2
✓ Branch 1 taken 490 times.
✗ Branch 2 not taken.
 490 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
1844 bool result;
1845
1/2
✓ Branch 1 taken 490 times.
✗ Branch 2 not taken.
 490 ManagedReadHalfPipe(pipe_reserve[0], &result, sizeof(result));
1846
1/2
✓ Branch 1 taken 490 times.
✗ Branch 2 not taken.
 490 CloseReturnPipe(pipe_reserve);
1847
1848
2/2
✓ Branch 0 taken 35 times.
✓ Branch 1 taken 455 times.
 490 if (!result)
1849 35 return false;
1850
3/4
✓ Branch 0 taken 70 times.
✓ Branch 1 taken 385 times.
✓ Branch 3 taken 455 times.
✗ Branch 4 not taken.
 455 DoInsert(hash, size, description, is_catalog ? kPin : kPinRegular);
1851
1852 455 return true;
1853 877 }
1854
1855
1856 2473 PosixQuotaManager::PosixQuotaManager(const uint64_t limit,
1857 const uint64_t cleanup_threshold,
1858 2473 const string &cache_workspace)
1859 2473 : shared_(false)
1860 2473 , spawned_(false)
1861 2473 , limit_(limit)
1862 2473 , cleanup_threshold_(cleanup_threshold)
1863 2473 , gauge_(0)
1864 2473 , pinned_(0)
1865 2473 , seq_(0)
1866 2473 , cache_dir_() // initialized in body
1867 2473 , workspace_dir_() // initialized in body
1868 2473 , fd_lock_cachedb_(-1)
1869 2473 , async_delete_(true)
1870 2473 , cachemgr_pid_(0)
1871 2473 , database_(NULL)
1872 2473 , stmt_touch_(NULL)
1873 2473 , stmt_unpin_(NULL)
1874 2473 , stmt_block_(NULL)
1875 2473 , stmt_unblock_(NULL)
1876 2473 , stmt_new_(NULL)
1877 2473 , stmt_lru_(NULL)
1878 2473 , stmt_size_(NULL)
1879 2473 , stmt_rm_(NULL)
1880 2473 , stmt_rm_batch_(NULL)
1881 2473 , stmt_list_(NULL)
1882 2473 , stmt_list_pinned_(NULL)
1883 2473 , stmt_list_catalogs_(NULL)
1884 2473 , stmt_list_volatile_(NULL)
1885 2473 , initialized_(false)
1886 4946 , cleanup_unused_first_(false) {
1887
2/4
✓ Branch 1 taken 2473 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2473 times.
✗ Branch 5 not taken.
 2473 ParseDirectories(cache_workspace, &cache_dir_, &workspace_dir_);
1888 2473 pipe_lru_[0] = pipe_lru_[1] = -1;
1889
1/2
✓ Branch 1 taken 2473 times.
✗ Branch 2 not taken.
 2473 cleanup_recorder_.AddRecorder(1, 90); // last 1.5 min with second resolution
1890 // last 1.5 h with minute resolution
1891
1/2
✓ Branch 1 taken 2473 times.
✗ Branch 2 not taken.
 2473 cleanup_recorder_.AddRecorder(60, 90 * 60);
1892 // last 18 hours with 20 min resolution
1893
1/2
✓ Branch 1 taken 2473 times.
✗ Branch 2 not taken.
 2473 cleanup_recorder_.AddRecorder(20 * 60, 60 * 60 * 18);
1894 // last 4 days with hour resolution
1895
1/2
✓ Branch 1 taken 2473 times.
✗ Branch 2 not taken.
 2473 cleanup_recorder_.AddRecorder(60 * 60, 60 * 60 * 24 * 4);
1896
1897 2473 lock_open_files_ = reinterpret_cast<pthread_mutex_t *>(
1898 2473 smalloc(sizeof(pthread_mutex_t)));
1899 2473 }
1900
1901
1902 9888 PosixQuotaManager::~PosixQuotaManager() {
1903 4944 free(lock_open_files_);
1904
1905
2/2
✓ Branch 0 taken 105 times.
✓ Branch 1 taken 2367 times.
 4944 if (!initialized_)
1906 210 return;
1907
1908
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2367 times.
 4734 if (shared_) {
1909 // Most of cleanup is done elsewhen by shared cache manager
1910 ✗ close(pipe_lru_[1]);
1911 ✗ return;
1912 }
1913
1914
2/2
✓ Branch 0 taken 1085 times.
✓ Branch 1 taken 1282 times.
 4734 if (spawned_) {
1915 2170 char fin = 0;
1916 2170 WritePipe(pipe_lru_[1], &fin, 1);
1917 2170 close(pipe_lru_[1]);
1918 2170 pthread_join(thread_lru_, NULL);
1919 } else {
1920 2564 ClosePipe(pipe_lru_);
1921 }
1922
1923 4734 CloseDatabase();
1924
14/14
✓ Branch 1 taken 2367 times.
✓ Branch 2 taken 105 times.
✓ Branch 4 taken 2367 times.
✓ Branch 5 taken 105 times.
✓ Branch 7 taken 2367 times.
✓ Branch 8 taken 105 times.
✓ Branch 10 taken 2367 times.
✓ Branch 11 taken 105 times.
✓ Branch 13 taken 2367 times.
✓ Branch 14 taken 105 times.
✓ Branch 16 taken 2367 times.
✓ Branch 17 taken 105 times.
✓ Branch 19 taken 2367 times.
✓ Branch 20 taken 105 times.
 11148 }
1925
1926
1927 167755 void PosixQuotaManager::ProcessCommandBunch(const unsigned num,
1928 const LruCommand *commands,
1929 const char *descriptions) {
1930 167755 int retval = sqlite3_exec(database_, "BEGIN", NULL, NULL, NULL);
1931
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 167755 times.
 167755 assert(retval == SQLITE_OK);
1932
1933
2/2
✓ Branch 0 taken 5251715 times.
✓ Branch 1 taken 167755 times.
 5419470 for (unsigned i = 0; i < num; ++i) {
1934
1/2
✓ Branch 1 taken 5251715 times.
✗ Branch 2 not taken.
 5251715 const shash::Any hash = commands[i].RetrieveHash();
1935
1/2
✓ Branch 1 taken 5251715 times.
✗ Branch 2 not taken.
 5251715 const string hash_str = hash.ToString();
1936 5251715 const unsigned size = commands[i].GetSize();
1937
1/2
✓ Branch 1 taken 5251715 times.
✗ Branch 2 not taken.
 5251715 LogCvmfs(kLogQuota, kLogDebug, "processing %s (%d)", hash_str.c_str(),
1938 5251715 commands[i].command_type);
1939
1940 bool exists;
1941
3/4
✓ Branch 0 taken 1750455 times.
✓ Branch 1 taken 70 times.
✓ Branch 2 taken 3501190 times.
✗ Branch 3 not taken.
 5251715 switch (commands[i].command_type) {
1942 1750455 case kTouch:
1943
1/2
✓ Branch 1 taken 1750455 times.
✗ Branch 2 not taken.
 1750455 sqlite3_bind_int64(stmt_touch_, 1, seq_++);
1944
1/2
✓ Branch 3 taken 1750455 times.
✗ Branch 4 not taken.
 1750455 sqlite3_bind_text(stmt_touch_, 2, &hash_str[0], hash_str.length(),
1945 SQLITE_STATIC);
1946
1/2
✓ Branch 1 taken 1750455 times.
✗ Branch 2 not taken.
 1750455 retval = sqlite3_step(stmt_touch_);
1947
1/2
✓ Branch 1 taken 1750455 times.
✗ Branch 2 not taken.
 1750455 LogCvmfs(kLogQuota, kLogDebug, "touching %s (%ld): %d",
1948 1750455 hash_str.c_str(), seq_ - 1, retval);
1949
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1750455 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 1750455 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
1950 ✗ PANIC(kLogSyslogErr, "failed to update %s in cachedb, error %d",
1951 hash_str.c_str(), retval);
1952 }
1953
1/2
✓ Branch 1 taken 1750455 times.
✗ Branch 2 not taken.
 1750455 sqlite3_reset(stmt_touch_);
1954 1750455 break;
1955 70 case kUnpin:
1956
1/2
✓ Branch 3 taken 70 times.
✗ Branch 4 not taken.
 70 sqlite3_bind_text(stmt_unpin_, 1, &hash_str[0], hash_str.length(),
1957 SQLITE_STATIC);
1958
1/2
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
 70 retval = sqlite3_step(stmt_unpin_);
1959
1/2
✓ Branch 2 taken 70 times.
✗ Branch 3 not taken.
 70 LogCvmfs(kLogQuota, kLogDebug, "unpinning %s: %d", hash_str.c_str(),
1960 retval);
1961
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 70 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
1962 ✗ PANIC(kLogSyslogErr, "failed to unpin %s in cachedb, error %d",
1963 hash_str.c_str(), retval);
1964 }
1965
1/2
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
 70 sqlite3_reset(stmt_unpin_);
1966 70 break;
1967 3501190 case kPin:
1968 case kPinRegular:
1969 case kInsert:
1970 case kInsertVolatile:
1971 // It could already be in, check
1972
1/2
✓ Branch 1 taken 3501190 times.
✗ Branch 2 not taken.
 3501190 exists = Contains(hash_str);
1973
1974 // Cleanup, move to trash and unlink
1975
3/4
✓ Branch 0 taken 3501015 times.
✓ Branch 1 taken 175 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 3501015 times.
 3501190 if (!exists && (gauge_ + size > limit_)) {
1976 ✗ LogCvmfs(kLogQuota, kLogDebug, "over limit, gauge %lu, file size %u",
1977 gauge_, size);
1978 ✗ retval = DoCleanup(cleanup_threshold_);
1979 ✗ assert(retval != 0);
1980 }
1981
1982 // Insert or replace
1983
1/2
✓ Branch 3 taken 3501190 times.
✗ Branch 4 not taken.
 3501190 sqlite3_bind_text(stmt_new_, 1, &hash_str[0], hash_str.length(),
1984 SQLITE_STATIC);
1985
1/2
✓ Branch 1 taken 3501190 times.
✗ Branch 2 not taken.
 3501190 sqlite3_bind_int64(stmt_new_, 2, size);
1986
2/2
✓ Branch 0 taken 140 times.
✓ Branch 1 taken 3501050 times.
 3501190 if (commands[i].command_type == kInsertVolatile) {
1987
1/2
✓ Branch 1 taken 140 times.
✗ Branch 2 not taken.
 140 sqlite3_bind_int64(stmt_new_, 3, (seq_++) | kVolatileFlag);
1988 } else {
1989
1/2
✓ Branch 1 taken 3501050 times.
✗ Branch 2 not taken.
 3501050 sqlite3_bind_int64(stmt_new_, 3, seq_++);
1990 }
1991 3501190 sqlite3_bind_text(stmt_new_, 4, &descriptions[i * kMaxDescription],
1992
1/2
✓ Branch 1 taken 3501190 times.
✗ Branch 2 not taken.
 3501190 commands[i].desc_length, SQLITE_STATIC);
1993
1/2
✓ Branch 1 taken 3501190 times.
✗ Branch 2 not taken.
 3501190 sqlite3_bind_int64(
1994 stmt_new_, 5,
1995
2/2
✓ Branch 0 taken 70 times.
✓ Branch 1 taken 3501120 times.
 3501190 (commands[i].command_type == kPin) ? kFileCatalog : kFileRegular);
1996
1/2
✓ Branch 1 taken 3501190 times.
✗ Branch 2 not taken.
 3501190 sqlite3_bind_int64(stmt_new_, 6,
1997
2/2
✓ Branch 0 taken 3501120 times.
✓ Branch 1 taken 70 times.
 3501190 ((commands[i].command_type == kPin)
1998
2/2
✓ Branch 0 taken 385 times.
✓ Branch 1 taken 3500735 times.
 3501120 || (commands[i].command_type == kPinRegular))
1999 ? 1
2000 : 0);
2001
1/2
✓ Branch 1 taken 3501190 times.
✗ Branch 2 not taken.
 3501190 retval = sqlite3_step(stmt_new_);
2002
1/2
✓ Branch 1 taken 3501190 times.
✗ Branch 2 not taken.
 3501190 LogCvmfs(kLogQuota, kLogDebug, "insert or replace %s, method %d: %d",
2003 3501190 hash_str.c_str(), commands[i].command_type, retval);
2004
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 3501190 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 3501190 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
2005 ✗ PANIC(kLogSyslogErr, "failed to insert %s in cachedb, error %d",
2006 hash_str.c_str(), retval);
2007 }
2008
1/2
✓ Branch 1 taken 3501190 times.
✗ Branch 2 not taken.
 3501190 sqlite3_reset(stmt_new_);
2009
2010
2/2
✓ Branch 0 taken 3501015 times.
✓ Branch 1 taken 175 times.
 3501190 if (!exists)
2011 3501015 gauge_ += size;
2012 3501190 break;
2013 ✗ default:
2014 // other types should have been taken care of by event loop
2015 ✗ PANIC(NULL);
2016 }
2017 5251715 }
2018
2019 167755 retval = sqlite3_exec(database_, "COMMIT", NULL, NULL, NULL);
2020
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 167755 times.
 167755 if (retval != SQLITE_OK) {
2021 ✗ PANIC(kLogSyslogErr, "failed to commit to cachedb, error %d", retval);
2022 }
2023 167755 }
2024
2025
2026 2403 bool PosixQuotaManager::RebuildDatabase() {
2027 2403 bool result = false;
2028 2403 string sql;
2029 2403 sqlite3_stmt *stmt_select = NULL;
2030 2403 sqlite3_stmt *stmt_insert = NULL;
2031 int sqlerr;
2032 2403 int seq = 0;
2033 char hex[4];
2034 struct stat info;
2035 platform_dirent64 *d;
2036 2403 DIR *dirp = NULL;
2037 2403 string path;
2038
2039
1/2
✓ Branch 1 taken 2403 times.
✗ Branch 2 not taken.
 2403 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug, "re-building cache database");
2040
2041 // Empty cache catalog and fscache
2042
1/2
✓ Branch 1 taken 2403 times.
✗ Branch 2 not taken.
 2403 sql = "DELETE FROM cache_catalog; DELETE FROM fscache;";
2043
1/2
✓ Branch 2 taken 2403 times.
✗ Branch 3 not taken.
 2403 sqlerr = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
2044
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2403 times.
 2403 if (sqlerr != SQLITE_OK) {
2045 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not clear cache database");
2046 ✗ goto build_return;
2047 }
2048
2049 2403 gauge_ = 0;
2050
2051 // Insert files from cache sub-directories 00 - ff
2052 // TODO(jblomer): fs_traversal
2053
1/2
✓ Branch 1 taken 2403 times.
✗ Branch 2 not taken.
 2403 sqlite3_prepare_v2(database_,
2054 "INSERT INTO fscache (sha1, size, actime) "
2055 "VALUES (:sha1, :s, :t);",
2056 -1, &stmt_insert, NULL);
2057
2058
2/2
✓ Branch 0 taken 588393 times.
✓ Branch 1 taken 2298 times.
 590691 for (int i = 0; i <= 0xff; i++) {
2059 588393 snprintf(hex, sizeof(hex), "%02x", i);
2060
3/6
✓ Branch 2 taken 588393 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 588393 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 588393 times.
✗ Branch 9 not taken.
 588393 path = cache_dir_ + "/" + string(hex);
2061
3/4
✓ Branch 2 taken 588393 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 105 times.
✓ Branch 5 taken 588288 times.
 588393 if ((dirp = opendir(path.c_str())) == NULL) {
2062
1/2
✓ Branch 2 taken 105 times.
✗ Branch 3 not taken.
 105 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
2063 "failed to open directory %s (tmpwatch interfering?)",
2064 path.c_str());
2065 105 goto build_return;
2066 }
2067
3/4
✓ Branch 1 taken 1764934 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1176646 times.
✓ Branch 4 taken 588288 times.
 1764934 while ((d = platform_readdir(dirp)) != NULL) {
2068
3/6
✓ Branch 2 taken 1176646 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1176646 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1176646 times.
✗ Branch 9 not taken.
 2353292 const string file_path = path + "/" + string(d->d_name);
2069
1/2
✓ Branch 2 taken 1176646 times.
✗ Branch 3 not taken.
 1176646 if (stat(file_path.c_str(), &info) == 0) {
2070
2/2
✓ Branch 0 taken 1176576 times.
✓ Branch 1 taken 70 times.
 1176646 if (!S_ISREG(info.st_mode))
2071 1176611 continue;
2072
2/2
✓ Branch 0 taken 35 times.
✓ Branch 1 taken 35 times.
 70 if (info.st_size == 0) {
2073
1/2
✓ Branch 2 taken 35 times.
✗ Branch 3 not taken.
 35 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug,
2074 "removing empty file %s during automatic cache db rebuild",
2075 file_path.c_str());
2076 35 unlink(file_path.c_str());
2077 35 continue;
2078 }
2079
2080
3/6
✓ Branch 2 taken 35 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 35 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 35 times.
✗ Branch 10 not taken.
 70 string hash = string(hex) + string(d->d_name);
2081
1/2
✓ Branch 3 taken 35 times.
✗ Branch 4 not taken.
 35 sqlite3_bind_text(stmt_insert, 1, hash.data(), hash.length(),
2082 SQLITE_STATIC);
2083
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 sqlite3_bind_int64(stmt_insert, 2, info.st_size);
2084
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 sqlite3_bind_int64(stmt_insert, 3, info.st_atime);
2085
2/4
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 35 times.
 35 if (sqlite3_step(stmt_insert) != SQLITE_DONE) {
2086 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not insert into temp table");
2087 ✗ goto build_return;
2088 }
2089
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 sqlite3_reset(stmt_insert);
2090
2091 35 gauge_ += info.st_size;
2092
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 } else {
2093 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not stat %s", file_path.c_str());
2094 }
2095
2/3
✓ Branch 1 taken 35 times.
✓ Branch 2 taken 1176611 times.
✗ Branch 3 not taken.
 1176646 }
2096
1/2
✓ Branch 1 taken 588288 times.
✗ Branch 2 not taken.
 588288 closedir(dirp);
2097 588288 dirp = NULL;
2098 }
2099
1/2
✓ Branch 1 taken 2298 times.
✗ Branch 2 not taken.
 2298 sqlite3_finalize(stmt_insert);
2100 2298 stmt_insert = NULL;
2101
2102 // Transfer from temp table in cache catalog
2103
1/2
✓ Branch 1 taken 2298 times.
✗ Branch 2 not taken.
 2298 sqlite3_prepare_v2(database_,
2104 "SELECT sha1, size FROM fscache ORDER BY actime;", -1,
2105 &stmt_select, NULL);
2106
1/2
✓ Branch 1 taken 2298 times.
✗ Branch 2 not taken.
 2298 sqlite3_prepare_v2(
2107 database_,
2108 "INSERT INTO cache_catalog (sha1, size, acseq, path, type, pinned) "
2109 "VALUES (:sha1, :s, :seq, 'unknown (automatic rebuild)', :t, 0);",
2110 -1, &stmt_insert, NULL);
2111
3/4
✓ Branch 1 taken 2333 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 35 times.
✓ Branch 4 taken 2298 times.
 2333 while (sqlite3_step(stmt_select) == SQLITE_ROW) {
2112 const string hash = string(
2113
2/4
✓ Branch 2 taken 35 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 35 times.
✗ Branch 6 not taken.
 35 reinterpret_cast<const char *>(sqlite3_column_text(stmt_select, 0)));
2114
1/2
✓ Branch 3 taken 35 times.
✗ Branch 4 not taken.
 35 sqlite3_bind_text(stmt_insert, 1, &hash[0], hash.length(), SQLITE_STATIC);
2115
2/4
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 35 times.
✗ Branch 5 not taken.
 35 sqlite3_bind_int64(stmt_insert, 2, sqlite3_column_int64(stmt_select, 1));
2116
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 sqlite3_bind_int64(stmt_insert, 3, seq++);
2117 // Might also be a catalog (information is lost)
2118
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 sqlite3_bind_int64(stmt_insert, 4, kFileRegular);
2119
2120
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 const int retval = sqlite3_step(stmt_insert);
2121
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 35 times.
 35 if (retval != SQLITE_DONE) {
2122 // If the file system hosting the cache is full, we'll likely notice here
2123 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
2124 "could not insert into cache catalog (%d - %s)", retval,
2125 sqlite3_errstr(retval));
2126 ✗ goto build_return;
2127 }
2128
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 sqlite3_reset(stmt_insert);
2129
1/2
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
 35 }
2130
2131 // Delete temporary table
2132
1/2
✓ Branch 1 taken 2298 times.
✗ Branch 2 not taken.
 2298 sql = "DELETE FROM fscache;";
2133
1/2
✓ Branch 2 taken 2298 times.
✗ Branch 3 not taken.
 2298 sqlerr = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
2134
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2298 times.
 2298 if (sqlerr != SQLITE_OK) {
2135 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not clear temporary table (%d)",
2136 sqlerr);
2137 ✗ goto build_return;
2138 }
2139
2140 2298 seq_ = seq;
2141 2298 result = true;
2142
1/2
✓ Branch 1 taken 2298 times.
✗ Branch 2 not taken.
 2298 LogCvmfs(kLogQuota, kLogDebug,
2143 "rebuilding finished, sequence %" PRIu64 ", gauge %" PRIu64, seq_,
2144 gauge_);
2145
2146 2403 build_return:
2147
1/2
✓ Branch 0 taken 2403 times.
✗ Branch 1 not taken.
 2403 if (stmt_insert)
2148
1/2
✓ Branch 1 taken 2403 times.
✗ Branch 2 not taken.
 2403 sqlite3_finalize(stmt_insert);
2149
2/2
✓ Branch 0 taken 2298 times.
✓ Branch 1 taken 105 times.
 2403 if (stmt_select)
2150
1/2
✓ Branch 1 taken 2298 times.
✗ Branch 2 not taken.
 2298 sqlite3_finalize(stmt_select);
2151
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2403 times.
 2403 if (dirp)
2152 ✗ closedir(dirp);
2153 2403 return result;
2154 2403 }
2155
2156
2157 /**
2158 * Register a channel that allows the cache manager to trigger action to its
2159 * clients. Currently used for releasing pinned catalogs.
2160 */
2161 140 void PosixQuotaManager::RegisterBackChannel(int back_channel[2],
2162 const string &channel_id) {
2163
1/2
✓ Branch 0 taken 140 times.
✗ Branch 1 not taken.
 140 if (protocol_revision_ >= 1) {
2164
1/2
✓ Branch 2 taken 140 times.
✗ Branch 3 not taken.
 140 shash::Md5 hash = shash::Md5(shash::AsciiPtr(channel_id));
2165
1/2
✓ Branch 1 taken 140 times.
✗ Branch 2 not taken.
 140 MakeReturnPipe(back_channel);
2166
2167 140 LruCommand cmd;
2168 140 cmd.command_type = kRegisterBackChannel;
2169 140 cmd.return_pipe = back_channel[1];
2170 // Not StoreHash(). This is an MD5 hash.
2171 140 memcpy(cmd.digest, hash.digest, hash.GetDigestSize());
2172
1/2
✓ Branch 1 taken 140 times.
✗ Branch 2 not taken.
 140 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2173
2174 char success;
2175
1/2
✓ Branch 1 taken 140 times.
✗ Branch 2 not taken.
 140 ManagedReadHalfPipe(back_channel[0], &success, sizeof(success));
2176 // At this point, the named FIFO is unlinked, so don't use CloseReturnPipe
2177
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 140 times.
 140 if (success != 'S') {
2178 ✗ PANIC(kLogDebug | kLogSyslogErr,
2179 "failed to register quota back channel (%c)", success);
2180 }
2181 } else {
2182 // Dummy pipe to return valid file descriptors
2183 ✗ MakePipe(back_channel);
2184 }
2185 140 }
2186
2187
2188 /**
2189 * Removes a chunk from cache, if it exists.
2190 */
2191 105 void PosixQuotaManager::Remove(const shash::Any &hash) {
2192
1/2
✓ Branch 1 taken 105 times.
✗ Branch 2 not taken.
 105 const string hash_str = hash.ToString();
2193
2194 int pipe_remove[2];
2195
1/2
✓ Branch 1 taken 105 times.
✗ Branch 2 not taken.
 105 MakeReturnPipe(pipe_remove);
2196
2197 105 LruCommand cmd;
2198 105 cmd.command_type = kRemove;
2199 105 cmd.return_pipe = pipe_remove[1];
2200
1/2
✓ Branch 1 taken 105 times.
✗ Branch 2 not taken.
 105 cmd.StoreHash(hash);
2201
1/2
✓ Branch 1 taken 105 times.
✗ Branch 2 not taken.
 105 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2202
2203 bool success;
2204
1/2
✓ Branch 1 taken 105 times.
✗ Branch 2 not taken.
 105 ManagedReadHalfPipe(pipe_remove[0], &success, sizeof(success));
2205
1/2
✓ Branch 1 taken 105 times.
✗ Branch 2 not taken.
 105 CloseReturnPipe(pipe_remove);
2206
2207
3/6
✓ Branch 1 taken 105 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 105 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 105 times.
✗ Branch 8 not taken.
 105 unlink((cache_dir_ + "/" + hash.MakePathWithoutSuffix()).c_str());
2208 105 }
2209
2210
2211 1155 void PosixQuotaManager::Spawn() {
2212
2/2
✓ Branch 0 taken 70 times.
✓ Branch 1 taken 1085 times.
 1155 if (spawned_)
2213 70 return;
2214
2215 1085 if (pthread_create(&thread_lru_, NULL, MainCommandServer,
2216 static_cast<void *>(this))
2217
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1085 times.
 1085 != 0) {
2218 ✗ PANIC(kLogDebug, "could not create lru thread");
2219 }
2220
2221 1085 spawned_ = true;
2222 }
2223
2224
2225 /**
2226 * Updates the sequence number of the file specified by the hash.
2227 */
2228 1750237 void PosixQuotaManager::Touch(const shash::Any &hash) {
2229 1750237 LruCommand cmd;
2230 1750237 cmd.command_type = kTouch;
2231
1/2
✓ Branch 1 taken 1750237 times.
✗ Branch 2 not taken.
 1750237 cmd.StoreHash(hash);
2232
1/2
✓ Branch 1 taken 1750237 times.
✗ Branch 2 not taken.
 1750237 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2233 1750237 }
2234
2235
2236 3010 void PosixQuotaManager::UnbindReturnPipe(int pipe_wronly) {
2237
2/2
✓ Branch 0 taken 35 times.
✓ Branch 1 taken 2975 times.
 3010 if (shared_)
2238 35 close(pipe_wronly);
2239 3010 }
2240
2241
2242 245 void PosixQuotaManager::UnlinkReturnPipe(int pipe_wronly) {
2243
2/2
✓ Branch 0 taken 105 times.
✓ Branch 1 taken 140 times.
 245 if (shared_)
2244
2/4
✓ Branch 2 taken 105 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 105 times.
✗ Branch 6 not taken.
 105 unlink((workspace_dir_ + "/pipe" + StringifyInt(pipe_wronly)).c_str());
2245 245 }
2246
2247
2248 261 void PosixQuotaManager::Unpin(const shash::Any &hash) {
2249
2/4
✓ Branch 1 taken 261 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 261 times.
✗ Branch 6 not taken.
 261 LogCvmfs(kLogQuota, kLogDebug, "Unpin %s", hash.ToString().c_str());
2250
2251 261 LruCommand cmd;
2252 261 cmd.command_type = kUnpin;
2253
1/2
✓ Branch 1 taken 261 times.
✗ Branch 2 not taken.
 261 cmd.StoreHash(hash);
2254
1/2
✓ Branch 1 taken 261 times.
✗ Branch 2 not taken.
 261 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2255 261 }
2256
2257
2258 70 void PosixQuotaManager::UnregisterBackChannel(int back_channel[2],
2259 const string &channel_id) {
2260
1/2
✓ Branch 0 taken 70 times.
✗ Branch 1 not taken.
 70 if (protocol_revision_ >= 1) {
2261
1/2
✓ Branch 2 taken 70 times.
✗ Branch 3 not taken.
 70 shash::Md5 hash = shash::Md5(shash::AsciiPtr(channel_id));
2262
2263 70 LruCommand cmd;
2264 70 cmd.command_type = kUnregisterBackChannel;
2265 // Not StoreHash(). This is an MD5 hash.
2266 70 memcpy(cmd.digest, hash.digest, hash.GetDigestSize());
2267
1/2
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
 70 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2268
2269 // Writer's end will be closed by cache manager, FIFO is already unlinked
2270
1/2
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
 70 close(back_channel[0]);
2271 } else {
2272 ✗ ClosePipe(back_channel);
2273 }
2274 70 }
2275
2276 3504445 void PosixQuotaManager::ManagedReadHalfPipe(int fd, void *buf, size_t nbyte) {
2277
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3504445 times.
 3504445 const unsigned timeout_ms = cachemgr_pid_ ? 1000 : 0;
2278 3504445 bool result = false;
2279 do {
2280 3504445 result = ReadHalfPipe(fd, buf, nbyte, timeout_ms);
2281 // try only as long as the cachemgr is still alive
2282
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 3504445 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 3504445 times.
 3504445 } while (!result && getpgid(cachemgr_pid_) >= 0);
2283
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3504445 times.
 3504445 if (!result) {
2284 ✗ PANIC(kLogStderr,
2285 "Error: quota manager could not read from cachemanager pipe");
2286 }
2287 3504445 }
2288
2289 ✗ void *PosixQuotaManager::CollectMountpointsHashes(void *data) {
2290 #ifndef __APPLE__
2291 ✗ pthread_setname_np(pthread_self(), "hash_collector");
2292 ✗ auto *handler = static_cast<CollectorHandler *>(data);
2293
2294 ✗ const std::string mountpoint = handler->mp[handler->i];
2295 ✗ ssize_t n = getxattr(mountpoint.c_str(), "user.list_open_hashes", nullptr, 0);
2296 ✗ if (n < 0) {
2297 ✗ pthread_exit(nullptr);
2298 }
2299 ✗ std::vector<char> buf((size_t)n);
2300 ✗ n = getxattr(mountpoint.c_str(), "user.list_open_hashes", buf.data(),
2301 buf.size());
2302 ✗ if (n < 0) {
2303 ✗ pthread_exit(nullptr);
2304 }
2305
2306 ✗ std::vector<std::string> hash_strs;
2307 ✗ std::string hash_str;
2308 ✗ for (const char c : buf) {
2309 ✗ if (c == '\n') {
2310 ✗ hash_strs.push_back(hash_str);
2311 ✗ hash_str.clear();
2312 } else {
2313 ✗ hash_str += c;
2314 }
2315 }
2316 ✗ const MutexLockGuard lock_guard(handler->l);
2317 ✗ for (auto hash_str : hash_strs) {
2318 ✗ handler->of.push_back(shash::Short( shash::MkFromHexPtr(shash::HexPtr(hash_str)) ));
2319 }
2320 #endif
2321 ✗ pthread_exit(nullptr);
2322 }
2323
2324 ✗ std::vector<shash::Short> PosixQuotaManager::CollectAllOpenHashes() {
2325 ✗ std::vector<CollectorHandler *> handlers;
2326 ✗ std::vector<pthread_t *> threads;
2327 ✗ open_files_.clear();
2328 #ifndef __APPLE__
2329 ✗ auto &&a_after_b = [](const struct timespec a, const struct timespec b) {
2330 ✗ return (a.tv_sec > b.tv_sec) ? true : false;
2331 };
2332
2333 ✗ for (size_t i = 0; i < mountpoints_.size(); ++i) {
2334 ✗ handlers.push_back(
2335 ✗ new CollectorHandler{open_files_, mountpoints_, lock_open_files_, i});
2336 ✗ threads.push_back(new pthread_t);
2337 }
2338
2339 ✗ const int retval = pthread_mutex_init(lock_open_files_, NULL);
2340 ✗ assert(retval == 0);
2341
2342 ✗ for (size_t i = 0; i < mountpoints_.size(); ++i) {
2343 ✗ pthread_create(threads[i], nullptr, CollectMountpointsHashes, handlers[i]);
2344 }
2345
2346 ✗ std::vector<bool> joined(handlers.size(), false);
2347 struct timespec reference, current;
2348 ✗ clock_gettime(CLOCK_REALTIME, &reference);
2349 ✗ clock_gettime(CLOCK_REALTIME, &current);
2350 ✗ reference.tv_sec += 10; // Give 10sec for hash collection
2351 ✗ size_t i = 0;
2352 ✗ while (
2353 ✗ (not std::all_of(joined.begin(), joined.end(), [](bool b) { return b; }))
2354 ✗ and a_after_b(reference, current)) {
2355 // as long as there are still threads that haven't joined yet
2356 // and for 10 seconds
2357 ✗ if (not joined[i]) {
2358 ✗ const int s = pthread_tryjoin_np(*threads[i], NULL);
2359 ✗ if (s == 0) {
2360 ✗ joined[i] = true;
2361 }
2362 }
2363 ✗ ++i;
2364 ✗ i = i % handlers.size();
2365 ✗ clock_gettime(CLOCK_REALTIME, &current);
2366 }
2367
2368 ✗ for (size_t i = 0; i < handlers.size(); ++i) {
2369 ✗ delete handlers[i];
2370 }
2371
2372 ✗ pthread_mutex_destroy(lock_open_files_);
2373 #endif
2374 ✗ return open_files_;
2375 }
2376
2377