GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/quota_posix.cc
Date: 2026-01-04 02:35:37
Exec Total Coverage
Lines: 955 1391 68.7%
Branches: 749 1920 39.0%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * This module implements a "managed local cache".
5 * This way, we are able to track access times of files in the cache
6 * and remove files based on least recently used strategy.
7 *
8 * We setup another SQLite catalog, a "cache catalog", that helps us
9 * in the bookkeeping of files, file sizes and access times.
10 *
11 * We might choose to not manage the local cache. This is indicated
12 * by limit == 0 and everything succeeds in that case.
13 */
14
15 #define __STDC_LIMIT_MACROS
16 #define __STDC_FORMAT_MACROS
17
18
19 #include "quota_posix.h"
20
21 #include <dirent.h>
22 #include <errno.h>
23 #include <fcntl.h>
24 #include <inttypes.h>
25 #include <pthread.h>
26 #include <signal.h>
27 #include <stdint.h>
28 #include <sys/dir.h>
29 #include <sys/stat.h>
30 #include <sys/xattr.h>
31
32 #ifndef __APPLE__
33 #include <sys/statfs.h>
34 #endif
35 #include <sys/statvfs.h>
36 #include <sys/types.h>
37 #include <sys/wait.h>
38 #include <unistd.h>
39
40 #include <algorithm>
41 #include <cassert>
42 #include <cstdio>
43 #include <cstdlib>
44 #include <cstring>
45 #include <limits>
46 #include <map>
47 #include <set>
48 #include <string>
49 #include <vector>
50
51 #include "crypto/hash.h"
52 #include "duplex_sqlite3.h"
53 #include "monitor.h"
54 #include "statistics.h"
55 #include "util/concurrency.h"
56 #include "util/exception.h"
57 #include "util/logging.h"
58 #include "util/platform.h"
59 #include "util/pointer.h"
60 #include "util/posix.h"
61 #include "util/smalloc.h"
62 #include "util/string.h"
63
64 using namespace std; // NOLINT
65
66
67 3128 int PosixQuotaManager::BindReturnPipe(int pipe_wronly) {
68
2/2
✓ Branch 0 taken 3060 times.
✓ Branch 1 taken 68 times.
 3128 if (!shared_)
69 3060 return pipe_wronly;
70
71 // Connect writer's end
72
1/2
✓ Branch 2 taken 68 times.
✗ Branch 3 not taken.
 68 const int result = open(
73
2/4
✓ Branch 2 taken 68 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 68 times.
✗ Branch 6 not taken.
 136 (workspace_dir_ + "/pipe" + StringifyInt(pipe_wronly)).c_str(),
74 O_WRONLY | O_NONBLOCK);
75
2/2
✓ Branch 0 taken 34 times.
✓ Branch 1 taken 34 times.
 68 if (result >= 0) {
76 34 Nonblock2Block(result);
77 } else {
78 34 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
79 34 "failed to bind return pipe (%d)", errno);
80 }
81 68 return result;
82 }
83
84
85 869 void PosixQuotaManager::CheckHighPinWatermark() {
86 869 const uint64_t watermark = kHighPinWatermark * cleanup_threshold_ / 100;
87
3/4
✓ Branch 0 taken 869 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 102 times.
✓ Branch 3 taken 767 times.
 869 if ((cleanup_threshold_ > 0) && (pinned_ > watermark)) {
88 102 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
89 "high watermark of pinned files (%" PRIu64 "M > %" PRIu64 "M)",
90 102 pinned_ / (1024 * 1024), watermark / (1024 * 1024));
91
2/4
✓ Branch 2 taken 102 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 102 times.
✗ Branch 6 not taken.
 102 BroadcastBackchannels("R"); // clients: please release pinned catalogs
92 }
93 869 }
94
95
96 ✗ void PosixQuotaManager::CleanupPipes() {
97 ✗ DIR *dirp = opendir(workspace_dir_.c_str());
98 ✗ assert(dirp != NULL);
99
100 platform_dirent64 *dent;
101 ✗ bool found_leftovers = false;
102 ✗ while ((dent = platform_readdir(dirp)) != NULL) {
103 ✗ const string name = dent->d_name;
104 ✗ const string path = workspace_dir_ + "/" + name;
105 platform_stat64 info;
106 ✗ const int retval = platform_stat(path.c_str(), &info);
107 ✗ if (retval != 0)
108 ✗ continue;
109 ✗ if (S_ISFIFO(info.st_mode) && (name.substr(0, 4) == "pipe")) {
110 ✗ if (!found_leftovers) {
111 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogWarn,
112 "removing left-over FIFOs from cache directory");
113 }
114 ✗ found_leftovers = true;
115 ✗ unlink(path.c_str());
116 }
117 }
118 ✗ closedir(dirp);
119 }
120
121
122 /**
123 * Cleans up in data cache, until cache size is below leave_size.
124 * The actual unlinking is done in a separate process (fork).
125 *
126 * \return True on success, false otherwise
127 */
128 306 bool PosixQuotaManager::Cleanup(const uint64_t leave_size) {
129
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 306 times.
 306 if (!spawned_)
130 ✗ return DoCleanup(leave_size);
131
132 bool result;
133 int pipe_cleanup[2];
134
1/2
✓ Branch 1 taken 306 times.
✗ Branch 2 not taken.
 306 MakeReturnPipe(pipe_cleanup);
135
136 306 LruCommand cmd;
137 306 cmd.command_type = kCleanup;
138 306 cmd.size = leave_size;
139 306 cmd.return_pipe = pipe_cleanup[1];
140
141
1/2
✓ Branch 1 taken 306 times.
✗ Branch 2 not taken.
 306 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
142
1/2
✓ Branch 1 taken 306 times.
✗ Branch 2 not taken.
 306 ManagedReadHalfPipe(pipe_cleanup[0], &result, sizeof(result));
143
1/2
✓ Branch 1 taken 306 times.
✗ Branch 2 not taken.
 306 CloseReturnPipe(pipe_cleanup);
144
145 306 return result;
146 }
147
148
149 2707 void PosixQuotaManager::CloseDatabase() {
150
1/2
✓ Branch 0 taken 2707 times.
✗ Branch 1 not taken.
 2707 if (stmt_list_catalogs_)
151 2707 sqlite3_finalize(stmt_list_catalogs_);
152
1/2
✓ Branch 0 taken 2707 times.
✗ Branch 1 not taken.
 2707 if (stmt_list_pinned_)
153 2707 sqlite3_finalize(stmt_list_pinned_);
154
1/2
✓ Branch 0 taken 2707 times.
✗ Branch 1 not taken.
 2707 if (stmt_list_volatile_)
155 2707 sqlite3_finalize(stmt_list_volatile_);
156
1/2
✓ Branch 0 taken 2707 times.
✗ Branch 1 not taken.
 2707 if (stmt_list_)
157 2707 sqlite3_finalize(stmt_list_);
158
1/2
✓ Branch 0 taken 2707 times.
✗ Branch 1 not taken.
 2707 if (stmt_lru_)
159 2707 sqlite3_finalize(stmt_lru_);
160
1/2
✓ Branch 0 taken 2707 times.
✗ Branch 1 not taken.
 2707 if (stmt_rm_)
161 2707 sqlite3_finalize(stmt_rm_);
162
1/2
✓ Branch 0 taken 2707 times.
✗ Branch 1 not taken.
 2707 if (stmt_rm_batch_)
163 2707 sqlite3_finalize(stmt_rm_batch_);
164
1/2
✓ Branch 0 taken 2707 times.
✗ Branch 1 not taken.
 2707 if (stmt_size_)
165 2707 sqlite3_finalize(stmt_size_);
166
1/2
✓ Branch 0 taken 2707 times.
✗ Branch 1 not taken.
 2707 if (stmt_touch_)
167 2707 sqlite3_finalize(stmt_touch_);
168
1/2
✓ Branch 0 taken 2707 times.
✗ Branch 1 not taken.
 2707 if (stmt_unpin_)
169 2707 sqlite3_finalize(stmt_unpin_);
170
1/2
✓ Branch 0 taken 2707 times.
✗ Branch 1 not taken.
 2707 if (stmt_block_)
171 2707 sqlite3_finalize(stmt_block_);
172
1/2
✓ Branch 0 taken 2707 times.
✗ Branch 1 not taken.
 2707 if (stmt_unblock_)
173 2707 sqlite3_finalize(stmt_unblock_);
174
1/2
✓ Branch 0 taken 2707 times.
✗ Branch 1 not taken.
 2707 if (stmt_new_)
175 2707 sqlite3_finalize(stmt_new_);
176
1/2
✓ Branch 0 taken 2707 times.
✗ Branch 1 not taken.
 2707 if (database_)
177 2707 sqlite3_close(database_);
178 2707 UnlockFile(fd_lock_cachedb_);
179
180 2707 stmt_list_catalogs_ = NULL;
181 2707 stmt_list_pinned_ = NULL;
182 2707 stmt_list_volatile_ = NULL;
183 2707 stmt_list_ = NULL;
184 2707 stmt_rm_ = NULL;
185 2707 stmt_rm_batch_ = NULL;
186 2707 stmt_size_ = NULL;
187 2707 stmt_touch_ = NULL;
188 2707 stmt_unpin_ = NULL;
189 2707 stmt_block_ = NULL;
190 2707 stmt_unblock_ = NULL;
191 2707 stmt_new_ = NULL;
192 2707 database_ = NULL;
193
194 2707 pinned_chunks_.clear();
195 2707 }
196
197
198 2958 void PosixQuotaManager::CloseReturnPipe(int pipe[2]) {
199
2/2
✓ Branch 0 taken 68 times.
✓ Branch 1 taken 2890 times.
 2958 if (shared_) {
200 68 close(pipe[0]);
201 68 UnlinkReturnPipe(pipe[1]);
202 } else {
203 2890 ClosePipe(pipe);
204 }
205 2958 }
206
207
208 3402166 bool PosixQuotaManager::Contains(const string &hash_str) {
209 3402166 bool result = false;
210
211 3402166 sqlite3_bind_text(stmt_size_, 1, &hash_str[0], hash_str.length(),
212 SQLITE_STATIC);
213
2/2
✓ Branch 1 taken 651 times.
✓ Branch 2 taken 3401515 times.
 3402166 if (sqlite3_step(stmt_size_) == SQLITE_ROW)
214 651 result = true;
215 3402166 sqlite3_reset(stmt_size_);
216 3402166 LogCvmfs(kLogQuota, kLogDebug, "contains %s returns %d", hash_str.c_str(),
217 result);
218
219 3402166 return result;
220 }
221
222
223 2640 void PosixQuotaManager::CheckFreeSpace() {
224
3/4
✓ Branch 0 taken 2640 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 34 times.
✓ Branch 3 taken 2606 times.
 2640 if ((limit_ == 0) || (gauge_ >= limit_))
225 34 return;
226
227 struct statvfs vfs_info;
228
1/2
✓ Branch 1 taken 2606 times.
✗ Branch 2 not taken.
 2606 const int retval = statvfs((cache_dir_ + "/cachedb").c_str(), &vfs_info);
229
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2606 times.
 2606 if (retval != 0) {
230 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
231 "failed to query %s for free space (%d)", cache_dir_.c_str(),
232 ✗ errno);
233 ✗ return;
234 }
235 2606 const int64_t free_space_byte = vfs_info.f_bavail * vfs_info.f_bsize;
236
1/2
✓ Branch 1 taken 2606 times.
✗ Branch 2 not taken.
 2606 LogCvmfs(kLogQuota, kLogDebug, "free space: %" PRId64 " MB",
237 free_space_byte / (1024 * 1024));
238
239 2606 const int64_t required_byte = limit_ - gauge_;
240
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2606 times.
 2606 if (free_space_byte < required_byte) {
241 ✗ LogCvmfs(kLogQuota, kLogSyslogWarn,
242 "too little free space on the file system hosting the cache,"
243 " %" PRId64 " MB available",
244 free_space_byte / (1024 * 1024));
245 }
246 }
247
248
249 2742 PosixQuotaManager *PosixQuotaManager::Create(const string &cache_workspace,
250 const uint64_t limit,
251 const uint64_t cleanup_threshold,
252 const bool rebuild_database) {
253
2/2
✓ Branch 0 taken 68 times.
✓ Branch 1 taken 2674 times.
 2742 if (cleanup_threshold >= limit) {
254 68 LogCvmfs(kLogQuota, kLogDebug,
255 "invalid parameters: limit %" PRIu64 ", "
256 "cleanup_threshold %" PRIu64,
257 limit, cleanup_threshold);
258 68 return NULL;
259 }
260
261 PosixQuotaManager *quota_manager = new PosixQuotaManager(
262
1/2
✓ Branch 2 taken 2674 times.
✗ Branch 3 not taken.
 2674 limit, cleanup_threshold, cache_workspace);
263
264 // Initialize cache catalog
265
2/2
✓ Branch 1 taken 34 times.
✓ Branch 2 taken 2640 times.
 2674 if (!quota_manager->InitDatabase(rebuild_database)) {
266
1/2
✓ Branch 0 taken 34 times.
✗ Branch 1 not taken.
 34 delete quota_manager;
267 34 return NULL;
268 }
269 2640 quota_manager->CheckFreeSpace();
270 2640 MakePipe(quota_manager->pipe_lru_);
271
272 2640 quota_manager->protocol_revision_ = kProtocolRevision;
273 2640 quota_manager->initialized_ = true;
274 2640 return quota_manager;
275 }
276
277
278 /**
279 * Connects to a running shared local quota manager. Creates one if necessary.
280 */
281 68 PosixQuotaManager *PosixQuotaManager::CreateShared(
282 const std::string &exe_path,
283 const std::string &cache_workspace,
284 const uint64_t limit,
285 const uint64_t cleanup_threshold,
286 bool foreground) {
287 68 string cache_dir;
288 68 string workspace_dir;
289
2/4
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 68 times.
✗ Branch 5 not taken.
 68 ParseDirectories(cache_workspace, &cache_dir, &workspace_dir);
290
291 pid_t new_cachemgr_pid;
292
293 // Create lock file: only one fuse client at a time
294
2/4
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 68 times.
✗ Branch 5 not taken.
 68 const int fd_lockfile = LockFile(workspace_dir + "/lock_cachemgr");
295
2/2
✓ Branch 0 taken 34 times.
✓ Branch 1 taken 34 times.
 68 if (fd_lockfile < 0) {
296
1/2
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
 34 LogCvmfs(kLogQuota, kLogDebug, "could not open lock file %s (%d)",
297
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 68 (workspace_dir + "/lock_cachemgr").c_str(), errno);
298 34 return NULL;
299 }
300
301 PosixQuotaManager *quota_mgr = new PosixQuotaManager(limit, cleanup_threshold,
302
2/4
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 34 times.
✗ Branch 5 not taken.
 34 cache_workspace);
303 34 quota_mgr->shared_ = true;
304 34 quota_mgr->spawned_ = true;
305
306 // Try to connect to pipe
307
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 const string fifo_path = workspace_dir + "/cachemgr";
308
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 LogCvmfs(kLogQuota, kLogDebug, "trying to connect to existing pipe");
309
1/2
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
 34 quota_mgr->pipe_lru_[1] = open(fifo_path.c_str(), O_WRONLY | O_NONBLOCK);
310
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 34 times.
 34 if (quota_mgr->pipe_lru_[1] >= 0) {
311 ✗ const int fd_lockfile_rw = open((workspace_dir + "/lock_cachemgr").c_str(),
312 O_RDWR, 0600);
313 ✗ unsigned lockfile_magicnumber = 0;
314 ✗ const ssize_t result_mn = SafeRead(fd_lockfile_rw, &lockfile_magicnumber,
315 sizeof(lockfile_magicnumber));
316 ✗ const ssize_t result = SafeRead(fd_lockfile_rw, &new_cachemgr_pid,
317 sizeof(new_cachemgr_pid));
318 ✗ close(fd_lockfile_rw);
319
320 ✗ if ((lockfile_magicnumber != kLockFileMagicNumber) || (result < 0)
321 ✗ || (result_mn < 0)
322 ✗ || (static_cast<size_t>(result) < sizeof(new_cachemgr_pid))) {
323 ✗ if (result != 0) {
324 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
325 "could not read cache manager pid from lockfile");
326 ✗ UnlockFile(fd_lockfile);
327 ✗ delete quota_mgr;
328 ✗ return NULL;
329 } else {
330 // support reload from old versions of the cache manager
331 // lock file is empty in this case, try a plain ReadHalfPipe to get pid
332 ✗ quota_mgr->SetCacheMgrPid(quota_mgr->GetPid());
333 }
334 } else {
335 ✗ quota_mgr->SetCacheMgrPid(new_cachemgr_pid);
336 }
337
338
339 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to existing cache manager pipe");
340 ✗ quota_mgr->initialized_ = true;
341 ✗ Nonblock2Block(quota_mgr->pipe_lru_[1]);
342 ✗ UnlockFile(fd_lockfile);
343 ✗ quota_mgr->GetLimits(&quota_mgr->limit_, &quota_mgr->cleanup_threshold_);
344 ✗ LogCvmfs(kLogQuota, kLogDebug,
345 "received limit %" PRIu64 ", threshold %" PRIu64,
346 quota_mgr->limit_, quota_mgr->cleanup_threshold_);
347 ✗ if (FileExists(workspace_dir + "/cachemgr.protocol")) {
348 ✗ quota_mgr->protocol_revision_ = quota_mgr->GetProtocolRevision();
349 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected protocol revision %u",
350 quota_mgr->protocol_revision_);
351 } else {
352 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to ancient cache manager");
353 }
354 ✗ return quota_mgr;
355 }
356 34 const int connect_error = errno;
357
358 // Lock file: let existing cache manager finish first
359
2/4
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 34 times.
✗ Branch 5 not taken.
 34 const int fd_lockfile_fifo = LockFile(workspace_dir + "/lock_cachemgr.fifo");
360
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 34 times.
 34 if (fd_lockfile_fifo < 0) {
361 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not open lock file %s (%d)",
362 ✗ (workspace_dir + "/lock_cachemgr.fifo").c_str(), errno);
363 ✗ UnlockFile(fd_lockfile);
364 ✗ delete quota_mgr;
365 ✗ return NULL;
366 }
367
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 UnlockFile(fd_lockfile_fifo);
368
369
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 34 times.
 34 if (connect_error == ENXIO) {
370 ✗ LogCvmfs(kLogQuota, kLogDebug, "left-over FIFO found, unlinking");
371 ✗ unlink(fifo_path.c_str());
372 }
373
374 // Creating a new FIFO for the cache manager (to be bound later)
375 34 int retval = mkfifo(fifo_path.c_str(), 0600);
376
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 34 times.
 34 if (retval != 0) {
377 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to create cache manager FIFO (%d)",
378 ✗ errno);
379 ✗ UnlockFile(fd_lockfile);
380 ✗ delete quota_mgr;
381 ✗ return NULL;
382 }
383
384 // Create new cache manager
385 int pipe_boot[2];
386 int pipe_handshake[2];
387
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 MakePipe(pipe_boot);
388
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 MakePipe(pipe_handshake);
389
390 34 vector<string> command_line;
391
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 command_line.push_back(exe_path);
392
2/4
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 34 times.
✗ Branch 6 not taken.
 34 command_line.push_back("__cachemgr__");
393
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 command_line.push_back(cache_workspace);
394
2/4
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 34 times.
✗ Branch 5 not taken.
 34 command_line.push_back(StringifyInt(pipe_boot[1]));
395
2/4
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 34 times.
✗ Branch 5 not taken.
 34 command_line.push_back(StringifyInt(pipe_handshake[0]));
396
2/4
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 34 times.
✗ Branch 5 not taken.
 34 command_line.push_back(StringifyInt(limit));
397
2/4
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 34 times.
✗ Branch 5 not taken.
 34 command_line.push_back(StringifyInt(cleanup_threshold));
398 // do not propagate foreground in order to reliably get pid from exec
399 // instead, daemonize right here
400
2/4
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 34 times.
✗ Branch 5 not taken.
 34 command_line.push_back(StringifyInt(true)); // foreground
401
3/6
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 34 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 34 times.
✗ Branch 8 not taken.
 34 command_line.push_back(StringifyInt(GetLogSyslogLevel()));
402
3/6
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 34 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 34 times.
✗ Branch 8 not taken.
 34 command_line.push_back(StringifyInt(GetLogSyslogFacility()));
403
5/14
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 34 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 34 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 34 times.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 34 times.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
 34 command_line.push_back(GetLogDebugFile() + ":" + GetLogMicroSyslog());
404
405 34 set<int> preserve_filedes;
406
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 preserve_filedes.insert(0);
407
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 preserve_filedes.insert(1);
408
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 preserve_filedes.insert(2);
409
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 preserve_filedes.insert(pipe_boot[1]);
410
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 preserve_filedes.insert(pipe_handshake[0]);
411
412
1/2
✓ Branch 0 taken 34 times.
✗ Branch 1 not taken.
 34 if (foreground) {
413
1/2
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
 34 retval = ManagedExec(command_line, preserve_filedes, map<int, int>(),
414 /*drop_credentials*/ false,
415 /*clear_env*/ false,
416 /*double_fork*/ true, &new_cachemgr_pid);
417 } else {
418 ✗ retval = ExecAsDaemon(command_line, &new_cachemgr_pid);
419 }
420
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 34 times.
 34 if (!retval) {
421 ✗ UnlockFile(fd_lockfile);
422 ✗ ClosePipe(pipe_boot);
423 ✗ ClosePipe(pipe_handshake);
424 ✗ delete quota_mgr;
425 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to start cache manager");
426 ✗ return NULL;
427 }
428
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 LogCvmfs(kLogQuota, kLogDebug, "new cache manager pid: %d", new_cachemgr_pid);
429 34 quota_mgr->SetCacheMgrPid(new_cachemgr_pid);
430
2/4
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 34 times.
✗ Branch 6 not taken.
 34 const int fd_lockfile_rw = open((workspace_dir + "/lock_cachemgr").c_str(),
431 O_RDWR | O_TRUNC, 0600);
432 34 const unsigned magic_number = PosixQuotaManager::kLockFileMagicNumber;
433
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 const bool result_mn = SafeWrite(fd_lockfile_rw, &magic_number,
434 sizeof(magic_number));
435
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 const bool result = SafeWrite(fd_lockfile_rw, &new_cachemgr_pid,
436 sizeof(new_cachemgr_pid));
437
2/4
✓ Branch 0 taken 34 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 34 times.
 34 if (!result || !result_mn) {
438 ✗ PANIC(kLogSyslogErr, "could not write cache manager pid to lockfile");
439 }
440
441
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 close(fd_lockfile_rw);
442 // Wait for cache manager to be ready
443
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 close(pipe_boot[1]);
444
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 close(pipe_handshake[0]);
445 char buf;
446
2/4
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 34 times.
✗ Branch 4 not taken.
 34 if (read(pipe_boot[0], &buf, 1) != 1) {
447
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 UnlockFile(fd_lockfile);
448
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 close(pipe_boot[0]);
449
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 close(pipe_handshake[1]);
450
1/2
✓ Branch 0 taken 34 times.
✗ Branch 1 not taken.
 34 delete quota_mgr;
451
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
452 "cache manager did not start");
453 34 return NULL;
454 }
455 ✗ close(pipe_boot[0]);
456
457 // Connect write end
458 ✗ quota_mgr->pipe_lru_[1] = open(fifo_path.c_str(), O_WRONLY | O_NONBLOCK);
459 ✗ if (quota_mgr->pipe_lru_[1] < 0) {
460 ✗ LogCvmfs(kLogQuota, kLogDebug,
461 ✗ "failed to connect to newly created FIFO (%d)", errno);
462 ✗ close(pipe_handshake[1]);
463 ✗ UnlockFile(fd_lockfile);
464 ✗ delete quota_mgr;
465 ✗ return NULL;
466 }
467
468 // Finalize handshake
469 ✗ buf = 'C';
470 ✗ if (write(pipe_handshake[1], &buf, 1) != 1) {
471 ✗ UnlockFile(fd_lockfile);
472 ✗ close(pipe_handshake[1]);
473 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not finalize handshake");
474 ✗ delete quota_mgr;
475 ✗ return NULL;
476 }
477 ✗ close(pipe_handshake[1]);
478
479 ✗ Nonblock2Block(quota_mgr->pipe_lru_[1]);
480 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to a new cache manager");
481 ✗ quota_mgr->protocol_revision_ = kProtocolRevision;
482
483 ✗ UnlockFile(fd_lockfile);
484
485 ✗ quota_mgr->initialized_ = true;
486 ✗ quota_mgr->GetLimits(&quota_mgr->limit_, &quota_mgr->cleanup_threshold_);
487 ✗ LogCvmfs(kLogQuota, kLogDebug,
488 "received limit %" PRIu64 ", "
489 "threshold %" PRIu64,
490 quota_mgr->limit_, quota_mgr->cleanup_threshold_);
491 ✗ return quota_mgr;
492 68 }
493
494
495 340 bool PosixQuotaManager::DoCleanup(const uint64_t leave_size) {
496
2/2
✓ Branch 0 taken 68 times.
✓ Branch 1 taken 272 times.
 340 if (gauge_ <= leave_size)
497 68 return true;
498
499 // TODO(jblomer) transaction
500
1/2
✓ Branch 1 taken 272 times.
✗ Branch 2 not taken.
 272 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug,
501 "clean up cache until at most %lu KB is used", leave_size / 1024);
502
1/2
✓ Branch 1 taken 272 times.
✗ Branch 2 not taken.
 272 LogCvmfs(kLogQuota, kLogDebug, "gauge %" PRIu64, gauge_);
503
1/2
✓ Branch 1 taken 272 times.
✗ Branch 2 not taken.
 272 cleanup_recorder_.Tick();
504
505 bool result;
506 272 vector<string> trash;
507
508 // Note that volatile files start counting from the smallest int64 number:
509 // the absolute sequence number with the first bit set in two's complement.
510 // So -1 can be a marker that will never appear in the database.
511 272 int64_t max_acseq = -1;
512 272 std::vector<EvictCandidate> lru_ordered_open;
513
514 do {
515
1/2
✓ Branch 1 taken 1972 times.
✗ Branch 2 not taken.
 1972 sqlite3_reset(stmt_lru_);
516
3/4
✓ Branch 0 taken 272 times.
✓ Branch 1 taken 1700 times.
✓ Branch 3 taken 1972 times.
✗ Branch 4 not taken.
 2244 sqlite3_bind_int64(stmt_lru_, 1,
517 272 (max_acseq == -1) ? std::numeric_limits<int64_t>::min()
518 : (max_acseq + 1));
519
520 1972 std::vector<EvictCandidate> candidates;
521
1/2
✓ Branch 1 taken 1972 times.
✗ Branch 2 not taken.
 1972 candidates.reserve(kEvictBatchSize);
522 1972 string hash_str;
523 1972 unsigned i = 0;
524
3/4
✓ Branch 1 taken 1770312 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1768340 times.
✓ Branch 4 taken 1972 times.
 1770312 while (sqlite3_step(stmt_lru_) == SQLITE_ROW) {
525 hash_str = reinterpret_cast<const char *>(
526
2/4
✓ Branch 1 taken 1768340 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1768340 times.
✗ Branch 5 not taken.
 1768340 sqlite3_column_text(stmt_lru_, 0));
527
1/2
✓ Branch 2 taken 1768340 times.
✗ Branch 3 not taken.
 1768340 LogCvmfs(kLogQuota, kLogDebug, "add %s to candidates for eviction",
528 hash_str.c_str());
529
1/2
✓ Branch 1 taken 1768340 times.
✗ Branch 2 not taken.
 1768340 candidates.push_back(
530
1/2
✓ Branch 1 taken 1768340 times.
✗ Branch 2 not taken.
 1768340 EvictCandidate(shash::MkFromHexPtr(shash::HexPtr(hash_str)),
531 1768340 sqlite3_column_int64(stmt_lru_, 1),
532
2/4
✓ Branch 1 taken 1768340 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1768340 times.
✗ Branch 5 not taken.
 1768340 sqlite3_column_int64(stmt_lru_, 2)));
533 1768340 i++;
534 }
535
2/2
✓ Branch 1 taken 34 times.
✓ Branch 2 taken 1938 times.
 1972 if (candidates.empty()) {
536
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 LogCvmfs(kLogQuota, kLogDebug, "no more entries to evict");
537 34 break;
538 }
539
540 1938 const unsigned N = candidates.size();
541
542 1938 open_files_.clear();
543
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1938 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 3876 open_files_ = (cleanup_unused_first_) ? CollectAllOpenHashes()
544 1938 : std::vector<shash::Any>();
545
546
2/2
✓ Branch 0 taken 1700306 times.
✓ Branch 1 taken 1700 times.
 1702006 for (i = 0; i < N; ++i) {
547 // That's a critical condition. We must not delete a not yet inserted
548 // pinned file as it is already reserved (but will be inserted later).
549 // Instead, set the pin bit in the db to not run into an endless loop
550
1/2
✓ Branch 2 taken 1700306 times.
✗ Branch 3 not taken.
 1700306 const bool is_pinned = pinned_chunks_.find(candidates[i].hash)
551 3400612 != pinned_chunks_.end();
552
553 // Avoid evicting open files hopping there are enough more recently used
554 // files to satisfy the cleanup request
555
1/2
✓ Branch 2 taken 1700306 times.
✗ Branch 3 not taken.
 1700306 const bool is_open = std::find(open_files_.begin(), open_files_.end(),
556 1700306 candidates[i].hash)
557 3400612 != open_files_.end();
558
559
2/2
✓ Branch 0 taken 34 times.
✓ Branch 1 taken 1700272 times.
 1700306 if (is_pinned) {
560
1/2
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
 34 SkipEviction(candidates[i]);
561 34 continue;
562 }
563
564
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1700272 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 1700272 if (cleanup_unused_first_ and is_open) {
565 ✗ SkipEviction(candidates[i]);
566 ✗ lru_ordered_open.push_back(candidates[i]);
567 ✗ continue;
568 }
569
570
2/4
✓ Branch 1 taken 1700272 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1700272 times.
✗ Branch 5 not taken.
 3400544 trash.push_back(cache_dir_ + "/"
571
2/4
✓ Branch 2 taken 1700272 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1700272 times.
✗ Branch 6 not taken.
 5100816 + candidates[i].hash.MakePathWithoutSuffix());
572 1700272 gauge_ -= candidates[i].size;
573 1700272 max_acseq = candidates[i].acseq;
574
1/2
✓ Branch 2 taken 1700272 times.
✗ Branch 3 not taken.
 1700272 LogCvmfs(kLogQuota, kLogDebug, "lru cleanup %s, new gauge %" PRIu64,
575
1/2
✓ Branch 2 taken 1700272 times.
✗ Branch 3 not taken.
 3400544 candidates[i].hash.ToString().c_str(), gauge_);
576
577
2/2
✓ Branch 0 taken 238 times.
✓ Branch 1 taken 1700034 times.
 1700272 if (gauge_ <= leave_size)
578 238 break;
579 }
580
6/6
✓ Branch 1 taken 1938 times.
✓ Branch 2 taken 34 times.
✓ Branch 4 taken 1938 times.
✓ Branch 5 taken 34 times.
✓ Branch 6 taken 1700 times.
✓ Branch 7 taken 238 times.
 3944 } while (gauge_ > leave_size);
581
582
1/2
✓ Branch 0 taken 272 times.
✗ Branch 1 not taken.
 272 if (max_acseq != -1) {
583
1/2
✓ Branch 1 taken 272 times.
✗ Branch 2 not taken.
 272 sqlite3_bind_int64(stmt_rm_batch_, 1, max_acseq);
584
1/2
✓ Branch 1 taken 272 times.
✗ Branch 2 not taken.
 272 result = (sqlite3_step(stmt_rm_batch_) == SQLITE_DONE);
585
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 272 times.
 272 assert(result);
586
1/2
✓ Branch 1 taken 272 times.
✗ Branch 2 not taken.
 272 sqlite3_reset(stmt_rm_batch_);
587
588
1/2
✓ Branch 1 taken 272 times.
✗ Branch 2 not taken.
 272 result = (sqlite3_step(stmt_unblock_) == SQLITE_DONE);
589
1/2
✓ Branch 1 taken 272 times.
✗ Branch 2 not taken.
 272 sqlite3_reset(stmt_unblock_);
590
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 272 times.
 272 assert(result);
591 }
592
593
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 272 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 272 times.
 272 while (!lru_ordered_open.empty() and gauge_ > leave_size) {
594 // cleanup files in use
595 ✗ auto &candidate = lru_ordered_open[0];
596 ✗ trash.push_back(cache_dir_ + "/" + candidate.hash.MakePathWithoutSuffix());
597 ✗ gauge_ -= candidate.size;
598 ✗ max_acseq = candidate.acseq;
599 ✗ LogCvmfs(kLogQuota, kLogDebug, "lru cleanup %s, new gauge %" PRIu64,
600 ✗ candidate.hash.ToString().c_str(), gauge_);
601 ✗ lru_ordered_open.erase(lru_ordered_open.begin());
602 }
603
604
2/4
✓ Branch 1 taken 272 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 272 times.
 272 if (!EmptyTrash(trash))
605 ✗ return false;
606
607
2/2
✓ Branch 0 taken 34 times.
✓ Branch 1 taken 238 times.
 272 if (gauge_ > leave_size) {
608
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
609 "request to clean until %" PRIu64 ", "
610 "but effective gauge is %" PRIu64,
611 leave_size, gauge_);
612 34 return false;
613 }
614 238 return true;
615 272 }
616
617 272 bool PosixQuotaManager::EmptyTrash(const std::vector<std::string> &trash) {
618
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 272 times.
 272 if (trash.empty())
619 ✗ return true;
620
621
2/2
✓ Branch 0 taken 204 times.
✓ Branch 1 taken 68 times.
 272 if (async_delete_) {
622 // Double fork avoids zombie, forked removal process must not flush file
623 // buffers
624 pid_t pid;
625 int statloc;
626
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 204 times.
 204 if ((pid = fork()) == 0) {
627 // TODO(jblomer): eviciting files in the cache should perhaps become a
628 // thread. This would also allow to block the chunks and prevent the
629 // race with re-insertion. Then again, a thread can block umount.
630 #ifndef DEBUGMSG
631 ✗ CloseAllFildes(std::set<int>());
632 #endif
633 ✗ if (fork() == 0) {
634 ✗ for (unsigned i = 0, iEnd = trash.size(); i < iEnd; ++i) {
635 ✗ LogCvmfs(kLogQuota, kLogDebug, "unlink %s", trash[i].c_str());
636 ✗ unlink(trash[i].c_str());
637 }
638 ✗ _exit(0);
639 }
640 ✗ _exit(0);
641 } else {
642
1/2
✓ Branch 0 taken 204 times.
✗ Branch 1 not taken.
 204 if (pid > 0)
643
1/2
✓ Branch 1 taken 204 times.
✗ Branch 2 not taken.
 204 waitpid(pid, &statloc, 0);
644 else
645 ✗ return false;
646 }
647 } else { // !async_delete_
648
2/2
✓ Branch 1 taken 102 times.
✓ Branch 2 taken 68 times.
 170 for (unsigned i = 0, iEnd = trash.size(); i < iEnd; ++i) {
649 102 LogCvmfs(kLogQuota, kLogDebug, "unlink %s", trash[i].c_str());
650 102 unlink(trash[i].c_str());
651 }
652 }
653 272 return true;
654 }
655
656
657 3401417 void PosixQuotaManager::DoInsert(const shash::Any &hash,
658 const uint64_t size,
659 const string &description,
660 const CommandType command_type) {
661
1/2
✓ Branch 1 taken 3401417 times.
✗ Branch 2 not taken.
 3401417 const string hash_str = hash.ToString();
662
1/2
✓ Branch 3 taken 3401417 times.
✗ Branch 4 not taken.
 3401417 LogCvmfs(kLogQuota, kLogDebug, "insert into lru %s, path %s, method %d",
663 hash_str.c_str(), description.c_str(), command_type);
664 3401417 const unsigned desc_length = (description.length() > kMaxDescription)
665 ? kMaxDescription
666
1/2
✓ Branch 0 taken 3401417 times.
✗ Branch 1 not taken.
 3401417 : description.length();
667
668 LruCommand *cmd = reinterpret_cast<LruCommand *>(
669 3401417 alloca(sizeof(LruCommand) + desc_length));
670 3401417 new (cmd) LruCommand;
671 3401417 cmd->command_type = command_type;
672 3401417 cmd->SetSize(size);
673
1/2
✓ Branch 1 taken 3401417 times.
✗ Branch 2 not taken.
 3401417 cmd->StoreHash(hash);
674 3401417 cmd->desc_length = desc_length;
675 3401417 memcpy(reinterpret_cast<char *>(cmd) + sizeof(LruCommand), &description[0],
676 desc_length);
677
1/2
✓ Branch 1 taken 3401417 times.
✗ Branch 2 not taken.
 3401417 WritePipe(pipe_lru_[1], cmd, sizeof(LruCommand) + desc_length);
678 3401417 }
679
680
681 1190 vector<string> PosixQuotaManager::DoList(const CommandType list_command) {
682 1190 vector<string> result;
683
684 int pipe_list[2];
685
1/2
✓ Branch 1 taken 1190 times.
✗ Branch 2 not taken.
 1190 MakeReturnPipe(pipe_list);
686 char description_buffer[kMaxDescription];
687
688 1190 LruCommand cmd;
689 1190 cmd.command_type = list_command;
690 1190 cmd.return_pipe = pipe_list[1];
691
1/2
✓ Branch 1 taken 1190 times.
✗ Branch 2 not taken.
 1190 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
692
693 int length;
694 do {
695
1/2
✓ Branch 1 taken 3402516 times.
✗ Branch 2 not taken.
 3402516 ManagedReadHalfPipe(pipe_list[0], &length, sizeof(length));
696
2/2
✓ Branch 0 taken 3401326 times.
✓ Branch 1 taken 1190 times.
 3402516 if (length > 0) {
697
1/2
✓ Branch 1 taken 3401326 times.
✗ Branch 2 not taken.
 3401326 ReadPipe(pipe_list[0], description_buffer, length);
698
2/4
✓ Branch 2 taken 3401326 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 3401326 times.
✗ Branch 6 not taken.
 3401326 result.push_back(string(description_buffer, length));
699 }
700
2/2
✓ Branch 0 taken 3401326 times.
✓ Branch 1 taken 1190 times.
 3402516 } while (length >= 0);
701
702
1/2
✓ Branch 1 taken 1190 times.
✗ Branch 2 not taken.
 1190 CloseReturnPipe(pipe_list);
703 2380 return result;
704 }
705
706
707 872 uint64_t PosixQuotaManager::GetCapacity() {
708
1/2
✓ Branch 0 taken 872 times.
✗ Branch 1 not taken.
 872 if (limit_ != (uint64_t)(-1))
709 872 return limit_;
710
711 // Unrestricted cache, look at free space on cache dir fs
712 struct statfs info;
713 ✗ if (statfs(".", &info) == 0) {
714 ✗ return info.f_bavail * info.f_bsize;
715 } else {
716 ✗ LogCvmfs(kLogQuota, kLogSyslogErr | kLogDebug,
717 ✗ "failed to query file system info of cache (%d)", errno);
718 ✗ return limit_;
719 }
720 }
721
722
723 ✗ void PosixQuotaManager::GetLimits(uint64_t *limit,
724 uint64_t *cleanup_threshold) {
725 int pipe_limits[2];
726 ✗ MakeReturnPipe(pipe_limits);
727
728 ✗ LruCommand cmd;
729 ✗ cmd.command_type = kLimits;
730 ✗ cmd.return_pipe = pipe_limits[1];
731 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
732 ✗ ManagedReadHalfPipe(pipe_limits[0], limit, sizeof(*limit));
733 ✗ ReadPipe(pipe_limits[0], cleanup_threshold, sizeof(*cleanup_threshold));
734 ✗ CloseReturnPipe(pipe_limits);
735 }
736
737
738 /**
739 * Since we only cleanup until cleanup_threshold, we can only add
740 * files smaller than limit-cleanup_threshold.
741 */
742 237 uint64_t PosixQuotaManager::GetMaxFileSize() {
743 237 return limit_ - cleanup_threshold_;
744 }
745
746
747 34 pid_t PosixQuotaManager::GetPid() {
748
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 34 if (!shared_ || !spawned_) {
749 34 return getpid();
750 }
751 ✗ if (cachemgr_pid_) {
752 ✗ return cachemgr_pid_;
753 }
754
755 pid_t result;
756 int pipe_pid[2];
757 ✗ MakeReturnPipe(pipe_pid);
758
759 ✗ LruCommand cmd;
760 ✗ cmd.command_type = kPid;
761 ✗ cmd.return_pipe = pipe_pid[1];
762 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
763 ✗ ReadHalfPipe(pipe_pid[0], &result, sizeof(result));
764 ✗ CloseReturnPipe(pipe_pid);
765 ✗ return result;
766 }
767
768
769 34 uint32_t PosixQuotaManager::GetProtocolRevision() {
770 int pipe_revision[2];
771
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 MakeReturnPipe(pipe_revision);
772
773 34 LruCommand cmd;
774 34 cmd.command_type = kGetProtocolRevision;
775 34 cmd.return_pipe = pipe_revision[1];
776
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
777
778 uint32_t revision;
779
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 ManagedReadHalfPipe(pipe_revision[0], &revision, sizeof(revision));
780
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 CloseReturnPipe(pipe_revision);
781 34 return revision;
782 }
783
784 838 void PosixQuotaManager::SetCleanupPolicy(bool cleanup_unused_first) {
785 838 LruCommand cmd;
786 838 cmd.command_type = kSetCleanupPolicy;
787
1/2
✓ Branch 1 taken 838 times.
✗ Branch 2 not taken.
 838 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
788
789
1/2
✓ Branch 1 taken 838 times.
✗ Branch 2 not taken.
 838 WritePipe(pipe_lru_[1], &cleanup_unused_first, sizeof(bool));
790 838 }
791
792 ✗ void PosixQuotaManager::RegisterMountpoint(const std::string &mountpoint) {
793 ✗ LruCommand cmd;
794 ✗ cmd.command_type = kRegisterMountpoint;
795 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
796
797 ✗ size_t mp_size = mountpoint.size();
798 ✗ WritePipe(pipe_lru_[1], &mp_size, sizeof(size_t));
799 ✗ WritePipe(pipe_lru_[1], mountpoint.data(), mp_size);
800 }
801
802 ✗ std::string PosixQuotaManager::GetMountpoints() {
803 int pipe_mp[2];
804 ✗ MakeReturnPipe(pipe_mp);
805
806 ✗ LruCommand cmd;
807 ✗ cmd.command_type = kGetMountpoints;
808 ✗ cmd.return_pipe = pipe_mp[1];
809 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
810 ✗ size_t mp_str_size = 0;
811 ✗ ManagedReadHalfPipe(pipe_mp[0], &mp_str_size, sizeof(size_t));
812 ✗ char *buf = (char *)malloc(mp_str_size * sizeof(char));
813 ✗ ManagedReadHalfPipe(pipe_mp[0], buf, mp_str_size);
814 ✗ return std::string(buf);
815 }
816
817 ✗ std::string PosixQuotaManager::GetGroupHashes() {
818 int pipe_gh[2];
819 ✗ MakeReturnPipe(pipe_gh);
820
821 ✗ LruCommand cmd;
822 ✗ cmd.command_type = kGetGroupHashes;
823 ✗ cmd.return_pipe = pipe_gh[1];
824 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
825 ✗ size_t mp_str_size = 0;
826 ✗ ManagedReadHalfPipe(pipe_gh[0], &mp_str_size, sizeof(size_t));
827 ✗ char *buf = (char *)malloc(mp_str_size * sizeof(char));
828 ✗ ManagedReadHalfPipe(pipe_gh[0], buf, mp_str_size);
829 ✗ return std::string(buf);
830 }
831
832 /**
833 * Queries the shared local hard disk quota manager.
834 */
835 612 void PosixQuotaManager::GetSharedStatus(uint64_t *gauge, uint64_t *pinned) {
836 int pipe_status[2];
837
1/2
✓ Branch 1 taken 612 times.
✗ Branch 2 not taken.
 612 MakeReturnPipe(pipe_status);
838
839 612 LruCommand cmd;
840 612 cmd.command_type = kStatus;
841 612 cmd.return_pipe = pipe_status[1];
842
1/2
✓ Branch 1 taken 612 times.
✗ Branch 2 not taken.
 612 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
843
1/2
✓ Branch 1 taken 612 times.
✗ Branch 2 not taken.
 612 ManagedReadHalfPipe(pipe_status[0], gauge, sizeof(*gauge));
844
1/2
✓ Branch 1 taken 612 times.
✗ Branch 2 not taken.
 612 ReadPipe(pipe_status[0], pinned, sizeof(*pinned));
845
1/2
✓ Branch 1 taken 612 times.
✗ Branch 2 not taken.
 612 CloseReturnPipe(pipe_status);
846 612 }
847
848 34 bool PosixQuotaManager::SetSharedLimit(uint64_t limit) {
849 int pipe_set_limit[2];
850 bool result;
851
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 MakeReturnPipe(pipe_set_limit);
852
853 34 LruCommand cmd;
854 34 cmd.command_type = kSetLimit;
855 34 cmd.size = limit;
856 34 cmd.return_pipe = pipe_set_limit[1];
857
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
858
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 ReadHalfPipe(pipe_set_limit[0], &result, sizeof(result));
859
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 CloseReturnPipe(pipe_set_limit);
860 34 return result;
861 }
862
863
864 34 bool PosixQuotaManager::SetLimit(uint64_t size) {
865
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 34 times.
 34 if (!spawned_) {
866 ✗ limit_ = size;
867 ✗ cleanup_threshold_ = size / 2;
868 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
869 "Quota limit set to %lu / threshold %lu", limit_,
870 cleanup_threshold_);
871 ✗ return true;
872 }
873 34 return SetSharedLimit(size);
874 }
875
876 2220 uint64_t PosixQuotaManager::GetSize() {
877
2/2
✓ Branch 0 taken 1676 times.
✓ Branch 1 taken 544 times.
 2220 if (!spawned_)
878 1676 return gauge_;
879 uint64_t gauge, size_pinned;
880
1/2
✓ Branch 1 taken 544 times.
✗ Branch 2 not taken.
 544 GetSharedStatus(&gauge, &size_pinned);
881 544 return gauge;
882 }
883
884
885 68 uint64_t PosixQuotaManager::GetSizePinned() {
886
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 68 times.
 68 if (!spawned_)
887 ✗ return pinned_;
888 uint64_t gauge, size_pinned;
889
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 GetSharedStatus(&gauge, &size_pinned);
890 68 return size_pinned;
891 }
892
893
894 136 uint64_t PosixQuotaManager::GetCleanupRate(uint64_t period_s) {
895
2/4
✓ Branch 0 taken 136 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 136 times.
 136 if (!spawned_ || (protocol_revision_ < 2))
896 ✗ return 0;
897 uint64_t cleanup_rate;
898
899 int pipe_cleanup_rate[2];
900
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 MakeReturnPipe(pipe_cleanup_rate);
901 136 LruCommand cmd;
902 136 cmd.command_type = kCleanupRate;
903 136 cmd.size = period_s;
904 136 cmd.return_pipe = pipe_cleanup_rate[1];
905
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
906
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 ManagedReadHalfPipe(pipe_cleanup_rate[0], &cleanup_rate,
907 sizeof(cleanup_rate));
908
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 CloseReturnPipe(pipe_cleanup_rate);
909
910 136 return cleanup_rate;
911 }
912
913
914 2844 bool PosixQuotaManager::InitDatabase(const bool rebuild_database) {
915 2844 string sql;
916 sqlite3_stmt *stmt;
917
918
2/4
✓ Branch 1 taken 2844 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2844 times.
✗ Branch 5 not taken.
 2844 fd_lock_cachedb_ = LockFile(workspace_dir_ + "/lock_cachedb");
919
2/2
✓ Branch 0 taken 34 times.
✓ Branch 1 taken 2810 times.
 2844 if (fd_lock_cachedb_ < 0) {
920
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 LogCvmfs(kLogQuota, kLogDebug, "failed to create cachedb lock");
921 34 return false;
922 }
923
924 2810 bool retry = false;
925
1/2
✓ Branch 1 taken 2810 times.
✗ Branch 2 not taken.
 2810 const string db_file = cache_dir_ + "/cachedb";
926
2/2
✓ Branch 0 taken 2650 times.
✓ Branch 1 taken 160 times.
 2810 if (rebuild_database) {
927
1/2
✓ Branch 2 taken 160 times.
✗ Branch 3 not taken.
 160 LogCvmfs(kLogQuota, kLogDebug, "rebuild database, unlinking existing (%s)",
928 db_file.c_str());
929 160 unlink(db_file.c_str());
930
1/2
✓ Branch 1 taken 160 times.
✗ Branch 2 not taken.
 160 unlink((db_file + "-journal").c_str());
931 }
932
933 2650 init_recover:
934
1/2
✓ Branch 2 taken 2810 times.
✗ Branch 3 not taken.
 2810 int err = sqlite3_open(db_file.c_str(), &database_);
935
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2810 times.
 2810 if (err != SQLITE_OK) {
936 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not open cache database (%d)", err);
937 ✗ goto init_database_fail;
938 }
939 // TODO(reneme): make this a `QuotaDatabase : public sqlite::Database`
940 sql = "PRAGMA synchronous=0; PRAGMA locking_mode=EXCLUSIVE; "
941 "PRAGMA auto_vacuum=1; "
942 "CREATE TABLE IF NOT EXISTS cache_catalog (sha1 TEXT, size INTEGER, "
943 " acseq INTEGER, path TEXT, type INTEGER, pinned INTEGER, "
944 "CONSTRAINT pk_cache_catalog PRIMARY KEY (sha1)); "
945 "CREATE UNIQUE INDEX IF NOT EXISTS idx_cache_catalog_acseq "
946 " ON cache_catalog (acseq); "
947 "CREATE TEMP TABLE fscache (sha1 TEXT, size INTEGER, actime INTEGER, "
948 "CONSTRAINT pk_fscache PRIMARY KEY (sha1)); "
949 "CREATE INDEX idx_fscache_actime ON fscache (actime); "
950 "CREATE TABLE IF NOT EXISTS properties (key TEXT, value TEXT, "
951
1/2
✓ Branch 1 taken 2810 times.
✗ Branch 2 not taken.
 2810 " CONSTRAINT pk_properties PRIMARY KEY(key));";
952
1/2
✓ Branch 2 taken 2810 times.
✗ Branch 3 not taken.
 2810 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
953
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2810 times.
 2810 if (err != SQLITE_OK) {
954 ✗ if (!retry) {
955 ✗ retry = true;
956 ✗ sqlite3_close(database_);
957 ✗ unlink(db_file.c_str());
958 ✗ unlink((db_file + "-journal").c_str());
959 ✗ LogCvmfs(kLogQuota, kLogSyslogWarn,
960 "LRU database corrupted, re-building");
961 ✗ goto init_recover;
962 }
963 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
964 sql.c_str());
965 ✗ goto init_database_fail;
966 }
967
968 // If this an old cache catalog,
969 // add and initialize new columns to cache_catalog
970 sql = "ALTER TABLE cache_catalog ADD type INTEGER; "
971
1/2
✓ Branch 1 taken 2810 times.
✗ Branch 2 not taken.
 2810 "ALTER TABLE cache_catalog ADD pinned INTEGER";
972
1/2
✓ Branch 2 taken 2810 times.
✗ Branch 3 not taken.
 2810 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
973
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2810 times.
 2810 if (err == SQLITE_OK) {
974 ✗ sql = "UPDATE cache_catalog SET type=" + StringifyInt(kFileRegular) + ";";
975 ✗ err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
976 ✗ if (err != SQLITE_OK) {
977 ✗ LogCvmfs(kLogQuota, kLogDebug,
978 "could not init cache database (failed: %s)", sql.c_str());
979 ✗ goto init_database_fail;
980 }
981 }
982
983 // Set pinned back
984
1/2
✓ Branch 1 taken 2810 times.
✗ Branch 2 not taken.
 2810 sql = "UPDATE cache_catalog SET pinned=0;";
985
1/2
✓ Branch 2 taken 2810 times.
✗ Branch 3 not taken.
 2810 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
986
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2810 times.
 2810 if (err != SQLITE_OK) {
987 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
988 sql.c_str());
989 ✗ goto init_database_fail;
990 }
991
992 // Set schema version
993 sql = "INSERT OR REPLACE INTO properties (key, value) "
994
1/2
✓ Branch 1 taken 2810 times.
✗ Branch 2 not taken.
 2810 "VALUES ('schema', '1.0')";
995
1/2
✓ Branch 2 taken 2810 times.
✗ Branch 3 not taken.
 2810 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
996
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2810 times.
 2810 if (err != SQLITE_OK) {
997 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
998 sql.c_str());
999 ✗ goto init_database_fail;
1000 }
1001
1002 // If cache catalog is empty, recreate from file system
1003
1/2
✓ Branch 1 taken 2810 times.
✗ Branch 2 not taken.
 2810 sql = "SELECT count(*) FROM cache_catalog;";
1004
1/2
✓ Branch 2 taken 2810 times.
✗ Branch 3 not taken.
 2810 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1005
2/4
✓ Branch 1 taken 2810 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2810 times.
✗ Branch 4 not taken.
 2810 if (sqlite3_step(stmt) == SQLITE_ROW) {
1006
6/8
✓ Branch 1 taken 2810 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 136 times.
✓ Branch 4 taken 2674 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 136 times.
✓ Branch 7 taken 2674 times.
✓ Branch 8 taken 136 times.
 2810 if ((sqlite3_column_int64(stmt, 0)) == 0 || rebuild_database) {
1007
1/2
✓ Branch 1 taken 2674 times.
✗ Branch 2 not taken.
 2674 LogCvmfs(kLogCvmfs, kLogDebug,
1008 "CernVM-FS: building lru cache database...");
1009
3/4
✓ Branch 1 taken 2674 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 102 times.
✓ Branch 4 taken 2572 times.
 2674 if (!RebuildDatabase()) {
1010
1/2
✓ Branch 1 taken 102 times.
✗ Branch 2 not taken.
 102 LogCvmfs(kLogQuota, kLogDebug,
1011 "could not build cache database from file system");
1012
1/2
✓ Branch 1 taken 102 times.
✗ Branch 2 not taken.
 102 sqlite3_finalize(stmt);
1013 102 goto init_database_fail;
1014 }
1015 }
1016
1/2
✓ Branch 1 taken 2708 times.
✗ Branch 2 not taken.
 2708 sqlite3_finalize(stmt);
1017 } else {
1018 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not select on cache catalog");
1019 ✗ sqlite3_finalize(stmt);
1020 ✗ goto init_database_fail;
1021 }
1022
1023 // How many bytes do we already have in cache?
1024
1/2
✓ Branch 1 taken 2708 times.
✗ Branch 2 not taken.
 2708 sql = "SELECT sum(size) FROM cache_catalog;";
1025
1/2
✓ Branch 2 taken 2708 times.
✗ Branch 3 not taken.
 2708 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1026
2/4
✓ Branch 1 taken 2708 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2708 times.
✗ Branch 4 not taken.
 2708 if (sqlite3_step(stmt) == SQLITE_ROW) {
1027
1/2
✓ Branch 1 taken 2708 times.
✗ Branch 2 not taken.
 2708 gauge_ = sqlite3_column_int64(stmt, 0);
1028 } else {
1029 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not determine cache size");
1030 ✗ sqlite3_finalize(stmt);
1031 ✗ goto init_database_fail;
1032 }
1033
1/2
✓ Branch 1 taken 2708 times.
✗ Branch 2 not taken.
 2708 sqlite3_finalize(stmt);
1034
1035 // Highest seq-no?
1036
1/2
✓ Branch 1 taken 2708 times.
✗ Branch 2 not taken.
 2708 sql = "SELECT coalesce(max(acseq & (~(1<<63))), 0) FROM cache_catalog;";
1037
1/2
✓ Branch 2 taken 2708 times.
✗ Branch 3 not taken.
 2708 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1038
2/4
✓ Branch 1 taken 2708 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2708 times.
✗ Branch 4 not taken.
 2708 if (sqlite3_step(stmt) == SQLITE_ROW) {
1039
1/2
✓ Branch 1 taken 2708 times.
✗ Branch 2 not taken.
 2708 seq_ = sqlite3_column_int64(stmt, 0) + 1;
1040 } else {
1041 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not determine highest seq-no");
1042 ✗ sqlite3_finalize(stmt);
1043 ✗ goto init_database_fail;
1044 }
1045
1/2
✓ Branch 1 taken 2708 times.
✗ Branch 2 not taken.
 2708 sqlite3_finalize(stmt);
1046
1047 // Prepare touch, new, remove statements
1048
1/2
✓ Branch 1 taken 2708 times.
✗ Branch 2 not taken.
 2708 sqlite3_prepare_v2(database_,
1049 "UPDATE cache_catalog SET acseq=:seq | (acseq&(1<<63)) "
1050 "WHERE sha1=:sha1;",
1051 -1, &stmt_touch_, NULL);
1052
1/2
✓ Branch 1 taken 2708 times.
✗ Branch 2 not taken.
 2708 sqlite3_prepare_v2(database_,
1053 "UPDATE cache_catalog SET pinned=0 "
1054 "WHERE sha1=:sha1;",
1055 -1, &stmt_unpin_, NULL);
1056
1/2
✓ Branch 1 taken 2708 times.
✗ Branch 2 not taken.
 2708 sqlite3_prepare_v2(database_,
1057 "UPDATE cache_catalog SET pinned=2 "
1058 "WHERE sha1=:sha1;",
1059 -1, &stmt_block_, NULL);
1060
1/2
✓ Branch 1 taken 2708 times.
✗ Branch 2 not taken.
 2708 sqlite3_prepare_v2(database_,
1061 "UPDATE cache_catalog SET pinned=1 "
1062 "WHERE pinned=2;",
1063 -1, &stmt_unblock_, NULL);
1064
1/2
✓ Branch 1 taken 2708 times.
✗ Branch 2 not taken.
 2708 sqlite3_prepare_v2(database_,
1065 "INSERT OR REPLACE INTO cache_catalog "
1066 "(sha1, size, acseq, path, type, pinned) "
1067 "VALUES (:sha1, :s, :seq, :p, :t, :pin);",
1068 -1, &stmt_new_, NULL);
1069
1/2
✓ Branch 1 taken 2708 times.
✗ Branch 2 not taken.
 2708 sqlite3_prepare_v2(database_,
1070 "SELECT size, pinned FROM cache_catalog WHERE sha1=:sha1;",
1071 -1, &stmt_size_, NULL);
1072
1/2
✓ Branch 1 taken 2708 times.
✗ Branch 2 not taken.
 2708 sqlite3_prepare_v2(database_, "DELETE FROM cache_catalog WHERE sha1=:sha1;",
1073 -1, &stmt_rm_, NULL);
1074
1/2
✓ Branch 1 taken 2708 times.
✗ Branch 2 not taken.
 2708 sqlite3_prepare_v2(database_,
1075 "DELETE FROM cache_catalog WHERE acseq<=:a AND pinned<>2;",
1076 -1, &stmt_rm_batch_, NULL);
1077
1/2
✓ Branch 2 taken 2708 times.
✗ Branch 3 not taken.
 2708 sqlite3_prepare_v2(database_,
1078
1/2
✓ Branch 2 taken 2708 times.
✗ Branch 3 not taken.
 5416 (std::string("SELECT sha1, size, acseq FROM cache_catalog "
1079 "WHERE pinned<>2 AND acseq>=:a "
1080 "ORDER BY acseq ASC "
1081 "LIMIT ")
1082
3/6
✓ Branch 1 taken 2708 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2708 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 2708 times.
✗ Branch 8 not taken.
 10832 + StringifyInt(kEvictBatchSize) + ";")
1083 .c_str(),
1084 -1, &stmt_lru_, NULL);
1085
1/2
✓ Branch 2 taken 2708 times.
✗ Branch 3 not taken.
 2708 sqlite3_prepare_v2(database_,
1086 ("SELECT path FROM cache_catalog WHERE type="
1087
3/6
✓ Branch 1 taken 2708 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2708 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 2708 times.
✗ Branch 8 not taken.
 5416 + StringifyInt(kFileRegular) + ";")
1088 .c_str(),
1089 -1, &stmt_list_, NULL);
1090
1/2
✓ Branch 1 taken 2708 times.
✗ Branch 2 not taken.
 2708 sqlite3_prepare_v2(database_,
1091 "SELECT path FROM cache_catalog WHERE pinned<>0;", -1,
1092 &stmt_list_pinned_, NULL);
1093
1/2
✓ Branch 1 taken 2708 times.
✗ Branch 2 not taken.
 2708 sqlite3_prepare_v2(database_,
1094 "SELECT path FROM cache_catalog WHERE acseq < 0;", -1,
1095 &stmt_list_volatile_, NULL);
1096
1/2
✓ Branch 2 taken 2708 times.
✗ Branch 3 not taken.
 2708 sqlite3_prepare_v2(database_,
1097 ("SELECT path FROM cache_catalog WHERE type="
1098
3/6
✓ Branch 1 taken 2708 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2708 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 2708 times.
✗ Branch 8 not taken.
 5416 + StringifyInt(kFileCatalog) + ";")
1099 .c_str(),
1100 -1, &stmt_list_catalogs_, NULL);
1101 2708 return true;
1102
1103 102 init_database_fail:
1104
1/2
✓ Branch 1 taken 102 times.
✗ Branch 2 not taken.
 102 sqlite3_close(database_);
1105 102 database_ = NULL;
1106
1/2
✓ Branch 1 taken 102 times.
✗ Branch 2 not taken.
 102 UnlockFile(fd_lock_cachedb_);
1107 102 return false;
1108 2844 }
1109
1110
1111 /**
1112 * Inserts a new file into cache catalog. This file gets a new,
1113 * highest sequence number. Does cache cleanup if necessary.
1114 */
1115 3400839 void PosixQuotaManager::Insert(const shash::Any &any_hash,
1116 const uint64_t size,
1117 const string &description) {
1118 3400839 DoInsert(any_hash, size, description, kInsert);
1119 3400839 }
1120
1121
1122 /**
1123 * Inserts a new file into cache catalog. This file is marked as volatile
1124 * and gets a new highest sequence number with the first bit set. Cache cleanup
1125 * treats these files with priority.
1126 */
1127 136 void PosixQuotaManager::InsertVolatile(const shash::Any &any_hash,
1128 const uint64_t size,
1129 const string &description) {
1130 136 DoInsert(any_hash, size, description, kInsertVolatile);
1131 136 }
1132
1133
1134 /**
1135 * Lists all path names from the cache db.
1136 */
1137 714 vector<string> PosixQuotaManager::List() { return DoList(kList); }
1138
1139
1140 /**
1141 * Lists all pinned files from the cache db.
1142 */
1143 272 vector<string> PosixQuotaManager::ListPinned() { return DoList(kListPinned); }
1144
1145
1146 /**
1147 * Lists all sqlite catalog files from the cache db.
1148 */
1149 102 vector<string> PosixQuotaManager::ListCatalogs() {
1150 102 return DoList(kListCatalogs);
1151 }
1152
1153
1154 /**
1155 * Lists only files flagged as volatile (priority removal)
1156 */
1157 102 vector<string> PosixQuotaManager::ListVolatile() {
1158 102 return DoList(kListVolatile);
1159 }
1160
1161
1162 /**
1163 * Entry point for the shared cache manager process
1164 */
1165 ✗ int PosixQuotaManager::MainCacheManager(int argc, char **argv) {
1166 ✗ LogCvmfs(kLogQuota, kLogDebug, "starting quota manager");
1167 int retval;
1168
1169 ✗ PosixQuotaManager shared_manager(0, 0, "");
1170 ✗ shared_manager.shared_ = true;
1171 ✗ shared_manager.spawned_ = true;
1172 ✗ shared_manager.pinned_ = 0;
1173
1174 // Process command line arguments
1175 ✗ ParseDirectories(string(argv[2]),
1176 &shared_manager.cache_dir_,
1177 &shared_manager.workspace_dir_);
1178 ✗ const int pipe_boot = String2Int64(argv[3]);
1179 ✗ const int pipe_handshake = String2Int64(argv[4]);
1180 ✗ shared_manager.limit_ = String2Int64(argv[5]);
1181 ✗ shared_manager.cleanup_threshold_ = String2Int64(argv[6]);
1182 ✗ const int foreground = String2Int64(argv[7]);
1183 ✗ const int syslog_level = String2Int64(argv[8]);
1184 ✗ const int syslog_facility = String2Int64(argv[9]);
1185 ✗ vector<string> logfiles = SplitString(argv[10], ':');
1186
1187 ✗ SetLogSyslogLevel(syslog_level);
1188 ✗ SetLogSyslogFacility(syslog_facility);
1189 ✗ if ((logfiles.size() > 0) && (logfiles[0] != ""))
1190 ✗ SetLogDebugFile(logfiles[0] + ".cachemgr");
1191 ✗ if (logfiles.size() > 1)
1192 ✗ SetLogMicroSyslog(logfiles[1]);
1193
1194 ✗ if (!foreground)
1195 ✗ Daemonize();
1196
1197 ✗ const UniquePtr<Watchdog> watchdog(Watchdog::Create(NULL));
1198 ✗ assert(watchdog.IsValid());
1199 ✗ watchdog->Spawn("./stacktrace.cachemgr");
1200
1201 // Initialize pipe, open non-blocking as cvmfs is not yet connected
1202 ✗ const int fd_lockfile_fifo = LockFile(shared_manager.workspace_dir_
1203 ✗ + "/lock_cachemgr.fifo");
1204 ✗ if (fd_lockfile_fifo < 0) {
1205 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1206 "could not open lock file "
1207 "%s (%d)",
1208 ✗ (shared_manager.workspace_dir_ + "/lock_cachemgr.fifo").c_str(),
1209 ✗ errno);
1210 ✗ return 1;
1211 }
1212 ✗ const string crash_guard = shared_manager.cache_dir_ + "/cachemgr.running";
1213 ✗ const bool rebuild = FileExists(crash_guard);
1214 ✗ retval = open(crash_guard.c_str(), O_RDONLY | O_CREAT, 0600);
1215 ✗ if (retval < 0) {
1216 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1217 "failed to create shared cache manager crash guard");
1218 ✗ UnlockFile(fd_lockfile_fifo);
1219 ✗ return 1;
1220 }
1221 ✗ close(retval);
1222
1223 // Redirect SQlite temp directory to cache (global variable)
1224 ✗ const string tmp_dir = shared_manager.workspace_dir_;
1225 ✗ sqlite3_temp_directory = static_cast<char *>(
1226 ✗ sqlite3_malloc(tmp_dir.length() + 1));
1227 ✗ snprintf(sqlite3_temp_directory, tmp_dir.length() + 1, "%s", tmp_dir.c_str());
1228
1229 // Cleanup leftover named pipes
1230 ✗ shared_manager.CleanupPipes();
1231
1232 ✗ if (!shared_manager.InitDatabase(rebuild)) {
1233 ✗ UnlockFile(fd_lockfile_fifo);
1234 ✗ return 1;
1235 }
1236 ✗ shared_manager.CheckFreeSpace();
1237
1238 // Save protocol revision to file. If the file is not found, it indicates
1239 // to the client that the cache manager is from times before the protocol
1240 // was versioned.
1241 const string protocol_revision_path = shared_manager.workspace_dir_
1242 ✗ + "/cachemgr.protocol";
1243 ✗ retval = open(protocol_revision_path.c_str(), O_WRONLY | O_CREAT, 0600);
1244 ✗ if (retval < 0) {
1245 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1246 ✗ "failed to open protocol revision file (%d)", errno);
1247 ✗ UnlockFile(fd_lockfile_fifo);
1248 ✗ return 1;
1249 }
1250 ✗ const string revision = StringifyInt(kProtocolRevision);
1251 ✗ const int written = write(retval, revision.data(), revision.length());
1252 ✗ close(retval);
1253 ✗ if ((written < 0) || static_cast<unsigned>(written) != revision.length()) {
1254 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1255 ✗ "failed to write protocol revision (%d)", errno);
1256 ✗ UnlockFile(fd_lockfile_fifo);
1257 ✗ return 1;
1258 }
1259
1260 ✗ const string fifo_path = shared_manager.workspace_dir_ + "/cachemgr";
1261 ✗ shared_manager.pipe_lru_[0] = open(fifo_path.c_str(), O_RDONLY | O_NONBLOCK);
1262 ✗ if (shared_manager.pipe_lru_[0] < 0) {
1263 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to listen on FIFO %s (%d)",
1264 ✗ fifo_path.c_str(), errno);
1265 ✗ UnlockFile(fd_lockfile_fifo);
1266 ✗ return 1;
1267 }
1268 ✗ Nonblock2Block(shared_manager.pipe_lru_[0]);
1269 ✗ LogCvmfs(kLogQuota, kLogDebug, "shared cache manager listening");
1270
1271 ✗ char buf = 'C';
1272 ✗ WritePipe(pipe_boot, &buf, 1);
1273 ✗ close(pipe_boot);
1274
1275 ✗ ReadPipe(pipe_handshake, &buf, 1);
1276 ✗ close(pipe_handshake);
1277 ✗ LogCvmfs(kLogQuota, kLogDebug, "shared cache manager handshake done");
1278
1279 // Ensure that broken pipes from clients do not kill the cache manager
1280 ✗ signal(SIGPIPE, SIG_IGN);
1281 // Don't let Ctrl-C ungracefully kill interactive session
1282 ✗ signal(SIGINT, SIG_IGN);
1283
1284 ✗ shared_manager.MainCommandServer(&shared_manager);
1285 ✗ unlink(fifo_path.c_str());
1286 ✗ unlink(protocol_revision_path.c_str());
1287 ✗ shared_manager.CloseDatabase();
1288 ✗ unlink(crash_guard.c_str());
1289 ✗ UnlockFile(fd_lockfile_fifo);
1290
1291 ✗ if (sqlite3_temp_directory) {
1292 ✗ sqlite3_free(sqlite3_temp_directory);
1293 ✗ sqlite3_temp_directory = NULL;
1294 }
1295
1296 ✗ return 0;
1297 }
1298
1299
1300 1054 void *PosixQuotaManager::MainCommandServer(void *data) {
1301 1054 PosixQuotaManager *quota_mgr = static_cast<PosixQuotaManager *>(data);
1302
1303
1/2
✓ Branch 1 taken 1054 times.
✗ Branch 2 not taken.
 1054 LogCvmfs(kLogQuota, kLogDebug, "starting quota manager");
1304
1/2
✓ Branch 1 taken 1054 times.
✗ Branch 2 not taken.
 1054 sqlite3_soft_heap_limit(quota_mgr->kSqliteMemPerThread);
1305
1306
2/2
✓ Branch 1 taken 33728 times.
✓ Branch 2 taken 1054 times.
 34782 LruCommand command_buffer[kCommandBufferSize];
1307 char description_buffer[kCommandBufferSize * kMaxDescription];
1308 1054 unsigned num_commands = 0;
1309
1310
1/2
✓ Branch 1 taken 5105474 times.
✗ Branch 2 not taken.
 5105474 while (read(quota_mgr->pipe_lru_[0], &command_buffer[num_commands],
1311 sizeof(command_buffer[0]))
1312
2/2
✓ Branch 0 taken 5104420 times.
✓ Branch 1 taken 1054 times.
 5105474 == sizeof(command_buffer[0])) {
1313 5104420 const CommandType command_type = command_buffer[num_commands].command_type;
1314
1/2
✓ Branch 1 taken 5104420 times.
✗ Branch 2 not taken.
 5104420 LogCvmfs(kLogQuota, kLogDebug, "received command %d", command_type);
1315 5104420 const uint64_t size = command_buffer[num_commands].GetSize();
1316
1317 // Inserts and pins come with a description (usually a path)
1318
4/4
✓ Branch 0 taken 1703842 times.
✓ Branch 1 taken 3400578 times.
✓ Branch 2 taken 1703706 times.
✓ Branch 3 taken 136 times.
 5104420 if ((command_type == kInsert) || (command_type == kInsertVolatile)
1319
4/4
✓ Branch 0 taken 1703638 times.
✓ Branch 1 taken 68 times.
✓ Branch 2 taken 374 times.
✓ Branch 3 taken 1703264 times.
 1703706 || (command_type == kPin) || (command_type == kPinRegular)) {
1320 3401156 const int desc_length = command_buffer[num_commands].desc_length;
1321 3401156 ReadPipe(quota_mgr->pipe_lru_[0],
1322
1/2
✓ Branch 1 taken 3401156 times.
✗ Branch 2 not taken.
 3401156 &description_buffer[kMaxDescription * num_commands],
1323 desc_length);
1324 }
1325
1326 // The protocol revision is returned immediately
1327
2/2
✓ Branch 0 taken 34 times.
✓ Branch 1 taken 5104386 times.
 5104420 if (command_type == kGetProtocolRevision) {
1328
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 const int return_pipe = quota_mgr->BindReturnPipe(
1329 command_buffer[num_commands].return_pipe);
1330
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 34 times.
 34 if (return_pipe < 0)
1331 ✗ continue;
1332
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 WritePipe(return_pipe, &quota_mgr->kProtocolRevision,
1333 sizeof(quota_mgr->kProtocolRevision));
1334
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 quota_mgr->UnbindReturnPipe(return_pipe);
1335 34 continue;
1336 34 }
1337
1338 // Register a new mountpoint
1339
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5104386 times.
 5104386 if (command_type == kRegisterMountpoint) {
1340 ✗ size_t mp_size = 0;
1341 ✗ ReadPipe(quota_mgr->pipe_lru_[0], &mp_size, sizeof(size_t));
1342 ✗ std::string mountpoint(mp_size, '\0');
1343 ✗ ReadPipe(quota_mgr->pipe_lru_[0], (void *)mountpoint.data(), mp_size);
1344 ✗ quota_mgr->mountpoints_.push_back(mountpoint);
1345 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
1346 "Mountpoint %s registered in the group", mountpoint.c_str());
1347 ✗ continue;
1348 }
1349
1350 // Set Cleanup Policy
1351
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5104386 times.
 5104386 if (command_type == kSetCleanupPolicy) {
1352 ✗ bool policy = false;
1353 ✗ ReadPipe(quota_mgr->pipe_lru_[0], &policy, sizeof(bool));
1354 ✗ quota_mgr->cleanup_unused_first_ = policy;
1355 ✗ continue;
1356 }
1357 // Mountpoints are returned immediately
1358
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5104386 times.
 5104386 if (command_type == kGetMountpoints) {
1359 ✗ const int return_pipe = quota_mgr->BindReturnPipe(
1360 command_buffer[num_commands].return_pipe);
1361 ✗ if (return_pipe < 0)
1362 ✗ continue;
1363
1364 ✗ std::string mps;
1365 ✗ for (auto it = quota_mgr->mountpoints_.begin();
1366 ✗ it != quota_mgr->mountpoints_.end();
1367 ✗ ++it) {
1368 ✗ mps += *it + "\n";
1369 }
1370 ✗ size_t mp_size = mps.size();
1371 ✗ WritePipe(return_pipe, &mp_size, sizeof(size_t));
1372 ✗ WritePipe(return_pipe, mps.c_str(), mp_size);
1373 ✗ quota_mgr->UnbindReturnPipe(return_pipe);
1374 ✗ continue;
1375 }
1376
1377 // Group hashes are returned immediately
1378
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5104386 times.
 5104386 if (command_type == kGetGroupHashes) {
1379 ✗ const int return_pipe = quota_mgr->BindReturnPipe(
1380 command_buffer[num_commands].return_pipe);
1381 ✗ if (return_pipe < 0)
1382 ✗ continue;
1383
1384 ✗ std::vector<shash::Any> gh = quota_mgr->CollectAllOpenHashes();
1385 ✗ std::string result;
1386 ✗ for (auto it = gh.begin(); it != gh.end(); ++it) {
1387 ✗ result += (*it).ToString() + "\n";
1388 }
1389 ✗ size_t result_size = result.size();
1390 ✗ WritePipe(return_pipe, &result_size, sizeof(size_t));
1391 ✗ WritePipe(return_pipe, result.c_str(), result_size);
1392 ✗ quota_mgr->UnbindReturnPipe(return_pipe);
1393 ✗ continue;
1394 }
1395
1396 // The cleanup rate is returned immediately
1397
2/2
✓ Branch 0 taken 136 times.
✓ Branch 1 taken 5104250 times.
 5104386 if (command_type == kCleanupRate) {
1398
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 const int return_pipe = quota_mgr->BindReturnPipe(
1399 command_buffer[num_commands].return_pipe);
1400
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 136 times.
 136 if (return_pipe < 0)
1401 ✗ continue;
1402 const uint64_t
1403 136 period_s = size; // use the size field to transmit the period
1404
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 uint64_t rate = quota_mgr->cleanup_recorder_.GetNoTicks(period_s);
1405
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 WritePipe(return_pipe, &rate, sizeof(rate));
1406
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 quota_mgr->UnbindReturnPipe(return_pipe);
1407 136 continue;
1408 136 }
1409
1410
2/2
✓ Branch 0 taken 34 times.
✓ Branch 1 taken 5104216 times.
 5104250 if (command_type == kSetLimit) {
1411
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 const int return_pipe = quota_mgr->BindReturnPipe(
1412 command_buffer[num_commands].return_pipe);
1413
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 34 times.
 34 if (return_pipe < 0)
1414 ✗ continue;
1415 34 quota_mgr->limit_ = size; // use the size field to transmit the size
1416 34 quota_mgr->cleanup_threshold_ = size / 2;
1417
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
1418 "Quota limit set to %lu / threshold %lu", quota_mgr->limit_,
1419 quota_mgr->cleanup_threshold_);
1420 34 bool ret = true;
1421
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 WritePipe(return_pipe, &ret, sizeof(ret));
1422
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 quota_mgr->UnbindReturnPipe(return_pipe);
1423 34 continue;
1424 34 }
1425
1426 // Reservations are handled immediately and "out of band"
1427
2/2
✓ Branch 0 taken 476 times.
✓ Branch 1 taken 5103740 times.
 5104216 if (command_type == kReserve) {
1428 476 bool success = true;
1429
1/2
✓ Branch 1 taken 476 times.
✗ Branch 2 not taken.
 476 const int return_pipe = quota_mgr->BindReturnPipe(
1430 command_buffer[num_commands].return_pipe);
1431
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 476 times.
 476 if (return_pipe < 0)
1432 ✗ continue;
1433
1434
1/2
✓ Branch 1 taken 476 times.
✗ Branch 2 not taken.
 476 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1435
1/2
✓ Branch 1 taken 476 times.
✗ Branch 2 not taken.
 476 const string hash_str(hash.ToString());
1436
1/2
✓ Branch 2 taken 476 times.
✗ Branch 3 not taken.
 476 LogCvmfs(kLogQuota, kLogDebug, "reserve %lu bytes for %s", size,
1437 hash_str.c_str());
1438
1439
1/2
✓ Branch 1 taken 476 times.
✗ Branch 2 not taken.
 476 if (quota_mgr->pinned_chunks_.find(hash)
1440
2/2
✓ Branch 2 taken 408 times.
✓ Branch 3 taken 68 times.
 952 == quota_mgr->pinned_chunks_.end()) {
1441
2/2
✓ Branch 0 taken 34 times.
✓ Branch 1 taken 374 times.
 408 if ((quota_mgr->pinned_ + size) > quota_mgr->cleanup_threshold_) {
1442
1/2
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
 34 LogCvmfs(kLogQuota, kLogDebug,
1443 "failed to insert %s (pinned), no space", hash_str.c_str());
1444 34 success = false;
1445 } else {
1446
1/2
✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
 374 quota_mgr->pinned_chunks_[hash] = size;
1447 374 quota_mgr->pinned_ += size;
1448
1/2
✓ Branch 1 taken 374 times.
✗ Branch 2 not taken.
 374 quota_mgr->CheckHighPinWatermark();
1449 }
1450 }
1451
1452
1/2
✓ Branch 1 taken 476 times.
✗ Branch 2 not taken.
 476 WritePipe(return_pipe, &success, sizeof(success));
1453
1/2
✓ Branch 1 taken 476 times.
✗ Branch 2 not taken.
 476 quota_mgr->UnbindReturnPipe(return_pipe);
1454 476 continue;
1455 476 }
1456
1457 // Back channels are also handled out of band
1458
2/2
✓ Branch 0 taken 136 times.
✓ Branch 1 taken 5103604 times.
 5103740 if (command_type == kRegisterBackChannel) {
1459
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 const int return_pipe = quota_mgr->BindReturnPipe(
1460 command_buffer[num_commands].return_pipe);
1461
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 136 times.
 136 if (return_pipe < 0)
1462 ✗ continue;
1463
1464
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 quota_mgr->UnlinkReturnPipe(command_buffer[num_commands].return_pipe);
1465
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 Block2Nonblock(return_pipe); // back channels are opportunistic
1466
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 shash::Md5 hash;
1467 136 memcpy(hash.digest, command_buffer[num_commands].digest,
1468 136 shash::kDigestSizes[shash::kMd5]);
1469
1470 136 quota_mgr->LockBackChannels();
1471 const map<shash::Md5, int>::const_iterator
1472
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 iter = quota_mgr->back_channels_.find(hash);
1473
1/2
✗ Branch 3 not taken.
✓ Branch 4 taken 136 times.
 136 if (iter != quota_mgr->back_channels_.end()) {
1474 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
1475 ✗ "closing left-over back channel %s", hash.ToString().c_str());
1476 ✗ close(iter->second);
1477 }
1478
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 quota_mgr->back_channels_[hash] = return_pipe;
1479 136 quota_mgr->UnlockBackChannels();
1480
1481 136 char success = 'S';
1482
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 WritePipe(return_pipe, &success, sizeof(success));
1483
1/2
✓ Branch 2 taken 136 times.
✗ Branch 3 not taken.
 136 LogCvmfs(kLogQuota, kLogDebug, "register back channel %s on fd %d",
1484
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 272 hash.ToString().c_str(), return_pipe);
1485
1486 136 continue;
1487 136 }
1488
1489
2/2
✓ Branch 0 taken 68 times.
✓ Branch 1 taken 5103536 times.
 5103604 if (command_type == kUnregisterBackChannel) {
1490
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 shash::Md5 hash;
1491 68 memcpy(hash.digest, command_buffer[num_commands].digest,
1492 68 shash::kDigestSizes[shash::kMd5]);
1493
1494 68 quota_mgr->LockBackChannels();
1495 const map<shash::Md5, int>::iterator iter = quota_mgr->back_channels_
1496
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 .find(hash);
1497
1/2
✓ Branch 2 taken 68 times.
✗ Branch 3 not taken.
 68 if (iter != quota_mgr->back_channels_.end()) {
1498
1/2
✓ Branch 2 taken 68 times.
✗ Branch 3 not taken.
 68 LogCvmfs(kLogQuota, kLogDebug, "closing back channel %s",
1499
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 136 hash.ToString().c_str());
1500
1/2
✓ Branch 2 taken 68 times.
✗ Branch 3 not taken.
 68 close(iter->second);
1501
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 quota_mgr->back_channels_.erase(iter);
1502 } else {
1503 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
1504 ✗ "did not find back channel %s", hash.ToString().c_str());
1505 }
1506 68 quota_mgr->UnlockBackChannels();
1507
1508 68 continue;
1509 68 }
1510
1511 // Unpinnings are also handled immediately with respect to the pinned gauge
1512
2/2
✓ Branch 0 taken 68 times.
✓ Branch 1 taken 5103468 times.
 5103536 if (command_type == kUnpin) {
1513
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1514
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 const string hash_str(hash.ToString());
1515
1516 const map<shash::Any, uint64_t>::iterator iter = quota_mgr->pinned_chunks_
1517
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 .find(hash);
1518
1/2
✓ Branch 2 taken 68 times.
✗ Branch 3 not taken.
 68 if (iter != quota_mgr->pinned_chunks_.end()) {
1519 68 quota_mgr->pinned_ -= iter->second;
1520
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 quota_mgr->pinned_chunks_.erase(iter);
1521 // It can happen that files get pinned that were removed from the cache
1522 // (see cache.cc). We fix this at this point, where we remove such
1523 // entries from the cache database.
1524
2/4
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 68 times.
✗ Branch 5 not taken.
 136 if (!FileExists(quota_mgr->cache_dir_ + "/"
1525
4/6
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 68 times.
✗ Branch 5 not taken.
✓ Branch 9 taken 34 times.
✓ Branch 10 taken 34 times.
 204 + hash.MakePathWithoutSuffix())) {
1526
1/2
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
 34 LogCvmfs(kLogQuota, kLogDebug,
1527 "remove orphaned pinned hash %s from cache database",
1528 hash_str.c_str());
1529
1/2
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
 34 sqlite3_bind_text(quota_mgr->stmt_size_, 1, &hash_str[0],
1530 34 hash_str.length(), SQLITE_STATIC);
1531 int retval;
1532
2/4
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 34 times.
✗ Branch 4 not taken.
 34 if ((retval = sqlite3_step(quota_mgr->stmt_size_)) == SQLITE_ROW) {
1533
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 const uint64_t size = sqlite3_column_int64(quota_mgr->stmt_size_,
1534 34 0);
1535
1/2
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
 34 sqlite3_bind_text(quota_mgr->stmt_rm_, 1, &(hash_str[0]),
1536 34 hash_str.length(), SQLITE_STATIC);
1537
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 retval = sqlite3_step(quota_mgr->stmt_rm_);
1538
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 34 if ((retval == SQLITE_DONE) || (retval == SQLITE_OK)) {
1539 34 quota_mgr->gauge_ -= size;
1540 } else {
1541 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1542 "failed to delete %s (%d)", hash_str.c_str(), retval);
1543 }
1544
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 sqlite3_reset(quota_mgr->stmt_rm_);
1545 }
1546
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 sqlite3_reset(quota_mgr->stmt_size_);
1547 }
1548 } else {
1549 ✗ LogCvmfs(kLogQuota, kLogDebug, "this chunk was not pinned");
1550 }
1551 68 }
1552
1553 // Immediate commands trigger flushing of the buffer
1554 5103536 const bool immediate_command = (command_type == kCleanup)
1555
2/2
✓ Branch 0 taken 5102516 times.
✓ Branch 1 taken 714 times.
 5103230 || (command_type == kList)
1556
2/2
✓ Branch 0 taken 5102244 times.
✓ Branch 1 taken 272 times.
 5102516 || (command_type == kListPinned)
1557
2/2
✓ Branch 0 taken 5102142 times.
✓ Branch 1 taken 102 times.
 5102244 || (command_type == kListCatalogs)
1558
2/2
✓ Branch 0 taken 5102040 times.
✓ Branch 1 taken 102 times.
 5102142 || (command_type == kListVolatile)
1559
2/2
✓ Branch 0 taken 5101938 times.
✓ Branch 1 taken 102 times.
 5102040 || (command_type == kRemove)
1560
2/2
✓ Branch 0 taken 5101326 times.
✓ Branch 1 taken 612 times.
 5101938 || (command_type == kStatus)
1561
1/2
✓ Branch 0 taken 5101326 times.
✗ Branch 1 not taken.
 5101326 || (command_type == kLimits)
1562
3/4
✓ Branch 0 taken 5103230 times.
✓ Branch 1 taken 306 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 5101326 times.
 10206766 || (command_type == kPid);
1563
2/2
✓ Branch 0 taken 5101326 times.
✓ Branch 1 taken 2210 times.
 5103536 if (!immediate_command)
1564 5101326 num_commands++;
1565
1566
4/4
✓ Branch 0 taken 4944178 times.
✓ Branch 1 taken 159358 times.
✓ Branch 2 taken 2210 times.
✓ Branch 3 taken 4941968 times.
 5103536 if ((num_commands == kCommandBufferSize) || immediate_command) {
1567
1/2
✓ Branch 1 taken 161568 times.
✗ Branch 2 not taken.
 161568 quota_mgr->ProcessCommandBunch(num_commands, command_buffer,
1568 description_buffer);
1569
2/2
✓ Branch 0 taken 159358 times.
✓ Branch 1 taken 2210 times.
 161568 if (!immediate_command)
1570 159358 num_commands = 0;
1571 }
1572
1573
2/2
✓ Branch 0 taken 2210 times.
✓ Branch 1 taken 5101326 times.
 5103536 if (immediate_command) {
1574 // Process cleanup, listings
1575
1/2
✓ Branch 1 taken 2210 times.
✗ Branch 2 not taken.
 2210 const int return_pipe = quota_mgr->BindReturnPipe(
1576 command_buffer[num_commands].return_pipe);
1577
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2210 times.
 2210 if (return_pipe < 0) {
1578 ✗ num_commands = 0;
1579 ✗ continue;
1580 }
1581
1582 int retval;
1583 2210 sqlite3_stmt *this_stmt_list = NULL;
1584
7/10
✓ Branch 0 taken 102 times.
✓ Branch 1 taken 306 times.
✓ Branch 2 taken 714 times.
✓ Branch 3 taken 272 times.
✓ Branch 4 taken 102 times.
✓ Branch 5 taken 102 times.
✓ Branch 6 taken 612 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
 2210 switch (command_type) {
1585 102 case kRemove: {
1586
1/2
✓ Branch 1 taken 102 times.
✗ Branch 2 not taken.
 102 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1587
1/2
✓ Branch 1 taken 102 times.
✗ Branch 2 not taken.
 102 const string hash_str = hash.ToString();
1588
1/2
✓ Branch 2 taken 102 times.
✗ Branch 3 not taken.
 102 LogCvmfs(kLogQuota, kLogDebug, "manually removing %s",
1589 hash_str.c_str());
1590 102 bool success = false;
1591
1592
1/2
✓ Branch 2 taken 102 times.
✗ Branch 3 not taken.
 102 sqlite3_bind_text(quota_mgr->stmt_size_, 1, &hash_str[0],
1593 102 hash_str.length(), SQLITE_STATIC);
1594 int retval;
1595
3/4
✓ Branch 1 taken 102 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 68 times.
✓ Branch 4 taken 34 times.
 102 if ((retval = sqlite3_step(quota_mgr->stmt_size_)) == SQLITE_ROW) {
1596
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 const uint64_t size = sqlite3_column_int64(quota_mgr->stmt_size_,
1597 68 0);
1598
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 const uint64_t is_pinned = sqlite3_column_int64(
1599 68 quota_mgr->stmt_size_, 1);
1600
1601
1/2
✓ Branch 2 taken 68 times.
✗ Branch 3 not taken.
 68 sqlite3_bind_text(quota_mgr->stmt_rm_, 1, &(hash_str[0]),
1602 68 hash_str.length(), SQLITE_STATIC);
1603
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 retval = sqlite3_step(quota_mgr->stmt_rm_);
1604
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 68 if ((retval == SQLITE_DONE) || (retval == SQLITE_OK)) {
1605 68 success = true;
1606 68 quota_mgr->gauge_ -= size;
1607
2/2
✓ Branch 0 taken 34 times.
✓ Branch 1 taken 34 times.
 68 if (is_pinned) {
1608
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 quota_mgr->pinned_chunks_.erase(hash);
1609 34 quota_mgr->pinned_ -= size;
1610 }
1611 } else {
1612 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1613 "failed to delete %s (%d)", hash_str.c_str(), retval);
1614 }
1615
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 sqlite3_reset(quota_mgr->stmt_rm_);
1616 } else {
1617 // File does not exist
1618 34 success = true;
1619 }
1620
1/2
✓ Branch 1 taken 102 times.
✗ Branch 2 not taken.
 102 sqlite3_reset(quota_mgr->stmt_size_);
1621
1622
1/2
✓ Branch 1 taken 102 times.
✗ Branch 2 not taken.
 102 WritePipe(return_pipe, &success, sizeof(success));
1623 102 break;
1624 102 }
1625 306 case kCleanup:
1626
1/2
✓ Branch 1 taken 306 times.
✗ Branch 2 not taken.
 306 retval = quota_mgr->DoCleanup(size);
1627
1/2
✓ Branch 1 taken 306 times.
✗ Branch 2 not taken.
 306 WritePipe(return_pipe, &retval, sizeof(retval));
1628 306 break;
1629 714 case kList:
1630
1/2
✓ Branch 0 taken 714 times.
✗ Branch 1 not taken.
 714 if (!this_stmt_list)
1631 714 this_stmt_list = quota_mgr->stmt_list_;
1632 case kListPinned:
1633
2/2
✓ Branch 0 taken 272 times.
✓ Branch 1 taken 714 times.
 986 if (!this_stmt_list)
1634 272 this_stmt_list = quota_mgr->stmt_list_pinned_;
1635 case kListCatalogs:
1636
2/2
✓ Branch 0 taken 102 times.
✓ Branch 1 taken 986 times.
 1088 if (!this_stmt_list)
1637 102 this_stmt_list = quota_mgr->stmt_list_catalogs_;
1638 case kListVolatile:
1639
2/2
✓ Branch 0 taken 102 times.
✓ Branch 1 taken 1088 times.
 1190 if (!this_stmt_list)
1640 102 this_stmt_list = quota_mgr->stmt_list_volatile_;
1641
1642 // Pipe back the list, one by one
1643 int length;
1644
3/4
✓ Branch 1 taken 3402516 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 3401326 times.
✓ Branch 4 taken 1190 times.
 3402516 while (sqlite3_step(this_stmt_list) == SQLITE_ROW) {
1645
1/2
✓ Branch 2 taken 3401326 times.
✗ Branch 3 not taken.
 3401326 string path = "(NULL)";
1646
2/4
✓ Branch 1 taken 3401326 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 3401326 times.
✗ Branch 4 not taken.
 3401326 if (sqlite3_column_type(this_stmt_list, 0) != SQLITE_NULL) {
1647 6802652 path = string(reinterpret_cast<const char *>(
1648
2/4
✓ Branch 1 taken 3401326 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3401326 times.
✗ Branch 5 not taken.
 3401326 sqlite3_column_text(this_stmt_list, 0)));
1649 }
1650 3401326 length = path.length();
1651
1/2
✓ Branch 1 taken 3401326 times.
✗ Branch 2 not taken.
 3401326 WritePipe(return_pipe, &length, sizeof(length));
1652
1/2
✓ Branch 0 taken 3401326 times.
✗ Branch 1 not taken.
 3401326 if (length > 0)
1653
2/4
✓ Branch 1 taken 3401326 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 3401326 times.
✗ Branch 5 not taken.
 3401326 WritePipe(return_pipe, &path[0], length);
1654 3401326 }
1655 1190 length = -1;
1656
1/2
✓ Branch 1 taken 1190 times.
✗ Branch 2 not taken.
 1190 WritePipe(return_pipe, &length, sizeof(length));
1657
1/2
✓ Branch 1 taken 1190 times.
✗ Branch 2 not taken.
 1190 sqlite3_reset(this_stmt_list);
1658 1190 break;
1659 612 case kStatus:
1660
1/2
✓ Branch 1 taken 612 times.
✗ Branch 2 not taken.
 612 WritePipe(return_pipe, &quota_mgr->gauge_, sizeof(quota_mgr->gauge_));
1661
1/2
✓ Branch 1 taken 612 times.
✗ Branch 2 not taken.
 612 WritePipe(return_pipe, &quota_mgr->pinned_,
1662 sizeof(quota_mgr->pinned_));
1663 612 break;
1664 ✗ case kLimits:
1665 ✗ WritePipe(return_pipe, &quota_mgr->limit_, sizeof(quota_mgr->limit_));
1666 ✗ WritePipe(return_pipe, &quota_mgr->cleanup_threshold_,
1667 sizeof(quota_mgr->cleanup_threshold_));
1668 ✗ break;
1669 ✗ case kPid: {
1670 ✗ pid_t pid = getpid();
1671 ✗ WritePipe(return_pipe, &pid, sizeof(pid));
1672 ✗ break;
1673 }
1674 ✗ default:
1675 ✗ PANIC(NULL); // other types are handled by the bunch processor
1676 }
1677
1/2
✓ Branch 1 taken 2210 times.
✗ Branch 2 not taken.
 2210 quota_mgr->UnbindReturnPipe(return_pipe);
1678 2210 num_commands = 0;
1679 }
1680 }
1681
1682
1/2
✓ Branch 1 taken 1054 times.
✗ Branch 2 not taken.
 1054 LogCvmfs(kLogQuota, kLogDebug, "stopping cache manager (%d)", errno);
1683
1/2
✓ Branch 1 taken 1054 times.
✗ Branch 2 not taken.
 1054 close(quota_mgr->pipe_lru_[0]);
1684
1/2
✓ Branch 1 taken 1054 times.
✗ Branch 2 not taken.
 1054 quota_mgr->ProcessCommandBunch(num_commands, command_buffer,
1685 description_buffer);
1686
1687 // Unpin
1688 1054 command_buffer[0].command_type = kTouch;
1689 1054 for (map<shash::Any, uint64_t>::const_iterator
1690 1054 i = quota_mgr->pinned_chunks_.begin(),
1691 1054 iEnd = quota_mgr->pinned_chunks_.end();
1692
2/2
✓ Branch 1 taken 340 times.
✓ Branch 2 taken 1054 times.
 1394 i != iEnd;
1693 340 ++i) {
1694
1/2
✓ Branch 2 taken 340 times.
✗ Branch 3 not taken.
 340 command_buffer[0].StoreHash(i->first);
1695
1/2
✓ Branch 1 taken 340 times.
✗ Branch 2 not taken.
 340 quota_mgr->ProcessCommandBunch(1, command_buffer, description_buffer);
1696 }
1697
1698 1054 return NULL;
1699 }
1700
1701
1702 3128 void PosixQuotaManager::MakeReturnPipe(int pipe[2]) {
1703
2/2
✓ Branch 0 taken 3026 times.
✓ Branch 1 taken 102 times.
 3128 if (!shared_) {
1704 3026 MakePipe(pipe);
1705 3026 return;
1706 }
1707
1708 // Create FIFO in cache directory, store path name (number) in pipe write end
1709 102 int i = 0;
1710 int retval;
1711 do {
1712
2/4
✓ Branch 2 taken 136 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 136 times.
✗ Branch 6 not taken.
 136 retval = mkfifo((workspace_dir_ + "/pipe" + StringifyInt(i)).c_str(), 0600);
1713 136 pipe[1] = i;
1714 136 i++;
1715
3/4
✓ Branch 0 taken 34 times.
✓ Branch 1 taken 102 times.
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
 136 } while ((retval == -1) && (errno == EEXIST));
1716
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 102 times.
 102 assert(retval == 0);
1717
1718 // Connect reader's end
1719
3/6
✓ Branch 2 taken 102 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 102 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 102 times.
✗ Branch 10 not taken.
 102 pipe[0] = open((workspace_dir_ + "/pipe" + StringifyInt(pipe[1])).c_str(),
1720 O_RDONLY | O_NONBLOCK);
1721
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 102 times.
 102 assert(pipe[0] >= 0);
1722 102 Nonblock2Block(pipe[0]);
1723 }
1724
1725
1726 2810 void PosixQuotaManager::ParseDirectories(const std::string cache_workspace,
1727 std::string *cache_dir,
1728 std::string *workspace_dir) {
1729
1/2
✓ Branch 1 taken 2810 times.
✗ Branch 2 not taken.
 2810 vector<string> dir_tokens(SplitString(cache_workspace, ':'));
1730
2/3
✓ Branch 1 taken 2742 times.
✓ Branch 2 taken 68 times.
✗ Branch 3 not taken.
 2810 switch (dir_tokens.size()) {
1731 2742 case 1:
1732
2/4
✓ Branch 2 taken 2742 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2742 times.
✗ Branch 6 not taken.
 2742 *cache_dir = *workspace_dir = dir_tokens[0];
1733 2742 break;
1734 68 case 2:
1735
1/2
✓ Branch 2 taken 68 times.
✗ Branch 3 not taken.
 68 *cache_dir = dir_tokens[0];
1736
1/2
✓ Branch 2 taken 68 times.
✗ Branch 3 not taken.
 68 *workspace_dir = dir_tokens[1];
1737 68 break;
1738 ✗ default:
1739 ✗ PANIC(NULL);
1740 }
1741 2810 }
1742
1743 34 void PosixQuotaManager::SkipEviction(const EvictCandidate &candidate) {
1744 34 bool res = true;
1745
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 std::string hash_str = candidate.hash.ToString();
1746
1/2
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
 34 LogCvmfs(kLogQuota, kLogDebug, "Exclude %s from eviction", hash_str.c_str());
1747
2/4
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 34 times.
✗ Branch 6 not taken.
 34 sqlite3_bind_text(stmt_block_, 1, &hash_str[0], hash_str.length(),
1748 SQLITE_STATIC);
1749
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 res = (sqlite3_step(stmt_block_) == SQLITE_DONE);
1750
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 sqlite3_reset(stmt_block_);
1751
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 34 times.
 34 assert(res);
1752 34 }
1753
1754 /**
1755 * Immediately inserts a new pinned catalog. Does cache cleanup if necessary.
1756 *
1757 * \return True on success, false otherwise
1758 */
1759 1452 bool PosixQuotaManager::Pin(const shash::Any &hash,
1760 const uint64_t size,
1761 const string &description,
1762 const bool is_catalog) {
1763
3/4
✓ Branch 0 taken 68 times.
✓ Branch 1 taken 1384 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 68 times.
 1452 assert((size > 0) || !is_catalog);
1764
1765
1/2
✓ Branch 1 taken 1452 times.
✗ Branch 2 not taken.
 1452 const string hash_str = hash.ToString();
1766
1/2
✓ Branch 3 taken 1452 times.
✗ Branch 4 not taken.
 1452 LogCvmfs(kLogQuota, kLogDebug, "pin into lru %s, path %s", hash_str.c_str(),
1767 description.c_str());
1768
1769 // Has to run when not yet spawned (cvmfs initialization)
1770
2/2
✓ Branch 0 taken 976 times.
✓ Branch 1 taken 476 times.
 1452 if (!spawned_) {
1771 // Code duplication here
1772
3/4
✓ Branch 2 taken 976 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 563 times.
✓ Branch 6 taken 413 times.
 976 if (pinned_chunks_.find(hash) == pinned_chunks_.end()) {
1773
2/2
✓ Branch 0 taken 68 times.
✓ Branch 1 taken 495 times.
 563 if (pinned_ + size > cleanup_threshold_) {
1774
1/2
✓ Branch 2 taken 68 times.
✗ Branch 3 not taken.
 68 LogCvmfs(kLogQuota, kLogDebug, "failed to insert %s (pinned), no space",
1775 hash_str.c_str());
1776 68 return false;
1777 } else {
1778
1/2
✓ Branch 1 taken 495 times.
✗ Branch 2 not taken.
 495 pinned_chunks_[hash] = size;
1779 495 pinned_ += size;
1780
1/2
✓ Branch 1 taken 495 times.
✗ Branch 2 not taken.
 495 CheckHighPinWatermark();
1781 }
1782 }
1783
1/2
✓ Branch 1 taken 908 times.
✗ Branch 2 not taken.
 908 const bool exists = Contains(hash_str);
1784
4/4
✓ Branch 0 taken 495 times.
✓ Branch 1 taken 413 times.
✓ Branch 2 taken 34 times.
✓ Branch 3 taken 461 times.
 908 if (!exists && (gauge_ + size > limit_)) {
1785
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 LogCvmfs(kLogQuota, kLogDebug, "over limit, gauge %lu, file size %lu",
1786 gauge_, size);
1787
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 const int retval = DoCleanup(cleanup_threshold_);
1788
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 34 times.
 34 assert(retval != 0);
1789 }
1790
1/2
✓ Branch 3 taken 908 times.
✗ Branch 4 not taken.
 908 sqlite3_bind_text(stmt_new_, 1, &hash_str[0], hash_str.length(),
1791 SQLITE_STATIC);
1792
1/2
✓ Branch 1 taken 908 times.
✗ Branch 2 not taken.
 908 sqlite3_bind_int64(stmt_new_, 2, size);
1793
1/2
✓ Branch 1 taken 908 times.
✗ Branch 2 not taken.
 908 sqlite3_bind_int64(stmt_new_, 3, seq_++);
1794
1/2
✓ Branch 3 taken 908 times.
✗ Branch 4 not taken.
 908 sqlite3_bind_text(stmt_new_, 4, &description[0], description.length(),
1795 SQLITE_STATIC);
1796
3/4
✓ Branch 0 taken 806 times.
✓ Branch 1 taken 102 times.
✓ Branch 3 taken 908 times.
✗ Branch 4 not taken.
 908 sqlite3_bind_int64(stmt_new_, 5, is_catalog ? kFileCatalog : kFileRegular);
1797
1/2
✓ Branch 1 taken 908 times.
✗ Branch 2 not taken.
 908 sqlite3_bind_int64(stmt_new_, 6, 1);
1798
1/2
✓ Branch 1 taken 908 times.
✗ Branch 2 not taken.
 908 const int retval = sqlite3_step(stmt_new_);
1799
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 908 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 908 assert((retval == SQLITE_DONE) || (retval == SQLITE_OK));
1800
1/2
✓ Branch 1 taken 908 times.
✗ Branch 2 not taken.
 908 sqlite3_reset(stmt_new_);
1801
2/2
✓ Branch 0 taken 495 times.
✓ Branch 1 taken 413 times.
 908 if (!exists)
1802 495 gauge_ += size;
1803 908 return true;
1804 }
1805
1806 int pipe_reserve[2];
1807
1/2
✓ Branch 1 taken 476 times.
✗ Branch 2 not taken.
 476 MakeReturnPipe(pipe_reserve);
1808
1809 476 LruCommand cmd;
1810 476 cmd.command_type = kReserve;
1811 476 cmd.SetSize(size);
1812
1/2
✓ Branch 1 taken 476 times.
✗ Branch 2 not taken.
 476 cmd.StoreHash(hash);
1813 476 cmd.return_pipe = pipe_reserve[1];
1814
1/2
✓ Branch 1 taken 476 times.
✗ Branch 2 not taken.
 476 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
1815 bool result;
1816
1/2
✓ Branch 1 taken 476 times.
✗ Branch 2 not taken.
 476 ManagedReadHalfPipe(pipe_reserve[0], &result, sizeof(result));
1817
1/2
✓ Branch 1 taken 476 times.
✗ Branch 2 not taken.
 476 CloseReturnPipe(pipe_reserve);
1818
1819
2/2
✓ Branch 0 taken 34 times.
✓ Branch 1 taken 442 times.
 476 if (!result)
1820 34 return false;
1821
3/4
✓ Branch 0 taken 68 times.
✓ Branch 1 taken 374 times.
✓ Branch 3 taken 442 times.
✗ Branch 4 not taken.
 442 DoInsert(hash, size, description, is_catalog ? kPin : kPinRegular);
1822
1823 442 return true;
1824 1452 }
1825
1826
1827 2742 PosixQuotaManager::PosixQuotaManager(const uint64_t limit,
1828 const uint64_t cleanup_threshold,
1829 2742 const string &cache_workspace)
1830 2742 : shared_(false)
1831 2742 , spawned_(false)
1832 2742 , limit_(limit)
1833 2742 , cleanup_threshold_(cleanup_threshold)
1834 2742 , gauge_(0)
1835 2742 , pinned_(0)
1836 2742 , seq_(0)
1837 2742 , cache_dir_() // initialized in body
1838 2742 , workspace_dir_() // initialized in body
1839 2742 , fd_lock_cachedb_(-1)
1840 2742 , async_delete_(true)
1841 2742 , cachemgr_pid_(0)
1842 2742 , database_(NULL)
1843 2742 , stmt_touch_(NULL)
1844 2742 , stmt_unpin_(NULL)
1845 2742 , stmt_block_(NULL)
1846 2742 , stmt_unblock_(NULL)
1847 2742 , stmt_new_(NULL)
1848 2742 , stmt_lru_(NULL)
1849 2742 , stmt_size_(NULL)
1850 2742 , stmt_rm_(NULL)
1851 2742 , stmt_rm_batch_(NULL)
1852 2742 , stmt_list_(NULL)
1853 2742 , stmt_list_pinned_(NULL)
1854 2742 , stmt_list_catalogs_(NULL)
1855 2742 , stmt_list_volatile_(NULL)
1856 2742 , initialized_(false)
1857 5484 , cleanup_unused_first_(false) {
1858
2/4
✓ Branch 1 taken 2742 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2742 times.
✗ Branch 5 not taken.
 2742 ParseDirectories(cache_workspace, &cache_dir_, &workspace_dir_);
1859 2742 pipe_lru_[0] = pipe_lru_[1] = -1;
1860
1/2
✓ Branch 1 taken 2742 times.
✗ Branch 2 not taken.
 2742 cleanup_recorder_.AddRecorder(1, 90); // last 1.5 min with second resolution
1861 // last 1.5 h with minute resolution
1862
1/2
✓ Branch 1 taken 2742 times.
✗ Branch 2 not taken.
 2742 cleanup_recorder_.AddRecorder(60, 90 * 60);
1863 // last 18 hours with 20 min resolution
1864
1/2
✓ Branch 1 taken 2742 times.
✗ Branch 2 not taken.
 2742 cleanup_recorder_.AddRecorder(20 * 60, 60 * 60 * 18);
1865 // last 4 days with hour resolution
1866
1/2
✓ Branch 1 taken 2742 times.
✗ Branch 2 not taken.
 2742 cleanup_recorder_.AddRecorder(60 * 60, 60 * 60 * 24 * 4);
1867
1868 2742 lock_open_files_ = reinterpret_cast<pthread_mutex_t *>(
1869 2742 smalloc(sizeof(pthread_mutex_t)));
1870 2742 }
1871
1872
1873 10964 PosixQuotaManager::~PosixQuotaManager() {
1874 5482 free(lock_open_files_);
1875
1876
2/2
✓ Branch 0 taken 102 times.
✓ Branch 1 taken 2639 times.
 5482 if (!initialized_)
1877 204 return;
1878
1879
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2639 times.
 5278 if (shared_) {
1880 // Most of cleanup is done elsewhen by shared cache manager
1881 ✗ close(pipe_lru_[1]);
1882 ✗ return;
1883 }
1884
1885
2/2
✓ Branch 0 taken 1054 times.
✓ Branch 1 taken 1585 times.
 5278 if (spawned_) {
1886 2108 char fin = 0;
1887 2108 WritePipe(pipe_lru_[1], &fin, 1);
1888 2108 close(pipe_lru_[1]);
1889 2108 pthread_join(thread_lru_, NULL);
1890 } else {
1891 3170 ClosePipe(pipe_lru_);
1892 }
1893
1894 5278 CloseDatabase();
1895
14/14
✓ Branch 1 taken 2639 times.
✓ Branch 2 taken 102 times.
✓ Branch 4 taken 2639 times.
✓ Branch 5 taken 102 times.
✓ Branch 7 taken 2639 times.
✓ Branch 8 taken 102 times.
✓ Branch 10 taken 2639 times.
✓ Branch 11 taken 102 times.
✓ Branch 13 taken 2639 times.
✓ Branch 14 taken 102 times.
✓ Branch 16 taken 2639 times.
✓ Branch 17 taken 102 times.
✓ Branch 19 taken 2639 times.
✓ Branch 20 taken 102 times.
 12188 }
1896
1897
1898 162962 void PosixQuotaManager::ProcessCommandBunch(const unsigned num,
1899 const LruCommand *commands,
1900 const char *descriptions) {
1901 162962 int retval = sqlite3_exec(database_, "BEGIN", NULL, NULL, NULL);
1902
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 162962 times.
 162962 assert(retval == SQLITE_OK);
1903
1904
2/2
✓ Branch 0 taken 5101666 times.
✓ Branch 1 taken 162962 times.
 5264628 for (unsigned i = 0; i < num; ++i) {
1905
1/2
✓ Branch 1 taken 5101666 times.
✗ Branch 2 not taken.
 5101666 const shash::Any hash = commands[i].RetrieveHash();
1906
1/2
✓ Branch 1 taken 5101666 times.
✗ Branch 2 not taken.
 5101666 const string hash_str = hash.ToString();
1907 5101666 const unsigned size = commands[i].GetSize();
1908
1/2
✓ Branch 1 taken 5101666 times.
✗ Branch 2 not taken.
 5101666 LogCvmfs(kLogQuota, kLogDebug, "processing %s (%d)", hash_str.c_str(),
1909 5101666 commands[i].command_type);
1910
1911 bool exists;
1912
3/4
✓ Branch 0 taken 1700442 times.
✓ Branch 1 taken 68 times.
✓ Branch 2 taken 3401156 times.
✗ Branch 3 not taken.
 5101666 switch (commands[i].command_type) {
1913 1700442 case kTouch:
1914
1/2
✓ Branch 1 taken 1700442 times.
✗ Branch 2 not taken.
 1700442 sqlite3_bind_int64(stmt_touch_, 1, seq_++);
1915
1/2
✓ Branch 3 taken 1700442 times.
✗ Branch 4 not taken.
 1700442 sqlite3_bind_text(stmt_touch_, 2, &hash_str[0], hash_str.length(),
1916 SQLITE_STATIC);
1917
1/2
✓ Branch 1 taken 1700442 times.
✗ Branch 2 not taken.
 1700442 retval = sqlite3_step(stmt_touch_);
1918
1/2
✓ Branch 1 taken 1700442 times.
✗ Branch 2 not taken.
 1700442 LogCvmfs(kLogQuota, kLogDebug, "touching %s (%ld): %d",
1919 1700442 hash_str.c_str(), seq_ - 1, retval);
1920
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1700442 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 1700442 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
1921 ✗ PANIC(kLogSyslogErr, "failed to update %s in cachedb, error %d",
1922 hash_str.c_str(), retval);
1923 }
1924
1/2
✓ Branch 1 taken 1700442 times.
✗ Branch 2 not taken.
 1700442 sqlite3_reset(stmt_touch_);
1925 1700442 break;
1926 68 case kUnpin:
1927
1/2
✓ Branch 3 taken 68 times.
✗ Branch 4 not taken.
 68 sqlite3_bind_text(stmt_unpin_, 1, &hash_str[0], hash_str.length(),
1928 SQLITE_STATIC);
1929
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 retval = sqlite3_step(stmt_unpin_);
1930
1/2
✓ Branch 2 taken 68 times.
✗ Branch 3 not taken.
 68 LogCvmfs(kLogQuota, kLogDebug, "unpinning %s: %d", hash_str.c_str(),
1931 retval);
1932
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 68 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
1933 ✗ PANIC(kLogSyslogErr, "failed to unpin %s in cachedb, error %d",
1934 hash_str.c_str(), retval);
1935 }
1936
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 sqlite3_reset(stmt_unpin_);
1937 68 break;
1938 3401156 case kPin:
1939 case kPinRegular:
1940 case kInsert:
1941 case kInsertVolatile:
1942 // It could already be in, check
1943
1/2
✓ Branch 1 taken 3401156 times.
✗ Branch 2 not taken.
 3401156 exists = Contains(hash_str);
1944
1945 // Cleanup, move to trash and unlink
1946
3/4
✓ Branch 0 taken 3400986 times.
✓ Branch 1 taken 170 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 3400986 times.
 3401156 if (!exists && (gauge_ + size > limit_)) {
1947 ✗ LogCvmfs(kLogQuota, kLogDebug, "over limit, gauge %lu, file size %u",
1948 gauge_, size);
1949 ✗ retval = DoCleanup(cleanup_threshold_);
1950 ✗ assert(retval != 0);
1951 }
1952
1953 // Insert or replace
1954
1/2
✓ Branch 3 taken 3401156 times.
✗ Branch 4 not taken.
 3401156 sqlite3_bind_text(stmt_new_, 1, &hash_str[0], hash_str.length(),
1955 SQLITE_STATIC);
1956
1/2
✓ Branch 1 taken 3401156 times.
✗ Branch 2 not taken.
 3401156 sqlite3_bind_int64(stmt_new_, 2, size);
1957
2/2
✓ Branch 0 taken 136 times.
✓ Branch 1 taken 3401020 times.
 3401156 if (commands[i].command_type == kInsertVolatile) {
1958
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 sqlite3_bind_int64(stmt_new_, 3, (seq_++) | kVolatileFlag);
1959 } else {
1960
1/2
✓ Branch 1 taken 3401020 times.
✗ Branch 2 not taken.
 3401020 sqlite3_bind_int64(stmt_new_, 3, seq_++);
1961 }
1962 3401156 sqlite3_bind_text(stmt_new_, 4, &descriptions[i * kMaxDescription],
1963
1/2
✓ Branch 1 taken 3401156 times.
✗ Branch 2 not taken.
 3401156 commands[i].desc_length, SQLITE_STATIC);
1964
1/2
✓ Branch 1 taken 3401156 times.
✗ Branch 2 not taken.
 3401156 sqlite3_bind_int64(
1965 stmt_new_, 5,
1966
2/2
✓ Branch 0 taken 68 times.
✓ Branch 1 taken 3401088 times.
 3401156 (commands[i].command_type == kPin) ? kFileCatalog : kFileRegular);
1967
1/2
✓ Branch 1 taken 3401156 times.
✗ Branch 2 not taken.
 3401156 sqlite3_bind_int64(stmt_new_, 6,
1968
2/2
✓ Branch 0 taken 3401088 times.
✓ Branch 1 taken 68 times.
 3401156 ((commands[i].command_type == kPin)
1969
2/2
✓ Branch 0 taken 374 times.
✓ Branch 1 taken 3400714 times.
 3401088 || (commands[i].command_type == kPinRegular))
1970 ? 1
1971 : 0);
1972
1/2
✓ Branch 1 taken 3401156 times.
✗ Branch 2 not taken.
 3401156 retval = sqlite3_step(stmt_new_);
1973
1/2
✓ Branch 1 taken 3401156 times.
✗ Branch 2 not taken.
 3401156 LogCvmfs(kLogQuota, kLogDebug, "insert or replace %s, method %d: %d",
1974 3401156 hash_str.c_str(), commands[i].command_type, retval);
1975
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 3401156 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 3401156 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
1976 ✗ PANIC(kLogSyslogErr, "failed to insert %s in cachedb, error %d",
1977 hash_str.c_str(), retval);
1978 }
1979
1/2
✓ Branch 1 taken 3401156 times.
✗ Branch 2 not taken.
 3401156 sqlite3_reset(stmt_new_);
1980
1981
2/2
✓ Branch 0 taken 3400986 times.
✓ Branch 1 taken 170 times.
 3401156 if (!exists)
1982 3400986 gauge_ += size;
1983 3401156 break;
1984 ✗ default:
1985 // other types should have been taken care of by event loop
1986 ✗ PANIC(NULL);
1987 }
1988 5101666 }
1989
1990 162962 retval = sqlite3_exec(database_, "COMMIT", NULL, NULL, NULL);
1991
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 162962 times.
 162962 if (retval != SQLITE_OK) {
1992 ✗ PANIC(kLogSyslogErr, "failed to commit to cachedb, error %d", retval);
1993 }
1994 162962 }
1995
1996
1997 2674 bool PosixQuotaManager::RebuildDatabase() {
1998 2674 bool result = false;
1999 2674 string sql;
2000 2674 sqlite3_stmt *stmt_select = NULL;
2001 2674 sqlite3_stmt *stmt_insert = NULL;
2002 int sqlerr;
2003 2674 int seq = 0;
2004 char hex[4];
2005 struct stat info;
2006 platform_dirent64 *d;
2007 2674 DIR *dirp = NULL;
2008 2674 string path;
2009
2010
1/2
✓ Branch 1 taken 2674 times.
✗ Branch 2 not taken.
 2674 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug, "re-building cache database");
2011
2012 // Empty cache catalog and fscache
2013
1/2
✓ Branch 1 taken 2674 times.
✗ Branch 2 not taken.
 2674 sql = "DELETE FROM cache_catalog; DELETE FROM fscache;";
2014
1/2
✓ Branch 2 taken 2674 times.
✗ Branch 3 not taken.
 2674 sqlerr = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
2015
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2674 times.
 2674 if (sqlerr != SQLITE_OK) {
2016 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not clear cache database");
2017 ✗ goto build_return;
2018 }
2019
2020 2674 gauge_ = 0;
2021
2022 // Insert files from cache sub-directories 00 - ff
2023 // TODO(jblomer): fs_traversal
2024
1/2
✓ Branch 1 taken 2674 times.
✗ Branch 2 not taken.
 2674 sqlite3_prepare_v2(database_,
2025 "INSERT INTO fscache (sha1, size, actime) "
2026 "VALUES (:sha1, :s, :t);",
2027 -1, &stmt_insert, NULL);
2028
2029
2/2
✓ Branch 0 taken 658534 times.
✓ Branch 1 taken 2572 times.
 661106 for (int i = 0; i <= 0xff; i++) {
2030 658534 snprintf(hex, sizeof(hex), "%02x", i);
2031
3/6
✓ Branch 2 taken 658534 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 658534 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 658534 times.
✗ Branch 9 not taken.
 658534 path = cache_dir_ + "/" + string(hex);
2032
3/4
✓ Branch 2 taken 658534 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 102 times.
✓ Branch 5 taken 658432 times.
 658534 if ((dirp = opendir(path.c_str())) == NULL) {
2033
1/2
✓ Branch 2 taken 102 times.
✗ Branch 3 not taken.
 102 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
2034 "failed to open directory %s (tmpwatch interfering?)",
2035 path.c_str());
2036 102 goto build_return;
2037 }
2038
3/4
✓ Branch 1 taken 1975364 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1316932 times.
✓ Branch 4 taken 658432 times.
 1975364 while ((d = platform_readdir(dirp)) != NULL) {
2039
3/6
✓ Branch 2 taken 1316932 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1316932 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1316932 times.
✗ Branch 9 not taken.
 2633864 const string file_path = path + "/" + string(d->d_name);
2040
1/2
✓ Branch 2 taken 1316932 times.
✗ Branch 3 not taken.
 1316932 if (stat(file_path.c_str(), &info) == 0) {
2041
2/2
✓ Branch 0 taken 1316864 times.
✓ Branch 1 taken 68 times.
 1316932 if (!S_ISREG(info.st_mode))
2042 1316898 continue;
2043
2/2
✓ Branch 0 taken 34 times.
✓ Branch 1 taken 34 times.
 68 if (info.st_size == 0) {
2044
1/2
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
 34 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug,
2045 "removing empty file %s during automatic cache db rebuild",
2046 file_path.c_str());
2047 34 unlink(file_path.c_str());
2048 34 continue;
2049 }
2050
2051
3/6
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 34 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 34 times.
✗ Branch 10 not taken.
 68 string hash = string(hex) + string(d->d_name);
2052
1/2
✓ Branch 3 taken 34 times.
✗ Branch 4 not taken.
 34 sqlite3_bind_text(stmt_insert, 1, hash.data(), hash.length(),
2053 SQLITE_STATIC);
2054
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 sqlite3_bind_int64(stmt_insert, 2, info.st_size);
2055
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 sqlite3_bind_int64(stmt_insert, 3, info.st_atime);
2056
2/4
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 34 times.
 34 if (sqlite3_step(stmt_insert) != SQLITE_DONE) {
2057 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not insert into temp table");
2058 ✗ goto build_return;
2059 }
2060
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 sqlite3_reset(stmt_insert);
2061
2062 34 gauge_ += info.st_size;
2063
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 } else {
2064 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not stat %s", file_path.c_str());
2065 }
2066
2/3
✓ Branch 1 taken 34 times.
✓ Branch 2 taken 1316898 times.
✗ Branch 3 not taken.
 1316932 }
2067
1/2
✓ Branch 1 taken 658432 times.
✗ Branch 2 not taken.
 658432 closedir(dirp);
2068 658432 dirp = NULL;
2069 }
2070
1/2
✓ Branch 1 taken 2572 times.
✗ Branch 2 not taken.
 2572 sqlite3_finalize(stmt_insert);
2071 2572 stmt_insert = NULL;
2072
2073 // Transfer from temp table in cache catalog
2074
1/2
✓ Branch 1 taken 2572 times.
✗ Branch 2 not taken.
 2572 sqlite3_prepare_v2(database_,
2075 "SELECT sha1, size FROM fscache ORDER BY actime;", -1,
2076 &stmt_select, NULL);
2077
1/2
✓ Branch 1 taken 2572 times.
✗ Branch 2 not taken.
 2572 sqlite3_prepare_v2(
2078 database_,
2079 "INSERT INTO cache_catalog (sha1, size, acseq, path, type, pinned) "
2080 "VALUES (:sha1, :s, :seq, 'unknown (automatic rebuild)', :t, 0);",
2081 -1, &stmt_insert, NULL);
2082
3/4
✓ Branch 1 taken 2606 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 34 times.
✓ Branch 4 taken 2572 times.
 2606 while (sqlite3_step(stmt_select) == SQLITE_ROW) {
2083 const string hash = string(
2084
2/4
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 34 times.
✗ Branch 6 not taken.
 34 reinterpret_cast<const char *>(sqlite3_column_text(stmt_select, 0)));
2085
1/2
✓ Branch 3 taken 34 times.
✗ Branch 4 not taken.
 34 sqlite3_bind_text(stmt_insert, 1, &hash[0], hash.length(), SQLITE_STATIC);
2086
2/4
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 34 times.
✗ Branch 5 not taken.
 34 sqlite3_bind_int64(stmt_insert, 2, sqlite3_column_int64(stmt_select, 1));
2087
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 sqlite3_bind_int64(stmt_insert, 3, seq++);
2088 // Might also be a catalog (information is lost)
2089
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 sqlite3_bind_int64(stmt_insert, 4, kFileRegular);
2090
2091
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 const int retval = sqlite3_step(stmt_insert);
2092
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 34 times.
 34 if (retval != SQLITE_DONE) {
2093 // If the file system hosting the cache is full, we'll likely notice here
2094 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
2095 "could not insert into cache catalog (%d - %s)", retval,
2096 sqlite3_errstr(retval));
2097 ✗ goto build_return;
2098 }
2099
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 sqlite3_reset(stmt_insert);
2100
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 }
2101
2102 // Delete temporary table
2103
1/2
✓ Branch 1 taken 2572 times.
✗ Branch 2 not taken.
 2572 sql = "DELETE FROM fscache;";
2104
1/2
✓ Branch 2 taken 2572 times.
✗ Branch 3 not taken.
 2572 sqlerr = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
2105
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2572 times.
 2572 if (sqlerr != SQLITE_OK) {
2106 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not clear temporary table (%d)",
2107 sqlerr);
2108 ✗ goto build_return;
2109 }
2110
2111 2572 seq_ = seq;
2112 2572 result = true;
2113
1/2
✓ Branch 1 taken 2572 times.
✗ Branch 2 not taken.
 2572 LogCvmfs(kLogQuota, kLogDebug,
2114 "rebuilding finished, sequence %" PRIu64 ", gauge %" PRIu64, seq_,
2115 gauge_);
2116
2117 2674 build_return:
2118
1/2
✓ Branch 0 taken 2674 times.
✗ Branch 1 not taken.
 2674 if (stmt_insert)
2119
1/2
✓ Branch 1 taken 2674 times.
✗ Branch 2 not taken.
 2674 sqlite3_finalize(stmt_insert);
2120
2/2
✓ Branch 0 taken 2572 times.
✓ Branch 1 taken 102 times.
 2674 if (stmt_select)
2121
1/2
✓ Branch 1 taken 2572 times.
✗ Branch 2 not taken.
 2572 sqlite3_finalize(stmt_select);
2122
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2674 times.
 2674 if (dirp)
2123 ✗ closedir(dirp);
2124 2674 return result;
2125 2674 }
2126
2127
2128 /**
2129 * Register a channel that allows the cache manager to trigger action to its
2130 * clients. Currently used for releasing pinned catalogs.
2131 */
2132 136 void PosixQuotaManager::RegisterBackChannel(int back_channel[2],
2133 const string &channel_id) {
2134
1/2
✓ Branch 0 taken 136 times.
✗ Branch 1 not taken.
 136 if (protocol_revision_ >= 1) {
2135
1/2
✓ Branch 2 taken 136 times.
✗ Branch 3 not taken.
 136 shash::Md5 hash = shash::Md5(shash::AsciiPtr(channel_id));
2136
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 MakeReturnPipe(back_channel);
2137
2138 136 LruCommand cmd;
2139 136 cmd.command_type = kRegisterBackChannel;
2140 136 cmd.return_pipe = back_channel[1];
2141 // Not StoreHash(). This is an MD5 hash.
2142 136 memcpy(cmd.digest, hash.digest, hash.GetDigestSize());
2143
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2144
2145 char success;
2146
1/2
✓ Branch 1 taken 136 times.
✗ Branch 2 not taken.
 136 ManagedReadHalfPipe(back_channel[0], &success, sizeof(success));
2147 // At this point, the named FIFO is unlinked, so don't use CloseReturnPipe
2148
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 136 times.
 136 if (success != 'S') {
2149 ✗ PANIC(kLogDebug | kLogSyslogErr,
2150 "failed to register quota back channel (%c)", success);
2151 }
2152 } else {
2153 // Dummy pipe to return valid file descriptors
2154 ✗ MakePipe(back_channel);
2155 }
2156 136 }
2157
2158
2159 /**
2160 * Removes a chunk from cache, if it exists.
2161 */
2162 102 void PosixQuotaManager::Remove(const shash::Any &hash) {
2163
1/2
✓ Branch 1 taken 102 times.
✗ Branch 2 not taken.
 102 const string hash_str = hash.ToString();
2164
2165 int pipe_remove[2];
2166
1/2
✓ Branch 1 taken 102 times.
✗ Branch 2 not taken.
 102 MakeReturnPipe(pipe_remove);
2167
2168 102 LruCommand cmd;
2169 102 cmd.command_type = kRemove;
2170 102 cmd.return_pipe = pipe_remove[1];
2171
1/2
✓ Branch 1 taken 102 times.
✗ Branch 2 not taken.
 102 cmd.StoreHash(hash);
2172
1/2
✓ Branch 1 taken 102 times.
✗ Branch 2 not taken.
 102 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2173
2174 bool success;
2175
1/2
✓ Branch 1 taken 102 times.
✗ Branch 2 not taken.
 102 ManagedReadHalfPipe(pipe_remove[0], &success, sizeof(success));
2176
1/2
✓ Branch 1 taken 102 times.
✗ Branch 2 not taken.
 102 CloseReturnPipe(pipe_remove);
2177
2178
3/6
✓ Branch 1 taken 102 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 102 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 102 times.
✗ Branch 8 not taken.
 102 unlink((cache_dir_ + "/" + hash.MakePathWithoutSuffix()).c_str());
2179 102 }
2180
2181
2182 1122 void PosixQuotaManager::Spawn() {
2183
2/2
✓ Branch 0 taken 68 times.
✓ Branch 1 taken 1054 times.
 1122 if (spawned_)
2184 68 return;
2185
2186 1054 if (pthread_create(&thread_lru_, NULL, MainCommandServer,
2187 static_cast<void *>(this))
2188
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1054 times.
 1054 != 0) {
2189 ✗ PANIC(kLogDebug, "could not create lru thread");
2190 }
2191
2192 1054 spawned_ = true;
2193 }
2194
2195
2196 /**
2197 * Updates the sequence number of the file specified by the hash.
2198 */
2199 1700553 void PosixQuotaManager::Touch(const shash::Any &hash) {
2200 1700553 LruCommand cmd;
2201 1700553 cmd.command_type = kTouch;
2202
1/2
✓ Branch 1 taken 1700553 times.
✗ Branch 2 not taken.
 1700553 cmd.StoreHash(hash);
2203
1/2
✓ Branch 1 taken 1700553 times.
✗ Branch 2 not taken.
 1700553 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2204 1700553 }
2205
2206
2207 2924 void PosixQuotaManager::UnbindReturnPipe(int pipe_wronly) {
2208
2/2
✓ Branch 0 taken 34 times.
✓ Branch 1 taken 2890 times.
 2924 if (shared_)
2209 34 close(pipe_wronly);
2210 2924 }
2211
2212
2213 238 void PosixQuotaManager::UnlinkReturnPipe(int pipe_wronly) {
2214
2/2
✓ Branch 0 taken 102 times.
✓ Branch 1 taken 136 times.
 238 if (shared_)
2215
2/4
✓ Branch 2 taken 102 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 102 times.
✗ Branch 6 not taken.
 102 unlink((workspace_dir_ + "/pipe" + StringifyInt(pipe_wronly)).c_str());
2216 238 }
2217
2218
2219 545 void PosixQuotaManager::Unpin(const shash::Any &hash) {
2220
2/4
✓ Branch 1 taken 545 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 545 times.
✗ Branch 6 not taken.
 545 LogCvmfs(kLogQuota, kLogDebug, "Unpin %s", hash.ToString().c_str());
2221
2222 545 LruCommand cmd;
2223 545 cmd.command_type = kUnpin;
2224
1/2
✓ Branch 1 taken 545 times.
✗ Branch 2 not taken.
 545 cmd.StoreHash(hash);
2225
1/2
✓ Branch 1 taken 545 times.
✗ Branch 2 not taken.
 545 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2226 545 }
2227
2228
2229 68 void PosixQuotaManager::UnregisterBackChannel(int back_channel[2],
2230 const string &channel_id) {
2231
1/2
✓ Branch 0 taken 68 times.
✗ Branch 1 not taken.
 68 if (protocol_revision_ >= 1) {
2232
1/2
✓ Branch 2 taken 68 times.
✗ Branch 3 not taken.
 68 shash::Md5 hash = shash::Md5(shash::AsciiPtr(channel_id));
2233
2234 68 LruCommand cmd;
2235 68 cmd.command_type = kUnregisterBackChannel;
2236 // Not StoreHash(). This is an MD5 hash.
2237 68 memcpy(cmd.digest, hash.digest, hash.GetDigestSize());
2238
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2239
2240 // Writer's end will be closed by cache manager, FIFO is already unlinked
2241
1/2
✓ Branch 1 taken 68 times.
✗ Branch 2 not taken.
 68 close(back_channel[0]);
2242 } else {
2243 ✗ ClosePipe(back_channel);
2244 }
2245 68 }
2246
2247 3404318 void PosixQuotaManager::ManagedReadHalfPipe(int fd, void *buf, size_t nbyte) {
2248
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3404318 times.
 3404318 const unsigned timeout_ms = cachemgr_pid_ ? 1000 : 0;
2249 3404318 bool result = false;
2250 do {
2251 3404318 result = ReadHalfPipe(fd, buf, nbyte, timeout_ms);
2252 // try only as long as the cachemgr is still alive
2253
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 3404318 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 3404318 times.
 3404318 } while (!result && getpgid(cachemgr_pid_) >= 0);
2254
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3404318 times.
 3404318 if (!result) {
2255 ✗ PANIC(kLogStderr,
2256 "Error: quota manager could not read from cachemanager pipe");
2257 }
2258 3404318 }
2259
2260 ✗ void *PosixQuotaManager::CollectMountpointsHashes(void *data) {
2261 #ifndef __APPLE__
2262 ✗ pthread_setname_np(pthread_self(), "hash_collector");
2263 ✗ auto *handler = static_cast<CollectorHandler *>(data);
2264
2265 ✗ const std::string mountpoint = handler->mp[handler->i];
2266 ✗ ssize_t n = getxattr(mountpoint.c_str(), "user.list_open_hashes", nullptr, 0);
2267 ✗ if (n < 0) {
2268 ✗ pthread_exit(nullptr);
2269 }
2270 ✗ std::vector<char> buf((size_t)n);
2271 ✗ n = getxattr(mountpoint.c_str(), "user.list_open_hashes", buf.data(),
2272 buf.size());
2273 ✗ if (n < 0) {
2274 ✗ pthread_exit(nullptr);
2275 }
2276
2277 ✗ std::vector<std::string> hash_strs;
2278 ✗ std::string hash_str;
2279 ✗ for (const char c : buf) {
2280 ✗ if (c == '\n') {
2281 ✗ hash_strs.push_back(hash_str);
2282 ✗ hash_str.clear();
2283 } else {
2284 ✗ hash_str += c;
2285 }
2286 }
2287 ✗ const MutexLockGuard lock_guard(handler->l);
2288 ✗ for (auto hash_str : hash_strs) {
2289 ✗ handler->of.push_back(shash::MkFromHexPtr(shash::HexPtr(hash_str)));
2290 }
2291 #endif
2292 ✗ pthread_exit(nullptr);
2293 }
2294
2295 ✗ std::vector<shash::Any> PosixQuotaManager::CollectAllOpenHashes() {
2296 ✗ std::vector<CollectorHandler *> handlers;
2297 ✗ std::vector<pthread_t *> threads;
2298 ✗ open_files_.clear();
2299 #ifndef __APPLE__
2300 ✗ auto &&a_after_b = [](const struct timespec a, const struct timespec b) {
2301 ✗ return (a.tv_sec > b.tv_sec) ? true : false;
2302 };
2303
2304 ✗ for (size_t i = 0; i < mountpoints_.size(); ++i) {
2305 ✗ handlers.push_back(
2306 ✗ new CollectorHandler{open_files_, mountpoints_, lock_open_files_, i});
2307 ✗ threads.push_back(new pthread_t);
2308 }
2309
2310 ✗ const int retval = pthread_mutex_init(lock_open_files_, NULL);
2311 ✗ assert(retval == 0);
2312
2313 ✗ for (size_t i = 0; i < mountpoints_.size(); ++i) {
2314 ✗ pthread_create(threads[i], nullptr, CollectMountpointsHashes, handlers[i]);
2315 }
2316
2317 ✗ std::vector<bool> joined(handlers.size(), false);
2318 struct timespec reference, current;
2319 ✗ clock_gettime(CLOCK_REALTIME, &reference);
2320 ✗ clock_gettime(CLOCK_REALTIME, &current);
2321 ✗ reference.tv_sec += 10; // Give 10sec for hash collection
2322 ✗ size_t i = 0;
2323 ✗ while (
2324 ✗ (not std::all_of(joined.begin(), joined.end(), [](bool b) { return b; }))
2325 ✗ and a_after_b(reference, current)) {
2326 // as long as there are still threads that haven't joined yet
2327 // and for 10 seconds
2328 ✗ if (not joined[i]) {
2329 ✗ const int s = pthread_tryjoin_np(*threads[i], NULL);
2330 ✗ if (s == 0) {
2331 ✗ joined[i] = true;
2332 }
2333 }
2334 ✗ ++i;
2335 ✗ i = i % handlers.size();
2336 ✗ clock_gettime(CLOCK_REALTIME, &current);
2337 }
2338
2339 ✗ for (size_t i = 0; i < handlers.size(); ++i) {
2340 ✗ delete handlers[i];
2341 }
2342
2343 ✗ pthread_mutex_destroy(lock_open_files_);
2344 #endif
2345 ✗ return open_files_;
2346 }
2347
2348