GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/quota_posix.cc
Date: 2025-12-07 02:35:36
Exec Total Coverage
Lines: 955 1391 68.7%
Branches: 749 1920 39.0%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * This module implements a "managed local cache".
5 * This way, we are able to track access times of files in the cache
6 * and remove files based on least recently used strategy.
7 *
8 * We setup another SQLite catalog, a "cache catalog", that helps us
9 * in the bookkeeping of files, file sizes and access times.
10 *
11 * We might choose to not manage the local cache. This is indicated
12 * by limit == 0 and everything succeeds in that case.
13 */
14
15 #define __STDC_LIMIT_MACROS
16 #define __STDC_FORMAT_MACROS
17
18
19 #include "quota_posix.h"
20
21 #include <dirent.h>
22 #include <errno.h>
23 #include <fcntl.h>
24 #include <inttypes.h>
25 #include <pthread.h>
26 #include <signal.h>
27 #include <stdint.h>
28 #include <sys/dir.h>
29 #include <sys/stat.h>
30 #include <sys/xattr.h>
31
32 #ifndef __APPLE__
33 #include <sys/statfs.h>
34 #endif
35 #include <sys/statvfs.h>
36 #include <sys/types.h>
37 #include <sys/wait.h>
38 #include <unistd.h>
39
40 #include <algorithm>
41 #include <cassert>
42 #include <cstdio>
43 #include <cstdlib>
44 #include <cstring>
45 #include <limits>
46 #include <map>
47 #include <set>
48 #include <string>
49 #include <vector>
50
51 #include "crypto/hash.h"
52 #include "duplex_sqlite3.h"
53 #include "monitor.h"
54 #include "statistics.h"
55 #include "util/concurrency.h"
56 #include "util/exception.h"
57 #include "util/logging.h"
58 #include "util/platform.h"
59 #include "util/pointer.h"
60 #include "util/posix.h"
61 #include "util/smalloc.h"
62 #include "util/string.h"
63
64 using namespace std; // NOLINT
65
66
67 644 int PosixQuotaManager::BindReturnPipe(int pipe_wronly) {
68
2/2
✓ Branch 0 taken 630 times.
✓ Branch 1 taken 14 times.
 644 if (!shared_)
69 630 return pipe_wronly;
70
71 // Connect writer's end
72
1/2
✓ Branch 2 taken 14 times.
✗ Branch 3 not taken.
 14 const int result = open(
73
2/4
✓ Branch 2 taken 14 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 14 times.
✗ Branch 6 not taken.
 28 (workspace_dir_ + "/pipe" + StringifyInt(pipe_wronly)).c_str(),
74 O_WRONLY | O_NONBLOCK);
75
2/2
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 7 times.
 14 if (result >= 0) {
76 7 Nonblock2Block(result);
77 } else {
78 7 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
79 7 "failed to bind return pipe (%d)", errno);
80 }
81 14 return result;
82 }
83
84
85 122 void PosixQuotaManager::CheckHighPinWatermark() {
86 122 const uint64_t watermark = kHighPinWatermark * cleanup_threshold_ / 100;
87
3/4
✓ Branch 0 taken 122 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 21 times.
✓ Branch 3 taken 101 times.
 122 if ((cleanup_threshold_ > 0) && (pinned_ > watermark)) {
88 21 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
89 "high watermark of pinned files (%" PRIu64 "M > %" PRIu64 "M)",
90 21 pinned_ / (1024 * 1024), watermark / (1024 * 1024));
91
2/4
✓ Branch 2 taken 21 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 21 times.
✗ Branch 6 not taken.
 21 BroadcastBackchannels("R"); // clients: please release pinned catalogs
92 }
93 122 }
94
95
96 ✗ void PosixQuotaManager::CleanupPipes() {
97 ✗ DIR *dirp = opendir(workspace_dir_.c_str());
98 ✗ assert(dirp != NULL);
99
100 platform_dirent64 *dent;
101 ✗ bool found_leftovers = false;
102 ✗ while ((dent = platform_readdir(dirp)) != NULL) {
103 ✗ const string name = dent->d_name;
104 ✗ const string path = workspace_dir_ + "/" + name;
105 platform_stat64 info;
106 ✗ const int retval = platform_stat(path.c_str(), &info);
107 ✗ if (retval != 0)
108 ✗ continue;
109 ✗ if (S_ISFIFO(info.st_mode) && (name.substr(0, 4) == "pipe")) {
110 ✗ if (!found_leftovers) {
111 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogWarn,
112 "removing left-over FIFOs from cache directory");
113 }
114 ✗ found_leftovers = true;
115 ✗ unlink(path.c_str());
116 }
117 }
118 ✗ closedir(dirp);
119 }
120
121
122 /**
123 * Cleans up in data cache, until cache size is below leave_size.
124 * The actual unlinking is done in a separate process (fork).
125 *
126 * \return True on success, false otherwise
127 */
128 63 bool PosixQuotaManager::Cleanup(const uint64_t leave_size) {
129
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 63 times.
 63 if (!spawned_)
130 ✗ return DoCleanup(leave_size);
131
132 bool result;
133 int pipe_cleanup[2];
134
1/2
✓ Branch 1 taken 63 times.
✗ Branch 2 not taken.
 63 MakeReturnPipe(pipe_cleanup);
135
136 63 LruCommand cmd;
137 63 cmd.command_type = kCleanup;
138 63 cmd.size = leave_size;
139 63 cmd.return_pipe = pipe_cleanup[1];
140
141
1/2
✓ Branch 1 taken 63 times.
✗ Branch 2 not taken.
 63 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
142
1/2
✓ Branch 1 taken 63 times.
✗ Branch 2 not taken.
 63 ManagedReadHalfPipe(pipe_cleanup[0], &result, sizeof(result));
143
1/2
✓ Branch 1 taken 63 times.
✗ Branch 2 not taken.
 63 CloseReturnPipe(pipe_cleanup);
144
145 63 return result;
146 }
147
148
149 632 void PosixQuotaManager::CloseDatabase() {
150
1/2
✓ Branch 0 taken 632 times.
✗ Branch 1 not taken.
 632 if (stmt_list_catalogs_)
151 632 sqlite3_finalize(stmt_list_catalogs_);
152
1/2
✓ Branch 0 taken 632 times.
✗ Branch 1 not taken.
 632 if (stmt_list_pinned_)
153 632 sqlite3_finalize(stmt_list_pinned_);
154
1/2
✓ Branch 0 taken 632 times.
✗ Branch 1 not taken.
 632 if (stmt_list_volatile_)
155 632 sqlite3_finalize(stmt_list_volatile_);
156
1/2
✓ Branch 0 taken 632 times.
✗ Branch 1 not taken.
 632 if (stmt_list_)
157 632 sqlite3_finalize(stmt_list_);
158
1/2
✓ Branch 0 taken 632 times.
✗ Branch 1 not taken.
 632 if (stmt_lru_)
159 632 sqlite3_finalize(stmt_lru_);
160
1/2
✓ Branch 0 taken 632 times.
✗ Branch 1 not taken.
 632 if (stmt_rm_)
161 632 sqlite3_finalize(stmt_rm_);
162
1/2
✓ Branch 0 taken 632 times.
✗ Branch 1 not taken.
 632 if (stmt_rm_batch_)
163 632 sqlite3_finalize(stmt_rm_batch_);
164
1/2
✓ Branch 0 taken 632 times.
✗ Branch 1 not taken.
 632 if (stmt_size_)
165 632 sqlite3_finalize(stmt_size_);
166
1/2
✓ Branch 0 taken 632 times.
✗ Branch 1 not taken.
 632 if (stmt_touch_)
167 632 sqlite3_finalize(stmt_touch_);
168
1/2
✓ Branch 0 taken 632 times.
✗ Branch 1 not taken.
 632 if (stmt_unpin_)
169 632 sqlite3_finalize(stmt_unpin_);
170
1/2
✓ Branch 0 taken 632 times.
✗ Branch 1 not taken.
 632 if (stmt_block_)
171 632 sqlite3_finalize(stmt_block_);
172
1/2
✓ Branch 0 taken 632 times.
✗ Branch 1 not taken.
 632 if (stmt_unblock_)
173 632 sqlite3_finalize(stmt_unblock_);
174
1/2
✓ Branch 0 taken 632 times.
✗ Branch 1 not taken.
 632 if (stmt_new_)
175 632 sqlite3_finalize(stmt_new_);
176
1/2
✓ Branch 0 taken 632 times.
✗ Branch 1 not taken.
 632 if (database_)
177 632 sqlite3_close(database_);
178 632 UnlockFile(fd_lock_cachedb_);
179
180 632 stmt_list_catalogs_ = NULL;
181 632 stmt_list_pinned_ = NULL;
182 632 stmt_list_volatile_ = NULL;
183 632 stmt_list_ = NULL;
184 632 stmt_rm_ = NULL;
185 632 stmt_rm_batch_ = NULL;
186 632 stmt_size_ = NULL;
187 632 stmt_touch_ = NULL;
188 632 stmt_unpin_ = NULL;
189 632 stmt_block_ = NULL;
190 632 stmt_unblock_ = NULL;
191 632 stmt_new_ = NULL;
192 632 database_ = NULL;
193
194 632 pinned_chunks_.clear();
195 632 }
196
197
198 609 void PosixQuotaManager::CloseReturnPipe(int pipe[2]) {
199
2/2
✓ Branch 0 taken 14 times.
✓ Branch 1 taken 595 times.
 609 if (shared_) {
200 14 close(pipe[0]);
201 14 UnlinkReturnPipe(pipe[1]);
202 } else {
203 595 ClosePipe(pipe);
204 }
205 609 }
206
207
208 700338 bool PosixQuotaManager::Contains(const string &hash_str) {
209 700338 bool result = false;
210
211 700338 sqlite3_bind_text(stmt_size_, 1, &hash_str[0], hash_str.length(),
212 SQLITE_STATIC);
213
2/2
✓ Branch 1 taken 83 times.
✓ Branch 2 taken 700255 times.
 700338 if (sqlite3_step(stmt_size_) == SQLITE_ROW)
214 83 result = true;
215 700338 sqlite3_reset(stmt_size_);
216 700338 LogCvmfs(kLogQuota, kLogDebug, "contains %s returns %d", hash_str.c_str(),
217 result);
218
219 700338 return result;
220 }
221
222
223 619 void PosixQuotaManager::CheckFreeSpace() {
224
3/4
✓ Branch 0 taken 619 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 7 times.
✓ Branch 3 taken 612 times.
 619 if ((limit_ == 0) || (gauge_ >= limit_))
225 7 return;
226
227 struct statvfs vfs_info;
228
1/2
✓ Branch 1 taken 612 times.
✗ Branch 2 not taken.
 612 const int retval = statvfs((cache_dir_ + "/cachedb").c_str(), &vfs_info);
229
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 612 times.
 612 if (retval != 0) {
230 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
231 "failed to query %s for free space (%d)", cache_dir_.c_str(),
232 ✗ errno);
233 ✗ return;
234 }
235 612 const int64_t free_space_byte = vfs_info.f_bavail * vfs_info.f_bsize;
236
1/2
✓ Branch 1 taken 612 times.
✗ Branch 2 not taken.
 612 LogCvmfs(kLogQuota, kLogDebug, "free space: %" PRId64 " MB",
237 free_space_byte / (1024 * 1024));
238
239 612 const int64_t required_byte = limit_ - gauge_;
240
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 612 times.
 612 if (free_space_byte < required_byte) {
241 ✗ LogCvmfs(kLogQuota, kLogSyslogWarn,
242 "too little free space on the file system hosting the cache,"
243 " %" PRId64 " MB available",
244 free_space_byte / (1024 * 1024));
245 }
246 }
247
248
249 640 PosixQuotaManager *PosixQuotaManager::Create(const string &cache_workspace,
250 const uint64_t limit,
251 const uint64_t cleanup_threshold,
252 const bool rebuild_database) {
253
2/2
✓ Branch 0 taken 14 times.
✓ Branch 1 taken 626 times.
 640 if (cleanup_threshold >= limit) {
254 14 LogCvmfs(kLogQuota, kLogDebug,
255 "invalid parameters: limit %" PRIu64 ", "
256 "cleanup_threshold %" PRIu64,
257 limit, cleanup_threshold);
258 14 return NULL;
259 }
260
261 PosixQuotaManager *quota_manager = new PosixQuotaManager(
262
1/2
✓ Branch 2 taken 626 times.
✗ Branch 3 not taken.
 626 limit, cleanup_threshold, cache_workspace);
263
264 // Initialize cache catalog
265
2/2
✓ Branch 1 taken 7 times.
✓ Branch 2 taken 619 times.
 626 if (!quota_manager->InitDatabase(rebuild_database)) {
266
1/2
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
 7 delete quota_manager;
267 7 return NULL;
268 }
269 619 quota_manager->CheckFreeSpace();
270 619 MakePipe(quota_manager->pipe_lru_);
271
272 619 quota_manager->protocol_revision_ = kProtocolRevision;
273 619 quota_manager->initialized_ = true;
274 619 return quota_manager;
275 }
276
277
278 /**
279 * Connects to a running shared local quota manager. Creates one if necessary.
280 */
281 14 PosixQuotaManager *PosixQuotaManager::CreateShared(
282 const std::string &exe_path,
283 const std::string &cache_workspace,
284 const uint64_t limit,
285 const uint64_t cleanup_threshold,
286 bool foreground) {
287 14 string cache_dir;
288 14 string workspace_dir;
289
2/4
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 14 times.
✗ Branch 5 not taken.
 14 ParseDirectories(cache_workspace, &cache_dir, &workspace_dir);
290
291 pid_t new_cachemgr_pid;
292
293 // Create lock file: only one fuse client at a time
294
2/4
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 14 times.
✗ Branch 5 not taken.
 14 const int fd_lockfile = LockFile(workspace_dir + "/lock_cachemgr");
295
2/2
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 7 times.
 14 if (fd_lockfile < 0) {
296
1/2
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
 7 LogCvmfs(kLogQuota, kLogDebug, "could not open lock file %s (%d)",
297
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 14 (workspace_dir + "/lock_cachemgr").c_str(), errno);
298 7 return NULL;
299 }
300
301 PosixQuotaManager *quota_mgr = new PosixQuotaManager(limit, cleanup_threshold,
302
2/4
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 7 times.
✗ Branch 5 not taken.
 7 cache_workspace);
303 7 quota_mgr->shared_ = true;
304 7 quota_mgr->spawned_ = true;
305
306 // Try to connect to pipe
307
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 const string fifo_path = workspace_dir + "/cachemgr";
308
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 LogCvmfs(kLogQuota, kLogDebug, "trying to connect to existing pipe");
309
1/2
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
 7 quota_mgr->pipe_lru_[1] = open(fifo_path.c_str(), O_WRONLY | O_NONBLOCK);
310
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
 7 if (quota_mgr->pipe_lru_[1] >= 0) {
311 ✗ const int fd_lockfile_rw = open((workspace_dir + "/lock_cachemgr").c_str(),
312 O_RDWR, 0600);
313 ✗ unsigned lockfile_magicnumber = 0;
314 ✗ const ssize_t result_mn = SafeRead(fd_lockfile_rw, &lockfile_magicnumber,
315 sizeof(lockfile_magicnumber));
316 ✗ const ssize_t result = SafeRead(fd_lockfile_rw, &new_cachemgr_pid,
317 sizeof(new_cachemgr_pid));
318 ✗ close(fd_lockfile_rw);
319
320 ✗ if ((lockfile_magicnumber != kLockFileMagicNumber) || (result < 0)
321 ✗ || (result_mn < 0)
322 ✗ || (static_cast<size_t>(result) < sizeof(new_cachemgr_pid))) {
323 ✗ if (result != 0) {
324 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
325 "could not read cache manager pid from lockfile");
326 ✗ UnlockFile(fd_lockfile);
327 ✗ delete quota_mgr;
328 ✗ return NULL;
329 } else {
330 // support reload from old versions of the cache manager
331 // lock file is empty in this case, try a plain ReadHalfPipe to get pid
332 ✗ quota_mgr->SetCacheMgrPid(quota_mgr->GetPid());
333 }
334 } else {
335 ✗ quota_mgr->SetCacheMgrPid(new_cachemgr_pid);
336 }
337
338
339 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to existing cache manager pipe");
340 ✗ quota_mgr->initialized_ = true;
341 ✗ Nonblock2Block(quota_mgr->pipe_lru_[1]);
342 ✗ UnlockFile(fd_lockfile);
343 ✗ quota_mgr->GetLimits(&quota_mgr->limit_, &quota_mgr->cleanup_threshold_);
344 ✗ LogCvmfs(kLogQuota, kLogDebug,
345 "received limit %" PRIu64 ", threshold %" PRIu64,
346 quota_mgr->limit_, quota_mgr->cleanup_threshold_);
347 ✗ if (FileExists(workspace_dir + "/cachemgr.protocol")) {
348 ✗ quota_mgr->protocol_revision_ = quota_mgr->GetProtocolRevision();
349 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected protocol revision %u",
350 quota_mgr->protocol_revision_);
351 } else {
352 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to ancient cache manager");
353 }
354 ✗ return quota_mgr;
355 }
356 7 const int connect_error = errno;
357
358 // Lock file: let existing cache manager finish first
359
2/4
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 7 times.
✗ Branch 5 not taken.
 7 const int fd_lockfile_fifo = LockFile(workspace_dir + "/lock_cachemgr.fifo");
360
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
 7 if (fd_lockfile_fifo < 0) {
361 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not open lock file %s (%d)",
362 ✗ (workspace_dir + "/lock_cachemgr.fifo").c_str(), errno);
363 ✗ UnlockFile(fd_lockfile);
364 ✗ delete quota_mgr;
365 ✗ return NULL;
366 }
367
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 UnlockFile(fd_lockfile_fifo);
368
369
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
 7 if (connect_error == ENXIO) {
370 ✗ LogCvmfs(kLogQuota, kLogDebug, "left-over FIFO found, unlinking");
371 ✗ unlink(fifo_path.c_str());
372 }
373
374 // Creating a new FIFO for the cache manager (to be bound later)
375 7 int retval = mkfifo(fifo_path.c_str(), 0600);
376
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
 7 if (retval != 0) {
377 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to create cache manager FIFO (%d)",
378 ✗ errno);
379 ✗ UnlockFile(fd_lockfile);
380 ✗ delete quota_mgr;
381 ✗ return NULL;
382 }
383
384 // Create new cache manager
385 int pipe_boot[2];
386 int pipe_handshake[2];
387
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 MakePipe(pipe_boot);
388
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 MakePipe(pipe_handshake);
389
390 7 vector<string> command_line;
391
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 command_line.push_back(exe_path);
392
2/4
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 7 times.
✗ Branch 6 not taken.
 7 command_line.push_back("__cachemgr__");
393
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 command_line.push_back(cache_workspace);
394
2/4
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 7 times.
✗ Branch 5 not taken.
 7 command_line.push_back(StringifyInt(pipe_boot[1]));
395
2/4
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 7 times.
✗ Branch 5 not taken.
 7 command_line.push_back(StringifyInt(pipe_handshake[0]));
396
2/4
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 7 times.
✗ Branch 5 not taken.
 7 command_line.push_back(StringifyInt(limit));
397
2/4
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 7 times.
✗ Branch 5 not taken.
 7 command_line.push_back(StringifyInt(cleanup_threshold));
398 // do not propagate foreground in order to reliably get pid from exec
399 // instead, daemonize right here
400
2/4
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 7 times.
✗ Branch 5 not taken.
 7 command_line.push_back(StringifyInt(true)); // foreground
401
3/6
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 7 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 7 times.
✗ Branch 8 not taken.
 7 command_line.push_back(StringifyInt(GetLogSyslogLevel()));
402
3/6
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 7 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 7 times.
✗ Branch 8 not taken.
 7 command_line.push_back(StringifyInt(GetLogSyslogFacility()));
403
5/14
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 7 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 7 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 7 times.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 7 times.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
 7 command_line.push_back(GetLogDebugFile() + ":" + GetLogMicroSyslog());
404
405 7 set<int> preserve_filedes;
406
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 preserve_filedes.insert(0);
407
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 preserve_filedes.insert(1);
408
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 preserve_filedes.insert(2);
409
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 preserve_filedes.insert(pipe_boot[1]);
410
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 preserve_filedes.insert(pipe_handshake[0]);
411
412
1/2
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
 7 if (foreground) {
413
1/2
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
 7 retval = ManagedExec(command_line, preserve_filedes, map<int, int>(),
414 /*drop_credentials*/ false,
415 /*clear_env*/ false,
416 /*double_fork*/ true, &new_cachemgr_pid);
417 } else {
418 ✗ retval = ExecAsDaemon(command_line, &new_cachemgr_pid);
419 }
420
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
 7 if (!retval) {
421 ✗ UnlockFile(fd_lockfile);
422 ✗ ClosePipe(pipe_boot);
423 ✗ ClosePipe(pipe_handshake);
424 ✗ delete quota_mgr;
425 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to start cache manager");
426 ✗ return NULL;
427 }
428
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 LogCvmfs(kLogQuota, kLogDebug, "new cache manager pid: %d", new_cachemgr_pid);
429 7 quota_mgr->SetCacheMgrPid(new_cachemgr_pid);
430
2/4
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 7 times.
✗ Branch 6 not taken.
 7 const int fd_lockfile_rw = open((workspace_dir + "/lock_cachemgr").c_str(),
431 O_RDWR | O_TRUNC, 0600);
432 7 const unsigned magic_number = PosixQuotaManager::kLockFileMagicNumber;
433
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 const bool result_mn = SafeWrite(fd_lockfile_rw, &magic_number,
434 sizeof(magic_number));
435
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 const bool result = SafeWrite(fd_lockfile_rw, &new_cachemgr_pid,
436 sizeof(new_cachemgr_pid));
437
2/4
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 7 times.
 7 if (!result || !result_mn) {
438 ✗ PANIC(kLogSyslogErr, "could not write cache manager pid to lockfile");
439 }
440
441
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 close(fd_lockfile_rw);
442 // Wait for cache manager to be ready
443
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 close(pipe_boot[1]);
444
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 close(pipe_handshake[0]);
445 char buf;
446
2/4
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 7 times.
✗ Branch 4 not taken.
 7 if (read(pipe_boot[0], &buf, 1) != 1) {
447
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 UnlockFile(fd_lockfile);
448
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 close(pipe_boot[0]);
449
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 close(pipe_handshake[1]);
450
1/2
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
 7 delete quota_mgr;
451
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
452 "cache manager did not start");
453 7 return NULL;
454 }
455 ✗ close(pipe_boot[0]);
456
457 // Connect write end
458 ✗ quota_mgr->pipe_lru_[1] = open(fifo_path.c_str(), O_WRONLY | O_NONBLOCK);
459 ✗ if (quota_mgr->pipe_lru_[1] < 0) {
460 ✗ LogCvmfs(kLogQuota, kLogDebug,
461 ✗ "failed to connect to newly created FIFO (%d)", errno);
462 ✗ close(pipe_handshake[1]);
463 ✗ UnlockFile(fd_lockfile);
464 ✗ delete quota_mgr;
465 ✗ return NULL;
466 }
467
468 // Finalize handshake
469 ✗ buf = 'C';
470 ✗ if (write(pipe_handshake[1], &buf, 1) != 1) {
471 ✗ UnlockFile(fd_lockfile);
472 ✗ close(pipe_handshake[1]);
473 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not finalize handshake");
474 ✗ delete quota_mgr;
475 ✗ return NULL;
476 }
477 ✗ close(pipe_handshake[1]);
478
479 ✗ Nonblock2Block(quota_mgr->pipe_lru_[1]);
480 ✗ LogCvmfs(kLogQuota, kLogDebug, "connected to a new cache manager");
481 ✗ quota_mgr->protocol_revision_ = kProtocolRevision;
482
483 ✗ UnlockFile(fd_lockfile);
484
485 ✗ quota_mgr->initialized_ = true;
486 ✗ quota_mgr->GetLimits(&quota_mgr->limit_, &quota_mgr->cleanup_threshold_);
487 ✗ LogCvmfs(kLogQuota, kLogDebug,
488 "received limit %" PRIu64 ", "
489 "threshold %" PRIu64,
490 quota_mgr->limit_, quota_mgr->cleanup_threshold_);
491 ✗ return quota_mgr;
492 14 }
493
494
495 70 bool PosixQuotaManager::DoCleanup(const uint64_t leave_size) {
496
2/2
✓ Branch 0 taken 14 times.
✓ Branch 1 taken 56 times.
 70 if (gauge_ <= leave_size)
497 14 return true;
498
499 // TODO(jblomer) transaction
500
1/2
✓ Branch 1 taken 56 times.
✗ Branch 2 not taken.
 56 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug,
501 "clean up cache until at most %lu KB is used", leave_size / 1024);
502
1/2
✓ Branch 1 taken 56 times.
✗ Branch 2 not taken.
 56 LogCvmfs(kLogQuota, kLogDebug, "gauge %" PRIu64, gauge_);
503
1/2
✓ Branch 1 taken 56 times.
✗ Branch 2 not taken.
 56 cleanup_recorder_.Tick();
504
505 bool result;
506 56 vector<string> trash;
507
508 // Note that volatile files start counting from the smallest int64 number:
509 // the absolute sequence number with the first bit set in two's complement.
510 // So -1 can be a marker that will never appear in the database.
511 56 int64_t max_acseq = -1;
512 56 std::vector<EvictCandidate> lru_ordered_open;
513
514 do {
515
1/2
✓ Branch 1 taken 406 times.
✗ Branch 2 not taken.
 406 sqlite3_reset(stmt_lru_);
516
3/4
✓ Branch 0 taken 56 times.
✓ Branch 1 taken 350 times.
✓ Branch 3 taken 406 times.
✗ Branch 4 not taken.
 462 sqlite3_bind_int64(stmt_lru_, 1,
517 56 (max_acseq == -1) ? std::numeric_limits<int64_t>::min()
518 : (max_acseq + 1));
519
520 406 std::vector<EvictCandidate> candidates;
521
1/2
✓ Branch 1 taken 406 times.
✗ Branch 2 not taken.
 406 candidates.reserve(kEvictBatchSize);
522 406 string hash_str;
523 406 unsigned i = 0;
524
3/4
✓ Branch 1 taken 364476 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 364070 times.
✓ Branch 4 taken 406 times.
 364476 while (sqlite3_step(stmt_lru_) == SQLITE_ROW) {
525 hash_str = reinterpret_cast<const char *>(
526
2/4
✓ Branch 1 taken 364070 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 364070 times.
✗ Branch 5 not taken.
 364070 sqlite3_column_text(stmt_lru_, 0));
527
1/2
✓ Branch 2 taken 364070 times.
✗ Branch 3 not taken.
 364070 LogCvmfs(kLogQuota, kLogDebug, "add %s to candidates for eviction",
528 hash_str.c_str());
529
1/2
✓ Branch 1 taken 364070 times.
✗ Branch 2 not taken.
 364070 candidates.push_back(
530
1/2
✓ Branch 1 taken 364070 times.
✗ Branch 2 not taken.
 364070 EvictCandidate(shash::MkFromHexPtr(shash::HexPtr(hash_str)),
531 364070 sqlite3_column_int64(stmt_lru_, 1),
532
2/4
✓ Branch 1 taken 364070 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 364070 times.
✗ Branch 5 not taken.
 364070 sqlite3_column_int64(stmt_lru_, 2)));
533 364070 i++;
534 }
535
2/2
✓ Branch 1 taken 7 times.
✓ Branch 2 taken 399 times.
 406 if (candidates.empty()) {
536
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 LogCvmfs(kLogQuota, kLogDebug, "no more entries to evict");
537 7 break;
538 }
539
540 399 const unsigned N = candidates.size();
541
542 399 open_files_.clear();
543
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 399 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 798 open_files_ = (cleanup_unused_first_) ? CollectAllOpenHashes()
544 399 : std::vector<shash::Any>();
545
546
2/2
✓ Branch 0 taken 350063 times.
✓ Branch 1 taken 350 times.
 350413 for (i = 0; i < N; ++i) {
547 // That's a critical condition. We must not delete a not yet inserted
548 // pinned file as it is already reserved (but will be inserted later).
549 // Instead, set the pin bit in the db to not run into an endless loop
550
1/2
✓ Branch 2 taken 350063 times.
✗ Branch 3 not taken.
 350063 const bool is_pinned = pinned_chunks_.find(candidates[i].hash)
551 700126 != pinned_chunks_.end();
552
553 // Avoid evicting open files hopping there are enough more recently used
554 // files to satisfy the cleanup request
555
1/2
✓ Branch 2 taken 350063 times.
✗ Branch 3 not taken.
 350063 const bool is_open = std::find(open_files_.begin(), open_files_.end(),
556 350063 candidates[i].hash)
557 700126 != open_files_.end();
558
559
2/2
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 350056 times.
 350063 if (is_pinned) {
560
1/2
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
 7 SkipEviction(candidates[i]);
561 7 continue;
562 }
563
564
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 350056 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 350056 if (cleanup_unused_first_ and is_open) {
565 ✗ SkipEviction(candidates[i]);
566 ✗ lru_ordered_open.push_back(candidates[i]);
567 ✗ continue;
568 }
569
570
2/4
✓ Branch 1 taken 350056 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 350056 times.
✗ Branch 5 not taken.
 700112 trash.push_back(cache_dir_ + "/"
571
2/4
✓ Branch 2 taken 350056 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 350056 times.
✗ Branch 6 not taken.
 1050168 + candidates[i].hash.MakePathWithoutSuffix());
572 350056 gauge_ -= candidates[i].size;
573 350056 max_acseq = candidates[i].acseq;
574
1/2
✓ Branch 2 taken 350056 times.
✗ Branch 3 not taken.
 350056 LogCvmfs(kLogQuota, kLogDebug, "lru cleanup %s, new gauge %" PRIu64,
575
1/2
✓ Branch 2 taken 350056 times.
✗ Branch 3 not taken.
 700112 candidates[i].hash.ToString().c_str(), gauge_);
576
577
2/2
✓ Branch 0 taken 49 times.
✓ Branch 1 taken 350007 times.
 350056 if (gauge_ <= leave_size)
578 49 break;
579 }
580
6/6
✓ Branch 1 taken 399 times.
✓ Branch 2 taken 7 times.
✓ Branch 4 taken 399 times.
✓ Branch 5 taken 7 times.
✓ Branch 6 taken 350 times.
✓ Branch 7 taken 49 times.
 812 } while (gauge_ > leave_size);
581
582
1/2
✓ Branch 0 taken 56 times.
✗ Branch 1 not taken.
 56 if (max_acseq != -1) {
583
1/2
✓ Branch 1 taken 56 times.
✗ Branch 2 not taken.
 56 sqlite3_bind_int64(stmt_rm_batch_, 1, max_acseq);
584
1/2
✓ Branch 1 taken 56 times.
✗ Branch 2 not taken.
 56 result = (sqlite3_step(stmt_rm_batch_) == SQLITE_DONE);
585
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 56 times.
 56 assert(result);
586
1/2
✓ Branch 1 taken 56 times.
✗ Branch 2 not taken.
 56 sqlite3_reset(stmt_rm_batch_);
587
588
1/2
✓ Branch 1 taken 56 times.
✗ Branch 2 not taken.
 56 result = (sqlite3_step(stmt_unblock_) == SQLITE_DONE);
589
1/2
✓ Branch 1 taken 56 times.
✗ Branch 2 not taken.
 56 sqlite3_reset(stmt_unblock_);
590
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 56 times.
 56 assert(result);
591 }
592
593
2/6
✗ Branch 1 not taken.
✓ Branch 2 taken 56 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 56 times.
 56 while (!lru_ordered_open.empty() and gauge_ > leave_size) {
594 // cleanup files in use
595 ✗ auto &candidate = lru_ordered_open[0];
596 ✗ trash.push_back(cache_dir_ + "/" + candidate.hash.MakePathWithoutSuffix());
597 ✗ gauge_ -= candidate.size;
598 ✗ max_acseq = candidate.acseq;
599 ✗ LogCvmfs(kLogQuota, kLogDebug, "lru cleanup %s, new gauge %" PRIu64,
600 ✗ candidate.hash.ToString().c_str(), gauge_);
601 ✗ lru_ordered_open.erase(lru_ordered_open.begin());
602 }
603
604
2/4
✓ Branch 1 taken 56 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 56 times.
 56 if (!EmptyTrash(trash))
605 ✗ return false;
606
607
2/2
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 49 times.
 56 if (gauge_ > leave_size) {
608
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
609 "request to clean until %" PRIu64 ", "
610 "but effective gauge is %" PRIu64,
611 leave_size, gauge_);
612 7 return false;
613 }
614 49 return true;
615 56 }
616
617 56 bool PosixQuotaManager::EmptyTrash(const std::vector<std::string> &trash) {
618
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 56 times.
 56 if (trash.empty())
619 ✗ return true;
620
621
2/2
✓ Branch 0 taken 42 times.
✓ Branch 1 taken 14 times.
 56 if (async_delete_) {
622 // Double fork avoids zombie, forked removal process must not flush file
623 // buffers
624 pid_t pid;
625 int statloc;
626
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 42 times.
 42 if ((pid = fork()) == 0) {
627 // TODO(jblomer): eviciting files in the cache should perhaps become a
628 // thread. This would also allow to block the chunks and prevent the
629 // race with re-insertion. Then again, a thread can block umount.
630 #ifndef DEBUGMSG
631 ✗ CloseAllFildes(std::set<int>());
632 #endif
633 ✗ if (fork() == 0) {
634 ✗ for (unsigned i = 0, iEnd = trash.size(); i < iEnd; ++i) {
635 ✗ LogCvmfs(kLogQuota, kLogDebug, "unlink %s", trash[i].c_str());
636 ✗ unlink(trash[i].c_str());
637 }
638 ✗ _exit(0);
639 }
640 ✗ _exit(0);
641 } else {
642
1/2
✓ Branch 0 taken 42 times.
✗ Branch 1 not taken.
 42 if (pid > 0)
643
1/2
✓ Branch 1 taken 42 times.
✗ Branch 2 not taken.
 42 waitpid(pid, &statloc, 0);
644 else
645 ✗ return false;
646 }
647 } else { // !async_delete_
648
2/2
✓ Branch 1 taken 21 times.
✓ Branch 2 taken 14 times.
 35 for (unsigned i = 0, iEnd = trash.size(); i < iEnd; ++i) {
649 21 LogCvmfs(kLogQuota, kLogDebug, "unlink %s", trash[i].c_str());
650 21 unlink(trash[i].c_str());
651 }
652 }
653 56 return true;
654 }
655
656
657 700257 void PosixQuotaManager::DoInsert(const shash::Any &hash,
658 const uint64_t size,
659 const string &description,
660 const CommandType command_type) {
661
1/2
✓ Branch 1 taken 700257 times.
✗ Branch 2 not taken.
 700257 const string hash_str = hash.ToString();
662
1/2
✓ Branch 3 taken 700257 times.
✗ Branch 4 not taken.
 700257 LogCvmfs(kLogQuota, kLogDebug, "insert into lru %s, path %s, method %d",
663 hash_str.c_str(), description.c_str(), command_type);
664 700257 const unsigned desc_length = (description.length() > kMaxDescription)
665 ? kMaxDescription
666
1/2
✓ Branch 0 taken 700257 times.
✗ Branch 1 not taken.
 700257 : description.length();
667
668 LruCommand *cmd = reinterpret_cast<LruCommand *>(
669 700257 alloca(sizeof(LruCommand) + desc_length));
670 700257 new (cmd) LruCommand;
671 700257 cmd->command_type = command_type;
672 700257 cmd->SetSize(size);
673
1/2
✓ Branch 1 taken 700257 times.
✗ Branch 2 not taken.
 700257 cmd->StoreHash(hash);
674 700257 cmd->desc_length = desc_length;
675 700257 memcpy(reinterpret_cast<char *>(cmd) + sizeof(LruCommand), &description[0],
676 desc_length);
677
1/2
✓ Branch 1 taken 700257 times.
✗ Branch 2 not taken.
 700257 WritePipe(pipe_lru_[1], cmd, sizeof(LruCommand) + desc_length);
678 700257 }
679
680
681 245 vector<string> PosixQuotaManager::DoList(const CommandType list_command) {
682 245 vector<string> result;
683
684 int pipe_list[2];
685
1/2
✓ Branch 1 taken 245 times.
✗ Branch 2 not taken.
 245 MakeReturnPipe(pipe_list);
686 char description_buffer[kMaxDescription];
687
688 245 LruCommand cmd;
689 245 cmd.command_type = list_command;
690 245 cmd.return_pipe = pipe_list[1];
691
1/2
✓ Branch 1 taken 245 times.
✗ Branch 2 not taken.
 245 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
692
693 int length;
694 do {
695
1/2
✓ Branch 1 taken 700518 times.
✗ Branch 2 not taken.
 700518 ManagedReadHalfPipe(pipe_list[0], &length, sizeof(length));
696
2/2
✓ Branch 0 taken 700273 times.
✓ Branch 1 taken 245 times.
 700518 if (length > 0) {
697
1/2
✓ Branch 1 taken 700273 times.
✗ Branch 2 not taken.
 700273 ReadPipe(pipe_list[0], description_buffer, length);
698
2/4
✓ Branch 2 taken 700273 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 700273 times.
✗ Branch 6 not taken.
 700273 result.push_back(string(description_buffer, length));
699 }
700
2/2
✓ Branch 0 taken 700273 times.
✓ Branch 1 taken 245 times.
 700518 } while (length >= 0);
701
702
1/2
✓ Branch 1 taken 245 times.
✗ Branch 2 not taken.
 245 CloseReturnPipe(pipe_list);
703 490 return result;
704 }
705
706
707 255 uint64_t PosixQuotaManager::GetCapacity() {
708
1/2
✓ Branch 0 taken 255 times.
✗ Branch 1 not taken.
 255 if (limit_ != (uint64_t)(-1))
709 255 return limit_;
710
711 // Unrestricted cache, look at free space on cache dir fs
712 struct statfs info;
713 ✗ if (statfs(".", &info) == 0) {
714 ✗ return info.f_bavail * info.f_bsize;
715 } else {
716 ✗ LogCvmfs(kLogQuota, kLogSyslogErr | kLogDebug,
717 ✗ "failed to query file system info of cache (%d)", errno);
718 ✗ return limit_;
719 }
720 }
721
722
723 ✗ void PosixQuotaManager::GetLimits(uint64_t *limit,
724 uint64_t *cleanup_threshold) {
725 int pipe_limits[2];
726 ✗ MakeReturnPipe(pipe_limits);
727
728 ✗ LruCommand cmd;
729 ✗ cmd.command_type = kLimits;
730 ✗ cmd.return_pipe = pipe_limits[1];
731 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
732 ✗ ManagedReadHalfPipe(pipe_limits[0], limit, sizeof(*limit));
733 ✗ ReadPipe(pipe_limits[0], cleanup_threshold, sizeof(*cleanup_threshold));
734 ✗ CloseReturnPipe(pipe_limits);
735 }
736
737
738 /**
739 * Since we only cleanup until cleanup_threshold, we can only add
740 * files smaller than limit-cleanup_threshold.
741 */
742 17 uint64_t PosixQuotaManager::GetMaxFileSize() {
743 17 return limit_ - cleanup_threshold_;
744 }
745
746
747 7 pid_t PosixQuotaManager::GetPid() {
748
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 7 if (!shared_ || !spawned_) {
749 7 return getpid();
750 }
751 ✗ if (cachemgr_pid_) {
752 ✗ return cachemgr_pid_;
753 }
754
755 pid_t result;
756 int pipe_pid[2];
757 ✗ MakeReturnPipe(pipe_pid);
758
759 ✗ LruCommand cmd;
760 ✗ cmd.command_type = kPid;
761 ✗ cmd.return_pipe = pipe_pid[1];
762 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
763 ✗ ReadHalfPipe(pipe_pid[0], &result, sizeof(result));
764 ✗ CloseReturnPipe(pipe_pid);
765 ✗ return result;
766 }
767
768
769 7 uint32_t PosixQuotaManager::GetProtocolRevision() {
770 int pipe_revision[2];
771
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 MakeReturnPipe(pipe_revision);
772
773 7 LruCommand cmd;
774 7 cmd.command_type = kGetProtocolRevision;
775 7 cmd.return_pipe = pipe_revision[1];
776
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
777
778 uint32_t revision;
779
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 ManagedReadHalfPipe(pipe_revision[0], &revision, sizeof(revision));
780
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 CloseReturnPipe(pipe_revision);
781 7 return revision;
782 }
783
784 248 void PosixQuotaManager::SetCleanupPolicy(bool cleanup_unused_first) {
785 248 LruCommand cmd;
786 248 cmd.command_type = kSetCleanupPolicy;
787
1/2
✓ Branch 1 taken 248 times.
✗ Branch 2 not taken.
 248 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
788
789
1/2
✓ Branch 1 taken 248 times.
✗ Branch 2 not taken.
 248 WritePipe(pipe_lru_[1], &cleanup_unused_first, sizeof(bool));
790 248 }
791
792 ✗ void PosixQuotaManager::RegisterMountpoint(const std::string &mountpoint) {
793 ✗ LruCommand cmd;
794 ✗ cmd.command_type = kRegisterMountpoint;
795 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
796
797 ✗ size_t mp_size = mountpoint.size();
798 ✗ WritePipe(pipe_lru_[1], &mp_size, sizeof(size_t));
799 ✗ WritePipe(pipe_lru_[1], mountpoint.data(), mp_size);
800 }
801
802 ✗ std::string PosixQuotaManager::GetMountpoints() {
803 int pipe_mp[2];
804 ✗ MakeReturnPipe(pipe_mp);
805
806 ✗ LruCommand cmd;
807 ✗ cmd.command_type = kGetMountpoints;
808 ✗ cmd.return_pipe = pipe_mp[1];
809 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
810 ✗ size_t mp_str_size = 0;
811 ✗ ManagedReadHalfPipe(pipe_mp[0], &mp_str_size, sizeof(size_t));
812 ✗ char *buf = (char *)malloc(mp_str_size * sizeof(char));
813 ✗ ManagedReadHalfPipe(pipe_mp[0], buf, mp_str_size);
814 ✗ return std::string(buf);
815 }
816
817 ✗ std::string PosixQuotaManager::GetGroupHashes() {
818 int pipe_gh[2];
819 ✗ MakeReturnPipe(pipe_gh);
820
821 ✗ LruCommand cmd;
822 ✗ cmd.command_type = kGetGroupHashes;
823 ✗ cmd.return_pipe = pipe_gh[1];
824 ✗ WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
825 ✗ size_t mp_str_size = 0;
826 ✗ ManagedReadHalfPipe(pipe_gh[0], &mp_str_size, sizeof(size_t));
827 ✗ char *buf = (char *)malloc(mp_str_size * sizeof(char));
828 ✗ ManagedReadHalfPipe(pipe_gh[0], buf, mp_str_size);
829 ✗ return std::string(buf);
830 }
831
832 /**
833 * Queries the shared local hard disk quota manager.
834 */
835 126 void PosixQuotaManager::GetSharedStatus(uint64_t *gauge, uint64_t *pinned) {
836 int pipe_status[2];
837
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 MakeReturnPipe(pipe_status);
838
839 126 LruCommand cmd;
840 126 cmd.command_type = kStatus;
841 126 cmd.return_pipe = pipe_status[1];
842
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
843
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 ManagedReadHalfPipe(pipe_status[0], gauge, sizeof(*gauge));
844
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 ReadPipe(pipe_status[0], pinned, sizeof(*pinned));
845
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 CloseReturnPipe(pipe_status);
846 126 }
847
848 7 bool PosixQuotaManager::SetSharedLimit(uint64_t limit) {
849 int pipe_set_limit[2];
850 bool result;
851
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 MakeReturnPipe(pipe_set_limit);
852
853 7 LruCommand cmd;
854 7 cmd.command_type = kSetLimit;
855 7 cmd.size = limit;
856 7 cmd.return_pipe = pipe_set_limit[1];
857
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
858
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 ReadHalfPipe(pipe_set_limit[0], &result, sizeof(result));
859
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 CloseReturnPipe(pipe_set_limit);
860 7 return result;
861 }
862
863
864 7 bool PosixQuotaManager::SetLimit(uint64_t size) {
865
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
 7 if (!spawned_) {
866 ✗ limit_ = size;
867 ✗ cleanup_threshold_ = size / 2;
868 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
869 "Quota limit set to %lu / threshold %lu", limit_,
870 cleanup_threshold_);
871 ✗ return true;
872 }
873 7 return SetSharedLimit(size);
874 }
875
876 608 uint64_t PosixQuotaManager::GetSize() {
877
2/2
✓ Branch 0 taken 496 times.
✓ Branch 1 taken 112 times.
 608 if (!spawned_)
878 496 return gauge_;
879 uint64_t gauge, size_pinned;
880
1/2
✓ Branch 1 taken 112 times.
✗ Branch 2 not taken.
 112 GetSharedStatus(&gauge, &size_pinned);
881 112 return gauge;
882 }
883
884
885 14 uint64_t PosixQuotaManager::GetSizePinned() {
886
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 14 times.
 14 if (!spawned_)
887 ✗ return pinned_;
888 uint64_t gauge, size_pinned;
889
1/2
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
 14 GetSharedStatus(&gauge, &size_pinned);
890 14 return size_pinned;
891 }
892
893
894 28 uint64_t PosixQuotaManager::GetCleanupRate(uint64_t period_s) {
895
2/4
✓ Branch 0 taken 28 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 28 times.
 28 if (!spawned_ || (protocol_revision_ < 2))
896 ✗ return 0;
897 uint64_t cleanup_rate;
898
899 int pipe_cleanup_rate[2];
900
1/2
✓ Branch 1 taken 28 times.
✗ Branch 2 not taken.
 28 MakeReturnPipe(pipe_cleanup_rate);
901 28 LruCommand cmd;
902 28 cmd.command_type = kCleanupRate;
903 28 cmd.size = period_s;
904 28 cmd.return_pipe = pipe_cleanup_rate[1];
905
1/2
✓ Branch 1 taken 28 times.
✗ Branch 2 not taken.
 28 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
906
1/2
✓ Branch 1 taken 28 times.
✗ Branch 2 not taken.
 28 ManagedReadHalfPipe(pipe_cleanup_rate[0], &cleanup_rate,
907 sizeof(cleanup_rate));
908
1/2
✓ Branch 1 taken 28 times.
✗ Branch 2 not taken.
 28 CloseReturnPipe(pipe_cleanup_rate);
909
910 28 return cleanup_rate;
911 }
912
913
914 661 bool PosixQuotaManager::InitDatabase(const bool rebuild_database) {
915 661 string sql;
916 sqlite3_stmt *stmt;
917
918
2/4
✓ Branch 1 taken 661 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 661 times.
✗ Branch 5 not taken.
 661 fd_lock_cachedb_ = LockFile(workspace_dir_ + "/lock_cachedb");
919
2/2
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 654 times.
 661 if (fd_lock_cachedb_ < 0) {
920
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 LogCvmfs(kLogQuota, kLogDebug, "failed to create cachedb lock");
921 7 return false;
922 }
923
924 654 bool retry = false;
925
1/2
✓ Branch 1 taken 654 times.
✗ Branch 2 not taken.
 654 const string db_file = cache_dir_ + "/cachedb";
926
2/2
✓ Branch 0 taken 624 times.
✓ Branch 1 taken 30 times.
 654 if (rebuild_database) {
927
1/2
✓ Branch 2 taken 30 times.
✗ Branch 3 not taken.
 30 LogCvmfs(kLogQuota, kLogDebug, "rebuild database, unlinking existing (%s)",
928 db_file.c_str());
929 30 unlink(db_file.c_str());
930
1/2
✓ Branch 1 taken 30 times.
✗ Branch 2 not taken.
 30 unlink((db_file + "-journal").c_str());
931 }
932
933 624 init_recover:
934
1/2
✓ Branch 2 taken 654 times.
✗ Branch 3 not taken.
 654 int err = sqlite3_open(db_file.c_str(), &database_);
935
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 654 times.
 654 if (err != SQLITE_OK) {
936 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not open cache database (%d)", err);
937 ✗ goto init_database_fail;
938 }
939 // TODO(reneme): make this a `QuotaDatabase : public sqlite::Database`
940 sql = "PRAGMA synchronous=0; PRAGMA locking_mode=EXCLUSIVE; "
941 "PRAGMA auto_vacuum=1; "
942 "CREATE TABLE IF NOT EXISTS cache_catalog (sha1 TEXT, size INTEGER, "
943 " acseq INTEGER, path TEXT, type INTEGER, pinned INTEGER, "
944 "CONSTRAINT pk_cache_catalog PRIMARY KEY (sha1)); "
945 "CREATE UNIQUE INDEX IF NOT EXISTS idx_cache_catalog_acseq "
946 " ON cache_catalog (acseq); "
947 "CREATE TEMP TABLE fscache (sha1 TEXT, size INTEGER, actime INTEGER, "
948 "CONSTRAINT pk_fscache PRIMARY KEY (sha1)); "
949 "CREATE INDEX idx_fscache_actime ON fscache (actime); "
950 "CREATE TABLE IF NOT EXISTS properties (key TEXT, value TEXT, "
951
1/2
✓ Branch 1 taken 654 times.
✗ Branch 2 not taken.
 654 " CONSTRAINT pk_properties PRIMARY KEY(key));";
952
1/2
✓ Branch 2 taken 654 times.
✗ Branch 3 not taken.
 654 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
953
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 654 times.
 654 if (err != SQLITE_OK) {
954 ✗ if (!retry) {
955 ✗ retry = true;
956 ✗ sqlite3_close(database_);
957 ✗ unlink(db_file.c_str());
958 ✗ unlink((db_file + "-journal").c_str());
959 ✗ LogCvmfs(kLogQuota, kLogSyslogWarn,
960 "LRU database corrupted, re-building");
961 ✗ goto init_recover;
962 }
963 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
964 sql.c_str());
965 ✗ goto init_database_fail;
966 }
967
968 // If this an old cache catalog,
969 // add and initialize new columns to cache_catalog
970 sql = "ALTER TABLE cache_catalog ADD type INTEGER; "
971
1/2
✓ Branch 1 taken 654 times.
✗ Branch 2 not taken.
 654 "ALTER TABLE cache_catalog ADD pinned INTEGER";
972
1/2
✓ Branch 2 taken 654 times.
✗ Branch 3 not taken.
 654 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
973
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 654 times.
 654 if (err == SQLITE_OK) {
974 ✗ sql = "UPDATE cache_catalog SET type=" + StringifyInt(kFileRegular) + ";";
975 ✗ err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
976 ✗ if (err != SQLITE_OK) {
977 ✗ LogCvmfs(kLogQuota, kLogDebug,
978 "could not init cache database (failed: %s)", sql.c_str());
979 ✗ goto init_database_fail;
980 }
981 }
982
983 // Set pinned back
984
1/2
✓ Branch 1 taken 654 times.
✗ Branch 2 not taken.
 654 sql = "UPDATE cache_catalog SET pinned=0;";
985
1/2
✓ Branch 2 taken 654 times.
✗ Branch 3 not taken.
 654 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
986
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 654 times.
 654 if (err != SQLITE_OK) {
987 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
988 sql.c_str());
989 ✗ goto init_database_fail;
990 }
991
992 // Set schema version
993 sql = "INSERT OR REPLACE INTO properties (key, value) "
994
1/2
✓ Branch 1 taken 654 times.
✗ Branch 2 not taken.
 654 "VALUES ('schema', '1.0')";
995
1/2
✓ Branch 2 taken 654 times.
✗ Branch 3 not taken.
 654 err = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
996
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 654 times.
 654 if (err != SQLITE_OK) {
997 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not init cache database (failed: %s)",
998 sql.c_str());
999 ✗ goto init_database_fail;
1000 }
1001
1002 // If cache catalog is empty, recreate from file system
1003
1/2
✓ Branch 1 taken 654 times.
✗ Branch 2 not taken.
 654 sql = "SELECT count(*) FROM cache_catalog;";
1004
1/2
✓ Branch 2 taken 654 times.
✗ Branch 3 not taken.
 654 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1005
2/4
✓ Branch 1 taken 654 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 654 times.
✗ Branch 4 not taken.
 654 if (sqlite3_step(stmt) == SQLITE_ROW) {
1006
6/8
✓ Branch 1 taken 654 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 28 times.
✓ Branch 4 taken 626 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 28 times.
✓ Branch 7 taken 626 times.
✓ Branch 8 taken 28 times.
 654 if ((sqlite3_column_int64(stmt, 0)) == 0 || rebuild_database) {
1007
1/2
✓ Branch 1 taken 626 times.
✗ Branch 2 not taken.
 626 LogCvmfs(kLogCvmfs, kLogDebug,
1008 "CernVM-FS: building lru cache database...");
1009
3/4
✓ Branch 1 taken 626 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 21 times.
✓ Branch 4 taken 605 times.
 626 if (!RebuildDatabase()) {
1010
1/2
✓ Branch 1 taken 21 times.
✗ Branch 2 not taken.
 21 LogCvmfs(kLogQuota, kLogDebug,
1011 "could not build cache database from file system");
1012
1/2
✓ Branch 1 taken 21 times.
✗ Branch 2 not taken.
 21 sqlite3_finalize(stmt);
1013 21 goto init_database_fail;
1014 }
1015 }
1016
1/2
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
 633 sqlite3_finalize(stmt);
1017 } else {
1018 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not select on cache catalog");
1019 ✗ sqlite3_finalize(stmt);
1020 ✗ goto init_database_fail;
1021 }
1022
1023 // How many bytes do we already have in cache?
1024
1/2
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
 633 sql = "SELECT sum(size) FROM cache_catalog;";
1025
1/2
✓ Branch 2 taken 633 times.
✗ Branch 3 not taken.
 633 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1026
2/4
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 633 times.
✗ Branch 4 not taken.
 633 if (sqlite3_step(stmt) == SQLITE_ROW) {
1027
1/2
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
 633 gauge_ = sqlite3_column_int64(stmt, 0);
1028 } else {
1029 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not determine cache size");
1030 ✗ sqlite3_finalize(stmt);
1031 ✗ goto init_database_fail;
1032 }
1033
1/2
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
 633 sqlite3_finalize(stmt);
1034
1035 // Highest seq-no?
1036
1/2
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
 633 sql = "SELECT coalesce(max(acseq & (~(1<<63))), 0) FROM cache_catalog;";
1037
1/2
✓ Branch 2 taken 633 times.
✗ Branch 3 not taken.
 633 sqlite3_prepare_v2(database_, sql.c_str(), -1, &stmt, NULL);
1038
2/4
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 633 times.
✗ Branch 4 not taken.
 633 if (sqlite3_step(stmt) == SQLITE_ROW) {
1039
1/2
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
 633 seq_ = sqlite3_column_int64(stmt, 0) + 1;
1040 } else {
1041 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not determine highest seq-no");
1042 ✗ sqlite3_finalize(stmt);
1043 ✗ goto init_database_fail;
1044 }
1045
1/2
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
 633 sqlite3_finalize(stmt);
1046
1047 // Prepare touch, new, remove statements
1048
1/2
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
 633 sqlite3_prepare_v2(database_,
1049 "UPDATE cache_catalog SET acseq=:seq | (acseq&(1<<63)) "
1050 "WHERE sha1=:sha1;",
1051 -1, &stmt_touch_, NULL);
1052
1/2
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
 633 sqlite3_prepare_v2(database_,
1053 "UPDATE cache_catalog SET pinned=0 "
1054 "WHERE sha1=:sha1;",
1055 -1, &stmt_unpin_, NULL);
1056
1/2
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
 633 sqlite3_prepare_v2(database_,
1057 "UPDATE cache_catalog SET pinned=2 "
1058 "WHERE sha1=:sha1;",
1059 -1, &stmt_block_, NULL);
1060
1/2
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
 633 sqlite3_prepare_v2(database_,
1061 "UPDATE cache_catalog SET pinned=1 "
1062 "WHERE pinned=2;",
1063 -1, &stmt_unblock_, NULL);
1064
1/2
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
 633 sqlite3_prepare_v2(database_,
1065 "INSERT OR REPLACE INTO cache_catalog "
1066 "(sha1, size, acseq, path, type, pinned) "
1067 "VALUES (:sha1, :s, :seq, :p, :t, :pin);",
1068 -1, &stmt_new_, NULL);
1069
1/2
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
 633 sqlite3_prepare_v2(database_,
1070 "SELECT size, pinned FROM cache_catalog WHERE sha1=:sha1;",
1071 -1, &stmt_size_, NULL);
1072
1/2
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
 633 sqlite3_prepare_v2(database_, "DELETE FROM cache_catalog WHERE sha1=:sha1;",
1073 -1, &stmt_rm_, NULL);
1074
1/2
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
 633 sqlite3_prepare_v2(database_,
1075 "DELETE FROM cache_catalog WHERE acseq<=:a AND pinned<>2;",
1076 -1, &stmt_rm_batch_, NULL);
1077
1/2
✓ Branch 2 taken 633 times.
✗ Branch 3 not taken.
 633 sqlite3_prepare_v2(database_,
1078
1/2
✓ Branch 2 taken 633 times.
✗ Branch 3 not taken.
 1266 (std::string("SELECT sha1, size, acseq FROM cache_catalog "
1079 "WHERE pinned<>2 AND acseq>=:a "
1080 "ORDER BY acseq ASC "
1081 "LIMIT ")
1082
3/6
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 633 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 633 times.
✗ Branch 8 not taken.
 2532 + StringifyInt(kEvictBatchSize) + ";")
1083 .c_str(),
1084 -1, &stmt_lru_, NULL);
1085
1/2
✓ Branch 2 taken 633 times.
✗ Branch 3 not taken.
 633 sqlite3_prepare_v2(database_,
1086 ("SELECT path FROM cache_catalog WHERE type="
1087
3/6
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 633 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 633 times.
✗ Branch 8 not taken.
 1266 + StringifyInt(kFileRegular) + ";")
1088 .c_str(),
1089 -1, &stmt_list_, NULL);
1090
1/2
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
 633 sqlite3_prepare_v2(database_,
1091 "SELECT path FROM cache_catalog WHERE pinned<>0;", -1,
1092 &stmt_list_pinned_, NULL);
1093
1/2
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
 633 sqlite3_prepare_v2(database_,
1094 "SELECT path FROM cache_catalog WHERE acseq < 0;", -1,
1095 &stmt_list_volatile_, NULL);
1096
1/2
✓ Branch 2 taken 633 times.
✗ Branch 3 not taken.
 633 sqlite3_prepare_v2(database_,
1097 ("SELECT path FROM cache_catalog WHERE type="
1098
3/6
✓ Branch 1 taken 633 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 633 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 633 times.
✗ Branch 8 not taken.
 1266 + StringifyInt(kFileCatalog) + ";")
1099 .c_str(),
1100 -1, &stmt_list_catalogs_, NULL);
1101 633 return true;
1102
1103 21 init_database_fail:
1104
1/2
✓ Branch 1 taken 21 times.
✗ Branch 2 not taken.
 21 sqlite3_close(database_);
1105 21 database_ = NULL;
1106
1/2
✓ Branch 1 taken 21 times.
✗ Branch 2 not taken.
 21 UnlockFile(fd_lock_cachedb_);
1107 21 return false;
1108 661 }
1109
1110
1111 /**
1112 * Inserts a new file into cache catalog. This file gets a new,
1113 * highest sequence number. Does cache cleanup if necessary.
1114 */
1115 700138 void PosixQuotaManager::Insert(const shash::Any &any_hash,
1116 const uint64_t size,
1117 const string &description) {
1118 700138 DoInsert(any_hash, size, description, kInsert);
1119 700138 }
1120
1121
1122 /**
1123 * Inserts a new file into cache catalog. This file is marked as volatile
1124 * and gets a new highest sequence number with the first bit set. Cache cleanup
1125 * treats these files with priority.
1126 */
1127 28 void PosixQuotaManager::InsertVolatile(const shash::Any &any_hash,
1128 const uint64_t size,
1129 const string &description) {
1130 28 DoInsert(any_hash, size, description, kInsertVolatile);
1131 28 }
1132
1133
1134 /**
1135 * Lists all path names from the cache db.
1136 */
1137 147 vector<string> PosixQuotaManager::List() { return DoList(kList); }
1138
1139
1140 /**
1141 * Lists all pinned files from the cache db.
1142 */
1143 56 vector<string> PosixQuotaManager::ListPinned() { return DoList(kListPinned); }
1144
1145
1146 /**
1147 * Lists all sqlite catalog files from the cache db.
1148 */
1149 21 vector<string> PosixQuotaManager::ListCatalogs() {
1150 21 return DoList(kListCatalogs);
1151 }
1152
1153
1154 /**
1155 * Lists only files flagged as volatile (priority removal)
1156 */
1157 21 vector<string> PosixQuotaManager::ListVolatile() {
1158 21 return DoList(kListVolatile);
1159 }
1160
1161
1162 /**
1163 * Entry point for the shared cache manager process
1164 */
1165 ✗ int PosixQuotaManager::MainCacheManager(int argc, char **argv) {
1166 ✗ LogCvmfs(kLogQuota, kLogDebug, "starting quota manager");
1167 int retval;
1168
1169 ✗ PosixQuotaManager shared_manager(0, 0, "");
1170 ✗ shared_manager.shared_ = true;
1171 ✗ shared_manager.spawned_ = true;
1172 ✗ shared_manager.pinned_ = 0;
1173
1174 // Process command line arguments
1175 ✗ ParseDirectories(string(argv[2]),
1176 &shared_manager.cache_dir_,
1177 &shared_manager.workspace_dir_);
1178 ✗ const int pipe_boot = String2Int64(argv[3]);
1179 ✗ const int pipe_handshake = String2Int64(argv[4]);
1180 ✗ shared_manager.limit_ = String2Int64(argv[5]);
1181 ✗ shared_manager.cleanup_threshold_ = String2Int64(argv[6]);
1182 ✗ const int foreground = String2Int64(argv[7]);
1183 ✗ const int syslog_level = String2Int64(argv[8]);
1184 ✗ const int syslog_facility = String2Int64(argv[9]);
1185 ✗ vector<string> logfiles = SplitString(argv[10], ':');
1186
1187 ✗ SetLogSyslogLevel(syslog_level);
1188 ✗ SetLogSyslogFacility(syslog_facility);
1189 ✗ if ((logfiles.size() > 0) && (logfiles[0] != ""))
1190 ✗ SetLogDebugFile(logfiles[0] + ".cachemgr");
1191 ✗ if (logfiles.size() > 1)
1192 ✗ SetLogMicroSyslog(logfiles[1]);
1193
1194 ✗ if (!foreground)
1195 ✗ Daemonize();
1196
1197 ✗ const UniquePtr<Watchdog> watchdog(Watchdog::Create(NULL));
1198 ✗ assert(watchdog.IsValid());
1199 ✗ watchdog->Spawn("./stacktrace.cachemgr");
1200
1201 // Initialize pipe, open non-blocking as cvmfs is not yet connected
1202 ✗ const int fd_lockfile_fifo = LockFile(shared_manager.workspace_dir_
1203 ✗ + "/lock_cachemgr.fifo");
1204 ✗ if (fd_lockfile_fifo < 0) {
1205 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1206 "could not open lock file "
1207 "%s (%d)",
1208 ✗ (shared_manager.workspace_dir_ + "/lock_cachemgr.fifo").c_str(),
1209 ✗ errno);
1210 ✗ return 1;
1211 }
1212 ✗ const string crash_guard = shared_manager.cache_dir_ + "/cachemgr.running";
1213 ✗ const bool rebuild = FileExists(crash_guard);
1214 ✗ retval = open(crash_guard.c_str(), O_RDONLY | O_CREAT, 0600);
1215 ✗ if (retval < 0) {
1216 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1217 "failed to create shared cache manager crash guard");
1218 ✗ UnlockFile(fd_lockfile_fifo);
1219 ✗ return 1;
1220 }
1221 ✗ close(retval);
1222
1223 // Redirect SQlite temp directory to cache (global variable)
1224 ✗ const string tmp_dir = shared_manager.workspace_dir_;
1225 ✗ sqlite3_temp_directory = static_cast<char *>(
1226 ✗ sqlite3_malloc(tmp_dir.length() + 1));
1227 ✗ snprintf(sqlite3_temp_directory, tmp_dir.length() + 1, "%s", tmp_dir.c_str());
1228
1229 // Cleanup leftover named pipes
1230 ✗ shared_manager.CleanupPipes();
1231
1232 ✗ if (!shared_manager.InitDatabase(rebuild)) {
1233 ✗ UnlockFile(fd_lockfile_fifo);
1234 ✗ return 1;
1235 }
1236 ✗ shared_manager.CheckFreeSpace();
1237
1238 // Save protocol revision to file. If the file is not found, it indicates
1239 // to the client that the cache manager is from times before the protocol
1240 // was versioned.
1241 const string protocol_revision_path = shared_manager.workspace_dir_
1242 ✗ + "/cachemgr.protocol";
1243 ✗ retval = open(protocol_revision_path.c_str(), O_WRONLY | O_CREAT, 0600);
1244 ✗ if (retval < 0) {
1245 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1246 ✗ "failed to open protocol revision file (%d)", errno);
1247 ✗ UnlockFile(fd_lockfile_fifo);
1248 ✗ return 1;
1249 }
1250 ✗ const string revision = StringifyInt(kProtocolRevision);
1251 ✗ const int written = write(retval, revision.data(), revision.length());
1252 ✗ close(retval);
1253 ✗ if ((written < 0) || static_cast<unsigned>(written) != revision.length()) {
1254 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogErr,
1255 ✗ "failed to write protocol revision (%d)", errno);
1256 ✗ UnlockFile(fd_lockfile_fifo);
1257 ✗ return 1;
1258 }
1259
1260 ✗ const string fifo_path = shared_manager.workspace_dir_ + "/cachemgr";
1261 ✗ shared_manager.pipe_lru_[0] = open(fifo_path.c_str(), O_RDONLY | O_NONBLOCK);
1262 ✗ if (shared_manager.pipe_lru_[0] < 0) {
1263 ✗ LogCvmfs(kLogQuota, kLogDebug, "failed to listen on FIFO %s (%d)",
1264 ✗ fifo_path.c_str(), errno);
1265 ✗ UnlockFile(fd_lockfile_fifo);
1266 ✗ return 1;
1267 }
1268 ✗ Nonblock2Block(shared_manager.pipe_lru_[0]);
1269 ✗ LogCvmfs(kLogQuota, kLogDebug, "shared cache manager listening");
1270
1271 ✗ char buf = 'C';
1272 ✗ WritePipe(pipe_boot, &buf, 1);
1273 ✗ close(pipe_boot);
1274
1275 ✗ ReadPipe(pipe_handshake, &buf, 1);
1276 ✗ close(pipe_handshake);
1277 ✗ LogCvmfs(kLogQuota, kLogDebug, "shared cache manager handshake done");
1278
1279 // Ensure that broken pipes from clients do not kill the cache manager
1280 ✗ signal(SIGPIPE, SIG_IGN);
1281 // Don't let Ctrl-C ungracefully kill interactive session
1282 ✗ signal(SIGINT, SIG_IGN);
1283
1284 ✗ shared_manager.MainCommandServer(&shared_manager);
1285 ✗ unlink(fifo_path.c_str());
1286 ✗ unlink(protocol_revision_path.c_str());
1287 ✗ shared_manager.CloseDatabase();
1288 ✗ unlink(crash_guard.c_str());
1289 ✗ UnlockFile(fd_lockfile_fifo);
1290
1291 ✗ if (sqlite3_temp_directory) {
1292 ✗ sqlite3_free(sqlite3_temp_directory);
1293 ✗ sqlite3_temp_directory = NULL;
1294 }
1295
1296 ✗ return 0;
1297 }
1298
1299
1300 217 void *PosixQuotaManager::MainCommandServer(void *data) {
1301 217 PosixQuotaManager *quota_mgr = static_cast<PosixQuotaManager *>(data);
1302
1303
1/2
✓ Branch 1 taken 217 times.
✗ Branch 2 not taken.
 217 LogCvmfs(kLogQuota, kLogDebug, "starting quota manager");
1304
1/2
✓ Branch 1 taken 217 times.
✗ Branch 2 not taken.
 217 sqlite3_soft_heap_limit(quota_mgr->kSqliteMemPerThread);
1305
1306
2/2
✓ Branch 1 taken 6944 times.
✓ Branch 2 taken 217 times.
 7161 LruCommand command_buffer[kCommandBufferSize];
1307 char description_buffer[kCommandBufferSize * kMaxDescription];
1308 217 unsigned num_commands = 0;
1309
1310
1/2
✓ Branch 1 taken 1051127 times.
✗ Branch 2 not taken.
 1051127 while (read(quota_mgr->pipe_lru_[0], &command_buffer[num_commands],
1311 sizeof(command_buffer[0]))
1312
2/2
✓ Branch 0 taken 1050910 times.
✓ Branch 1 taken 217 times.
 1051127 == sizeof(command_buffer[0])) {
1313 1050910 const CommandType command_type = command_buffer[num_commands].command_type;
1314
1/2
✓ Branch 1 taken 1050910 times.
✗ Branch 2 not taken.
 1050910 LogCvmfs(kLogQuota, kLogDebug, "received command %d", command_type);
1315 1050910 const uint64_t size = command_buffer[num_commands].GetSize();
1316
1317 // Inserts and pins come with a description (usually a path)
1318
4/4
✓ Branch 0 taken 350791 times.
✓ Branch 1 taken 700119 times.
✓ Branch 2 taken 350763 times.
✓ Branch 3 taken 28 times.
 1050910 if ((command_type == kInsert) || (command_type == kInsertVolatile)
1319
4/4
✓ Branch 0 taken 350749 times.
✓ Branch 1 taken 14 times.
✓ Branch 2 taken 77 times.
✓ Branch 3 taken 350672 times.
 350763 || (command_type == kPin) || (command_type == kPinRegular)) {
1320 700238 const int desc_length = command_buffer[num_commands].desc_length;
1321 700238 ReadPipe(quota_mgr->pipe_lru_[0],
1322
1/2
✓ Branch 1 taken 700238 times.
✗ Branch 2 not taken.
 700238 &description_buffer[kMaxDescription * num_commands],
1323 desc_length);
1324 }
1325
1326 // The protocol revision is returned immediately
1327
2/2
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 1050903 times.
 1050910 if (command_type == kGetProtocolRevision) {
1328
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 const int return_pipe = quota_mgr->BindReturnPipe(
1329 command_buffer[num_commands].return_pipe);
1330
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
 7 if (return_pipe < 0)
1331 ✗ continue;
1332
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 WritePipe(return_pipe, &quota_mgr->kProtocolRevision,
1333 sizeof(quota_mgr->kProtocolRevision));
1334
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 quota_mgr->UnbindReturnPipe(return_pipe);
1335 7 continue;
1336 7 }
1337
1338 // Register a new mountpoint
1339
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1050903 times.
 1050903 if (command_type == kRegisterMountpoint) {
1340 ✗ size_t mp_size = 0;
1341 ✗ ReadPipe(quota_mgr->pipe_lru_[0], &mp_size, sizeof(size_t));
1342 ✗ std::string mountpoint(mp_size, '\0');
1343 ✗ ReadPipe(quota_mgr->pipe_lru_[0], (void *)mountpoint.data(), mp_size);
1344 ✗ quota_mgr->mountpoints_.push_back(mountpoint);
1345 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
1346 "Mountpoint %s registered in the group", mountpoint.c_str());
1347 ✗ continue;
1348 }
1349
1350 // Set Cleanup Policy
1351
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1050903 times.
 1050903 if (command_type == kSetCleanupPolicy) {
1352 ✗ bool policy = false;
1353 ✗ ReadPipe(quota_mgr->pipe_lru_[0], &policy, sizeof(bool));
1354 ✗ quota_mgr->cleanup_unused_first_ = policy;
1355 ✗ continue;
1356 }
1357 // Mountpoints are returned immediately
1358
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1050903 times.
 1050903 if (command_type == kGetMountpoints) {
1359 ✗ const int return_pipe = quota_mgr->BindReturnPipe(
1360 command_buffer[num_commands].return_pipe);
1361 ✗ if (return_pipe < 0)
1362 ✗ continue;
1363
1364 ✗ std::string mps;
1365 ✗ for (auto it = quota_mgr->mountpoints_.begin();
1366 ✗ it != quota_mgr->mountpoints_.end();
1367 ✗ ++it) {
1368 ✗ mps += *it + "\n";
1369 }
1370 ✗ size_t mp_size = mps.size();
1371 ✗ WritePipe(return_pipe, &mp_size, sizeof(size_t));
1372 ✗ WritePipe(return_pipe, mps.c_str(), mp_size);
1373 ✗ quota_mgr->UnbindReturnPipe(return_pipe);
1374 ✗ continue;
1375 }
1376
1377 // Group hashes are returned immediately
1378
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1050903 times.
 1050903 if (command_type == kGetGroupHashes) {
1379 ✗ const int return_pipe = quota_mgr->BindReturnPipe(
1380 command_buffer[num_commands].return_pipe);
1381 ✗ if (return_pipe < 0)
1382 ✗ continue;
1383
1384 ✗ std::vector<shash::Any> gh = quota_mgr->CollectAllOpenHashes();
1385 ✗ std::string result;
1386 ✗ for (auto it = gh.begin(); it != gh.end(); ++it) {
1387 ✗ result += (*it).ToString() + "\n";
1388 }
1389 ✗ size_t result_size = result.size();
1390 ✗ WritePipe(return_pipe, &result_size, sizeof(size_t));
1391 ✗ WritePipe(return_pipe, result.c_str(), result_size);
1392 ✗ quota_mgr->UnbindReturnPipe(return_pipe);
1393 ✗ continue;
1394 }
1395
1396 // The cleanup rate is returned immediately
1397
2/2
✓ Branch 0 taken 28 times.
✓ Branch 1 taken 1050875 times.
 1050903 if (command_type == kCleanupRate) {
1398
1/2
✓ Branch 1 taken 28 times.
✗ Branch 2 not taken.
 28 const int return_pipe = quota_mgr->BindReturnPipe(
1399 command_buffer[num_commands].return_pipe);
1400
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 28 times.
 28 if (return_pipe < 0)
1401 ✗ continue;
1402 const uint64_t
1403 28 period_s = size; // use the size field to transmit the period
1404
1/2
✓ Branch 1 taken 28 times.
✗ Branch 2 not taken.
 28 uint64_t rate = quota_mgr->cleanup_recorder_.GetNoTicks(period_s);
1405
1/2
✓ Branch 1 taken 28 times.
✗ Branch 2 not taken.
 28 WritePipe(return_pipe, &rate, sizeof(rate));
1406
1/2
✓ Branch 1 taken 28 times.
✗ Branch 2 not taken.
 28 quota_mgr->UnbindReturnPipe(return_pipe);
1407 28 continue;
1408 28 }
1409
1410
2/2
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 1050868 times.
 1050875 if (command_type == kSetLimit) {
1411
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 const int return_pipe = quota_mgr->BindReturnPipe(
1412 command_buffer[num_commands].return_pipe);
1413
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
 7 if (return_pipe < 0)
1414 ✗ continue;
1415 7 quota_mgr->limit_ = size; // use the size field to transmit the size
1416 7 quota_mgr->cleanup_threshold_ = size / 2;
1417
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 LogCvmfs(kLogQuota, kLogDebug | kLogSyslog,
1418 "Quota limit set to %lu / threshold %lu", quota_mgr->limit_,
1419 quota_mgr->cleanup_threshold_);
1420 7 bool ret = true;
1421
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 WritePipe(return_pipe, &ret, sizeof(ret));
1422
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 quota_mgr->UnbindReturnPipe(return_pipe);
1423 7 continue;
1424 7 }
1425
1426 // Reservations are handled immediately and "out of band"
1427
2/2
✓ Branch 0 taken 98 times.
✓ Branch 1 taken 1050770 times.
 1050868 if (command_type == kReserve) {
1428 98 bool success = true;
1429
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 const int return_pipe = quota_mgr->BindReturnPipe(
1430 command_buffer[num_commands].return_pipe);
1431
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 98 times.
 98 if (return_pipe < 0)
1432 ✗ continue;
1433
1434
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1435
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 const string hash_str(hash.ToString());
1436
1/2
✓ Branch 2 taken 98 times.
✗ Branch 3 not taken.
 98 LogCvmfs(kLogQuota, kLogDebug, "reserve %lu bytes for %s", size,
1437 hash_str.c_str());
1438
1439
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 if (quota_mgr->pinned_chunks_.find(hash)
1440
2/2
✓ Branch 2 taken 84 times.
✓ Branch 3 taken 14 times.
 196 == quota_mgr->pinned_chunks_.end()) {
1441
2/2
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 77 times.
 84 if ((quota_mgr->pinned_ + size) > quota_mgr->cleanup_threshold_) {
1442
1/2
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
 7 LogCvmfs(kLogQuota, kLogDebug,
1443 "failed to insert %s (pinned), no space", hash_str.c_str());
1444 7 success = false;
1445 } else {
1446
1/2
✓ Branch 1 taken 77 times.
✗ Branch 2 not taken.
 77 quota_mgr->pinned_chunks_[hash] = size;
1447 77 quota_mgr->pinned_ += size;
1448
1/2
✓ Branch 1 taken 77 times.
✗ Branch 2 not taken.
 77 quota_mgr->CheckHighPinWatermark();
1449 }
1450 }
1451
1452
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 WritePipe(return_pipe, &success, sizeof(success));
1453
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 quota_mgr->UnbindReturnPipe(return_pipe);
1454 98 continue;
1455 98 }
1456
1457 // Back channels are also handled out of band
1458
2/2
✓ Branch 0 taken 28 times.
✓ Branch 1 taken 1050742 times.
 1050770 if (command_type == kRegisterBackChannel) {
1459
1/2
✓ Branch 1 taken 28 times.
✗ Branch 2 not taken.
 28 const int return_pipe = quota_mgr->BindReturnPipe(
1460 command_buffer[num_commands].return_pipe);
1461
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 28 times.
 28 if (return_pipe < 0)
1462 ✗ continue;
1463
1464
1/2
✓ Branch 1 taken 28 times.
✗ Branch 2 not taken.
 28 quota_mgr->UnlinkReturnPipe(command_buffer[num_commands].return_pipe);
1465
1/2
✓ Branch 1 taken 28 times.
✗ Branch 2 not taken.
 28 Block2Nonblock(return_pipe); // back channels are opportunistic
1466
1/2
✓ Branch 1 taken 28 times.
✗ Branch 2 not taken.
 28 shash::Md5 hash;
1467 28 memcpy(hash.digest, command_buffer[num_commands].digest,
1468 28 shash::kDigestSizes[shash::kMd5]);
1469
1470 28 quota_mgr->LockBackChannels();
1471 const map<shash::Md5, int>::const_iterator
1472
1/2
✓ Branch 1 taken 28 times.
✗ Branch 2 not taken.
 28 iter = quota_mgr->back_channels_.find(hash);
1473
1/2
✗ Branch 3 not taken.
✓ Branch 4 taken 28 times.
 28 if (iter != quota_mgr->back_channels_.end()) {
1474 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
1475 ✗ "closing left-over back channel %s", hash.ToString().c_str());
1476 ✗ close(iter->second);
1477 }
1478
1/2
✓ Branch 1 taken 28 times.
✗ Branch 2 not taken.
 28 quota_mgr->back_channels_[hash] = return_pipe;
1479 28 quota_mgr->UnlockBackChannels();
1480
1481 28 char success = 'S';
1482
1/2
✓ Branch 1 taken 28 times.
✗ Branch 2 not taken.
 28 WritePipe(return_pipe, &success, sizeof(success));
1483
1/2
✓ Branch 2 taken 28 times.
✗ Branch 3 not taken.
 28 LogCvmfs(kLogQuota, kLogDebug, "register back channel %s on fd %d",
1484
1/2
✓ Branch 1 taken 28 times.
✗ Branch 2 not taken.
 56 hash.ToString().c_str(), return_pipe);
1485
1486 28 continue;
1487 28 }
1488
1489
2/2
✓ Branch 0 taken 14 times.
✓ Branch 1 taken 1050728 times.
 1050742 if (command_type == kUnregisterBackChannel) {
1490
1/2
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
 14 shash::Md5 hash;
1491 14 memcpy(hash.digest, command_buffer[num_commands].digest,
1492 14 shash::kDigestSizes[shash::kMd5]);
1493
1494 14 quota_mgr->LockBackChannels();
1495 const map<shash::Md5, int>::iterator iter = quota_mgr->back_channels_
1496
1/2
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
 14 .find(hash);
1497
1/2
✓ Branch 2 taken 14 times.
✗ Branch 3 not taken.
 14 if (iter != quota_mgr->back_channels_.end()) {
1498
1/2
✓ Branch 2 taken 14 times.
✗ Branch 3 not taken.
 14 LogCvmfs(kLogQuota, kLogDebug, "closing back channel %s",
1499
1/2
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
 28 hash.ToString().c_str());
1500
1/2
✓ Branch 2 taken 14 times.
✗ Branch 3 not taken.
 14 close(iter->second);
1501
1/2
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
 14 quota_mgr->back_channels_.erase(iter);
1502 } else {
1503 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogWarn,
1504 ✗ "did not find back channel %s", hash.ToString().c_str());
1505 }
1506 14 quota_mgr->UnlockBackChannels();
1507
1508 14 continue;
1509 14 }
1510
1511 // Unpinnings are also handled immediately with respect to the pinned gauge
1512
2/2
✓ Branch 0 taken 14 times.
✓ Branch 1 taken 1050714 times.
 1050728 if (command_type == kUnpin) {
1513
1/2
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
 14 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1514
1/2
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
 14 const string hash_str(hash.ToString());
1515
1516 const map<shash::Any, uint64_t>::iterator iter = quota_mgr->pinned_chunks_
1517
1/2
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
 14 .find(hash);
1518
1/2
✓ Branch 2 taken 14 times.
✗ Branch 3 not taken.
 14 if (iter != quota_mgr->pinned_chunks_.end()) {
1519 14 quota_mgr->pinned_ -= iter->second;
1520
1/2
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
 14 quota_mgr->pinned_chunks_.erase(iter);
1521 // It can happen that files get pinned that were removed from the cache
1522 // (see cache.cc). We fix this at this point, where we remove such
1523 // entries from the cache database.
1524
2/4
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 14 times.
✗ Branch 5 not taken.
 28 if (!FileExists(quota_mgr->cache_dir_ + "/"
1525
4/6
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 14 times.
✗ Branch 5 not taken.
✓ Branch 9 taken 7 times.
✓ Branch 10 taken 7 times.
 42 + hash.MakePathWithoutSuffix())) {
1526
1/2
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
 7 LogCvmfs(kLogQuota, kLogDebug,
1527 "remove orphaned pinned hash %s from cache database",
1528 hash_str.c_str());
1529
1/2
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
 7 sqlite3_bind_text(quota_mgr->stmt_size_, 1, &hash_str[0],
1530 7 hash_str.length(), SQLITE_STATIC);
1531 int retval;
1532
2/4
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 7 times.
✗ Branch 4 not taken.
 7 if ((retval = sqlite3_step(quota_mgr->stmt_size_)) == SQLITE_ROW) {
1533
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 const uint64_t size = sqlite3_column_int64(quota_mgr->stmt_size_,
1534 7 0);
1535
1/2
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
 7 sqlite3_bind_text(quota_mgr->stmt_rm_, 1, &(hash_str[0]),
1536 7 hash_str.length(), SQLITE_STATIC);
1537
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 retval = sqlite3_step(quota_mgr->stmt_rm_);
1538
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 7 if ((retval == SQLITE_DONE) || (retval == SQLITE_OK)) {
1539 7 quota_mgr->gauge_ -= size;
1540 } else {
1541 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1542 "failed to delete %s (%d)", hash_str.c_str(), retval);
1543 }
1544
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 sqlite3_reset(quota_mgr->stmt_rm_);
1545 }
1546
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 sqlite3_reset(quota_mgr->stmt_size_);
1547 }
1548 } else {
1549 ✗ LogCvmfs(kLogQuota, kLogDebug, "this chunk was not pinned");
1550 }
1551 14 }
1552
1553 // Immediate commands trigger flushing of the buffer
1554 1050728 const bool immediate_command = (command_type == kCleanup)
1555
2/2
✓ Branch 0 taken 1050518 times.
✓ Branch 1 taken 147 times.
 1050665 || (command_type == kList)
1556
2/2
✓ Branch 0 taken 1050462 times.
✓ Branch 1 taken 56 times.
 1050518 || (command_type == kListPinned)
1557
2/2
✓ Branch 0 taken 1050441 times.
✓ Branch 1 taken 21 times.
 1050462 || (command_type == kListCatalogs)
1558
2/2
✓ Branch 0 taken 1050420 times.
✓ Branch 1 taken 21 times.
 1050441 || (command_type == kListVolatile)
1559
2/2
✓ Branch 0 taken 1050399 times.
✓ Branch 1 taken 21 times.
 1050420 || (command_type == kRemove)
1560
2/2
✓ Branch 0 taken 1050273 times.
✓ Branch 1 taken 126 times.
 1050399 || (command_type == kStatus)
1561
1/2
✓ Branch 0 taken 1050273 times.
✗ Branch 1 not taken.
 1050273 || (command_type == kLimits)
1562
3/4
✓ Branch 0 taken 1050665 times.
✓ Branch 1 taken 63 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1050273 times.
 2101393 || (command_type == kPid);
1563
2/2
✓ Branch 0 taken 1050273 times.
✓ Branch 1 taken 455 times.
 1050728 if (!immediate_command)
1564 1050273 num_commands++;
1565
1566
4/4
✓ Branch 0 taken 1017919 times.
✓ Branch 1 taken 32809 times.
✓ Branch 2 taken 455 times.
✓ Branch 3 taken 1017464 times.
 1050728 if ((num_commands == kCommandBufferSize) || immediate_command) {
1567
1/2
✓ Branch 1 taken 33264 times.
✗ Branch 2 not taken.
 33264 quota_mgr->ProcessCommandBunch(num_commands, command_buffer,
1568 description_buffer);
1569
2/2
✓ Branch 0 taken 32809 times.
✓ Branch 1 taken 455 times.
 33264 if (!immediate_command)
1570 32809 num_commands = 0;
1571 }
1572
1573
2/2
✓ Branch 0 taken 455 times.
✓ Branch 1 taken 1050273 times.
 1050728 if (immediate_command) {
1574 // Process cleanup, listings
1575
1/2
✓ Branch 1 taken 455 times.
✗ Branch 2 not taken.
 455 const int return_pipe = quota_mgr->BindReturnPipe(
1576 command_buffer[num_commands].return_pipe);
1577
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 455 times.
 455 if (return_pipe < 0) {
1578 ✗ num_commands = 0;
1579 ✗ continue;
1580 }
1581
1582 int retval;
1583 455 sqlite3_stmt *this_stmt_list = NULL;
1584
7/10
✓ Branch 0 taken 21 times.
✓ Branch 1 taken 63 times.
✓ Branch 2 taken 147 times.
✓ Branch 3 taken 56 times.
✓ Branch 4 taken 21 times.
✓ Branch 5 taken 21 times.
✓ Branch 6 taken 126 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
 455 switch (command_type) {
1585 21 case kRemove: {
1586
1/2
✓ Branch 1 taken 21 times.
✗ Branch 2 not taken.
 21 const shash::Any hash = command_buffer[num_commands].RetrieveHash();
1587
1/2
✓ Branch 1 taken 21 times.
✗ Branch 2 not taken.
 21 const string hash_str = hash.ToString();
1588
1/2
✓ Branch 2 taken 21 times.
✗ Branch 3 not taken.
 21 LogCvmfs(kLogQuota, kLogDebug, "manually removing %s",
1589 hash_str.c_str());
1590 21 bool success = false;
1591
1592
1/2
✓ Branch 2 taken 21 times.
✗ Branch 3 not taken.
 21 sqlite3_bind_text(quota_mgr->stmt_size_, 1, &hash_str[0],
1593 21 hash_str.length(), SQLITE_STATIC);
1594 int retval;
1595
3/4
✓ Branch 1 taken 21 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 14 times.
✓ Branch 4 taken 7 times.
 21 if ((retval = sqlite3_step(quota_mgr->stmt_size_)) == SQLITE_ROW) {
1596
1/2
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
 14 const uint64_t size = sqlite3_column_int64(quota_mgr->stmt_size_,
1597 14 0);
1598
1/2
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
 14 const uint64_t is_pinned = sqlite3_column_int64(
1599 14 quota_mgr->stmt_size_, 1);
1600
1601
1/2
✓ Branch 2 taken 14 times.
✗ Branch 3 not taken.
 14 sqlite3_bind_text(quota_mgr->stmt_rm_, 1, &(hash_str[0]),
1602 14 hash_str.length(), SQLITE_STATIC);
1603
1/2
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
 14 retval = sqlite3_step(quota_mgr->stmt_rm_);
1604
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 14 if ((retval == SQLITE_DONE) || (retval == SQLITE_OK)) {
1605 14 success = true;
1606 14 quota_mgr->gauge_ -= size;
1607
2/2
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 7 times.
 14 if (is_pinned) {
1608
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 quota_mgr->pinned_chunks_.erase(hash);
1609 7 quota_mgr->pinned_ -= size;
1610 }
1611 } else {
1612 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
1613 "failed to delete %s (%d)", hash_str.c_str(), retval);
1614 }
1615
1/2
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
 14 sqlite3_reset(quota_mgr->stmt_rm_);
1616 } else {
1617 // File does not exist
1618 7 success = true;
1619 }
1620
1/2
✓ Branch 1 taken 21 times.
✗ Branch 2 not taken.
 21 sqlite3_reset(quota_mgr->stmt_size_);
1621
1622
1/2
✓ Branch 1 taken 21 times.
✗ Branch 2 not taken.
 21 WritePipe(return_pipe, &success, sizeof(success));
1623 21 break;
1624 21 }
1625 63 case kCleanup:
1626
1/2
✓ Branch 1 taken 63 times.
✗ Branch 2 not taken.
 63 retval = quota_mgr->DoCleanup(size);
1627
1/2
✓ Branch 1 taken 63 times.
✗ Branch 2 not taken.
 63 WritePipe(return_pipe, &retval, sizeof(retval));
1628 63 break;
1629 147 case kList:
1630
1/2
✓ Branch 0 taken 147 times.
✗ Branch 1 not taken.
 147 if (!this_stmt_list)
1631 147 this_stmt_list = quota_mgr->stmt_list_;
1632 case kListPinned:
1633
2/2
✓ Branch 0 taken 56 times.
✓ Branch 1 taken 147 times.
 203 if (!this_stmt_list)
1634 56 this_stmt_list = quota_mgr->stmt_list_pinned_;
1635 case kListCatalogs:
1636
2/2
✓ Branch 0 taken 21 times.
✓ Branch 1 taken 203 times.
 224 if (!this_stmt_list)
1637 21 this_stmt_list = quota_mgr->stmt_list_catalogs_;
1638 case kListVolatile:
1639
2/2
✓ Branch 0 taken 21 times.
✓ Branch 1 taken 224 times.
 245 if (!this_stmt_list)
1640 21 this_stmt_list = quota_mgr->stmt_list_volatile_;
1641
1642 // Pipe back the list, one by one
1643 int length;
1644
3/4
✓ Branch 1 taken 700518 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 700273 times.
✓ Branch 4 taken 245 times.
 700518 while (sqlite3_step(this_stmt_list) == SQLITE_ROW) {
1645
1/2
✓ Branch 2 taken 700273 times.
✗ Branch 3 not taken.
 700273 string path = "(NULL)";
1646
2/4
✓ Branch 1 taken 700273 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 700273 times.
✗ Branch 4 not taken.
 700273 if (sqlite3_column_type(this_stmt_list, 0) != SQLITE_NULL) {
1647 1400546 path = string(reinterpret_cast<const char *>(
1648
2/4
✓ Branch 1 taken 700273 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 700273 times.
✗ Branch 5 not taken.
 700273 sqlite3_column_text(this_stmt_list, 0)));
1649 }
1650 700273 length = path.length();
1651
1/2
✓ Branch 1 taken 700273 times.
✗ Branch 2 not taken.
 700273 WritePipe(return_pipe, &length, sizeof(length));
1652
1/2
✓ Branch 0 taken 700273 times.
✗ Branch 1 not taken.
 700273 if (length > 0)
1653
2/4
✓ Branch 1 taken 700273 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 700273 times.
✗ Branch 5 not taken.
 700273 WritePipe(return_pipe, &path[0], length);
1654 700273 }
1655 245 length = -1;
1656
1/2
✓ Branch 1 taken 245 times.
✗ Branch 2 not taken.
 245 WritePipe(return_pipe, &length, sizeof(length));
1657
1/2
✓ Branch 1 taken 245 times.
✗ Branch 2 not taken.
 245 sqlite3_reset(this_stmt_list);
1658 245 break;
1659 126 case kStatus:
1660
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 WritePipe(return_pipe, &quota_mgr->gauge_, sizeof(quota_mgr->gauge_));
1661
1/2
✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
 126 WritePipe(return_pipe, &quota_mgr->pinned_,
1662 sizeof(quota_mgr->pinned_));
1663 126 break;
1664 ✗ case kLimits:
1665 ✗ WritePipe(return_pipe, &quota_mgr->limit_, sizeof(quota_mgr->limit_));
1666 ✗ WritePipe(return_pipe, &quota_mgr->cleanup_threshold_,
1667 sizeof(quota_mgr->cleanup_threshold_));
1668 ✗ break;
1669 ✗ case kPid: {
1670 ✗ pid_t pid = getpid();
1671 ✗ WritePipe(return_pipe, &pid, sizeof(pid));
1672 ✗ break;
1673 }
1674 ✗ default:
1675 ✗ PANIC(NULL); // other types are handled by the bunch processor
1676 }
1677
1/2
✓ Branch 1 taken 455 times.
✗ Branch 2 not taken.
 455 quota_mgr->UnbindReturnPipe(return_pipe);
1678 455 num_commands = 0;
1679 }
1680 }
1681
1682
1/2
✓ Branch 1 taken 217 times.
✗ Branch 2 not taken.
 217 LogCvmfs(kLogQuota, kLogDebug, "stopping cache manager (%d)", errno);
1683
1/2
✓ Branch 1 taken 217 times.
✗ Branch 2 not taken.
 217 close(quota_mgr->pipe_lru_[0]);
1684
1/2
✓ Branch 1 taken 217 times.
✗ Branch 2 not taken.
 217 quota_mgr->ProcessCommandBunch(num_commands, command_buffer,
1685 description_buffer);
1686
1687 // Unpin
1688 217 command_buffer[0].command_type = kTouch;
1689 217 for (map<shash::Any, uint64_t>::const_iterator
1690 217 i = quota_mgr->pinned_chunks_.begin(),
1691 217 iEnd = quota_mgr->pinned_chunks_.end();
1692
2/2
✓ Branch 1 taken 70 times.
✓ Branch 2 taken 217 times.
 287 i != iEnd;
1693 70 ++i) {
1694
1/2
✓ Branch 2 taken 70 times.
✗ Branch 3 not taken.
 70 command_buffer[0].StoreHash(i->first);
1695
1/2
✓ Branch 1 taken 70 times.
✗ Branch 2 not taken.
 70 quota_mgr->ProcessCommandBunch(1, command_buffer, description_buffer);
1696 }
1697
1698 217 return NULL;
1699 }
1700
1701
1702 644 void PosixQuotaManager::MakeReturnPipe(int pipe[2]) {
1703
2/2
✓ Branch 0 taken 623 times.
✓ Branch 1 taken 21 times.
 644 if (!shared_) {
1704 623 MakePipe(pipe);
1705 623 return;
1706 }
1707
1708 // Create FIFO in cache directory, store path name (number) in pipe write end
1709 21 int i = 0;
1710 int retval;
1711 do {
1712
2/4
✓ Branch 2 taken 28 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 28 times.
✗ Branch 6 not taken.
 28 retval = mkfifo((workspace_dir_ + "/pipe" + StringifyInt(i)).c_str(), 0600);
1713 28 pipe[1] = i;
1714 28 i++;
1715
3/4
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 21 times.
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
 28 } while ((retval == -1) && (errno == EEXIST));
1716
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 21 times.
 21 assert(retval == 0);
1717
1718 // Connect reader's end
1719
3/6
✓ Branch 2 taken 21 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 21 times.
✗ Branch 6 not taken.
✓ Branch 9 taken 21 times.
✗ Branch 10 not taken.
 21 pipe[0] = open((workspace_dir_ + "/pipe" + StringifyInt(pipe[1])).c_str(),
1720 O_RDONLY | O_NONBLOCK);
1721
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 21 times.
 21 assert(pipe[0] >= 0);
1722 21 Nonblock2Block(pipe[0]);
1723 }
1724
1725
1726 654 void PosixQuotaManager::ParseDirectories(const std::string cache_workspace,
1727 std::string *cache_dir,
1728 std::string *workspace_dir) {
1729
1/2
✓ Branch 1 taken 654 times.
✗ Branch 2 not taken.
 654 vector<string> dir_tokens(SplitString(cache_workspace, ':'));
1730
2/3
✓ Branch 1 taken 640 times.
✓ Branch 2 taken 14 times.
✗ Branch 3 not taken.
 654 switch (dir_tokens.size()) {
1731 640 case 1:
1732
2/4
✓ Branch 2 taken 640 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 640 times.
✗ Branch 6 not taken.
 640 *cache_dir = *workspace_dir = dir_tokens[0];
1733 640 break;
1734 14 case 2:
1735
1/2
✓ Branch 2 taken 14 times.
✗ Branch 3 not taken.
 14 *cache_dir = dir_tokens[0];
1736
1/2
✓ Branch 2 taken 14 times.
✗ Branch 3 not taken.
 14 *workspace_dir = dir_tokens[1];
1737 14 break;
1738 ✗ default:
1739 ✗ PANIC(NULL);
1740 }
1741 654 }
1742
1743 7 void PosixQuotaManager::SkipEviction(const EvictCandidate &candidate) {
1744 7 bool res = true;
1745
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 std::string hash_str = candidate.hash.ToString();
1746
1/2
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
 7 LogCvmfs(kLogQuota, kLogDebug, "Exclude %s from eviction", hash_str.c_str());
1747
2/4
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 7 times.
✗ Branch 6 not taken.
 7 sqlite3_bind_text(stmt_block_, 1, &hash_str[0], hash_str.length(),
1748 SQLITE_STATIC);
1749
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 res = (sqlite3_step(stmt_block_) == SQLITE_DONE);
1750
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 sqlite3_reset(stmt_block_);
1751
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
 7 assert(res);
1752 7 }
1753
1754 /**
1755 * Immediately inserts a new pinned catalog. Does cache cleanup if necessary.
1756 *
1757 * \return True on success, false otherwise
1758 */
1759 191 bool PosixQuotaManager::Pin(const shash::Any &hash,
1760 const uint64_t size,
1761 const string &description,
1762 const bool is_catalog) {
1763
3/4
✓ Branch 0 taken 14 times.
✓ Branch 1 taken 177 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 14 times.
 191 assert((size > 0) || !is_catalog);
1764
1765
1/2
✓ Branch 1 taken 191 times.
✗ Branch 2 not taken.
 191 const string hash_str = hash.ToString();
1766
1/2
✓ Branch 3 taken 191 times.
✗ Branch 4 not taken.
 191 LogCvmfs(kLogQuota, kLogDebug, "pin into lru %s, path %s", hash_str.c_str(),
1767 description.c_str());
1768
1769 // Has to run when not yet spawned (cvmfs initialization)
1770
2/2
✓ Branch 0 taken 93 times.
✓ Branch 1 taken 98 times.
 191 if (!spawned_) {
1771 // Code duplication here
1772
3/4
✓ Branch 2 taken 93 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 59 times.
✓ Branch 6 taken 34 times.
 93 if (pinned_chunks_.find(hash) == pinned_chunks_.end()) {
1773
2/2
✓ Branch 0 taken 14 times.
✓ Branch 1 taken 45 times.
 59 if (pinned_ + size > cleanup_threshold_) {
1774
1/2
✓ Branch 2 taken 14 times.
✗ Branch 3 not taken.
 14 LogCvmfs(kLogQuota, kLogDebug, "failed to insert %s (pinned), no space",
1775 hash_str.c_str());
1776 14 return false;
1777 } else {
1778
1/2
✓ Branch 1 taken 45 times.
✗ Branch 2 not taken.
 45 pinned_chunks_[hash] = size;
1779 45 pinned_ += size;
1780
1/2
✓ Branch 1 taken 45 times.
✗ Branch 2 not taken.
 45 CheckHighPinWatermark();
1781 }
1782 }
1783
1/2
✓ Branch 1 taken 79 times.
✗ Branch 2 not taken.
 79 const bool exists = Contains(hash_str);
1784
4/4
✓ Branch 0 taken 45 times.
✓ Branch 1 taken 34 times.
✓ Branch 2 taken 7 times.
✓ Branch 3 taken 38 times.
 79 if (!exists && (gauge_ + size > limit_)) {
1785
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 LogCvmfs(kLogQuota, kLogDebug, "over limit, gauge %lu, file size %lu",
1786 gauge_, size);
1787
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 const int retval = DoCleanup(cleanup_threshold_);
1788
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
 7 assert(retval != 0);
1789 }
1790
1/2
✓ Branch 3 taken 79 times.
✗ Branch 4 not taken.
 79 sqlite3_bind_text(stmt_new_, 1, &hash_str[0], hash_str.length(),
1791 SQLITE_STATIC);
1792
1/2
✓ Branch 1 taken 79 times.
✗ Branch 2 not taken.
 79 sqlite3_bind_int64(stmt_new_, 2, size);
1793
1/2
✓ Branch 1 taken 79 times.
✗ Branch 2 not taken.
 79 sqlite3_bind_int64(stmt_new_, 3, seq_++);
1794
1/2
✓ Branch 3 taken 79 times.
✗ Branch 4 not taken.
 79 sqlite3_bind_text(stmt_new_, 4, &description[0], description.length(),
1795 SQLITE_STATIC);
1796
3/4
✓ Branch 0 taken 58 times.
✓ Branch 1 taken 21 times.
✓ Branch 3 taken 79 times.
✗ Branch 4 not taken.
 79 sqlite3_bind_int64(stmt_new_, 5, is_catalog ? kFileCatalog : kFileRegular);
1797
1/2
✓ Branch 1 taken 79 times.
✗ Branch 2 not taken.
 79 sqlite3_bind_int64(stmt_new_, 6, 1);
1798
1/2
✓ Branch 1 taken 79 times.
✗ Branch 2 not taken.
 79 const int retval = sqlite3_step(stmt_new_);
1799
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 79 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 79 assert((retval == SQLITE_DONE) || (retval == SQLITE_OK));
1800
1/2
✓ Branch 1 taken 79 times.
✗ Branch 2 not taken.
 79 sqlite3_reset(stmt_new_);
1801
2/2
✓ Branch 0 taken 45 times.
✓ Branch 1 taken 34 times.
 79 if (!exists)
1802 45 gauge_ += size;
1803 79 return true;
1804 }
1805
1806 int pipe_reserve[2];
1807
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 MakeReturnPipe(pipe_reserve);
1808
1809 98 LruCommand cmd;
1810 98 cmd.command_type = kReserve;
1811 98 cmd.SetSize(size);
1812
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 cmd.StoreHash(hash);
1813 98 cmd.return_pipe = pipe_reserve[1];
1814
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
1815 bool result;
1816
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 ManagedReadHalfPipe(pipe_reserve[0], &result, sizeof(result));
1817
1/2
✓ Branch 1 taken 98 times.
✗ Branch 2 not taken.
 98 CloseReturnPipe(pipe_reserve);
1818
1819
2/2
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 91 times.
 98 if (!result)
1820 7 return false;
1821
3/4
✓ Branch 0 taken 14 times.
✓ Branch 1 taken 77 times.
✓ Branch 3 taken 91 times.
✗ Branch 4 not taken.
 91 DoInsert(hash, size, description, is_catalog ? kPin : kPinRegular);
1822
1823 91 return true;
1824 191 }
1825
1826
1827 640 PosixQuotaManager::PosixQuotaManager(const uint64_t limit,
1828 const uint64_t cleanup_threshold,
1829 640 const string &cache_workspace)
1830 640 : shared_(false)
1831 640 , spawned_(false)
1832 640 , limit_(limit)
1833 640 , cleanup_threshold_(cleanup_threshold)
1834 640 , gauge_(0)
1835 640 , pinned_(0)
1836 640 , seq_(0)
1837 640 , cache_dir_() // initialized in body
1838 640 , workspace_dir_() // initialized in body
1839 640 , fd_lock_cachedb_(-1)
1840 640 , async_delete_(true)
1841 640 , cachemgr_pid_(0)
1842 640 , database_(NULL)
1843 640 , stmt_touch_(NULL)
1844 640 , stmt_unpin_(NULL)
1845 640 , stmt_block_(NULL)
1846 640 , stmt_unblock_(NULL)
1847 640 , stmt_new_(NULL)
1848 640 , stmt_lru_(NULL)
1849 640 , stmt_size_(NULL)
1850 640 , stmt_rm_(NULL)
1851 640 , stmt_rm_batch_(NULL)
1852 640 , stmt_list_(NULL)
1853 640 , stmt_list_pinned_(NULL)
1854 640 , stmt_list_catalogs_(NULL)
1855 640 , stmt_list_volatile_(NULL)
1856 640 , initialized_(false)
1857 1280 , cleanup_unused_first_(false) {
1858
2/4
✓ Branch 1 taken 640 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 640 times.
✗ Branch 5 not taken.
 640 ParseDirectories(cache_workspace, &cache_dir_, &workspace_dir_);
1859 640 pipe_lru_[0] = pipe_lru_[1] = -1;
1860
1/2
✓ Branch 1 taken 640 times.
✗ Branch 2 not taken.
 640 cleanup_recorder_.AddRecorder(1, 90); // last 1.5 min with second resolution
1861 // last 1.5 h with minute resolution
1862
1/2
✓ Branch 1 taken 640 times.
✗ Branch 2 not taken.
 640 cleanup_recorder_.AddRecorder(60, 90 * 60);
1863 // last 18 hours with 20 min resolution
1864
1/2
✓ Branch 1 taken 640 times.
✗ Branch 2 not taken.
 640 cleanup_recorder_.AddRecorder(20 * 60, 60 * 60 * 18);
1865 // last 4 days with hour resolution
1866
1/2
✓ Branch 1 taken 640 times.
✗ Branch 2 not taken.
 640 cleanup_recorder_.AddRecorder(60 * 60, 60 * 60 * 24 * 4);
1867
1868 640 lock_open_files_ = reinterpret_cast<pthread_mutex_t *>(
1869 640 smalloc(sizeof(pthread_mutex_t)));
1870 640 }
1871
1872
1873 2556 PosixQuotaManager::~PosixQuotaManager() {
1874 1278 free(lock_open_files_);
1875
1876
2/2
✓ Branch 0 taken 21 times.
✓ Branch 1 taken 618 times.
 1278 if (!initialized_)
1877 42 return;
1878
1879
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 618 times.
 1236 if (shared_) {
1880 // Most of cleanup is done elsewhen by shared cache manager
1881 ✗ close(pipe_lru_[1]);
1882 ✗ return;
1883 }
1884
1885
2/2
✓ Branch 0 taken 217 times.
✓ Branch 1 taken 401 times.
 1236 if (spawned_) {
1886 434 char fin = 0;
1887 434 WritePipe(pipe_lru_[1], &fin, 1);
1888 434 close(pipe_lru_[1]);
1889 434 pthread_join(thread_lru_, NULL);
1890 } else {
1891 802 ClosePipe(pipe_lru_);
1892 }
1893
1894 1236 CloseDatabase();
1895
14/14
✓ Branch 1 taken 618 times.
✓ Branch 2 taken 21 times.
✓ Branch 4 taken 618 times.
✓ Branch 5 taken 21 times.
✓ Branch 7 taken 618 times.
✓ Branch 8 taken 21 times.
✓ Branch 10 taken 618 times.
✓ Branch 11 taken 21 times.
✓ Branch 13 taken 618 times.
✓ Branch 14 taken 21 times.
✓ Branch 16 taken 618 times.
✓ Branch 17 taken 21 times.
✓ Branch 19 taken 618 times.
✓ Branch 20 taken 21 times.
 2808 }
1896
1897
1898 33551 void PosixQuotaManager::ProcessCommandBunch(const unsigned num,
1899 const LruCommand *commands,
1900 const char *descriptions) {
1901 33551 int retval = sqlite3_exec(database_, "BEGIN", NULL, NULL, NULL);
1902
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 33551 times.
 33551 assert(retval == SQLITE_OK);
1903
1904
2/2
✓ Branch 0 taken 1050343 times.
✓ Branch 1 taken 33551 times.
 1083894 for (unsigned i = 0; i < num; ++i) {
1905
1/2
✓ Branch 1 taken 1050343 times.
✗ Branch 2 not taken.
 1050343 const shash::Any hash = commands[i].RetrieveHash();
1906
1/2
✓ Branch 1 taken 1050343 times.
✗ Branch 2 not taken.
 1050343 const string hash_str = hash.ToString();
1907 1050343 const unsigned size = commands[i].GetSize();
1908
1/2
✓ Branch 1 taken 1050343 times.
✗ Branch 2 not taken.
 1050343 LogCvmfs(kLogQuota, kLogDebug, "processing %s (%d)", hash_str.c_str(),
1909 1050343 commands[i].command_type);
1910
1911 bool exists;
1912
3/4
✓ Branch 0 taken 350091 times.
✓ Branch 1 taken 14 times.
✓ Branch 2 taken 700238 times.
✗ Branch 3 not taken.
 1050343 switch (commands[i].command_type) {
1913 350091 case kTouch:
1914
1/2
✓ Branch 1 taken 350091 times.
✗ Branch 2 not taken.
 350091 sqlite3_bind_int64(stmt_touch_, 1, seq_++);
1915
1/2
✓ Branch 3 taken 350091 times.
✗ Branch 4 not taken.
 350091 sqlite3_bind_text(stmt_touch_, 2, &hash_str[0], hash_str.length(),
1916 SQLITE_STATIC);
1917
1/2
✓ Branch 1 taken 350091 times.
✗ Branch 2 not taken.
 350091 retval = sqlite3_step(stmt_touch_);
1918
1/2
✓ Branch 1 taken 350091 times.
✗ Branch 2 not taken.
 350091 LogCvmfs(kLogQuota, kLogDebug, "touching %s (%ld): %d",
1919 350091 hash_str.c_str(), seq_ - 1, retval);
1920
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 350091 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 350091 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
1921 ✗ PANIC(kLogSyslogErr, "failed to update %s in cachedb, error %d",
1922 hash_str.c_str(), retval);
1923 }
1924
1/2
✓ Branch 1 taken 350091 times.
✗ Branch 2 not taken.
 350091 sqlite3_reset(stmt_touch_);
1925 350091 break;
1926 14 case kUnpin:
1927
1/2
✓ Branch 3 taken 14 times.
✗ Branch 4 not taken.
 14 sqlite3_bind_text(stmt_unpin_, 1, &hash_str[0], hash_str.length(),
1928 SQLITE_STATIC);
1929
1/2
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
 14 retval = sqlite3_step(stmt_unpin_);
1930
1/2
✓ Branch 2 taken 14 times.
✗ Branch 3 not taken.
 14 LogCvmfs(kLogQuota, kLogDebug, "unpinning %s: %d", hash_str.c_str(),
1931 retval);
1932
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 14 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
1933 ✗ PANIC(kLogSyslogErr, "failed to unpin %s in cachedb, error %d",
1934 hash_str.c_str(), retval);
1935 }
1936
1/2
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
 14 sqlite3_reset(stmt_unpin_);
1937 14 break;
1938 700238 case kPin:
1939 case kPinRegular:
1940 case kInsert:
1941 case kInsertVolatile:
1942 // It could already be in, check
1943
1/2
✓ Branch 1 taken 700238 times.
✗ Branch 2 not taken.
 700238 exists = Contains(hash_str);
1944
1945 // Cleanup, move to trash and unlink
1946
3/4
✓ Branch 0 taken 700203 times.
✓ Branch 1 taken 35 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 700203 times.
 700238 if (!exists && (gauge_ + size > limit_)) {
1947 ✗ LogCvmfs(kLogQuota, kLogDebug, "over limit, gauge %lu, file size %u",
1948 gauge_, size);
1949 ✗ retval = DoCleanup(cleanup_threshold_);
1950 ✗ assert(retval != 0);
1951 }
1952
1953 // Insert or replace
1954
1/2
✓ Branch 3 taken 700238 times.
✗ Branch 4 not taken.
 700238 sqlite3_bind_text(stmt_new_, 1, &hash_str[0], hash_str.length(),
1955 SQLITE_STATIC);
1956
1/2
✓ Branch 1 taken 700238 times.
✗ Branch 2 not taken.
 700238 sqlite3_bind_int64(stmt_new_, 2, size);
1957
2/2
✓ Branch 0 taken 28 times.
✓ Branch 1 taken 700210 times.
 700238 if (commands[i].command_type == kInsertVolatile) {
1958
1/2
✓ Branch 1 taken 28 times.
✗ Branch 2 not taken.
 28 sqlite3_bind_int64(stmt_new_, 3, (seq_++) | kVolatileFlag);
1959 } else {
1960
1/2
✓ Branch 1 taken 700210 times.
✗ Branch 2 not taken.
 700210 sqlite3_bind_int64(stmt_new_, 3, seq_++);
1961 }
1962 700238 sqlite3_bind_text(stmt_new_, 4, &descriptions[i * kMaxDescription],
1963
1/2
✓ Branch 1 taken 700238 times.
✗ Branch 2 not taken.
 700238 commands[i].desc_length, SQLITE_STATIC);
1964
1/2
✓ Branch 1 taken 700238 times.
✗ Branch 2 not taken.
 700238 sqlite3_bind_int64(
1965 stmt_new_, 5,
1966
2/2
✓ Branch 0 taken 14 times.
✓ Branch 1 taken 700224 times.
 700238 (commands[i].command_type == kPin) ? kFileCatalog : kFileRegular);
1967
1/2
✓ Branch 1 taken 700238 times.
✗ Branch 2 not taken.
 700238 sqlite3_bind_int64(stmt_new_, 6,
1968
2/2
✓ Branch 0 taken 700224 times.
✓ Branch 1 taken 14 times.
 700238 ((commands[i].command_type == kPin)
1969
2/2
✓ Branch 0 taken 77 times.
✓ Branch 1 taken 700147 times.
 700224 || (commands[i].command_type == kPinRegular))
1970 ? 1
1971 : 0);
1972
1/2
✓ Branch 1 taken 700238 times.
✗ Branch 2 not taken.
 700238 retval = sqlite3_step(stmt_new_);
1973
1/2
✓ Branch 1 taken 700238 times.
✗ Branch 2 not taken.
 700238 LogCvmfs(kLogQuota, kLogDebug, "insert or replace %s, method %d: %d",
1974 700238 hash_str.c_str(), commands[i].command_type, retval);
1975
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 700238 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 700238 if ((retval != SQLITE_DONE) && (retval != SQLITE_OK)) {
1976 ✗ PANIC(kLogSyslogErr, "failed to insert %s in cachedb, error %d",
1977 hash_str.c_str(), retval);
1978 }
1979
1/2
✓ Branch 1 taken 700238 times.
✗ Branch 2 not taken.
 700238 sqlite3_reset(stmt_new_);
1980
1981
2/2
✓ Branch 0 taken 700203 times.
✓ Branch 1 taken 35 times.
 700238 if (!exists)
1982 700203 gauge_ += size;
1983 700238 break;
1984 ✗ default:
1985 // other types should have been taken care of by event loop
1986 ✗ PANIC(NULL);
1987 }
1988 1050343 }
1989
1990 33551 retval = sqlite3_exec(database_, "COMMIT", NULL, NULL, NULL);
1991
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 33551 times.
 33551 if (retval != SQLITE_OK) {
1992 ✗ PANIC(kLogSyslogErr, "failed to commit to cachedb, error %d", retval);
1993 }
1994 33551 }
1995
1996
1997 626 bool PosixQuotaManager::RebuildDatabase() {
1998 626 bool result = false;
1999 626 string sql;
2000 626 sqlite3_stmt *stmt_select = NULL;
2001 626 sqlite3_stmt *stmt_insert = NULL;
2002 int sqlerr;
2003 626 int seq = 0;
2004 char hex[4];
2005 struct stat info;
2006 platform_dirent64 *d;
2007 626 DIR *dirp = NULL;
2008 626 string path;
2009
2010
1/2
✓ Branch 1 taken 626 times.
✗ Branch 2 not taken.
 626 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug, "re-building cache database");
2011
2012 // Empty cache catalog and fscache
2013
1/2
✓ Branch 1 taken 626 times.
✗ Branch 2 not taken.
 626 sql = "DELETE FROM cache_catalog; DELETE FROM fscache;";
2014
1/2
✓ Branch 2 taken 626 times.
✗ Branch 3 not taken.
 626 sqlerr = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
2015
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 626 times.
 626 if (sqlerr != SQLITE_OK) {
2016 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not clear cache database");
2017 ✗ goto build_return;
2018 }
2019
2020 626 gauge_ = 0;
2021
2022 // Insert files from cache sub-directories 00 - ff
2023 // TODO(jblomer): fs_traversal
2024
1/2
✓ Branch 1 taken 626 times.
✗ Branch 2 not taken.
 626 sqlite3_prepare_v2(database_,
2025 "INSERT INTO fscache (sha1, size, actime) "
2026 "VALUES (:sha1, :s, :t);",
2027 -1, &stmt_insert, NULL);
2028
2029
2/2
✓ Branch 0 taken 154901 times.
✓ Branch 1 taken 605 times.
 155506 for (int i = 0; i <= 0xff; i++) {
2030 154901 snprintf(hex, sizeof(hex), "%02x", i);
2031
3/6
✓ Branch 2 taken 154901 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 154901 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 154901 times.
✗ Branch 9 not taken.
 154901 path = cache_dir_ + "/" + string(hex);
2032
3/4
✓ Branch 2 taken 154901 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 21 times.
✓ Branch 5 taken 154880 times.
 154901 if ((dirp = opendir(path.c_str())) == NULL) {
2033
1/2
✓ Branch 2 taken 21 times.
✗ Branch 3 not taken.
 21 LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
2034 "failed to open directory %s (tmpwatch interfering?)",
2035 path.c_str());
2036 21 goto build_return;
2037 }
2038
3/4
✓ Branch 1 taken 464654 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 309774 times.
✓ Branch 4 taken 154880 times.
 464654 while ((d = platform_readdir(dirp)) != NULL) {
2039
3/6
✓ Branch 2 taken 309774 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 309774 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 309774 times.
✗ Branch 9 not taken.
 619548 const string file_path = path + "/" + string(d->d_name);
2040
1/2
✓ Branch 2 taken 309774 times.
✗ Branch 3 not taken.
 309774 if (stat(file_path.c_str(), &info) == 0) {
2041
2/2
✓ Branch 0 taken 309760 times.
✓ Branch 1 taken 14 times.
 309774 if (!S_ISREG(info.st_mode))
2042 309767 continue;
2043
2/2
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 7 times.
 14 if (info.st_size == 0) {
2044
1/2
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
 7 LogCvmfs(kLogQuota, kLogSyslog | kLogDebug,
2045 "removing empty file %s during automatic cache db rebuild",
2046 file_path.c_str());
2047 7 unlink(file_path.c_str());
2048 7 continue;
2049 }
2050
2051
3/6
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 7 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 7 times.
✗ Branch 10 not taken.
 14 string hash = string(hex) + string(d->d_name);
2052
1/2
✓ Branch 3 taken 7 times.
✗ Branch 4 not taken.
 7 sqlite3_bind_text(stmt_insert, 1, hash.data(), hash.length(),
2053 SQLITE_STATIC);
2054
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 sqlite3_bind_int64(stmt_insert, 2, info.st_size);
2055
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 sqlite3_bind_int64(stmt_insert, 3, info.st_atime);
2056
2/4
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 7 times.
 7 if (sqlite3_step(stmt_insert) != SQLITE_DONE) {
2057 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not insert into temp table");
2058 ✗ goto build_return;
2059 }
2060
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 sqlite3_reset(stmt_insert);
2061
2062 7 gauge_ += info.st_size;
2063
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 } else {
2064 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not stat %s", file_path.c_str());
2065 }
2066
2/3
✓ Branch 1 taken 7 times.
✓ Branch 2 taken 309767 times.
✗ Branch 3 not taken.
 309774 }
2067
1/2
✓ Branch 1 taken 154880 times.
✗ Branch 2 not taken.
 154880 closedir(dirp);
2068 154880 dirp = NULL;
2069 }
2070
1/2
✓ Branch 1 taken 605 times.
✗ Branch 2 not taken.
 605 sqlite3_finalize(stmt_insert);
2071 605 stmt_insert = NULL;
2072
2073 // Transfer from temp table in cache catalog
2074
1/2
✓ Branch 1 taken 605 times.
✗ Branch 2 not taken.
 605 sqlite3_prepare_v2(database_,
2075 "SELECT sha1, size FROM fscache ORDER BY actime;", -1,
2076 &stmt_select, NULL);
2077
1/2
✓ Branch 1 taken 605 times.
✗ Branch 2 not taken.
 605 sqlite3_prepare_v2(
2078 database_,
2079 "INSERT INTO cache_catalog (sha1, size, acseq, path, type, pinned) "
2080 "VALUES (:sha1, :s, :seq, 'unknown (automatic rebuild)', :t, 0);",
2081 -1, &stmt_insert, NULL);
2082
3/4
✓ Branch 1 taken 612 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 7 times.
✓ Branch 4 taken 605 times.
 612 while (sqlite3_step(stmt_select) == SQLITE_ROW) {
2083 const string hash = string(
2084
2/4
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 7 times.
✗ Branch 6 not taken.
 7 reinterpret_cast<const char *>(sqlite3_column_text(stmt_select, 0)));
2085
1/2
✓ Branch 3 taken 7 times.
✗ Branch 4 not taken.
 7 sqlite3_bind_text(stmt_insert, 1, &hash[0], hash.length(), SQLITE_STATIC);
2086
2/4
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 7 times.
✗ Branch 5 not taken.
 7 sqlite3_bind_int64(stmt_insert, 2, sqlite3_column_int64(stmt_select, 1));
2087
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 sqlite3_bind_int64(stmt_insert, 3, seq++);
2088 // Might also be a catalog (information is lost)
2089
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 sqlite3_bind_int64(stmt_insert, 4, kFileRegular);
2090
2091
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 const int retval = sqlite3_step(stmt_insert);
2092
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
 7 if (retval != SQLITE_DONE) {
2093 // If the file system hosting the cache is full, we'll likely notice here
2094 ✗ LogCvmfs(kLogQuota, kLogDebug | kLogSyslogErr,
2095 "could not insert into cache catalog (%d - %s)", retval,
2096 sqlite3_errstr(retval));
2097 ✗ goto build_return;
2098 }
2099
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 sqlite3_reset(stmt_insert);
2100
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 }
2101
2102 // Delete temporary table
2103
1/2
✓ Branch 1 taken 605 times.
✗ Branch 2 not taken.
 605 sql = "DELETE FROM fscache;";
2104
1/2
✓ Branch 2 taken 605 times.
✗ Branch 3 not taken.
 605 sqlerr = sqlite3_exec(database_, sql.c_str(), NULL, NULL, NULL);
2105
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 605 times.
 605 if (sqlerr != SQLITE_OK) {
2106 ✗ LogCvmfs(kLogQuota, kLogDebug, "could not clear temporary table (%d)",
2107 sqlerr);
2108 ✗ goto build_return;
2109 }
2110
2111 605 seq_ = seq;
2112 605 result = true;
2113
1/2
✓ Branch 1 taken 605 times.
✗ Branch 2 not taken.
 605 LogCvmfs(kLogQuota, kLogDebug,
2114 "rebuilding finished, sequence %" PRIu64 ", gauge %" PRIu64, seq_,
2115 gauge_);
2116
2117 626 build_return:
2118
1/2
✓ Branch 0 taken 626 times.
✗ Branch 1 not taken.
 626 if (stmt_insert)
2119
1/2
✓ Branch 1 taken 626 times.
✗ Branch 2 not taken.
 626 sqlite3_finalize(stmt_insert);
2120
2/2
✓ Branch 0 taken 605 times.
✓ Branch 1 taken 21 times.
 626 if (stmt_select)
2121
1/2
✓ Branch 1 taken 605 times.
✗ Branch 2 not taken.
 605 sqlite3_finalize(stmt_select);
2122
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 626 times.
 626 if (dirp)
2123 ✗ closedir(dirp);
2124 626 return result;
2125 626 }
2126
2127
2128 /**
2129 * Register a channel that allows the cache manager to trigger action to its
2130 * clients. Currently used for releasing pinned catalogs.
2131 */
2132 28 void PosixQuotaManager::RegisterBackChannel(int back_channel[2],
2133 const string &channel_id) {
2134
1/2
✓ Branch 0 taken 28 times.
✗ Branch 1 not taken.
 28 if (protocol_revision_ >= 1) {
2135
1/2
✓ Branch 2 taken 28 times.
✗ Branch 3 not taken.
 28 shash::Md5 hash = shash::Md5(shash::AsciiPtr(channel_id));
2136
1/2
✓ Branch 1 taken 28 times.
✗ Branch 2 not taken.
 28 MakeReturnPipe(back_channel);
2137
2138 28 LruCommand cmd;
2139 28 cmd.command_type = kRegisterBackChannel;
2140 28 cmd.return_pipe = back_channel[1];
2141 // Not StoreHash(). This is an MD5 hash.
2142 28 memcpy(cmd.digest, hash.digest, hash.GetDigestSize());
2143
1/2
✓ Branch 1 taken 28 times.
✗ Branch 2 not taken.
 28 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2144
2145 char success;
2146
1/2
✓ Branch 1 taken 28 times.
✗ Branch 2 not taken.
 28 ManagedReadHalfPipe(back_channel[0], &success, sizeof(success));
2147 // At this point, the named FIFO is unlinked, so don't use CloseReturnPipe
2148
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 28 times.
 28 if (success != 'S') {
2149 ✗ PANIC(kLogDebug | kLogSyslogErr,
2150 "failed to register quota back channel (%c)", success);
2151 }
2152 } else {
2153 // Dummy pipe to return valid file descriptors
2154 ✗ MakePipe(back_channel);
2155 }
2156 28 }
2157
2158
2159 /**
2160 * Removes a chunk from cache, if it exists.
2161 */
2162 21 void PosixQuotaManager::Remove(const shash::Any &hash) {
2163
1/2
✓ Branch 1 taken 21 times.
✗ Branch 2 not taken.
 21 const string hash_str = hash.ToString();
2164
2165 int pipe_remove[2];
2166
1/2
✓ Branch 1 taken 21 times.
✗ Branch 2 not taken.
 21 MakeReturnPipe(pipe_remove);
2167
2168 21 LruCommand cmd;
2169 21 cmd.command_type = kRemove;
2170 21 cmd.return_pipe = pipe_remove[1];
2171
1/2
✓ Branch 1 taken 21 times.
✗ Branch 2 not taken.
 21 cmd.StoreHash(hash);
2172
1/2
✓ Branch 1 taken 21 times.
✗ Branch 2 not taken.
 21 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2173
2174 bool success;
2175
1/2
✓ Branch 1 taken 21 times.
✗ Branch 2 not taken.
 21 ManagedReadHalfPipe(pipe_remove[0], &success, sizeof(success));
2176
1/2
✓ Branch 1 taken 21 times.
✗ Branch 2 not taken.
 21 CloseReturnPipe(pipe_remove);
2177
2178
3/6
✓ Branch 1 taken 21 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 21 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 21 times.
✗ Branch 8 not taken.
 21 unlink((cache_dir_ + "/" + hash.MakePathWithoutSuffix()).c_str());
2179 21 }
2180
2181
2182 231 void PosixQuotaManager::Spawn() {
2183
2/2
✓ Branch 0 taken 14 times.
✓ Branch 1 taken 217 times.
 231 if (spawned_)
2184 14 return;
2185
2186 217 if (pthread_create(&thread_lru_, NULL, MainCommandServer,
2187 static_cast<void *>(this))
2188
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 217 times.
 217 != 0) {
2189 ✗ PANIC(kLogDebug, "could not create lru thread");
2190 }
2191
2192 217 spawned_ = true;
2193 }
2194
2195
2196 /**
2197 * Updates the sequence number of the file specified by the hash.
2198 */
2199 350054 void PosixQuotaManager::Touch(const shash::Any &hash) {
2200 350054 LruCommand cmd;
2201 350054 cmd.command_type = kTouch;
2202
1/2
✓ Branch 1 taken 350054 times.
✗ Branch 2 not taken.
 350054 cmd.StoreHash(hash);
2203
1/2
✓ Branch 1 taken 350054 times.
✗ Branch 2 not taken.
 350054 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2204 350054 }
2205
2206
2207 602 void PosixQuotaManager::UnbindReturnPipe(int pipe_wronly) {
2208
2/2
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 595 times.
 602 if (shared_)
2209 7 close(pipe_wronly);
2210 602 }
2211
2212
2213 49 void PosixQuotaManager::UnlinkReturnPipe(int pipe_wronly) {
2214
2/2
✓ Branch 0 taken 21 times.
✓ Branch 1 taken 28 times.
 49 if (shared_)
2215
2/4
✓ Branch 2 taken 21 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 21 times.
✗ Branch 6 not taken.
 21 unlink((workspace_dir_ + "/pipe" + StringifyInt(pipe_wronly)).c_str());
2216 49 }
2217
2218
2219 51 void PosixQuotaManager::Unpin(const shash::Any &hash) {
2220
2/4
✓ Branch 1 taken 51 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 51 times.
✗ Branch 6 not taken.
 51 LogCvmfs(kLogQuota, kLogDebug, "Unpin %s", hash.ToString().c_str());
2221
2222 51 LruCommand cmd;
2223 51 cmd.command_type = kUnpin;
2224
1/2
✓ Branch 1 taken 51 times.
✗ Branch 2 not taken.
 51 cmd.StoreHash(hash);
2225
1/2
✓ Branch 1 taken 51 times.
✗ Branch 2 not taken.
 51 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2226 51 }
2227
2228
2229 14 void PosixQuotaManager::UnregisterBackChannel(int back_channel[2],
2230 const string &channel_id) {
2231
1/2
✓ Branch 0 taken 14 times.
✗ Branch 1 not taken.
 14 if (protocol_revision_ >= 1) {
2232
1/2
✓ Branch 2 taken 14 times.
✗ Branch 3 not taken.
 14 shash::Md5 hash = shash::Md5(shash::AsciiPtr(channel_id));
2233
2234 14 LruCommand cmd;
2235 14 cmd.command_type = kUnregisterBackChannel;
2236 // Not StoreHash(). This is an MD5 hash.
2237 14 memcpy(cmd.digest, hash.digest, hash.GetDigestSize());
2238
1/2
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
 14 WritePipe(pipe_lru_[1], &cmd, sizeof(cmd));
2239
2240 // Writer's end will be closed by cache manager, FIFO is already unlinked
2241
1/2
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
 14 close(back_channel[0]);
2242 } else {
2243 ✗ ClosePipe(back_channel);
2244 }
2245 14 }
2246
2247 700889 void PosixQuotaManager::ManagedReadHalfPipe(int fd, void *buf, size_t nbyte) {
2248
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 700889 times.
 700889 const unsigned timeout_ms = cachemgr_pid_ ? 1000 : 0;
2249 700889 bool result = false;
2250 do {
2251 700889 result = ReadHalfPipe(fd, buf, nbyte, timeout_ms);
2252 // try only as long as the cachemgr is still alive
2253
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 700889 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 700889 times.
 700889 } while (!result && getpgid(cachemgr_pid_) >= 0);
2254
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 700889 times.
 700889 if (!result) {
2255 ✗ PANIC(kLogStderr,
2256 "Error: quota manager could not read from cachemanager pipe");
2257 }
2258 700889 }
2259
2260 ✗ void *PosixQuotaManager::CollectMountpointsHashes(void *data) {
2261 #ifndef __APPLE__
2262 ✗ pthread_setname_np(pthread_self(), "hash_collector");
2263 ✗ auto *handler = static_cast<CollectorHandler *>(data);
2264
2265 ✗ const std::string mountpoint = handler->mp[handler->i];
2266 ✗ ssize_t n = getxattr(mountpoint.c_str(), "user.list_open_hashes", nullptr, 0);
2267 ✗ if (n < 0) {
2268 ✗ pthread_exit(nullptr);
2269 }
2270 ✗ std::vector<char> buf((size_t)n);
2271 ✗ n = getxattr(mountpoint.c_str(), "user.list_open_hashes", buf.data(),
2272 buf.size());
2273 ✗ if (n < 0) {
2274 ✗ pthread_exit(nullptr);
2275 }
2276
2277 ✗ std::vector<std::string> hash_strs;
2278 ✗ std::string hash_str;
2279 ✗ for (const char c : buf) {
2280 ✗ if (c == '\n') {
2281 ✗ hash_strs.push_back(hash_str);
2282 ✗ hash_str.clear();
2283 } else {
2284 ✗ hash_str += c;
2285 }
2286 }
2287 ✗ const MutexLockGuard lock_guard(handler->l);
2288 ✗ for (auto hash_str : hash_strs) {
2289 ✗ handler->of.push_back(shash::MkFromHexPtr(shash::HexPtr(hash_str)));
2290 }
2291 #endif
2292 ✗ pthread_exit(nullptr);
2293 }
2294
2295 ✗ std::vector<shash::Any> PosixQuotaManager::CollectAllOpenHashes() {
2296 ✗ std::vector<CollectorHandler *> handlers;
2297 ✗ std::vector<pthread_t *> threads;
2298 ✗ open_files_.clear();
2299 #ifndef __APPLE__
2300 ✗ auto &&a_after_b = [](const struct timespec a, const struct timespec b) {
2301 ✗ return (a.tv_sec > b.tv_sec) ? true : false;
2302 };
2303
2304 ✗ for (size_t i = 0; i < mountpoints_.size(); ++i) {
2305 ✗ handlers.push_back(
2306 ✗ new CollectorHandler{open_files_, mountpoints_, lock_open_files_, i});
2307 ✗ threads.push_back(new pthread_t);
2308 }
2309
2310 ✗ const int retval = pthread_mutex_init(lock_open_files_, NULL);
2311 ✗ assert(retval == 0);
2312
2313 ✗ for (size_t i = 0; i < mountpoints_.size(); ++i) {
2314 ✗ pthread_create(threads[i], nullptr, CollectMountpointsHashes, handlers[i]);
2315 }
2316
2317 ✗ std::vector<bool> joined(handlers.size(), false);
2318 struct timespec reference, current;
2319 ✗ clock_gettime(CLOCK_REALTIME, &reference);
2320 ✗ clock_gettime(CLOCK_REALTIME, &current);
2321 ✗ reference.tv_sec += 10; // Give 10sec for hash collection
2322 ✗ size_t i = 0;
2323 ✗ while (
2324 ✗ (not std::all_of(joined.begin(), joined.end(), [](bool b) { return b; }))
2325 ✗ and a_after_b(reference, current)) {
2326 // as long as there are still threads that haven't joined yet
2327 // and for 10 seconds
2328 ✗ if (not joined[i]) {
2329 ✗ const int s = pthread_tryjoin_np(*threads[i], NULL);
2330 ✗ if (s == 0) {
2331 ✗ joined[i] = true;
2332 }
2333 }
2334 ✗ ++i;
2335 ✗ i = i % handlers.size();
2336 ✗ clock_gettime(CLOCK_REALTIME, &current);
2337 }
2338
2339 ✗ for (size_t i = 0; i < handlers.size(); ++i) {
2340 ✗ delete handlers[i];
2341 }
2342
2343 ✗ pthread_mutex_destroy(lock_open_files_);
2344 #endif
2345 ✗ return open_files_;
2346 }
2347
2348