CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
cvmfs_cache_posix.cc
Go to the documentation of this file.
1 
6 #include <signal.h>
7 #include <sys/types.h>
8 #include <sys/wait.h>
9 
10 #include <cstring>
11 #include <string>
12 
13 #include "atomic.h"
15 #include "cache_posix.h"
16 #include "logging.h"
17 #include "smallhash.h"
18 #include "util/posix.h"
19 #include "util/string.h"
20 
21 bool operator==(const cvmcache_hash &a, const cvmcache_hash &b) {
22  return memcmp(a.digest, b.digest, 20) == 0;
23 }
24 
25 bool operator!=(const cvmcache_hash &a, const cvmcache_hash &b) {
26  return memcmp(a.digest, b.digest, 20) != 0;
27 }
28 
29 namespace {
30 
31 struct CacheObject {
32  uint32_t refcnt;
33  int fd;
34  uint64_t size;
35 };
36 
37 struct Txn {
38  struct cvmcache_hash hash;
39  uint64_t size;
41  void *txn;
42  int fd;
43 };
44 
45 struct Listing {
47  std::vector<std::string> list;
48  std::vector<std::string>::iterator it;
49 };
50 
51 struct Settings {
53  is_alien(false), cache_base_defined(false),
54  cache_dir_defined(false), quota_limit(0)
55  { }
56 
57  bool IsValid() {
58  if (is_alien && quota_limit > 0) {
59  error_reason = "Alien cache cannot be managed (no quota manager allowed)";
60  return false;
61  }
62  if (is_alien && workspace == "") {
63  error_reason = "Workspace option needs to be set for alien cache";
64  return false;
65  }
66  if ((is_alien ? 1 : 0) +
67  (cache_base_defined ? 1 : 0) +
68  (cache_dir_defined ? 1 : 0) != 1) {
69  error_reason = "CVMFS_CACHE_DIR, CVMFS_CACHE_BASE and CVMFS_CACHE_ALIEN "
70  "are mutually exclusive. Exactly one needs to be defined.";
71  return false;
72  }
73  return true;
74  }
75 
76  bool is_alien;
83  int64_t quota_limit;
84  std::string cache_path;
89  std::string workspace;
90 
91  std::string error_reason;
92 };
93 
94 
97  char *optarg = NULL;
98 
99  if ((optarg = cvmcache_options_get(options, "CVMFS_CACHE_QUOTA_LIMIT"))) {
100  settings.quota_limit = String2Int64(optarg) * 1024 * 1024;
101  cvmcache_options_free(optarg);
102  }
103 
104  if ((optarg = cvmcache_options_get(options, "CVMFS_CACHE_BASE"))) {
105  settings.cache_base_defined = true;
106  settings.cache_path = MakeCanonicalPath(optarg);
107  settings.cache_path += "/shared"; // this cache is always shared
108  settings.workspace = settings.cache_path; // default value for workspace
109  cvmcache_options_free(optarg);
110  }
111 
112  if ((optarg = cvmcache_options_get(options, "CVMFS_CACHE_DIR"))) {
113  settings.cache_dir_defined = true;
114  settings.cache_path = optarg;
115  settings.workspace = settings.cache_path; // default value for workspace
116  cvmcache_options_free(optarg);
117  }
118  if ((optarg = cvmcache_options_get(options, "CVMFS_CACHE_ALIEN"))) {
119  settings.is_alien = true;
120  settings.cache_path = optarg;
121  cvmcache_options_free(optarg);
122  }
123 
124  if ((optarg = cvmcache_options_get(options, "CVMFS_CACHE_WORKSPACE"))) {
125  // Used for the shared quota manager
126  settings.workspace = optarg;
127  cvmcache_options_free(optarg);
128  }
129  return settings;
130 }
131 
132 uint32_t cvmcache_hash_hasher(const struct cvmcache_hash &key) {
133  return (uint32_t) *(reinterpret_cast<const uint32_t *>(key.digest) + 1);
134 }
135 
136 uint32_t uint64_hasher(const uint64_t &key) {
137  return (uint32_t) key;
138 }
139 
141  shash::Any hash;
142  memcpy(hash.digest, h->digest, sizeof(h->digest));
143  hash.algorithm = static_cast<shash::Algorithms>(h->algorithm);
144  return hash;
145 }
146 
147 struct cvmcache_hash Cpphash2Chash(const shash::Any &hash) {
148  struct cvmcache_hash h;
149  memset(h.digest, 0, 20); // ensure deterministic digest
150  memcpy(h.digest, hash.digest, sizeof(h.digest));
151  h.algorithm = hash.algorithm;
152  return h;
153 }
154 
161 uint64_t g_pinned_size;
162 uint64_t g_used_size;
163 uint64_t g_capacity;
164 
165 int posix_chrefcnt(struct cvmcache_hash *id, int32_t change_by) {
166  CacheObject object;
167  if (!g_opened_objects->Lookup(*id, &object)) {
168  if (change_by < 0) {
170  }
171  CacheManager::BlessedObject blessed_object(Chash2Cpphash(id));
172  int fd = g_cache_mgr->Open(blessed_object);
173  if (fd < 0) {
175  }
176  object.fd = fd;
177  object.size = g_cache_mgr->GetSize(fd);
178  object.refcnt = 0;
179  g_pinned_size += object.size;
180  } else if (static_cast<int32_t>(object.refcnt) + change_by < 0) {
182  }
183 
184  object.refcnt += change_by;
185  if (object.refcnt == 0) {
186  if (g_cache_mgr->Close(object.fd) != 0) {
187  return CVMCACHE_STATUS_IOERR;
188  }
189  g_pinned_size -= object.size;
190  g_opened_objects->Erase(*id);
191  } else {
192  g_opened_objects->Insert(*id, object);
193  }
194  return CVMCACHE_STATUS_OK;
195 }
196 
197 // Only gives info for opened objects.
198 // Should be fine, since cvmfs only requests info for opened objects.
200  struct cvmcache_object_info *info) {
201  CacheObject object;
202  if (!g_opened_objects->Lookup(*id, &object)) {
204  }
205  info->id = *id;
206  info->size = object.size;
207  return CVMCACHE_STATUS_OK;
208 }
209 
210 int posix_pread(struct cvmcache_hash *id, uint64_t offset,
211  uint32_t *size, unsigned char *buffer) {
212  CacheObject object;
213  if (!g_opened_objects->Lookup(*id, &object)) {
215  }
216  if (offset > object.size) {
218  }
219  int64_t bytes_read = g_cache_mgr->Pread(object.fd, buffer, *size, offset);
220  if (bytes_read < 0) {
221  return CVMCACHE_STATUS_IOERR;
222  }
223  *size = static_cast<uint32_t>(bytes_read);
224  return CVMCACHE_STATUS_OK;
225 }
226 
228  uint64_t txn_id,
229  struct cvmcache_object_info *info) {
230  // cachemgr deletes txn in commit_txn
231  void *txn = malloc(g_cache_mgr->SizeOfTxn());
232  int fd = g_cache_mgr->StartTxn(Chash2Cpphash(id), info->size, txn);
233  if (fd < 0) {
234  return CVMCACHE_STATUS_IOERR;
235  }
236  Txn transaction;
237  transaction.fd = fd;
238  transaction.hash = *id;
239  transaction.txn = txn;
240  transaction.size = info->size;
241  transaction.type = info->type;
242  g_transactions->Insert(txn_id, transaction);
243 
244  CacheManager::ObjectInfo object_info;
245  if (info->type == CVMCACHE_OBJECT_CATALOG) {
246  object_info.type = CacheManager::kTypeCatalog;
247  } else if (info->type == CVMCACHE_OBJECT_VOLATILE) {
248  object_info.type = CacheManager::kTypeVolatile;
249  } else {
250  object_info.type = CacheManager::kTypeRegular;
251  }
252  if (info->description) {
253  object_info.description = info->description;
254  }
255  g_cache_mgr->CtrlTxn(object_info, 0, txn);
256  return CVMCACHE_STATUS_OK;
257 }
258 
259 int posix_write_txn(uint64_t txn_id, unsigned char *buffer,
260  uint32_t size) {
261  Txn transaction;
262  if (!g_transactions->Lookup(txn_id, &transaction)) {
264  }
265  int64_t bytes_written = g_cache_mgr->Write(buffer, size, transaction.txn);
266  if ((bytes_written >= 0) && (static_cast<uint32_t>(bytes_written) == size)) {
267  return CVMCACHE_STATUS_OK;
268  } else {
269  return CVMCACHE_STATUS_IOERR;
270  }
271 }
272 
273 int posix_commit_txn(uint64_t txn_id) {
274  Txn transaction;
275  if (!g_transactions->Lookup(txn_id, &transaction)) {
277  }
278  CacheObject object;
279  if (!g_opened_objects->Lookup(transaction.hash, &object)) {
280  object.fd = g_cache_mgr->OpenFromTxn(transaction.txn);
281  if (object.fd < 0) {
282  return CVMCACHE_STATUS_IOERR;
283  }
284  int result = g_cache_mgr->CommitTxn(transaction.txn);
285  if (result) {
286  return CVMCACHE_STATUS_IOERR;
287  }
288  object.refcnt = 0;
289  object.size = g_cache_mgr->GetSize(object.fd);
290  g_pinned_size += object.size;
291  g_used_size += object.size;
292  } else {
293  if (g_cache_mgr->AbortTxn(transaction.txn) != 0) {
294  return CVMCACHE_STATUS_IOERR;
295  }
296  }
297  object.refcnt += 1;
298 
299  g_opened_objects->Insert(transaction.hash, object);
300  g_transactions->Erase(txn_id);
301  return CVMCACHE_STATUS_OK;
302 }
303 
304 int posix_abort_txn(uint64_t txn_id) {
305  Txn transaction;
306  if (!g_transactions->Lookup(txn_id, &transaction)) {
308  }
309  if (g_cache_mgr->AbortTxn(transaction.txn)) {
310  return CVMCACHE_STATUS_IOERR;
311  }
312  g_transactions->Erase(txn_id);
313  return CVMCACHE_STATUS_OK;
314 }
315 
316 int posix_info(struct cvmcache_info *info) {
317  info->no_shrink = -1;
318  info->size_bytes = g_capacity;
319  info->used_bytes = g_used_size;
320  info->pinned_bytes = g_pinned_size;
321  return CVMCACHE_STATUS_OK;
322 }
323 
324 int posix_breadcrumb_store(const char *fqrn,
325  const cvmcache_breadcrumb *breadcrumb) {
327  breadcrumb->timestamp);
328  if (!g_cache_mgr->StoreBreadcrumb(fqrn, bc)) {
329  return CVMCACHE_STATUS_IOERR;
330  }
331  return CVMCACHE_STATUS_OK;
332 }
333 
334 int posix_breadcrumb_load(const char *fqrn,
335  cvmcache_breadcrumb *breadcrumb) {
337  if (!bc.IsValid()) {
339  }
340  breadcrumb->catalog_hash = Cpphash2Chash(bc.catalog_hash);
341  breadcrumb->timestamp = bc.timestamp;
342  return CVMCACHE_STATUS_OK;
343 }
344 
345 void handle_sigint(int sig) {
347  atomic_inc32(&g_terminated);
348 }
349 
350 } // namespace
351 
352 int main(int argc, char **argv) {
353  if (argc < 2) {
354  fprintf(stderr, "Missing argument: path to config file\n");
355  return 1;
356  }
357 
359 
361  if (cvmcache_options_parse(options, argv[1]) != 0) {
363  "cannot parse options file %s", argv[1]);
364  return 1;
365  }
366  char *debug_log =
367  cvmcache_options_get(options, "CVMFS_CACHE_PLUGIN_DEBUGLOG");
368  if (debug_log != NULL) {
369  SetLogDebugFile(debug_log);
370  cvmcache_options_free(debug_log);
371  } else {
372  SetLogDebugFile("/dev/null");
373  }
374  char *locator = cvmcache_options_get(options, "CVMFS_CACHE_PLUGIN_LOCATOR");
375  if (locator == NULL) {
377  "CVMFS_CACHE_PLUGIN_LOCATOR missing");
378  cvmcache_options_fini(options);
379  return 1;
380  }
381  char *test_mode = cvmcache_options_get(options, "CVMFS_CACHE_PLUGIN_TEST");
382  if (!test_mode) {
383  char *watchdog_crash_dump_path =
384  cvmcache_options_get(options, "CVMFS_CACHE_PLUGIN_CRASH_DUMP");
385  cvmcache_spawn_watchdog(watchdog_crash_dump_path);
386  if (watchdog_crash_dump_path)
387  cvmcache_options_free(watchdog_crash_dump_path);
388  }
389 
390  Settings settings = GetSettings(options);
391  if (!settings.IsValid()) {
393  "Invalid config in file %s: %s", argv[1],
394  settings.error_reason.c_str());
395  return 1;
396  }
397 
398  g_cache_mgr = PosixCacheManager::Create(settings.cache_path,
399  settings.is_alien);
400 
401  cvmcache_hash empty_hash;
402  empty_hash.algorithm = 0;
403  memset(empty_hash.digest, 0, 20);
407  g_opened_objects->Init(32, empty_hash, cvmcache_hash_hasher);
408  g_transactions->Init(32, (uint64_t(-1)), uint64_hasher);
409  g_listings->Init(32, (uint64_t(-1)), uint64_hasher);
410  g_pinned_size = 0;
411  g_used_size = 0;
413 
414  struct cvmcache_callbacks callbacks;
415  memset(&callbacks, 0, sizeof(callbacks));
416  callbacks.cvmcache_chrefcnt = posix_chrefcnt;
417  callbacks.cvmcache_obj_info = posix_obj_info;
418  callbacks.cvmcache_pread = posix_pread;
423  callbacks.cvmcache_info = posix_info;
428 
429  g_ctx = cvmcache_init(&callbacks);
430  int retval = cvmcache_listen(g_ctx, locator);
431  if (!retval) {
433  "failed to listen on %s", locator);
434  return 1;
435  }
436 
437  if (test_mode) {
438  // Daemonize, print out PID
439  pid_t pid;
440  int statloc;
441  if ((pid = fork()) == 0) {
442  if ((pid = fork()) == 0) {
443  int null_read = open("/dev/null", O_RDONLY);
444  int null_write = open("/dev/null", O_WRONLY);
445  assert((null_read >= 0) && (null_write >= 0));
446  int retval = dup2(null_read, 0);
447  assert(retval == 0);
448  retval = dup2(null_write, 1);
449  assert(retval == 1);
450  retval = dup2(null_write, 2);
451  assert(retval == 2);
452  close(null_read);
453  close(null_write);
454  } else {
455  assert(pid > 0);
456  printf("%d\n", pid);
457  fflush(stdout);
458  fsync(1);
459  _exit(0);
460  }
461  } else {
462  assert(pid > 0);
463  waitpid(pid, &statloc, 0);
464  _exit(0);
465  }
466  }
467 
468  LogCvmfs(kLogCache, kLogStdout, "Listening for cvmfs clients on %s", locator);
469 
471 
472  if (!cvmcache_is_supervised()) {
474  "Running unsupervised. Quit by SIGINT (CTRL+C)");
475  atomic_init32(&g_terminated);
476  signal(SIGINT, handle_sigint);
477  while (atomic_read32(&g_terminated) == 0) sleep(1);
478  }
479 
481 
482  delete g_opened_objects;
483  g_opened_objects = NULL;
484  delete g_transactions;
485  g_transactions = NULL;
486  delete g_listings;
487  g_listings = NULL;
488  delete g_cache_mgr;
489  g_cache_mgr = NULL;
490 
491  cvmcache_options_free(locator);
492  cvmcache_options_fini(options);
493 
496  return 0;
497 }
int posix_breadcrumb_store(const char *fqrn, const cvmcache_breadcrumb *breadcrumb)
struct cvmcache_hash catalog_hash
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
int(* cvmcache_chrefcnt)(struct cvmcache_hash *id, int32_t change_by)
struct cvmcache_hash Cpphash2Chash(const shash::Any &hash)
virtual int64_t Write(const void *buf, uint64_t size, void *txn)
Definition: cache_posix.cc:550
SmallHashDynamic< uint64_t, Txn > * g_transactions
cvmcache_object_type
uint64_t timestamp
Definition: manifest.h:33
virtual int CommitTxn(void *txn)
Definition: cache_posix.cc:148
int(* cvmcache_info)(struct cvmcache_info *info)
shash::Any catalog_hash
Definition: manifest.h:32
void cvmcache_terminate(struct cvmcache_context *ctx)
uint32_t cvmcache_hash_hasher(const struct cvmcache_hash &key)
int cvmcache_listen(struct cvmcache_context *ctx, char *locator)
virtual int Open(const BlessedObject &object)
Definition: cache_posix.cc:356
Settings GetSettings(cvmcache_option_map *options)
struct cvmcache_hash id
void cvmcache_cleanup_global()
virtual int AbortTxn(void *txn)
Definition: cache_posix.cc:113
void cvmcache_terminate_watchdog()
int posix_pread(struct cvmcache_hash *id, uint64_t offset, uint32_t *size, unsigned char *buffer)
const SettingsPublisher & settings() const
Definition: repository.h:306
virtual int64_t GetSize(int fd)
Definition: cache_posix.cc:347
int posix_breadcrumb_load(const char *fqrn, cvmcache_breadcrumb *breadcrumb)
assert((mem||(size==0))&&"Out Of Memory")
uint64_t pinned_bytes
int cvmcache_is_supervised()
Algorithms algorithm
Definition: hash.h:123
int(* cvmcache_pread)(struct cvmcache_hash *id, uint64_t offset, uint32_t *size, unsigned char *buffer)
int(* cvmcache_breadcrumb_load)(const char *fqrn, cvmcache_breadcrumb *breadcrumb)
int posix_info(struct cvmcache_info *info)
std::string description
Definition: cache.h:97
#define SetLogDebugFile(filename)
unsigned char digest[digest_size_]
Definition: hash.h:122
virtual int OpenFromTxn(void *txn)
Definition: cache_posix.cc:372
void cvmcache_wait_for(struct cvmcache_context *ctx)
uint64_t used_bytes
Algorithms
Definition: hash.h:39
int main()
Definition: helper_allow.cc:16
int32_t atomic_int32
Definition: atomic.h:17
int64_t String2Int64(const string &value)
Definition: string.cc:221
int64_t no_shrink
virtual void CtrlTxn(const ObjectInfo &object_info, const int flags, void *txn)
Definition: cache_posix.cc:277
int(* cvmcache_write_txn)(uint64_t txn_id, unsigned char *buffer, uint32_t size)
ObjectType type
Definition: cache.h:93
virtual int StartTxn(const shash::Any &id, uint64_t size, void *txn)
Definition: cache_posix.cc:457
virtual manifest::Breadcrumb LoadBreadcrumb(const std::string &fqrn)
Definition: cache_posix.cc:522
SmallHashDynamic< struct cvmcache_hash, CacheObject > * g_opened_objects
unsigned char digest[20]
SmallHashDynamic< uint64_t, Listing > * g_listings
bool operator!=(const cvmcache_hash &a, const cvmcache_hash &b)
uint32_t uint64_hasher(const uint64_t &key)
void cvmcache_options_free(char *value)
void cvmcache_spawn_watchdog(const char *crash_dump_file)
void cvmcache_options_fini(cvmcache_option_map *opts)
int posix_start_txn(struct cvmcache_hash *id, uint64_t txn_id, struct cvmcache_object_info *info)
uint64_t size_bytes
enum cvmcache_object_type type
struct cvmcache_context * cvmcache_init(struct cvmcache_callbacks *callbacks)
void cvmcache_init_global()
int cvmcache_options_parse(cvmcache_option_map *opts, const char *path)
#define CVMCACHE_SIZE_UNKNOWN
bool operator==(const cvmcache_hash &a, const cvmcache_hash &b)
void cvmcache_process_requests(struct cvmcache_context *ctx, unsigned nworkers)
int(* cvmcache_breadcrumb_store)(const char *fqrn, const cvmcache_breadcrumb *breadcrumb)
int posix_write_txn(uint64_t txn_id, unsigned char *buffer, uint32_t size)
char * cvmcache_options_get(cvmcache_option_map *opts, const char *key)
int(* cvmcache_commit_txn)(uint64_t txn_id)
int(* cvmcache_start_txn)(struct cvmcache_hash *id, uint64_t txn_id, struct cvmcache_object_info *info)
bool IsValid() const
Definition: manifest.h:30
int(* cvmcache_abort_txn)(uint64_t txn_id)
int posix_chrefcnt(struct cvmcache_hash *id, int32_t change_by)
std::string MakeCanonicalPath(const std::string &path)
Definition: posix.cc:95
int posix_obj_info(struct cvmcache_hash *id, struct cvmcache_object_info *info)
virtual bool StoreBreadcrumb(const manifest::Manifest &manifest)
Definition: cache_posix.cc:528
int(* cvmcache_obj_info)(struct cvmcache_hash *id, struct cvmcache_object_info *info)
shash::Any Chash2Cpphash(const struct cvmcache_hash *h)
static void size_t size
Definition: smalloc.h:47
virtual int Close(int fd)
Definition: cache_posix.cc:140
virtual int64_t Pread(int fd, void *buf, uint64_t size, uint64_t offset)
Definition: cache_posix.cc:384
virtual uint32_t SizeOfTxn()
Definition: cache_posix.h:85
cvmcache_option_map * cvmcache_options_init()
static PosixCacheManager * Create(const std::string &cache_path, const bool alien_cache, const RenameWorkarounds rename_workaround=kRenameNormal)
Definition: cache_posix.cc:230