CernVM-FS  2.11.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
cvmfs_cache_posix.cc
Go to the documentation of this file.
1 
6 #include <signal.h>
7 #include <sys/types.h>
8 #include <sys/wait.h>
9 
10 #include <cstring>
11 #include <string>
12 
14 #include "cache_posix.h"
15 #include "smallhash.h"
16 #include "util/atomic.h"
17 #include "util/logging.h"
18 #include "util/posix.h"
19 #include "util/string.h"
20 
21 bool operator==(const cvmcache_hash &a, const cvmcache_hash &b) {
22  return memcmp(a.digest, b.digest, 20) == 0;
23 }
24 
25 bool operator!=(const cvmcache_hash &a, const cvmcache_hash &b) {
26  return memcmp(a.digest, b.digest, 20) != 0;
27 }
28 
29 namespace {
30 
31 struct CacheObject {
32  uint32_t refcnt;
33  int fd;
34  uint64_t size;
35 };
36 
37 struct Txn {
38  struct cvmcache_hash hash;
39  uint64_t size;
41  void *txn;
42  int fd;
43 };
44 
45 struct Listing {
47  std::vector<std::string> list;
48  std::vector<std::string>::iterator it;
49 };
50 
51 struct Settings {
53  is_alien(false), cache_base_defined(false),
54  cache_dir_defined(false), quota_limit(0)
55  { }
56 
57  bool IsValid() {
58  if (is_alien && quota_limit > 0) {
59  error_reason = "Alien cache cannot be managed (no quota manager allowed)";
60  return false;
61  }
62  if (is_alien && workspace == "") {
63  error_reason = "Workspace option needs to be set for alien cache";
64  return false;
65  }
66  if ((is_alien ? 1 : 0) +
67  (cache_base_defined ? 1 : 0) +
68  (cache_dir_defined ? 1 : 0) != 1) {
69  error_reason = "CVMFS_CACHE_DIR, CVMFS_CACHE_BASE and CVMFS_CACHE_ALIEN "
70  "are mutually exclusive. Exactly one needs to be defined.";
71  return false;
72  }
73  return true;
74  }
75 
76  bool is_alien;
83  int64_t quota_limit;
84  std::string cache_path;
89  std::string workspace;
90 
91  std::string error_reason;
92 };
93 
94 
97  char *optarg = NULL;
98 
99  if ((optarg = cvmcache_options_get(options, "CVMFS_CACHE_QUOTA_LIMIT"))) {
100  settings.quota_limit = String2Int64(optarg) * 1024 * 1024;
101  cvmcache_options_free(optarg);
102  }
103 
104  if ((optarg = cvmcache_options_get(options, "CVMFS_CACHE_BASE"))) {
105  settings.cache_base_defined = true;
106  settings.cache_path = MakeCanonicalPath(optarg);
107  settings.cache_path += "/shared"; // this cache is always shared
108  settings.workspace = settings.cache_path; // default value for workspace
109  cvmcache_options_free(optarg);
110  }
111 
112  if ((optarg = cvmcache_options_get(options, "CVMFS_CACHE_DIR"))) {
113  settings.cache_dir_defined = true;
114  settings.cache_path = optarg;
115  settings.workspace = settings.cache_path; // default value for workspace
116  cvmcache_options_free(optarg);
117  }
118  if ((optarg = cvmcache_options_get(options, "CVMFS_CACHE_ALIEN"))) {
119  settings.is_alien = true;
120  settings.cache_path = optarg;
121  cvmcache_options_free(optarg);
122  }
123 
124  if ((optarg = cvmcache_options_get(options, "CVMFS_CACHE_WORKSPACE"))) {
125  // Used for the shared quota manager
126  settings.workspace = optarg;
127  cvmcache_options_free(optarg);
128  }
129  return settings;
130 }
131 
132 uint32_t cvmcache_hash_hasher(const struct cvmcache_hash &key) {
133  return (uint32_t) *(reinterpret_cast<const uint32_t *>(key.digest) + 1);
134 }
135 
136 uint32_t uint64_hasher(const uint64_t &key) {
137  return (uint32_t) key;
138 }
139 
141  shash::Any hash;
142  memcpy(hash.digest, h->digest, sizeof(h->digest));
143  hash.algorithm = static_cast<shash::Algorithms>(h->algorithm);
144  return hash;
145 }
146 
147 struct cvmcache_hash Cpphash2Chash(const shash::Any &hash) {
148  struct cvmcache_hash h;
149  memset(h.digest, 0, 20); // ensure deterministic digest
150  memcpy(h.digest, hash.digest, sizeof(h.digest));
151  h.algorithm = hash.algorithm;
152  return h;
153 }
154 
161 uint64_t g_pinned_size;
162 uint64_t g_used_size;
163 uint64_t g_capacity;
164 
165 int posix_chrefcnt(struct cvmcache_hash *id, int32_t change_by) {
166  CacheObject object;
167  if (!g_opened_objects->Lookup(*id, &object)) {
168  if (change_by < 0) {
170  }
171  CacheManager::LabeledObject labeled_object(Chash2Cpphash(id));
172  int fd = g_cache_mgr->Open(labeled_object);
173  if (fd < 0) {
175  }
176  object.fd = fd;
177  object.size = g_cache_mgr->GetSize(fd);
178  object.refcnt = 0;
179  g_pinned_size += object.size;
180  } else if (static_cast<int32_t>(object.refcnt) + change_by < 0) {
182  }
183 
184  object.refcnt += change_by;
185  if (object.refcnt == 0) {
186  if (g_cache_mgr->Close(object.fd) != 0) {
187  return CVMCACHE_STATUS_IOERR;
188  }
189  g_pinned_size -= object.size;
190  g_opened_objects->Erase(*id);
191  } else {
192  g_opened_objects->Insert(*id, object);
193  }
194  return CVMCACHE_STATUS_OK;
195 }
196 
197 // Only gives info for opened objects.
198 // Should be fine, since cvmfs only requests info for opened objects.
200  struct cvmcache_object_info *info) {
201  CacheObject object;
202  if (!g_opened_objects->Lookup(*id, &object)) {
204  }
205  info->id = *id;
206  info->size = object.size;
207  return CVMCACHE_STATUS_OK;
208 }
209 
210 int posix_pread(struct cvmcache_hash *id, uint64_t offset,
211  uint32_t *size, unsigned char *buffer) {
212  CacheObject object;
213  if (!g_opened_objects->Lookup(*id, &object)) {
215  }
216  if (offset > object.size) {
218  }
219  int64_t bytes_read = g_cache_mgr->Pread(object.fd, buffer, *size, offset);
220  if (bytes_read < 0) {
221  return CVMCACHE_STATUS_IOERR;
222  }
223  *size = static_cast<uint32_t>(bytes_read);
224  return CVMCACHE_STATUS_OK;
225 }
226 
228  uint64_t txn_id,
229  struct cvmcache_object_info *info) {
230  // cachemgr deletes txn in commit_txn
231  void *txn = malloc(g_cache_mgr->SizeOfTxn());
232  int fd = g_cache_mgr->StartTxn(Chash2Cpphash(id), info->size, txn);
233  if (fd < 0) {
234  return CVMCACHE_STATUS_IOERR;
235  }
236  Txn transaction;
237  transaction.fd = fd;
238  transaction.hash = *id;
239  transaction.txn = txn;
240  transaction.size = info->size;
241  transaction.type = info->type;
242  g_transactions->Insert(txn_id, transaction);
243 
244  CacheManager::Label label;
245  if (info->type == CVMCACHE_OBJECT_CATALOG) {
247  } else if (info->type == CVMCACHE_OBJECT_VOLATILE) {
249  }
250  if (info->description) {
251  label.path = info->description;
252  }
253  g_cache_mgr->CtrlTxn(label, 0, txn);
254  return CVMCACHE_STATUS_OK;
255 }
256 
257 int posix_write_txn(uint64_t txn_id, unsigned char *buffer,
258  uint32_t size) {
259  Txn transaction;
260  if (!g_transactions->Lookup(txn_id, &transaction)) {
262  }
263  int64_t bytes_written = g_cache_mgr->Write(buffer, size, transaction.txn);
264  if ((bytes_written >= 0) && (static_cast<uint32_t>(bytes_written) == size)) {
265  return CVMCACHE_STATUS_OK;
266  } else {
267  return CVMCACHE_STATUS_IOERR;
268  }
269 }
270 
271 int posix_commit_txn(uint64_t txn_id) {
272  Txn transaction;
273  if (!g_transactions->Lookup(txn_id, &transaction)) {
275  }
276  CacheObject object;
277  if (!g_opened_objects->Lookup(transaction.hash, &object)) {
278  object.fd = g_cache_mgr->OpenFromTxn(transaction.txn);
279  if (object.fd < 0) {
280  return CVMCACHE_STATUS_IOERR;
281  }
282  int result = g_cache_mgr->CommitTxn(transaction.txn);
283  if (result) {
284  return CVMCACHE_STATUS_IOERR;
285  }
286  object.refcnt = 0;
287  object.size = g_cache_mgr->GetSize(object.fd);
288  g_pinned_size += object.size;
289  g_used_size += object.size;
290  } else {
291  if (g_cache_mgr->AbortTxn(transaction.txn) != 0) {
292  return CVMCACHE_STATUS_IOERR;
293  }
294  }
295  object.refcnt += 1;
296 
297  g_opened_objects->Insert(transaction.hash, object);
298  g_transactions->Erase(txn_id);
299  return CVMCACHE_STATUS_OK;
300 }
301 
302 int posix_abort_txn(uint64_t txn_id) {
303  Txn transaction;
304  if (!g_transactions->Lookup(txn_id, &transaction)) {
306  }
307  if (g_cache_mgr->AbortTxn(transaction.txn)) {
308  return CVMCACHE_STATUS_IOERR;
309  }
310  g_transactions->Erase(txn_id);
311  return CVMCACHE_STATUS_OK;
312 }
313 
314 int posix_info(struct cvmcache_info *info) {
315  info->no_shrink = -1;
316  info->size_bytes = g_capacity;
317  info->used_bytes = g_used_size;
318  info->pinned_bytes = g_pinned_size;
319  return CVMCACHE_STATUS_OK;
320 }
321 
322 int posix_breadcrumb_store(const char *fqrn,
323  const cvmcache_breadcrumb *breadcrumb) {
325  breadcrumb->timestamp);
326  if (!g_cache_mgr->StoreBreadcrumb(fqrn, bc)) {
327  return CVMCACHE_STATUS_IOERR;
328  }
329  return CVMCACHE_STATUS_OK;
330 }
331 
332 int posix_breadcrumb_load(const char *fqrn,
333  cvmcache_breadcrumb *breadcrumb) {
335  if (!bc.IsValid()) {
337  }
338  breadcrumb->catalog_hash = Cpphash2Chash(bc.catalog_hash);
339  breadcrumb->timestamp = bc.timestamp;
340  return CVMCACHE_STATUS_OK;
341 }
342 
343 void handle_sigint(int sig) {
345  atomic_inc32(&g_terminated);
346 }
347 
348 } // namespace
349 
350 int main(int argc, char **argv) {
351  if (argc < 2) {
352  fprintf(stderr, "Missing argument: path to config file\n");
353  return 1;
354  }
355 
357 
359  if (cvmcache_options_parse(options, argv[1]) != 0) {
361  "cannot parse options file %s", argv[1]);
362  return 1;
363  }
364  char *debug_log =
365  cvmcache_options_get(options, "CVMFS_CACHE_PLUGIN_DEBUGLOG");
366  if (debug_log != NULL) {
367  SetLogDebugFile(debug_log);
368  cvmcache_options_free(debug_log);
369  } else {
370  SetLogDebugFile("/dev/null");
371  }
372  char *locator = cvmcache_options_get(options, "CVMFS_CACHE_PLUGIN_LOCATOR");
373  if (locator == NULL) {
375  "CVMFS_CACHE_PLUGIN_LOCATOR missing");
376  cvmcache_options_fini(options);
377  return 1;
378  }
379  char *test_mode = cvmcache_options_get(options, "CVMFS_CACHE_PLUGIN_TEST");
380  if (!test_mode) {
381  char *watchdog_crash_dump_path =
382  cvmcache_options_get(options, "CVMFS_CACHE_PLUGIN_CRASH_DUMP");
383  cvmcache_spawn_watchdog(watchdog_crash_dump_path);
384  if (watchdog_crash_dump_path)
385  cvmcache_options_free(watchdog_crash_dump_path);
386  }
387 
388  Settings settings = GetSettings(options);
389  if (!settings.IsValid()) {
391  "Invalid config in file %s: %s", argv[1],
392  settings.error_reason.c_str());
393  return 1;
394  }
395 
396  g_cache_mgr = PosixCacheManager::Create(settings.cache_path,
397  settings.is_alien);
398 
399  cvmcache_hash empty_hash;
400  empty_hash.algorithm = 0;
401  memset(empty_hash.digest, 0, 20);
405  g_opened_objects->Init(32, empty_hash, cvmcache_hash_hasher);
406  g_transactions->Init(32, (uint64_t(-1)), uint64_hasher);
407  g_listings->Init(32, (uint64_t(-1)), uint64_hasher);
408  g_pinned_size = 0;
409  g_used_size = 0;
411 
412  struct cvmcache_callbacks callbacks;
413  memset(&callbacks, 0, sizeof(callbacks));
414  callbacks.cvmcache_chrefcnt = posix_chrefcnt;
415  callbacks.cvmcache_obj_info = posix_obj_info;
416  callbacks.cvmcache_pread = posix_pread;
421  callbacks.cvmcache_info = posix_info;
426 
427  g_ctx = cvmcache_init(&callbacks);
428  int retval = cvmcache_listen(g_ctx, locator);
429  if (!retval) {
431  "failed to listen on %s", locator);
432  return 1;
433  }
434 
435  if (test_mode) {
436  // Daemonize, print out PID
437  pid_t pid;
438  int statloc;
439  if ((pid = fork()) == 0) {
440  if ((pid = fork()) == 0) {
441  int null_read = open("/dev/null", O_RDONLY);
442  int null_write = open("/dev/null", O_WRONLY);
443  assert((null_read >= 0) && (null_write >= 0));
444  int retval = dup2(null_read, 0);
445  assert(retval == 0);
446  retval = dup2(null_write, 1);
447  assert(retval == 1);
448  retval = dup2(null_write, 2);
449  assert(retval == 2);
450  close(null_read);
451  close(null_write);
452  } else {
453  assert(pid > 0);
454  printf("%d\n", pid);
455  fflush(stdout);
456  fsync(1);
457  _exit(0);
458  }
459  } else {
460  assert(pid > 0);
461  waitpid(pid, &statloc, 0);
462  _exit(0);
463  }
464  }
465 
466  LogCvmfs(kLogCache, kLogStdout, "Listening for cvmfs clients on %s", locator);
467 
469 
470  if (!cvmcache_is_supervised()) {
472  "Running unsupervised. Quit by SIGINT (CTRL+C)");
473  atomic_init32(&g_terminated);
474  signal(SIGINT, handle_sigint);
475  while (atomic_read32(&g_terminated) == 0) sleep(1);
476  }
477 
479 
480  delete g_opened_objects;
481  g_opened_objects = NULL;
482  delete g_transactions;
483  g_transactions = NULL;
484  delete g_listings;
485  g_listings = NULL;
486  delete g_cache_mgr;
487  g_cache_mgr = NULL;
488 
489  cvmcache_options_free(locator);
490  cvmcache_options_fini(options);
491 
494  return 0;
495 }
int posix_breadcrumb_store(const char *fqrn, const cvmcache_breadcrumb *breadcrumb)
struct cvmcache_hash catalog_hash
#define LogCvmfs(source, mask,...)
Definition: logging.h:25
int(* cvmcache_chrefcnt)(struct cvmcache_hash *id, int32_t change_by)
struct cvmcache_hash Cpphash2Chash(const shash::Any &hash)
virtual int64_t Write(const void *buf, uint64_t size, void *txn)
Definition: cache_posix.cc:624
SmallHashDynamic< uint64_t, Txn > * g_transactions
virtual int Open(const LabeledObject &object)
Definition: cache_posix.cc:414
cvmcache_object_type
uint64_t timestamp
Definition: manifest.h:33
virtual int CommitTxn(void *txn)
Definition: cache_posix.cc:148
int(* cvmcache_info)(struct cvmcache_info *info)
shash::Any catalog_hash
Definition: manifest.h:32
void cvmcache_terminate(struct cvmcache_context *ctx)
static const int kLabelCatalog
Definition: cache.h:80
uint32_t cvmcache_hash_hasher(const struct cvmcache_hash &key)
int cvmcache_listen(struct cvmcache_context *ctx, char *locator)
Settings GetSettings(cvmcache_option_map *options)
struct cvmcache_hash id
void cvmcache_cleanup_global()
virtual int AbortTxn(void *txn)
Definition: cache_posix.cc:113
void cvmcache_terminate_watchdog()
int posix_pread(struct cvmcache_hash *id, uint64_t offset, uint32_t *size, unsigned char *buffer)
const SettingsPublisher & settings() const
Definition: repository.h:317
virtual int64_t GetSize(int fd)
Definition: cache_posix.cc:405
int posix_breadcrumb_load(const char *fqrn, cvmcache_breadcrumb *breadcrumb)
assert((mem||(size==0))&&"Out Of Memory")
uint64_t pinned_bytes
virtual void CtrlTxn(const Label &label, const int flags, void *txn)
Definition: cache_posix.cc:293
int cvmcache_is_supervised()
Algorithms algorithm
Definition: hash.h:125
int(* cvmcache_pread)(struct cvmcache_hash *id, uint64_t offset, uint32_t *size, unsigned char *buffer)
int(* cvmcache_breadcrumb_load)(const char *fqrn, cvmcache_breadcrumb *breadcrumb)
int posix_info(struct cvmcache_info *info)
#define SetLogDebugFile(filename)
unsigned char digest[digest_size_]
Definition: hash.h:124
virtual int OpenFromTxn(void *txn)
Definition: cache_posix.cc:434
void cvmcache_wait_for(struct cvmcache_context *ctx)
uint64_t used_bytes
Algorithms
Definition: hash.h:41
int main()
Definition: helper_allow.cc:16
int32_t atomic_int32
Definition: atomic.h:17
int64_t String2Int64(const string &value)
Definition: string.cc:222
int64_t no_shrink
int(* cvmcache_write_txn)(uint64_t txn_id, unsigned char *buffer, uint32_t size)
virtual int StartTxn(const shash::Any &id, uint64_t size, void *txn)
Definition: cache_posix.cc:531
virtual manifest::Breadcrumb LoadBreadcrumb(const std::string &fqrn)
Definition: cache_posix.cc:596
SmallHashDynamic< struct cvmcache_hash, CacheObject > * g_opened_objects
unsigned char digest[20]
SmallHashDynamic< uint64_t, Listing > * g_listings
bool operator!=(const cvmcache_hash &a, const cvmcache_hash &b)
uint32_t uint64_hasher(const uint64_t &key)
void cvmcache_options_free(char *value)
void cvmcache_spawn_watchdog(const char *crash_dump_file)
void cvmcache_options_fini(cvmcache_option_map *opts)
int posix_start_txn(struct cvmcache_hash *id, uint64_t txn_id, struct cvmcache_object_info *info)
uint64_t size_bytes
enum cvmcache_object_type type
struct cvmcache_context * cvmcache_init(struct cvmcache_callbacks *callbacks)
void cvmcache_init_global()
static const int kLabelVolatile
Definition: cache.h:82
int cvmcache_options_parse(cvmcache_option_map *opts, const char *path)
#define CVMCACHE_SIZE_UNKNOWN
bool operator==(const cvmcache_hash &a, const cvmcache_hash &b)
void cvmcache_process_requests(struct cvmcache_context *ctx, unsigned nworkers)
int(* cvmcache_breadcrumb_store)(const char *fqrn, const cvmcache_breadcrumb *breadcrumb)
int posix_write_txn(uint64_t txn_id, unsigned char *buffer, uint32_t size)
char * cvmcache_options_get(cvmcache_option_map *opts, const char *key)
int(* cvmcache_commit_txn)(uint64_t txn_id)
int(* cvmcache_start_txn)(struct cvmcache_hash *id, uint64_t txn_id, struct cvmcache_object_info *info)
bool IsValid() const
Definition: manifest.h:30
int(* cvmcache_abort_txn)(uint64_t txn_id)
std::string path
Definition: cache.h:133
static PosixCacheManager * Create(const std::string &cache_path, const bool alien_cache, const RenameWorkarounds rename_workaround=kRenameNormal, const bool do_refcount=false)
Definition: cache_posix.cc:272
int posix_chrefcnt(struct cvmcache_hash *id, int32_t change_by)
std::string MakeCanonicalPath(const std::string &path)
Definition: posix.cc:98
int posix_obj_info(struct cvmcache_hash *id, struct cvmcache_object_info *info)
virtual bool StoreBreadcrumb(const manifest::Manifest &manifest)
Definition: cache_posix.cc:602
int(* cvmcache_obj_info)(struct cvmcache_hash *id, struct cvmcache_object_info *info)
shash::Any Chash2Cpphash(const struct cvmcache_hash *h)
static void size_t size
Definition: smalloc.h:54
virtual int Close(int fd)
Definition: cache_posix.cc:140
virtual int64_t Pread(int fd, void *buf, uint64_t size, uint64_t offset)
Definition: cache_posix.cc:452
virtual uint32_t SizeOfTxn()
Definition: cache_posix.h:87
cvmcache_option_map * cvmcache_options_init()