CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
cvmfs_cache_null.cc
Go to the documentation of this file.
1 
8 #define __STDC_FORMAT_MACROS
9 
10 #include <alloca.h>
11 #include <fcntl.h>
12 #include <inttypes.h>
13 #include <stdint.h>
14 #include <sys/types.h>
15 #include <sys/wait.h>
16 #include <unistd.h>
17 
18 #include <cassert>
19 #include <cstdio>
20 #include <cstring>
21 #include <ctime>
22 #include <map>
23 #include <string>
24 #include <vector>
25 
26 #include "libcvmfs_cache.h"
27 
28 using namespace std; // NOLINT
29 
30 struct Object {
31  struct cvmcache_hash id;
32  string data;
34  int32_t refcnt;
35  string description;
36 };
37 
38 struct TxnInfo {
39  struct cvmcache_hash id;
41 };
42 
43 struct Listing {
44  Listing() : type(CVMCACHE_OBJECT_REGULAR), pos(0), elems(NULL) { }
46  uint64_t pos;
47  vector<Object> *elems;
48 };
49 
51  ComparableHash() { memset(&hash, 0, sizeof(hash)); }
52  explicit ComparableHash(const struct cvmcache_hash &h) : hash(h) { }
53  struct cvmcache_hash hash;
54  bool operator <(const ComparableHash &other) const {
55  return cvmcache_hash_cmp(const_cast<cvmcache_hash *>(&(this->hash)),
56  const_cast<cvmcache_hash *>(&(other.hash))) < 0;
57  }
58 };
59 
60 map<uint64_t, TxnInfo> transactions;
61 map<ComparableHash, Object> storage;
62 map<uint64_t, Listing> listings;
63 map<std::string, cvmcache_breadcrumb> breadcrumbs;
64 
66 
67 static int null_chrefcnt(struct cvmcache_hash *id, int32_t change_by) {
68  ComparableHash h(*id);
69  if (storage.find(h) == storage.end())
71 
72  Object obj = storage[h];
73  obj.refcnt += change_by;
74  if (obj.refcnt < 0)
76  storage[h] = obj;
77  return CVMCACHE_STATUS_OK;
78 }
79 
80 
81 static int null_obj_info(
82  struct cvmcache_hash *id,
83  struct cvmcache_object_info *info)
84 {
85  ComparableHash h(*id);
86  if (storage.find(h) == storage.end())
88 
89  Object obj = storage[h];
90  info->size = obj.data.length();
91  info->type = obj.type;
92  info->pinned = obj.refcnt > 0;
93  info->description = strdup(obj.description.c_str());
94  return CVMCACHE_STATUS_OK;
95 }
96 
97 
98 static int null_pread(struct cvmcache_hash *id,
99  uint64_t offset,
100  uint32_t *size,
101  unsigned char *buffer)
102 {
103  ComparableHash h(*id);
104  string data = storage[h].data;
105  if (offset > data.length())
107  unsigned nbytes =
108  std::min(*size, static_cast<uint32_t>(data.length() - offset));
109  memcpy(buffer, data.data() + offset, nbytes);
110  *size = nbytes;
111  return CVMCACHE_STATUS_OK;
112 }
113 
114 
115 static int null_start_txn(
116  struct cvmcache_hash *id,
117  uint64_t txn_id,
118  struct cvmcache_object_info *info)
119 {
120  Object partial_object;
121  partial_object.id = *id;
122  partial_object.type = info->type;
123  partial_object.refcnt = 1;
124  if (info->description != NULL)
125  partial_object.description = string(info->description);
126  TxnInfo txn;
127  txn.id = *id;
128  txn.partial_object = partial_object;
129  transactions[txn_id] = txn;
130  return CVMCACHE_STATUS_OK;
131 }
132 
133 
134 static int null_write_txn(
135  uint64_t txn_id,
136  unsigned char *buffer,
137  uint32_t size)
138 {
139  TxnInfo txn = transactions[txn_id];
140  txn.partial_object.data += string(reinterpret_cast<char *>(buffer), size);
141  transactions[txn_id] = txn;
142  return CVMCACHE_STATUS_OK;
143 }
144 
145 
146 static int null_commit_txn(uint64_t txn_id) {
147  TxnInfo txn = transactions[txn_id];
148  ComparableHash h(txn.id);
149  storage[h] = txn.partial_object;
150  transactions.erase(txn_id);
151  return CVMCACHE_STATUS_OK;
152 }
153 
154 static int null_abort_txn(uint64_t txn_id) {
155  transactions.erase(txn_id);
156  return CVMCACHE_STATUS_OK;
157 }
158 
159 static int null_info(struct cvmcache_info *info) {
160  info->size_bytes = uint64_t(-1);
161  info->used_bytes = info->pinned_bytes = 0;
162  for (map<ComparableHash, Object>::const_iterator i = storage.begin(),
163  i_end = storage.end(); i != i_end; ++i)
164  {
165  info->used_bytes += i->second.data.length();
166  if (i->second.refcnt > 0)
167  info->pinned_bytes += i->second.data.length();
168  }
169  info->no_shrink = 0;
170  return CVMCACHE_STATUS_OK;
171 }
172 
173 static int null_shrink(uint64_t shrink_to, uint64_t *used) {
174  struct cvmcache_info info;
175  null_info(&info);
176  *used = info.used_bytes;
177  if (info.used_bytes <= shrink_to)
178  return CVMCACHE_STATUS_OK;
179 
180  // Volatile objects
181  for (map<ComparableHash, Object>::iterator i = storage.begin(),
182  i_end = storage.end(); i != i_end; )
183  {
184  if ((i->second.refcnt > 0) || (i->second.type != CVMCACHE_OBJECT_VOLATILE))
185  {
186  ++i;
187  continue;
188  }
189  unsigned length = i->second.data.length();
190  map<ComparableHash, Object>::iterator delete_me = i++;
191  storage.erase(delete_me);
192  info.used_bytes -= length;
193  if (info.used_bytes <= shrink_to) {
194  *used = info.used_bytes;
195  return CVMCACHE_STATUS_OK;
196  }
197  }
198  // All other objects
199  for (map<ComparableHash, Object>::iterator i = storage.begin(),
200  i_end = storage.end(); i != i_end; )
201  {
202  if (i->second.refcnt > 0) {
203  ++i;
204  continue;
205  }
206  unsigned length = i->second.data.length();
207  map<ComparableHash, Object>::iterator delete_me = i++;
208  storage.erase(delete_me);
209  info.used_bytes -= length;
210  if (info.used_bytes <= shrink_to) {
211  *used = info.used_bytes;
212  return CVMCACHE_STATUS_OK;
213  }
214  }
215 
216  *used = info.used_bytes;
218 }
219 
221  uint64_t lst_id,
223 {
224  Listing lst;
225  lst.type = type;
226  lst.elems = new vector<Object>();
227  for (map<ComparableHash, Object>::const_iterator i = storage.begin(),
228  i_end = storage.end(); i != i_end; ++i)
229  {
230  lst.elems->push_back(i->second);
231  }
232  listings[lst_id] = lst;
233  return CVMCACHE_STATUS_OK;
234 }
235 
236 static int null_listing_next(
237  int64_t listing_id,
238  struct cvmcache_object_info *item)
239 {
240  Listing lst = listings[listing_id];
241  do {
242  if (lst.pos >= lst.elems->size())
244 
245  vector<Object> *elems = lst.elems;
246  if ((*elems)[lst.pos].type == lst.type) {
247  item->id = (*elems)[lst.pos].id;
248  item->size = (*elems)[lst.pos].data.length();
249  item->type = (*elems)[lst.pos].type;
250  item->pinned = (*elems)[lst.pos].refcnt > 0;
251  item->description = (*elems)[lst.pos].description.empty()
252  ? NULL
253  : strdup((*elems)[lst.pos].description.c_str());
254  break;
255  }
256  lst.pos++;
257  } while (true);
258  lst.pos++;
259  listings[listing_id] = lst;
260  return CVMCACHE_STATUS_OK;
261 }
262 
263 static int null_listing_end(int64_t listing_id) {
264  delete listings[listing_id].elems;
265  listings.erase(listing_id);
266  return CVMCACHE_STATUS_OK;
267 }
268 
270  const char *fqrn,
271  const cvmcache_breadcrumb *breadcrumb)
272 {
273  breadcrumbs[fqrn] = *breadcrumb;
274  return CVMCACHE_STATUS_OK;
275 }
276 
278  const char *fqrn,
279  cvmcache_breadcrumb *breadcrumb)
280 {
281  map<std::string, cvmcache_breadcrumb>::const_iterator itr =
282  breadcrumbs.find(fqrn);
283  if (itr == breadcrumbs.end())
285  *breadcrumb = itr->second;
286  return CVMCACHE_STATUS_OK;
287 }
288 
289 static void Usage(const char *progname) {
290  printf("%s <config file>\n", progname);
291 }
292 
293 
294 int main(int argc, char **argv) {
295  if (argc < 2) {
296  Usage(argv[0]);
297  return 1;
298  }
299 
301 
303  if (cvmcache_options_parse(options, argv[1]) != 0) {
304  printf("cannot parse options file %s\n", argv[1]);
305  return 1;
306  }
307  char *locator = cvmcache_options_get(options, "CVMFS_CACHE_PLUGIN_LOCATOR");
308  if (locator == NULL) {
309  printf("CVMFS_CACHE_PLUGIN_LOCATOR missing\n");
310  cvmcache_options_fini(options);
311  return 1;
312  }
313  char *test_mode = cvmcache_options_get(options, "CVMFS_CACHE_PLUGIN_TEST");
314 
315  if (!test_mode)
317 
318  struct cvmcache_callbacks callbacks;
319  memset(&callbacks, 0, sizeof(callbacks));
320  callbacks.cvmcache_chrefcnt = null_chrefcnt;
321  callbacks.cvmcache_obj_info = null_obj_info;
322  callbacks.cvmcache_pread = null_pread;
327  callbacks.cvmcache_info = null_info;
328  callbacks.cvmcache_shrink = null_shrink;
334  callbacks.capabilities = CVMCACHE_CAP_ALL_V2;
335 
336  ctx = cvmcache_init(&callbacks);
337  int retval = cvmcache_listen(ctx, locator);
338  if (!retval) {
339  fprintf(stderr, "failed to listen on %s\n", locator);
340  return 1;
341  }
342 
343  if (test_mode) {
344  // Daemonize, print out PID
345  pid_t pid;
346  int statloc;
347  if ((pid = fork()) == 0) {
348  if ((pid = fork()) == 0) {
349  int null_read = open("/dev/null", O_RDONLY);
350  int null_write = open("/dev/null", O_WRONLY);
351  assert((null_read >= 0) && (null_write >= 0));
352  int retval = dup2(null_read, 0);
353  assert(retval == 0);
354  retval = dup2(null_write, 1);
355  assert(retval == 1);
356  retval = dup2(null_write, 2);
357  assert(retval == 2);
358  close(null_read);
359  close(null_write);
360  } else {
361  assert(pid > 0);
362  printf("%d\n", pid);
363  fflush(stdout);
364  fsync(1);
365  _exit(0);
366  }
367  } else {
368  assert(pid > 0);
369  waitpid(pid, &statloc, 0);
370  _exit(0);
371  }
372  }
373 
374  printf("Listening for cvmfs clients on %s\n", locator);
375  printf("NOTE: this process needs to run as user cvmfs\n\n");
376 
377  // Starts the I/O processing thread
379 
380  if (test_mode)
381  while (true) sleep(1);
382 
383  if (!cvmcache_is_supervised()) {
384  printf("Press <R ENTER> to ask clients to release nested catalogs\n");
385  printf("Press <Ctrl+D> to quit\n");
386  while (true) {
387  char buf;
388  int retval = read(fileno(stdin), &buf, 1);
389  if (retval != 1)
390  break;
391  if (buf == 'R') {
392  printf(" ... asking clients to release nested catalogs\n");
394  }
395  }
397  }
398 
400  printf(" ... good bye\n");
401 
402  cvmcache_options_free(locator);
403  cvmcache_options_fini(options);
406  return 0;
407 }
static int null_write_txn(uint64_t txn_id, unsigned char *buffer, uint32_t size)
static int null_chrefcnt(struct cvmcache_hash *id, int32_t change_by)
int(* cvmcache_chrefcnt)(struct cvmcache_hash *id, int32_t change_by)
struct cvmcache_hash id
struct cvmcache_context * ctx
static int null_pread(struct cvmcache_hash *id, uint64_t offset, uint32_t *size, unsigned char *buffer)
struct cvmcache_hash hash
cvmcache_object_type
int(* cvmcache_info)(struct cvmcache_info *info)
static void Usage(const char *progname)
void cvmcache_terminate(struct cvmcache_context *ctx)
int cvmcache_listen(struct cvmcache_context *ctx, char *locator)
struct cvmcache_hash id
int(* cvmcache_listing_next)(int64_t lst_id, struct cvmcache_object_info *item)
void cvmcache_cleanup_global()
void cvmcache_terminate_watchdog()
string description
static int null_listing_begin(uint64_t lst_id, enum cvmcache_object_type type)
static int null_breadcrumb_store(const char *fqrn, const cvmcache_breadcrumb *breadcrumb)
assert((mem||(size==0))&&"Out Of Memory")
uint64_t pinned_bytes
int cvmcache_is_supervised()
map< uint64_t, TxnInfo > transactions
static int null_info(struct cvmcache_info *info)
int(* cvmcache_pread)(struct cvmcache_hash *id, uint64_t offset, uint32_t *size, unsigned char *buffer)
map< std::string, cvmcache_breadcrumb > breadcrumbs
int(* cvmcache_breadcrumb_load)(const char *fqrn, cvmcache_breadcrumb *breadcrumb)
static int null_start_txn(struct cvmcache_hash *id, uint64_t txn_id, struct cvmcache_object_info *info)
map< ComparableHash, Object > storage
void cvmcache_wait_for(struct cvmcache_context *ctx)
uint64_t used_bytes
int main()
Definition: helper_allow.cc:16
Object partial_object
static int null_listing_end(int64_t listing_id)
static int null_shrink(uint64_t shrink_to, uint64_t *used)
static int null_breadcrumb_load(const char *fqrn, cvmcache_breadcrumb *breadcrumb)
int64_t no_shrink
void cvmcache_ask_detach(struct cvmcache_context *ctx)
int(* cvmcache_write_txn)(uint64_t txn_id, unsigned char *buffer, uint32_t size)
map< uint64_t, Listing > listings
uint64_t pos
int cvmcache_hash_cmp(struct cvmcache_hash *a, struct cvmcache_hash *b)
bool operator<(SharedPtr< T > const &a, SharedPtr< U > const &b)
Definition: shared_ptr.h:141
cvmcache_object_type type
void cvmcache_options_free(char *value)
void cvmcache_spawn_watchdog(const char *crash_dump_file)
void cvmcache_options_fini(cvmcache_option_map *opts)
uint64_t size_bytes
enum cvmcache_object_type type
struct cvmcache_context * cvmcache_init(struct cvmcache_callbacks *callbacks)
void cvmcache_init_global()
vector< Object > * elems
int cvmcache_options_parse(cvmcache_option_map *opts, const char *path)
int(* cvmcache_shrink)(uint64_t shrink_to, uint64_t *used)
static int null_obj_info(struct cvmcache_hash *id, struct cvmcache_object_info *info)
int32_t refcnt
int(* cvmcache_listing_begin)(uint64_t lst_id, enum cvmcache_object_type type)
void cvmcache_process_requests(struct cvmcache_context *ctx, unsigned nworkers)
int(* cvmcache_breadcrumb_store)(const char *fqrn, const cvmcache_breadcrumb *breadcrumb)
char * cvmcache_options_get(cvmcache_option_map *opts, const char *key)
cvmcache_object_type type
int(* cvmcache_commit_txn)(uint64_t txn_id)
int(* cvmcache_start_txn)(struct cvmcache_hash *id, uint64_t txn_id, struct cvmcache_object_info *info)
int(* cvmcache_listing_end)(int64_t lst_id)
int(* cvmcache_abort_txn)(uint64_t txn_id)
string data
ComparableHash(const struct cvmcache_hash &h)
static int null_abort_txn(uint64_t txn_id)
struct cvmcache_hash id
int(* cvmcache_obj_info)(struct cvmcache_hash *id, struct cvmcache_object_info *info)
static void size_t size
Definition: smalloc.h:47
static int null_listing_next(int64_t listing_id, struct cvmcache_object_info *item)
static int null_commit_txn(uint64_t txn_id)
cvmcache_option_map * cvmcache_options_init()