CernVM-FS  2.12.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
libcvmfs_cache.cc
Go to the documentation of this file.
1 
5 #define _FILE_OFFSET_BITS 64
6 #define __STDC_FORMAT_MACROS
7 
8 #include "cvmfs_config.h"
9 #include "libcvmfs_cache.h"
10 
11 #include <unistd.h>
12 
13 #include <cassert>
14 #include <cstdlib>
15 #include <cstring>
16 #include <string>
17 
18 #include "cache_plugin/channel.h"
19 #include "cache_transport.h"
20 #include "crypto/hash.h"
21 #include "manifest.h"
22 #include "monitor.h"
23 #include "util/pointer.h"
24 
25 using namespace std; // NOLINT
26 
27 namespace {
28 
29 static shash::Any Chash2Cpphash(const struct cvmcache_hash *h) {
30  shash::Any hash;
31  memcpy(hash.digest, h->digest, sizeof(h->digest));
32  hash.algorithm = static_cast<shash::Algorithms>(h->algorithm);
33  return hash;
34 }
35 
36 static struct cvmcache_hash Cpphash2Chash(const shash::Any &hash) {
37  struct cvmcache_hash h;
38  memcpy(h.digest, hash.digest, sizeof(h.digest));
39  h.algorithm = hash.algorithm;
40  return h;
41 }
42 
43 static enum cvmcache_object_type ObjectType2CType(cvmfs::EnumObjectType type) {
44  switch (type) {
45  case cvmfs::OBJECT_REGULAR:
47  case cvmfs::OBJECT_CATALOG:
49  case cvmfs::OBJECT_VOLATILE:
51  }
52  abort();
53 }
54 
56  public:
57  explicit ForwardCachePlugin(struct cvmcache_callbacks *callbacks)
58  : CachePlugin(callbacks->capabilities)
59  , callbacks_(*callbacks)
60  {
61  assert(callbacks->cvmcache_chrefcnt != NULL);
62  assert(callbacks->cvmcache_obj_info != NULL);
63  assert(callbacks->cvmcache_pread != NULL);
64  if (callbacks->capabilities & CVMCACHE_CAP_WRITE) {
65  assert(callbacks->cvmcache_start_txn != NULL);
66  assert(callbacks->cvmcache_write_txn != NULL);
67  assert(callbacks->cvmcache_commit_txn != NULL);
68  assert(callbacks->cvmcache_abort_txn != NULL);
69  }
70  if (callbacks->capabilities & CVMCACHE_CAP_INFO)
71  assert(callbacks->cvmcache_info != NULL);
72  if (callbacks->capabilities & CVMCACHE_CAP_SHRINK_RATE)
74  if (callbacks->capabilities & CVMCACHE_CAP_SHRINK)
75  assert(callbacks->cvmcache_shrink != NULL);
76  if (callbacks->capabilities & CVMCACHE_CAP_LIST) {
77  assert(callbacks->cvmcache_listing_begin != NULL);
78  assert(callbacks->cvmcache_listing_next != NULL);
79  assert(callbacks->cvmcache_listing_end != NULL);
80  }
81  if (callbacks->capabilities & CVMCACHE_CAP_BREADCRUMB) {
82  assert(callbacks->cvmcache_breadcrumb_store != NULL);
83  assert(callbacks->cvmcache_breadcrumb_load != NULL);
84  }
85  }
86  virtual ~ForwardCachePlugin() { }
87 
88  protected:
89  virtual cvmfs::EnumStatus ChangeRefcount(
90  const shash::Any &id,
91  int32_t change_by)
92  {
93  struct cvmcache_hash c_hash = Cpphash2Chash(id);
94  int result = callbacks_.cvmcache_chrefcnt(&c_hash, change_by);
95  return static_cast<cvmfs::EnumStatus>(result);
96  }
97 
98  virtual cvmfs::EnumStatus GetObjectInfo(
99  const shash::Any &id,
100  ObjectInfo *info)
101  {
102  struct cvmcache_hash c_hash = Cpphash2Chash(id);
103  cvmcache_object_info c_info;
104  memset(&c_info, 0, sizeof(c_info));
106  int result = callbacks_.cvmcache_obj_info(&c_hash, &c_info);
107  info->size = c_info.size;
108  info->object_type = static_cast<cvmfs::EnumObjectType>(c_info.type);
109  info->pinned = c_info.pinned;
110  if (c_info.description) {
111  info->description = string(c_info.description);
112  free(c_info.description);
113  }
114  return static_cast<cvmfs::EnumStatus>(result);
115  }
116 
117  virtual cvmfs::EnumStatus Pread(
118  const shash::Any &id,
119  uint64_t offset,
120  uint32_t *size,
121  unsigned char *buffer)
122  {
123  struct cvmcache_hash c_hash = Cpphash2Chash(id);
124  int result = callbacks_.cvmcache_pread(&c_hash, offset, size, buffer);
125  return static_cast<cvmfs::EnumStatus>(result);
126  }
127 
128  virtual cvmfs::EnumStatus StartTxn(
129  const shash::Any &id,
130  const uint64_t txn_id,
131  const ObjectInfo &info)
132  {
133  if (!(callbacks_.capabilities & CVMCACHE_CAP_WRITE))
134  return cvmfs::STATUS_NOSUPPORT;
135 
136  struct cvmcache_hash c_hash = Cpphash2Chash(id);
137  cvmcache_object_info c_info;
138  memset(&c_info, 0, sizeof(c_info));
139  c_info.size = info.size;
140  c_info.type = ObjectType2CType(info.object_type);
141  if (info.description.empty()) {
142  c_info.description = NULL;
143  } else {
144  c_info.description = strdup(info.description.c_str());
145  }
146  int result = callbacks_.cvmcache_start_txn(&c_hash, txn_id, &c_info);
147  free(c_info.description);
148  return static_cast<cvmfs::EnumStatus>(result);
149  }
150 
151  virtual cvmfs::EnumStatus WriteTxn(
152  const uint64_t txn_id,
153  unsigned char *buffer,
154  uint32_t size)
155  {
156  if (!(callbacks_.capabilities & CVMCACHE_CAP_WRITE))
157  return cvmfs::STATUS_NOSUPPORT;
158 
159  int result = callbacks_.cvmcache_write_txn(txn_id, buffer, size);
160  return static_cast<cvmfs::EnumStatus>(result);
161  }
162 
163  virtual cvmfs::EnumStatus CommitTxn(const uint64_t txn_id) {
164  if (!(callbacks_.capabilities & CVMCACHE_CAP_WRITE))
165  return cvmfs::STATUS_NOSUPPORT;
166 
167  int result = callbacks_.cvmcache_commit_txn(txn_id);
168  return static_cast<cvmfs::EnumStatus>(result);
169  }
170 
171  virtual cvmfs::EnumStatus AbortTxn(const uint64_t txn_id) {
172  if (!(callbacks_.capabilities & CVMCACHE_CAP_WRITE))
173  return cvmfs::STATUS_NOSUPPORT;
174 
175  int result = callbacks_.cvmcache_abort_txn(txn_id);
176  return static_cast<cvmfs::EnumStatus>(result);
177  }
178 
179  virtual cvmfs::EnumStatus GetInfo(Info *info) {
180  if (!(callbacks_.capabilities & CVMCACHE_CAP_INFO))
181  return cvmfs::STATUS_NOSUPPORT;
182 
183  cvmcache_info c_info;
184  c_info.size_bytes = info->size_bytes;
185  c_info.used_bytes = info->used_bytes;
186  c_info.pinned_bytes = info->pinned_bytes;
187  c_info.no_shrink = info->no_shrink;
188  int result = callbacks_.cvmcache_info(&c_info);
189  if (result == CVMCACHE_STATUS_OK) {
190  info->size_bytes = c_info.size_bytes;
191  info->used_bytes = c_info.used_bytes;
192  info->pinned_bytes = c_info.pinned_bytes;
193  info->no_shrink = c_info.no_shrink;
194  }
195  return static_cast<cvmfs::EnumStatus>(result);
196  }
197 
198  virtual cvmfs::EnumStatus Shrink(uint64_t shrink_to, uint64_t *used) {
199  if (!(callbacks_.capabilities & CVMCACHE_CAP_SHRINK))
200  return cvmfs::STATUS_NOSUPPORT;
201 
202  int result = callbacks_.cvmcache_shrink(shrink_to, used);
203  return static_cast<cvmfs::EnumStatus>(result);
204  }
205 
206  virtual cvmfs::EnumStatus ListingBegin(
207  uint64_t lst_id,
208  cvmfs::EnumObjectType type)
209  {
210  if (!(callbacks_.capabilities & CVMCACHE_CAP_LIST))
211  return cvmfs::STATUS_NOSUPPORT;
212 
213  int result =
214  callbacks_.cvmcache_listing_begin(lst_id, ObjectType2CType(type));
215  return static_cast<cvmfs::EnumStatus>(result);
216  }
217 
218  virtual cvmfs::EnumStatus ListingNext(
219  int64_t lst_id,
220  ObjectInfo *item)
221  {
222  if (!(callbacks_.capabilities & CVMCACHE_CAP_LIST))
223  return cvmfs::STATUS_NOSUPPORT;
224 
225  struct cvmcache_object_info c_item;
226  memset(&c_item, 0, sizeof(c_item));
227  int result = callbacks_.cvmcache_listing_next(lst_id, &c_item);
228  if (result == CVMCACHE_STATUS_OK) {
229  item->id = Chash2Cpphash(&c_item.id);
230  item->size = c_item.size;
231  item->object_type = static_cast<cvmfs::EnumObjectType>(c_item.type);
232  item->pinned = c_item.pinned;
233  if (c_item.description) {
234  item->description = string(c_item.description);
235  free(c_item.description);
236  }
237  }
238  return static_cast<cvmfs::EnumStatus>(result);
239  }
240 
241  virtual cvmfs::EnumStatus ListingEnd(int64_t lst_id) {
242  if (!(callbacks_.capabilities & CVMCACHE_CAP_LIST))
243  return cvmfs::STATUS_NOSUPPORT;
244 
245  int result = callbacks_.cvmcache_listing_end(lst_id);
246  return static_cast<cvmfs::EnumStatus>(result);
247  }
248 
249  virtual cvmfs::EnumStatus LoadBreadcrumb(
250  const std::string &fqrn, manifest::Breadcrumb *breadcrumb)
251  {
252  if (!(callbacks_.capabilities & CVMCACHE_CAP_BREADCRUMB))
253  return cvmfs::STATUS_NOSUPPORT;
254 
255  cvmcache_breadcrumb c_breadcrumb;
256  int result =
257  callbacks_.cvmcache_breadcrumb_load(fqrn.c_str(), &c_breadcrumb);
258  if (result == CVMCACHE_STATUS_OK) {
259  breadcrumb->catalog_hash = Chash2Cpphash(&c_breadcrumb.catalog_hash);
260  breadcrumb->timestamp = c_breadcrumb.timestamp;
261  breadcrumb->revision = c_breadcrumb.revision;
262  }
263  return static_cast<cvmfs::EnumStatus>(result);
264  }
265 
266  virtual cvmfs::EnumStatus StoreBreadcrumb(
267  const std::string &fqrn, const manifest::Breadcrumb &breadcrumb)
268  {
269  if (!(callbacks_.capabilities & CVMCACHE_CAP_BREADCRUMB))
270  return cvmfs::STATUS_NOSUPPORT;
271 
272  cvmcache_breadcrumb c_breadcrumb;
273  c_breadcrumb.catalog_hash = Cpphash2Chash(breadcrumb.catalog_hash);
274  c_breadcrumb.timestamp = breadcrumb.timestamp;
275  c_breadcrumb.revision = breadcrumb.revision;
276  int result =
277  callbacks_.cvmcache_breadcrumb_store(fqrn.c_str(), &c_breadcrumb);
278  return static_cast<cvmfs::EnumStatus>(result);
279  }
280 
281  private:
282  struct cvmcache_callbacks callbacks_;
283 };
284 
286 
287 } // anonymous namespace
288 
289 
291  explicit cvmcache_context(ForwardCachePlugin *p) : plugin(p) { }
293 };
294 
295 
296 int cvmcache_hash_cmp(struct cvmcache_hash *a, struct cvmcache_hash *b) {
297  const shash::Any hash_a = Chash2Cpphash(a);
298  const shash::Any hash_b = Chash2Cpphash(b);
299  if (hash_a < hash_b)
300  return -1;
301  else if (hash_a == hash_b)
302  return 0;
303  else
304  return 1;
305 }
306 
307 char *cvmcache_hash_print(const struct cvmcache_hash *h) {
308  const shash::Any hash = Chash2Cpphash(h);
309  return strdup(hash.ToString().c_str());
310 }
311 
312 
314 
315 
317 
319  return getenv(CacheTransport::kEnvReadyNotifyFd) != NULL;
320 }
321 
323  return new cvmcache_context(new ForwardCachePlugin(callbacks));
324 }
325 
326 int cvmcache_listen(struct cvmcache_context *ctx, char *locator) {
327  return ctx->plugin->Listen(locator);
328 }
329 
330 void cvmcache_process_requests(struct cvmcache_context *ctx, unsigned nworkers)
331 {
332  ctx->plugin->ProcessRequests(nworkers);
333 }
334 
336  ctx->plugin->AskToDetach();
337 }
338 
340  ctx->plugin->WaitFor();
341  delete ctx;
342 }
343 
345  ctx->plugin->Terminate();
346 }
347 
349  return ctx->plugin->max_object_size();
350 }
351 
353  assert(session != NULL);
354  SessionCtx *session_ctx = SessionCtx::GetInstance();
355  assert(session_ctx);
356  session_ctx->Get(&(session->id),
357  &(session->repository_name),
358  &(session->client_instance));
359 }
360 
361 void cvmcache_spawn_watchdog(const char *crash_dump_file) {
362  if (g_watchdog != NULL)
363  return;
365  assert(g_watchdog != NULL);
366  g_watchdog->Spawn((crash_dump_file != NULL) ? string(crash_dump_file) : "");
367 }
368 
370  delete g_watchdog;
371  g_watchdog = NULL;
372 }
uint64_t pinned_bytes
Definition: channel.h:90
struct cvmcache_hash catalog_hash
void Get(uint64_t *id, char **reponame, char **client_instance)
Definition: channel.cc:71
int(* cvmcache_chrefcnt)(struct cvmcache_hash *id, int32_t change_by)
struct cvmcache_hash Cpphash2Chash(const shash::Any &hash)
struct cvmcache_context * ctx
Session * session() const
Definition: repository.h:320
cvmcache_object_type
uint64_t timestamp
Definition: manifest.h:40
static const uint64_t kSizeUnknown
Definition: channel.h:71
int(* cvmcache_info)(struct cvmcache_info *info)
shash::Any catalog_hash
Definition: manifest.h:39
void cvmcache_terminate(struct cvmcache_context *ctx)
int cvmcache_listen(struct cvmcache_context *ctx, char *locator)
std::string ToString(const bool with_suffix=false) const
Definition: hash.h:249
struct cvmcache_hash id
int(* cvmcache_listing_next)(int64_t lst_id, struct cvmcache_object_info *item)
void cvmcache_cleanup_global()
void cvmcache_terminate_watchdog()
virtual cvmfs::EnumStatus LoadBreadcrumb(const std::string &fqrn, manifest::Breadcrumb *breadcrumb)
cvmfs::EnumObjectType object_type
Definition: channel.h:81
assert((mem||(size==0))&&"Out Of Memory")
uint64_t pinned_bytes
int cvmcache_is_supervised()
Algorithms algorithm
Definition: hash.h:125
int(* cvmcache_pread)(struct cvmcache_hash *id, uint64_t offset, uint32_t *size, unsigned char *buffer)
int(* cvmcache_breadcrumb_load)(const char *fqrn, cvmcache_breadcrumb *breadcrumb)
unsigned char digest[digest_size_]
Definition: hash.h:124
void cvmcache_wait_for(struct cvmcache_context *ctx)
uint64_t used_bytes
virtual cvmfs::EnumStatus AbortTxn(const uint64_t txn_id)
Algorithms
Definition: hash.h:41
uint64_t revision
Definition: manifest.h:41
static Watchdog * Create(FnOnCrash on_crash)
Definition: monitor.cc:70
void cvmcache_ask_detach(struct cvmcache_context *ctx)
int(* cvmcache_write_txn)(uint64_t txn_id, unsigned char *buffer, uint32_t size)
virtual cvmfs::EnumStatus StartTxn(const shash::Any &id, const uint64_t txn_id, const ObjectInfo &info)
virtual cvmfs::EnumStatus CommitTxn(const uint64_t txn_id)
int cvmcache_hash_cmp(struct cvmcache_hash *a, struct cvmcache_hash *b)
unsigned char digest[20]
virtual cvmfs::EnumStatus StoreBreadcrumb(const std::string &fqrn, const manifest::Breadcrumb &breadcrumb)
void cvmcache_get_session(cvmcache_session *session)
cvmcache_context(ForwardCachePlugin *p)
int64_t no_shrink
Definition: channel.h:91
UniquePtr< ForwardCachePlugin > plugin
std::string description
Definition: channel.h:83
virtual cvmfs::EnumStatus WriteTxn(const uint64_t txn_id, unsigned char *buffer, uint32_t size)
void cvmcache_spawn_watchdog(const char *crash_dump_file)
char * cvmcache_hash_print(const struct cvmcache_hash *h)
uint64_t size_bytes
enum cvmcache_object_type type
virtual cvmfs::EnumStatus Shrink(uint64_t shrink_to, uint64_t *used)
struct cvmcache_context * cvmcache_init(struct cvmcache_callbacks *callbacks)
virtual cvmfs::EnumStatus ListingBegin(uint64_t lst_id, cvmfs::EnumObjectType type)
void cvmcache_init_global()
int(* cvmcache_shrink)(uint64_t shrink_to, uint64_t *used)
uint64_t used_bytes
Definition: channel.h:89
int(* cvmcache_listing_begin)(uint64_t lst_id, enum cvmcache_object_type type)
virtual cvmfs::EnumStatus ListingEnd(int64_t lst_id)
void cvmcache_process_requests(struct cvmcache_context *ctx, unsigned nworkers)
int(* cvmcache_breadcrumb_store)(const char *fqrn, const cvmcache_breadcrumb *breadcrumb)
static enum cvmcache_object_type ObjectType2CType(cvmfs::EnumObjectType type)
int(* cvmcache_commit_txn)(uint64_t txn_id)
int(* cvmcache_start_txn)(struct cvmcache_hash *id, uint64_t txn_id, struct cvmcache_object_info *info)
int(* cvmcache_listing_end)(int64_t lst_id)
int(* cvmcache_abort_txn)(uint64_t txn_id)
void Spawn(const std::string &crash_dump_path)
Definition: monitor.cc:510
ForwardCachePlugin(struct cvmcache_callbacks *callbacks)
virtual cvmfs::EnumStatus GetObjectInfo(const shash::Any &id, ObjectInfo *info)
virtual cvmfs::EnumStatus ChangeRefcount(const shash::Any &id, int32_t change_by)
int(* cvmcache_obj_info)(struct cvmcache_hash *id, struct cvmcache_object_info *info)
shash::Any Chash2Cpphash(const struct cvmcache_hash *h)
static void size_t size
Definition: smalloc.h:54
virtual cvmfs::EnumStatus Pread(const shash::Any &id, uint64_t offset, uint32_t *size, unsigned char *buffer)
virtual cvmfs::EnumStatus ListingNext(int64_t lst_id, ObjectInfo *item)
static const char * kEnvReadyNotifyFd
uint64_t size_bytes
Definition: channel.h:88
static SessionCtx * GetInstance()
Definition: channel.cc:59
uint32_t cvmcache_max_object_size(struct cvmcache_context *ctx)