GCC Code Coverage Report
Directory: cvmfs/ Exec Total Coverage
File: cvmfs/kvstore.h Lines: 19 19 100.0 %
Date: 2019-02-03 02:48:13 Branches: 0 0 - %

Line Branch Exec Source
1
/**
2
 * This file is part of the CernVM File System.
3
 */
4
5
#ifndef CVMFS_KVSTORE_H_
6
#define CVMFS_KVSTORE_H_
7
8
#include <pthread.h>
9
#include <unistd.h>
10
11
#include <string>
12
#include <vector>
13
14
#include "cache.h"
15
#include "lru.h"
16
#include "malloc_heap.h"
17
#include "statistics.h"
18
#include "util/async.h"
19
#include "util/single_copy.h"
20
21
using namespace std;  // NOLINT
22
23
24
/**
25
 * All objects in memory are prepended by an AllocHeader that allows the
26
 * Key-Value store to find the pointer when the heap memory manager compacts
27
 * the allocations.
28
 */
29
struct AllocHeader {
30
254
  AllocHeader() : version(0), id() { }
31
  uint8_t version;
32
  shash::Any id;
33
};
34
35
36
/**
37
 * A MmeoryBuffer is used in the staging phase of new objects until they are
38
 * committed.
39
 */
40
struct MemoryBuffer {
41
721122
  MemoryBuffer()
42
    : address(NULL)
43
    , size(0)
44
    , refcount(0)
45
    , object_type(CacheManager::kTypeRegular)
46
721122
    , id() {}
47
  void *address;
48
  size_t size;
49
  unsigned int refcount;
50
  CacheManager::ObjectType object_type;
51
  shash::Any id;
52
};
53
54
55
/**
56
 * @p MemoryKvStore provides a simple RAM-backed key/value store suited to
57
 * use with @ref RamCacheManager. To insert entries, the caller must allocate
58
 * some memory and can choose to set some metadata such as object type.
59
 * @p MemoryKvStore takes ownership of the passed-in memory and maintains
60
 * reference counts for all its objects. Callers must increment the reference
61
 * count on an entry before reading to ensure that the entry is not removed
62
 * mid-operation, and decrement the reference count when done. The store
63
 * can attempt to reduce its size by removing the least recently used
64
 * entries without any outstanding references.
65
 */
66
class MemoryKvStore : SingleCopy, public Callbackable<MallocHeap::BlockPtr> {
67
 public:
68
  enum MemoryAllocator {
69
    kMallocLibc,
70
    kMallocHeap,
71
  };
72
73
  struct Counters {
74
    perf::Counter *sz_size;
75
    perf::Counter *n_getsize;
76
    perf::Counter *n_getrefcount;
77
    perf::Counter *n_incref;
78
    perf::Counter *n_unref;
79
    perf::Counter *n_read;
80
    perf::Counter *n_commit;
81
    perf::Counter *n_delete;
82
    perf::Counter *n_shrinkto;
83
    perf::Counter *sz_read;
84
    perf::Counter *sz_committed;
85
    perf::Counter *sz_deleted;
86
    perf::Counter *sz_shrunk;
87
88
101
    explicit Counters(perf::StatisticsTemplate statistics) {
89
101
      sz_size = statistics.RegisterTemplated("sz_size", "Total size");
90
      n_getsize = statistics.RegisterTemplated("n_getsize",
91
101
        "Number of GetSize calls");
92
      n_getrefcount = statistics.RegisterTemplated("n_getrefcount",
93
101
        "Number of GetRefcount calls");
94
      n_incref = statistics.RegisterTemplated("n_incref",
95
101
        "Number of IncRef calls");
96
      n_unref = statistics.RegisterTemplated("n_unref",
97
101
        "Number of Unref calls");
98
101
      n_read = statistics.RegisterTemplated("n_read", "Number of Read calls");
99
      n_commit = statistics.RegisterTemplated("n_commit",
100
101
        "Number of Commit calls");
101
      n_delete = statistics.RegisterTemplated("n_delete",
102
101
        "Number of Delete calls");
103
      n_shrinkto = statistics.RegisterTemplated("n_shrinkto",
104
101
        "Number of ShrinkTo calls");
105
101
      sz_read = statistics.RegisterTemplated("sz_read", "Bytes read");
106
      sz_committed = statistics.RegisterTemplated("sz_committed",
107
101
        "Bytes committed");
108
101
      sz_deleted = statistics.RegisterTemplated("sz_deleted", "Bytes deleted");
109
101
      sz_shrunk = statistics.RegisterTemplated("sz_shrunk", "Bytes shrunk");
110
101
    }
111
  };
112
113
  MemoryKvStore(
114
    unsigned int cache_entries,
115
    MemoryAllocator alloc,
116
    unsigned alloc_size,
117
    perf::StatisticsTemplate statistics);
118
119
  ~MemoryKvStore();
120
121
  /**
122
   * Check for the existence of an entry
123
   * @param id The hash key
124
   * @returns True iff the entry exists
125
   */
126
  bool Contains(const shash::Any &id);
127
128
  /**
129
   * Get the size in bytes of the entry at id
130
   * @param id The hash key
131
   * @returns The entry's size
132
   * @retval -ENOENT The entry is absent
133
   */
134
  int64_t GetSize(const shash::Any &id);
135
136
  /**
137
   * Get the number of references to the entry at id
138
   * @param id The hash key
139
   * @returns A reference count
140
   * @retval -ENOENT The entry is absent
141
   */
142
  int64_t GetRefcount(const shash::Any &id);
143
144
  /**
145
   * Increase the reference count on the entry at id
146
   * @param id The hash key
147
   * @returns True if the entry exists and was updated
148
   */
149
  bool IncRef(const shash::Any &id);
150
151
  /**
152
   * Decrease the reference count on the entry at id. If the refcount is zero, no effect
153
   * @param id The hash key
154
   * @returns True if the entry exists and was updated
155
   */
156
  bool Unref(const shash::Any &id);
157
158
  /**
159
   * Copy a portion of the entry at id to the given address. See pread(2)
160
   * @param id The hash key
161
   * @param buf The address at which to write the data
162
   * @param size The number of bytes to copy
163
   * @param offset The offset within the entry to start the copy
164
   * @returns The number of bytes copied
165
   * @retval -ENOENT The entry is absent
166
   */
167
  int64_t Read(
168
    const shash::Any &id,
169
    void *buf,
170
    size_t size,
171
    size_t offset);
172
173
  /**
174
   * Insert a new memory buffer. The KvStore copies the referred memory, so
175
   * callers may free() their buffers after Commit returns
176
   * @param buf The memory buffer to insert
177
   * @returns -ENFILE if too many file handles are in use
178
   * @returns -EIO if memory allocation fails
179
   */
180
  int Commit(const MemoryBuffer &buf);
181
182
  /**
183
   * Delete an entry, free()ing its memory. Note that the entry not have any references
184
   * @param id The hash key
185
   * @returns True iff the entry was successfully deleted
186
   */
187
  bool Delete(const shash::Any &id);
188
189
  /**
190
   * Delete the oldest entries until the KvStore uses less than the given size.
191
   * Entries with nonzero refcount will not be deleted.
192
   * @param size The maximum size to make the KvStore
193
   * @returns True iff the shrink succeeds
194
   */
195
  bool ShrinkTo(size_t size);
196
197
  /**
198
   * Get the total space used for data
199
   */
200
177
  size_t GetUsed() { return used_bytes_; }
201
202
 private:
203
  // Compact memory once utilization falls below the threshold
204
  static const double kCompactThreshold;  // = 0.8
205
206
  bool DoDelete(const shash::Any &id);
207
  int DoMalloc(MemoryBuffer *buf);
208
  void DoFree(MemoryBuffer *buf);
209
  int DoCommit(const MemoryBuffer &buf);
210
  void OnBlockMove(const MallocHeap::BlockPtr &ptr);
211
  bool CompactMemory();
212
213
  MemoryAllocator allocator_;
214
  size_t used_bytes_;
215
  unsigned int entry_count_;
216
  unsigned int max_entries_;
217
  lru::LruCache<shash::Any, MemoryBuffer> entries_;
218
  MallocHeap *heap_;
219
  pthread_rwlock_t rwlock_;
220
  Counters counters_;
221
};
222
223
#endif  // CVMFS_KVSTORE_H_