Directory: | cvmfs/ |
---|---|
File: | cvmfs/kvstore.h |
Date: | 2025-06-29 02:35:41 |
Exec | Total | Coverage | |
---|---|---|---|
Lines: | 19 | 19 | 100.0% |
Branches: | 39 | 78 | 50.0% |
Line | Branch | Exec | Source |
---|---|---|---|
1 | /** | ||
2 | * This file is part of the CernVM File System. | ||
3 | */ | ||
4 | |||
5 | #ifndef CVMFS_KVSTORE_H_ | ||
6 | #define CVMFS_KVSTORE_H_ | ||
7 | |||
8 | #include <pthread.h> | ||
9 | #include <unistd.h> | ||
10 | |||
11 | #include <string> | ||
12 | #include <vector> | ||
13 | |||
14 | #include "cache.h" | ||
15 | #include "lru.h" | ||
16 | #include "malloc_heap.h" | ||
17 | #include "statistics.h" | ||
18 | #include "util/async.h" | ||
19 | #include "util/single_copy.h" | ||
20 | |||
21 | using namespace std; // NOLINT | ||
22 | |||
23 | |||
24 | /** | ||
25 | * All objects in memory are prepended by an AllocHeader that allows the | ||
26 | * Key-Value store to find the pointer when the heap memory manager compacts | ||
27 | * the allocations. | ||
28 | */ | ||
29 | struct AllocHeader { | ||
30 | 9962 | AllocHeader() : version(0), id() { } | |
31 | uint8_t version; | ||
32 | shash::Any id; | ||
33 | }; | ||
34 | |||
35 | |||
36 | /** | ||
37 | * A MmeoryBuffer is used in the staging phase of new objects until they are | ||
38 | * committed. | ||
39 | */ | ||
40 | struct MemoryBuffer { | ||
41 | 8418814 | MemoryBuffer() | |
42 | 8418814 | : address(NULL), size(0), refcount(0), object_flags(0), id() { } | |
43 | void *address; | ||
44 | size_t size; | ||
45 | unsigned int refcount; | ||
46 | int object_flags; | ||
47 | shash::Any id; | ||
48 | }; | ||
49 | |||
50 | |||
51 | /** | ||
52 | * @p MemoryKvStore provides a simple RAM-backed key/value store suited to | ||
53 | * use with @ref RamCacheManager. To insert entries, the caller must allocate | ||
54 | * some memory and can choose to set some metadata such as object type. | ||
55 | * @p MemoryKvStore takes ownership of the passed-in memory and maintains | ||
56 | * reference counts for all its objects. Callers must increment the reference | ||
57 | * count on an entry before reading to ensure that the entry is not removed | ||
58 | * mid-operation, and decrement the reference count when done. The store | ||
59 | * can attempt to reduce its size by removing the least recently used | ||
60 | * entries without any outstanding references. | ||
61 | */ | ||
62 | class MemoryKvStore : SingleCopy, public Callbackable<MallocHeap::BlockPtr> { | ||
63 | public: | ||
64 | enum MemoryAllocator { | ||
65 | kMallocLibc, | ||
66 | kMallocHeap, | ||
67 | }; | ||
68 | |||
69 | struct Counters { | ||
70 | perf::Counter *sz_size; | ||
71 | perf::Counter *n_getsize; | ||
72 | perf::Counter *n_getrefcount; | ||
73 | perf::Counter *n_incref; | ||
74 | perf::Counter *n_unref; | ||
75 | perf::Counter *n_read; | ||
76 | perf::Counter *n_commit; | ||
77 | perf::Counter *n_delete; | ||
78 | perf::Counter *n_shrinkto; | ||
79 | perf::Counter *sz_read; | ||
80 | perf::Counter *sz_committed; | ||
81 | perf::Counter *sz_deleted; | ||
82 | perf::Counter *sz_shrunk; | ||
83 | |||
84 | 2245 | explicit Counters(perf::StatisticsTemplate statistics) { | |
85 |
3/6✓ Branch 2 taken 2245 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 2245 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 2245 times.
✗ Branch 10 not taken.
|
2245 | sz_size = statistics.RegisterTemplated("sz_size", "Total size"); |
86 |
3/6✓ Branch 2 taken 2245 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 2245 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 2245 times.
✗ Branch 10 not taken.
|
2245 | n_getsize = statistics.RegisterTemplated("n_getsize", |
87 | "Number of GetSize calls"); | ||
88 |
3/6✓ Branch 2 taken 2245 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 2245 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 2245 times.
✗ Branch 10 not taken.
|
2245 | n_getrefcount = statistics.RegisterTemplated( |
89 | "n_getrefcount", "Number of GetRefcount calls"); | ||
90 |
3/6✓ Branch 2 taken 2245 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 2245 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 2245 times.
✗ Branch 10 not taken.
|
2245 | n_incref = statistics.RegisterTemplated("n_incref", |
91 | "Number of IncRef calls"); | ||
92 |
3/6✓ Branch 2 taken 2245 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 2245 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 2245 times.
✗ Branch 10 not taken.
|
2245 | n_unref = statistics.RegisterTemplated("n_unref", |
93 | "Number of Unref calls"); | ||
94 |
3/6✓ Branch 2 taken 2245 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 2245 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 2245 times.
✗ Branch 10 not taken.
|
2245 | n_read = statistics.RegisterTemplated("n_read", "Number of Read calls"); |
95 |
3/6✓ Branch 2 taken 2245 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 2245 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 2245 times.
✗ Branch 10 not taken.
|
2245 | n_commit = statistics.RegisterTemplated("n_commit", |
96 | "Number of Commit calls"); | ||
97 |
3/6✓ Branch 2 taken 2245 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 2245 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 2245 times.
✗ Branch 10 not taken.
|
2245 | n_delete = statistics.RegisterTemplated("n_delete", |
98 | "Number of Delete calls"); | ||
99 |
3/6✓ Branch 2 taken 2245 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 2245 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 2245 times.
✗ Branch 10 not taken.
|
2245 | n_shrinkto = statistics.RegisterTemplated("n_shrinkto", |
100 | "Number of ShrinkTo calls"); | ||
101 |
3/6✓ Branch 2 taken 2245 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 2245 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 2245 times.
✗ Branch 10 not taken.
|
2245 | sz_read = statistics.RegisterTemplated("sz_read", "Bytes read"); |
102 |
3/6✓ Branch 2 taken 2245 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 2245 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 2245 times.
✗ Branch 10 not taken.
|
2245 | sz_committed = statistics.RegisterTemplated("sz_committed", |
103 | "Bytes committed"); | ||
104 |
3/6✓ Branch 2 taken 2245 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 2245 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 2245 times.
✗ Branch 10 not taken.
|
2245 | sz_deleted = statistics.RegisterTemplated("sz_deleted", "Bytes deleted"); |
105 |
3/6✓ Branch 2 taken 2245 times.
✗ Branch 3 not taken.
✓ Branch 6 taken 2245 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 2245 times.
✗ Branch 10 not taken.
|
2245 | sz_shrunk = statistics.RegisterTemplated("sz_shrunk", "Bytes shrunk"); |
106 | 2245 | } | |
107 | }; | ||
108 | |||
109 | MemoryKvStore(unsigned int cache_entries, | ||
110 | MemoryAllocator alloc, | ||
111 | unsigned alloc_size, | ||
112 | perf::StatisticsTemplate statistics); | ||
113 | |||
114 | ~MemoryKvStore(); | ||
115 | |||
116 | /** | ||
117 | * Check for the existence of an entry | ||
118 | * @param id The hash key | ||
119 | * @returns True iff the entry exists | ||
120 | */ | ||
121 | bool Contains(const shash::Any &id); | ||
122 | |||
123 | /** | ||
124 | * Get the size in bytes of the entry at id | ||
125 | * @param id The hash key | ||
126 | * @returns The entry's size | ||
127 | * @retval -ENOENT The entry is absent | ||
128 | */ | ||
129 | int64_t GetSize(const shash::Any &id); | ||
130 | |||
131 | /** | ||
132 | * Get the number of references to the entry at id | ||
133 | * @param id The hash key | ||
134 | * @returns A reference count | ||
135 | * @retval -ENOENT The entry is absent | ||
136 | */ | ||
137 | int64_t GetRefcount(const shash::Any &id); | ||
138 | |||
139 | /** | ||
140 | * Increase the reference count on the entry at id | ||
141 | * @param id The hash key | ||
142 | * @returns True if the entry exists and was updated | ||
143 | */ | ||
144 | bool IncRef(const shash::Any &id); | ||
145 | |||
146 | /** | ||
147 | * Decrease the reference count on the entry at id. If the refcount is zero, | ||
148 | * no effect | ||
149 | * @param id The hash key | ||
150 | * @returns True if the entry exists and was updated | ||
151 | */ | ||
152 | bool Unref(const shash::Any &id); | ||
153 | |||
154 | /** | ||
155 | * Copy a portion of the entry at id to the given address. See pread(2) | ||
156 | * @param id The hash key | ||
157 | * @param buf The address at which to write the data | ||
158 | * @param size The number of bytes to copy | ||
159 | * @param offset The offset within the entry to start the copy | ||
160 | * @returns The number of bytes copied | ||
161 | * @retval -ENOENT The entry is absent | ||
162 | */ | ||
163 | int64_t Read(const shash::Any &id, void *buf, size_t size, size_t offset); | ||
164 | |||
165 | /** | ||
166 | * Insert a new memory buffer. The KvStore copies the referred memory, so | ||
167 | * callers may free() their buffers after Commit returns | ||
168 | * @param buf The memory buffer to insert | ||
169 | * @returns -ENFILE if too many file handles are in use | ||
170 | * @returns -EIO if memory allocation fails | ||
171 | */ | ||
172 | int Commit(const MemoryBuffer &buf); | ||
173 | |||
174 | /** | ||
175 | * Delete an entry, free()ing its memory. Note that the entry not have any | ||
176 | * references | ||
177 | * @param id The hash key | ||
178 | * @returns True iff the entry was successfully deleted | ||
179 | */ | ||
180 | bool Delete(const shash::Any &id); | ||
181 | |||
182 | /** | ||
183 | * Delete the oldest entries until the KvStore uses less than the given size. | ||
184 | * Entries with nonzero refcount will not be deleted. | ||
185 | * @param size The maximum size to make the KvStore | ||
186 | * @returns True iff the shrink succeeds | ||
187 | */ | ||
188 | bool ShrinkTo(size_t size); | ||
189 | |||
190 | /** | ||
191 | * Get the total space used for data | ||
192 | */ | ||
193 | 7099 | size_t GetUsed() { return used_bytes_; } | |
194 | |||
195 | private: | ||
196 | // Compact memory once utilization falls below the threshold | ||
197 | static const double kCompactThreshold; // = 0.8 | ||
198 | |||
199 | bool DoDelete(const shash::Any &id); | ||
200 | int DoMalloc(MemoryBuffer *buf); | ||
201 | void DoFree(MemoryBuffer *buf); | ||
202 | int DoCommit(const MemoryBuffer &buf); | ||
203 | void OnBlockMove(const MallocHeap::BlockPtr &ptr); | ||
204 | bool CompactMemory(); | ||
205 | |||
206 | MemoryAllocator allocator_; | ||
207 | size_t used_bytes_; | ||
208 | unsigned int entry_count_; | ||
209 | unsigned int max_entries_; | ||
210 | lru::LruCache<shash::Any, MemoryBuffer> entries_; | ||
211 | MallocHeap *heap_; | ||
212 | pthread_rwlock_t rwlock_; | ||
213 | Counters counters_; | ||
214 | }; | ||
215 | |||
216 | #endif // CVMFS_KVSTORE_H_ | ||
217 |