GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/sqlitemem.h
Date: 2025-02-02 02:34:22
Exec Total Coverage
Lines: 5 5 100.0%
Branches: 3 4 75.0%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 */
4
5 #ifndef CVMFS_SQLITEMEM_H_
6 #define CVMFS_SQLITEMEM_H_
7
8 #include <inttypes.h>
9 #include <pthread.h>
10 #include <stdint.h>
11
12 #include <cassert>
13 #include <cstdlib>
14 #include <vector>
15
16 #include "duplex_sqlite3.h"
17 #include "gtest/gtest_prod.h"
18
19 class MallocArena;
20
21
22 /**
23 * The MemoryManager uses the sqlite hooks to optimize memory allocations. It
24 * is tuned for reading the cvmfs file catalogs. It provides a page cache of
25 * a few MB and per-database lookaside buffers. It also contains a
26 * "general purpose" malloc/free implementation tailored to the behavior of
27 * sqlite memory allocation.
28 *
29 * It is implemented as a singleton. GetInstance() will reserve memory blocks,
30 * AssignGlobalArenas will set the global sqlite configuration and
31 * CleanupInstance frees the memory blocks. These three routines are not
32 * thread-safe. AssignGlobalArenas has to be called as first sqlite operation,
33 * CleanupInstance has to be called after sqlite3_shutdown. If the singleton
34 * is alive, read-only sqlite databases will automatically use it for their
35 * lookaside buffers (see sql.h). Assignment of lookaside buffers is
36 * thread-safe.
37 */
38 class SqliteMemoryManager {
39 FRIEND_TEST(T_Sqlitemem, LookasideBuffer);
40 FRIEND_TEST(T_Sqlitemem, Malloc);
41 FRIEND_TEST(T_Sqlitemem, Realloc);
42 FRIEND_TEST(T_Sqlitemem, ReallocStress);
43
44 public:
45 /**
46 * In practice, we hardly ever see scratch memory used. If it is used,
47 * the allocation size is <8kB.
48 * NOTE: The scratch memory allocator has been removed from sqlite as of
49 * version 3.21.
50 */
51 static const unsigned kScratchSlotSize = 8192;
52 /**
53 * Sqlite ensures that never more slots than 2 times the number of threads are
54 * used. In practice, it's much lower.
55 */
56 static const unsigned kScratchNoSlots = 8;
57 /**
58 * In total: 64kB scratch memory
59 */
60 static const unsigned kScratchSize = kScratchSlotSize * kScratchNoSlots;
61
62 /**
63 * Empricially, the largest page cache allocation is 1296B.
64 */
65 static const unsigned kPageCacheSlotSize = 1300;
66 /**
67 * Number of pages that can be cached.
68 */
69 static const unsigned kPageCacheNoSlots = 4000;
70 /**
71 * In total: ~5MB
72 */
73 static const unsigned kPageCacheSize = kPageCacheSlotSize * kPageCacheNoSlots;
74
75 /**
76 * 32 bytes per slot is an empirically good value so that memory is not wasted
77 * (too much) and many allocations fit within the boundary.
78 */
79 static const unsigned kLookasideSlotSize = 32;
80 /**
81 * 128 slots with 32bytes accumulate to 4kB. See LookasideBufferArena below.
82 */
83 static const unsigned kLookasideSlotsPerDb = 128;
84
85
86 /**
87 * A continuous chunk of memory from which fixed-sized chunks are given as
88 * lookaside buffers to SQlite database connections. A bitmap indicates used
89 * and free buffers.
90 * One arena serves 128 database connections and assuming less than 10000
91 * concurrently open catalogs, the number of mmap'd regions stays under 100.
92 */
93 class LookasideBufferArena {
94 public:
95 /**
96 * Accumulates to 4kB.
97 */
98 static const unsigned kBufferSize =
99 kLookasideSlotSize * kLookasideSlotsPerDb;
100
101 /**
102 * Has to be a multiple of the number of bits in the int type. One arena
103 * can serve 128 concurrent sqlite database connections.
104 */
105 static const unsigned kBuffersPerArena = 128;
106 /**
107 * 512kB = 128 4kB buffers.
108 */
109 static const unsigned kArenaSize = kBuffersPerArena * kBufferSize;
110 /**
111 * Number of bitmap ints for 128 buffers.
112 */
113 static const unsigned kNoBitmaps = kBuffersPerArena / (sizeof(int) * 8);
114
115 LookasideBufferArena();
116 ~LookasideBufferArena();
117
118 void *GetBuffer();
119 void PutBuffer(void *buffer);
120 bool IsEmpty();
121 bool Contains(void *buffer);
122
123 private:
124 /**
125 * An mmap'd memory block.
126 */
127 void *arena_;
128
129 /**
130 * Indicates free/used slots. A used slot has its bit set to zero, an
131 * unused slot has its bit set to 1.
132 */
133 int freemap_[kNoBitmaps];
134 };
135
136
137 149 static SqliteMemoryManager *GetInstance() {
138
2/2
✓ Branch 0 taken 81 times.
✓ Branch 1 taken 68 times.
 149 if (instance_ == NULL)
139
1/2
✓ Branch 2 taken 81 times.
✗ Branch 3 not taken.
 81 instance_ = new SqliteMemoryManager();
140 149 return instance_;
141 }
142 static void CleanupInstance();
143 101 static bool HasInstance() { return instance_ != NULL; }
144 ~SqliteMemoryManager();
145
146 void AssignGlobalArenas();
147 void *AssignLookasideBuffer(sqlite3 *db);
148 void ReleaseLookasideBuffer(void *buffer);
149
150 private:
151 /**
152 * Should be larger than 10 times the largest allocation, which for reading
153 * sqlite file catalogs is 64kB. An arena size of 8MB limits the total
154 * number of arenas (mapped blocks) to <40, given typical storage needs for
155 * 10,000 open catalogs. Has to be a power of 2MB (see MallocArena).
156 */
157 static const unsigned kArenaSize = 8 * 1024 * 1024;
158 static SqliteMemoryManager *instance_;
159
160 /**
161 * SQlite memory callbacks need to be static because they are referenced by
162 * C function pointers. See https://www.sqlite.org/c3ref/mem_methods.html
163 */
164 static void *xMalloc(int size);
165 static void xFree(void *ptr);
166 static void *xRealloc(void *ptr, int new_size);
167 static int xSize(void *ptr);
168 static int xRoundup(int size);
169 static int xInit(void *app_data);
170 static void xShutdown(void *app_data);
171
172
173 SqliteMemoryManager();
174
175 void *GetLookasideBuffer();
176 void PutLookasideBuffer(void *buffer);
177
178 void *GetMemory(int size);
179 void PutMemory(void *ptr);
180 int GetMemorySize(void *ptr);
181
182 pthread_mutex_t lock_;
183
184 /**
185 * True if AssignGlobalArenas was called and the memory manager is used by
186 * sqlite.
187 */
188 bool assigned_;
189
190 /**
191 * The standard memory allocator used by sqlite3. Saved in order to reset on
192 * destruction.
193 */
194 struct sqlite3_mem_methods sqlite3_mem_vanilla_;
195
196 struct sqlite3_mem_methods mem_methods_;
197 void *page_cache_memory_;
198 std::vector<LookasideBufferArena *> lookaside_buffer_arenas_;
199 std::vector<MallocArena *> malloc_arenas_;
200 /**
201 * Where the last successful allocation took place.
202 */
203 unsigned idx_last_arena_;
204 }; // class SqliteMemoryManager
205
206 #endif // CVMFS_SQLITEMEM_H_
207