GCC Code Coverage Report
Directory: cvmfs/ Exec Total Coverage
File: cvmfs/sqlitemem.h Lines: 5 5 100.0 %
Date: 2019-02-03 02:48:13 Branches: 2 2 100.0 %

Line Branch Exec Source
1
/**
2
 * This file is part of the CernVM File System.
3
 */
4
5
#ifndef CVMFS_SQLITEMEM_H_
6
#define CVMFS_SQLITEMEM_H_
7
8
#include <inttypes.h>
9
#include <pthread.h>
10
#include <stdint.h>
11
12
#include <cassert>
13
#include <cstdlib>
14
#include <vector>
15
16
#include "duplex_sqlite3.h"
17
#include "gtest/gtest_prod.h"
18
19
class MallocArena;
20
21
22
/**
23
 * The MemoryManager uses the sqlite hooks to optimize memory allocations.  It
24
 * is tuned for reading the cvmfs file catalogs.  It provides a page cache of
25
 * a few MB, a small scratch space and per-database lookaside buffers.  It also
26
 * contains a "general purpose" malloc/free implementation tailored to the
27
 * behavior of sqlite memory allocation.
28
 *
29
 * It is implemented as a singleton.  GetInstance() will reserve memory blocks,
30
 * AssignGlobalArenas will set the global sqlite configuration and
31
 * CleanupInstance frees the memory blocks.  These three routines are not
32
 * thread-safe.  AssignGlobalArenas has to be called as first sqlite operation,
33
 * CleanupInstance has to be called after sqlite3_shutdown.  If the singleton
34
 * is alive, read-only sqlite databases will automatically use it for their
35
 * lookaside buffers (see sql.h).  Assignment of lookaside buffers is
36
 * thread-safe.
37
 */
38
class SqliteMemoryManager {
39
  FRIEND_TEST(T_Sqlitemem, LookasideBuffer);
40
  FRIEND_TEST(T_Sqlitemem, Malloc);
41
  FRIEND_TEST(T_Sqlitemem, Realloc);
42
  FRIEND_TEST(T_Sqlitemem, ReallocStress);
43
44
 public:
45
  /**
46
   * In practice, we hardly ever see scratch memory used.  If it is used,
47
   * the allocation size is <8kB
48
   */
49
  static const unsigned kScratchSlotSize = 8192;
50
  /**
51
   * Sqlite ensures that never more slots than 2 times the number of threads are
52
   * used.  In practice, it's much lower.
53
   */
54
  static const unsigned kScratchNoSlots = 8;
55
  /**
56
   * In total: 64kB scratch memory
57
   */
58
  static const unsigned kScratchSize = kScratchSlotSize * kScratchNoSlots;
59
60
  /**
61
   * Empricially, the largest page cache allocation is 1296B.
62
   */
63
  static const unsigned kPageCacheSlotSize = 1300;
64
  /**
65
   * Number of pages that can be cached.
66
   */
67
  static const unsigned kPageCacheNoSlots = 4000;
68
  /**
69
   * In total: ~5MB
70
   */
71
  static const unsigned kPageCacheSize = kPageCacheSlotSize * kPageCacheNoSlots;
72
73
  /**
74
   * 32 bytes per slot is an empirically good value so that memory is not wasted
75
   * (too much) and many allocations fit within the boundary.
76
   */
77
  static const unsigned kLookasideSlotSize = 32;
78
  /**
79
   * 128 slots with 32bytes accumulate to 4kB.  See LookasideBufferArena below.
80
   */
81
  static const unsigned kLookasideSlotsPerDb = 128;
82
83
84
  /**
85
   * A continuous chunk of memory from which fixed-sized chunks are given as
86
   * lookaside buffers to SQlite database connections.  A bitmap indicates used
87
   * and free buffers.
88
   * One arena serves 128 database connections and assuming less than 10000
89
   * concurrently open catalogs, the number of mmap'd regions stays under 100.
90
   */
91
  class LookasideBufferArena {
92
   public:
93
    /**
94
     * Accumulates to 4kB.
95
     */
96
    static const unsigned kBufferSize =
97
      kLookasideSlotSize * kLookasideSlotsPerDb;
98
99
    /**
100
     * Has to be a multiple of the number of bits in the int type.  One arena
101
     * can serve 128 concurrent sqlite database connections.
102
     */
103
    static const unsigned kBuffersPerArena = 128;
104
    /**
105
     * 512kB = 128 4kB buffers.
106
     */
107
    static const unsigned kArenaSize = kBuffersPerArena * kBufferSize;
108
    /**
109
     * Number of bitmap ints for 128 buffers.
110
     */
111
    static const unsigned kNoBitmaps = kBuffersPerArena / (sizeof(int) * 8);
112
113
    LookasideBufferArena();
114
    ~LookasideBufferArena();
115
116
    void *GetBuffer();
117
    void PutBuffer(void *buffer);
118
    bool IsEmpty();
119
    bool Contains(void *buffer);
120
121
   private:
122
    /**
123
     * An mmap'd memory block.
124
     */
125
    void *arena_;
126
127
    /**
128
     * Indicates free/used slots.  A used slot has its bit set to zero, an
129
     * unused slot has its bit set to 1.
130
     */
131
    int freemap_[kNoBitmaps];
132
  };
133
134
135
261
  static SqliteMemoryManager *GetInstance() {
136
261
    if (instance_ == NULL)
137
157
      instance_ = new SqliteMemoryManager();
138
261
    return instance_;
139
  }
140
  static void CleanupInstance();
141
155
  static bool HasInstance() { return instance_ != NULL; }
142
  ~SqliteMemoryManager();
143
144
  void AssignGlobalArenas();
145
  void *AssignLookasideBuffer(sqlite3 *db);
146
  void ReleaseLookasideBuffer(void *buffer);
147
148
 private:
149
  /**
150
   * Should be larger than 10 times the largest allocation, which for reading
151
   * sqlite file catalogs is 64kB.  An arena size of 8MB limits the total
152
   * number of arenas (mapped blocks) to <40, given typical storage needs for
153
   * 10,000 open catalogs.  Has to be a power of 2MB (see MallocArena).
154
   */
155
  static const unsigned kArenaSize = 8 * 1024 * 1024;
156
  static SqliteMemoryManager *instance_;
157
158
  /**
159
   * SQlite memory callbacks need to be static because they are referenced by
160
   * C function pointers.  See https://www.sqlite.org/c3ref/mem_methods.html
161
   */
162
  static void *xMalloc(int size);
163
  static void xFree(void *ptr);
164
  static void *xRealloc(void *ptr, int new_size);
165
  static int xSize(void *ptr);
166
  static int xRoundup(int size);
167
  static int xInit(void *app_data);
168
  static void xShutdown(void *app_data);
169
170
171
  SqliteMemoryManager();
172
173
  void *GetLookasideBuffer();
174
  void PutLookasideBuffer(void *buffer);
175
176
  void *GetMemory(int size);
177
  void PutMemory(void *ptr);
178
  int GetMemorySize(void *ptr);
179
180
  pthread_mutex_t lock_;
181
182
  /**
183
   * True if AssignGlobalArenas was called and the memory manager is used by
184
   * sqlite.
185
   */
186
  bool assigned_;
187
188
  /**
189
   * The standard memory allocator used by sqlite3.  Saved in order to reset on
190
   * destruction.
191
   */
192
  struct sqlite3_mem_methods sqlite3_mem_vanilla_;
193
194
  struct sqlite3_mem_methods mem_methods_;
195
  void *scratch_memory_;
196
  void *page_cache_memory_;
197
  std::vector<LookasideBufferArena *> lookaside_buffer_arenas_;
198
  std::vector<MallocArena *> malloc_arenas_;
199
  /**
200
   * Where the last successful allocation took place.
201
   */
202
  unsigned idx_last_arena_;
203
};  // class SqliteMemoryManager
204
205
#endif  // CVMFS_SQLITEMEM_H_