Head

GCC Code Coverage Report

Directory:	cvmfs/		Exec	Total	Coverage
File:	cvmfs/cache_ram.h	Lines:	37	38	97.4 %
Date:	2019-02-03 02:48:13	Branches:	2	2	100.0 %


/**
 * This file is part of the CernVM File System.
 */
#ifndef CVMFS_CACHE_RAM_H_
#define CVMFS_CACHE_RAM_H_

#include <pthread.h>

#include <cassert>
#include <cstdlib>
#include <string>
#include <vector>

#include "cache.h"
#include "fd_table.h"
#include "hash.h"
#include "kvstore.h"
#include "statistics.h"
#include "util/pointer.h"


/**
 * The @p RamCacheManager class provides a cache backend that operates
 * entirely from the host's RAM. This backend does not require any
 * additional privileges on the host such as filesystem access.
 * This cache resides in a single process' memory, and does not
 * support sharing or persistence. This cache backend is a good
 * choice in highly restrictive environments, e.g. HPC resources,
 * where it is not possible to make use of a shared/local filesystem.
 *
 * To use this cache backend, set @p CVMFS_CACHE_PRIMARY=ram. There
 * are not many knobs to configure; at present only the size to be
 * used for cached objects. @p CVMFS_CACHE_RAM_SIZE sets the amount
 * of memory to use (in MB) for caching objects. If the size ends
 * with a percent sign, the cache size will be set to that
 * percentage of the system memory. If no size is specified, the
 * RAM cache size defaults to ~3% of the system memory.
 * The minimum cache size is 200 MB.
 *
 * RamCacheManager uses a custom heap allocator rather than
 * the system's libc @p malloc(). To switch to libc malloc, set
 * @p CVMFS_CACHE_RAM_MALLOC=libc
 */
class RamCacheManager : public CacheManager {
 public:
  struct Counters {
    perf::Counter *n_getsize;
    perf::Counter *n_close;
    perf::Counter *n_pread;
    perf::Counter *n_dup;
    perf::Counter *n_readahead;
    perf::Counter *n_starttxn;
    perf::Counter *n_write;
    perf::Counter *n_reset;
    perf::Counter *n_aborttxn;
    perf::Counter *n_committxn;
    perf::Counter *n_enfile;
    perf::Counter *n_openregular;
    perf::Counter *n_openvolatile;
    perf::Counter *n_openmiss;
    perf::Counter *n_overrun;
    perf::Counter *n_full;
    perf::Counter *n_realloc;

    explicit Counters(perf::StatisticsTemplate statistics) {
      n_getsize = statistics.RegisterTemplated("n_getsize",
        "Number of GetSize calls");
      n_close = statistics.RegisterTemplated("n_close",
        "Number of Close calls");
      n_pread = statistics.RegisterTemplated("n_pread",
        "Number of Pread calls");
      n_dup = statistics.RegisterTemplated("n_dup",
        "Number of Dup calls");
      n_readahead = statistics.RegisterTemplated("n_readahead",
        "Number of ReadAhead calls");
      n_starttxn = statistics.RegisterTemplated("n_starttxn",
        "Number of StartTxn calls");
      n_write = statistics.RegisterTemplated("n_write",
        "Number of Write calls");
      n_reset = statistics.RegisterTemplated("n_reset",
        "Number of Reset calls");
      n_aborttxn = statistics.RegisterTemplated("n_aborttxn",
        "Number of AbortTxn calls");
      n_committxn = statistics.RegisterTemplated("n_committxn",
        "Number of Commit calls");
      n_enfile = statistics.RegisterTemplated("n_enfile",
        "Number of times the limit on handles was reached");
      n_openregular = statistics.RegisterTemplated("n_openregular",
        "Number of opens from the regular cache");
      n_openvolatile = statistics.RegisterTemplated("n_openvolatile",
        "Number of opens from the volatile cache");
      n_openmiss = statistics.RegisterTemplated("n_openmiss",
        "Number of missed opens");
      n_realloc = statistics.RegisterTemplated("n_realloc",
        "Number of reallocs");
      n_overrun = statistics.RegisterTemplated("n_overrun",
        "Number of cache limit overruns");
      n_full = statistics.RegisterTemplated("n_full",
        "Number of overruns that could not be resolved");
    }
  };

  virtual CacheManagerIds id() { return kRamCacheManager; }
  virtual std::string Describe();

  RamCacheManager(
    uint64_t max_size,
    unsigned max_entries,
    MemoryKvStore::MemoryAllocator alloc,
    perf::StatisticsTemplate statistics);

  virtual ~RamCacheManager();

  virtual bool AcquireQuotaManager(QuotaManager *quota_mgr);

  /**
   * Open a new file descriptor into the cache. Note that opening entries
   * effectively pins them in the cache, so it may be necessary to close
   * unneeded file descriptors to allow eviction to make room in the cache
   * @param object The blessed hash key
   * @returns A nonnegative integer file descriptor
   * @retval -ENOENT The entry is not in the cache
   * @retval -ENFILE No handles are available
   */
  virtual int Open(const BlessedObject &object);

  /**
   * Look up the size in bytes of the open cache entry
   * @param id The hash key
   * @returns The size of the entry
   * @retval -EBADF @p fd is not valid
   */
  virtual int64_t GetSize(int fd);

  /**
   * Close the descriptor in the cache. Entries not marked as pinned will become
   * subject to eviction once closed
   * @param id The hash key
   * @retval -EBADF @p fd is not valid
   */
  virtual int Close(int fd);

  /**
   * Read a section from the cache entry. See pread(3) for a discussion of the
   * arguments
   * @param id The hash key
   * @returns The number of bytes copied
   * @retval -EBADF @p fd is not valid
   */
  virtual int64_t Pread(int fd, void *buf, uint64_t size, uint64_t offset);

  /**
   * Duplicates the open file descriptor, allowing the original and the new one
   * to be used independently
   * @param id The hash key
   * @returns A new, nonnegative integer fd
   * @retval -EBADF @p fd is not valid
   * @retval -ENFILE No handles are available
   */
  virtual int Dup(int fd);

  /**
   * No effect for in-memory caches
   * @param id The hash key
   * @retval -EBADF @p fd is not valid
   */
  virtual int Readahead(int fd);


  /**
   * Get the amount of space to be allocated for a call to @ref StartTxn
   */
  virtual uint32_t SizeOfTxn() { return sizeof(Transaction); }


  /**
   * Start a new transaction, allocating the required memory and initializing
   * the transaction parameters
   * @param id The hash key
   * @param size The total size of the new cache entry
   * @param txn A pointer to space allocated for storing the transaction details
   */
  virtual int StartTxn(const shash::Any &id, uint64_t size, void *txn);

  /**
   * Set the transaction parameters. At present, only @ref type is used for
   * pinned, volatile, etc.
   * @param description Unused
   * @param type The type of the entry, e.g. pinned
   * @param flags Unused
   * @param txn A pointer to space allocated for storing the transaction details
   */
  virtual void CtrlTxn(const ObjectInfo &object_info,
                       const int flags,
                       void *txn);

  /**
   * Copy the given memory region into the transaction buffer. Copying starts at
   * the transaction's current offset
   * @param buf The source address
   * @param size The number of bytes to copy
   * @param txn A pointer to space allocated for storing the transaction details
   */
  virtual int64_t Write(const void *buf, uint64_t size, void *txn);

  /**
   * Seek to the beginning of the transaction buffer
   * @param txn A pointer to space allocated for storing the transaction details
   */
  virtual int Reset(void *txn);

  /**
   * Commit a transaction and open the resulting cache entry. This is necessary
   * to avoid a race condition in which the cache entry is evicted between the
   * calls to CommitTxn and Open.
   * @param txn A pointer to space allocated for storing the transaction details
   * @returns A file descriptor to the new cache entry, or a negative error from
   *          @ref CommitTxn
   */
  virtual int OpenFromTxn(void *txn);

  /**
   * Free the resources allocated for the pending transaction
   * @param txn A pointer to space allocated for storing the transaction details
   */
  virtual int AbortTxn(void *txn);

  /**
   * Commit a transaction to the cache. If there is not enough free space in the
   * cache, first try to make room by evicting volatile entries. If there is
   * still not enough room, try evicting regular entries. If there is *still*
   * not enough space, give up an return failure.
   * @param txn A pointer to space allocated for storing the transaction details
   * @retval -ENOSPC The transaction would exceed the size of the cache
   * @retval -ENFILE No handles are available
   * @retval -EEXIST An entry with nonzero reference count already exists
   */
  virtual int CommitTxn(void *txn);

  virtual void Spawn() { }

 private:
  // The null hash (hashed output is all null bytes) serves as a marker for
  // an invalid handle
  static const shash::Any kInvalidHandle;

  struct ReadOnlyHandle {
    ReadOnlyHandle()
      : handle(kInvalidHandle)
      , is_volatile(false)
      { }
    ReadOnlyHandle(const shash::Any &h, bool v)
      : handle(h)
      , is_volatile(v)
      { }
    bool operator ==(const ReadOnlyHandle &other) const {
      return this->handle == other.handle;
    }
    bool operator !=(const ReadOnlyHandle &other) const {
      return this->handle != other.handle;
    }

    shash::Any handle;
    bool is_volatile;
  };

  struct Transaction {
    Transaction()
      : buffer()
      , expected_size(0)
      , pos(0) { }
    MemoryBuffer buffer;
    uint64_t expected_size;
    uint64_t pos;
    std::string description;
  };

  inline MemoryKvStore *GetStore(const ReadOnlyHandle &fd) {
    if (fd.is_volatile) {
      return &volatile_entries_;
    } else {
      return &regular_entries_;
    }
  }

  inline MemoryKvStore *GetTransactionStore(Transaction *txn) {
    if (txn->buffer.object_type == kTypeVolatile) {
      return &volatile_entries_;
    } else {
      return &regular_entries_;
    }
  }

  int AddFd(const ReadOnlyHandle &handle);
  int64_t CommitToKvStore(Transaction *transaction);
  virtual int DoOpen(const shash::Any &id);

  uint64_t max_size_;
  FdTable<ReadOnlyHandle> fd_table_;
  pthread_rwlock_t rwlock_;
  MemoryKvStore regular_entries_;
  MemoryKvStore volatile_entries_;
  Counters counters_;
};  // class RamCacheManager

#endif  // CVMFS_CACHE_RAM_H_


Generated by: GCOVR (Version 4.1)

Line	Branch	Exec	Source
1			/**
2			* This file is part of the CernVM File System.
3			*/
4			#ifndef CVMFS_CACHE_RAM_H_
5			#define CVMFS_CACHE_RAM_H_
6
7			#include <pthread.h>
8
9			#include <cassert>
10			#include <cstdlib>
11			#include <string>
12			#include <vector>
13
14			#include "cache.h"
15			#include "fd_table.h"
16			#include "hash.h"
17			#include "kvstore.h"
18			#include "statistics.h"
19			#include "util/pointer.h"
20
21
22			/**
23			* The @p RamCacheManager class provides a cache backend that operates
24			* entirely from the host's RAM. This backend does not require any
25			* additional privileges on the host such as filesystem access.
26			* This cache resides in a single process' memory, and does not
27			* support sharing or persistence. This cache backend is a good
28			* choice in highly restrictive environments, e.g. HPC resources,
29			* where it is not possible to make use of a shared/local filesystem.
30			*
31			* To use this cache backend, set @p CVMFS_CACHE_PRIMARY=ram. There
32			* are not many knobs to configure; at present only the size to be
33			* used for cached objects. @p CVMFS_CACHE_RAM_SIZE sets the amount
34			* of memory to use (in MB) for caching objects. If the size ends
35			* with a percent sign, the cache size will be set to that
36			* percentage of the system memory. If no size is specified, the
37			* RAM cache size defaults to ~3% of the system memory.
38			* The minimum cache size is 200 MB.
39			*
40			* RamCacheManager uses a custom heap allocator rather than
41			* the system's libc @p malloc(). To switch to libc malloc, set
42			* @p CVMFS_CACHE_RAM_MALLOC=libc
43			*/
44			class RamCacheManager : public CacheManager {
45			public:
46			struct Counters {
47			perf::Counter *n_getsize;
48			perf::Counter *n_close;
49			perf::Counter *n_pread;
50			perf::Counter *n_dup;
51			perf::Counter *n_readahead;
52			perf::Counter *n_starttxn;
53			perf::Counter *n_write;
54			perf::Counter *n_reset;
55			perf::Counter *n_aborttxn;
56			perf::Counter *n_committxn;
57			perf::Counter *n_enfile;
58			perf::Counter *n_openregular;
59			perf::Counter *n_openvolatile;
60			perf::Counter *n_openmiss;
61			perf::Counter *n_overrun;
62			perf::Counter *n_full;
63			perf::Counter *n_realloc;
64
65		48	explicit Counters(perf::StatisticsTemplate statistics) {
66			n_getsize = statistics.RegisterTemplated("n_getsize",
67		48	"Number of GetSize calls");
68			n_close = statistics.RegisterTemplated("n_close",
69		48	"Number of Close calls");
70			n_pread = statistics.RegisterTemplated("n_pread",
71		48	"Number of Pread calls");
72			n_dup = statistics.RegisterTemplated("n_dup",
73		48	"Number of Dup calls");
74			n_readahead = statistics.RegisterTemplated("n_readahead",
75		48	"Number of ReadAhead calls");
76			n_starttxn = statistics.RegisterTemplated("n_starttxn",
77		48	"Number of StartTxn calls");
78			n_write = statistics.RegisterTemplated("n_write",
79		48	"Number of Write calls");
80			n_reset = statistics.RegisterTemplated("n_reset",
81		48	"Number of Reset calls");
82			n_aborttxn = statistics.RegisterTemplated("n_aborttxn",
83		48	"Number of AbortTxn calls");
84			n_committxn = statistics.RegisterTemplated("n_committxn",
85		48	"Number of Commit calls");
86			n_enfile = statistics.RegisterTemplated("n_enfile",
87		48	"Number of times the limit on handles was reached");
88			n_openregular = statistics.RegisterTemplated("n_openregular",
89		48	"Number of opens from the regular cache");
90			n_openvolatile = statistics.RegisterTemplated("n_openvolatile",
91		48	"Number of opens from the volatile cache");
92			n_openmiss = statistics.RegisterTemplated("n_openmiss",
93		48	"Number of missed opens");
94			n_realloc = statistics.RegisterTemplated("n_realloc",
95		48	"Number of reallocs");
96			n_overrun = statistics.RegisterTemplated("n_overrun",
97		48	"Number of cache limit overruns");
98			n_full = statistics.RegisterTemplated("n_full",
99		48	"Number of overruns that could not be resolved");
100		48	}
101			};
102
103		11	virtual CacheManagerIds id() { return kRamCacheManager; }
104			virtual std::string Describe();
105
106			RamCacheManager(
107			uint64_t max_size,
108			unsigned max_entries,
109			MemoryKvStore::MemoryAllocator alloc,
110			perf::StatisticsTemplate statistics);
111
112			virtual ~RamCacheManager();
113
114			virtual bool AcquireQuotaManager(QuotaManager *quota_mgr);
115
116			/**
117			* Open a new file descriptor into the cache. Note that opening entries
118			* effectively pins them in the cache, so it may be necessary to close
119			* unneeded file descriptors to allow eviction to make room in the cache
120			* @param object The blessed hash key
121			* @returns A nonnegative integer file descriptor
122			* @retval -ENOENT The entry is not in the cache
123			* @retval -ENFILE No handles are available
124			*/
125			virtual int Open(const BlessedObject &object);
126
127			/**
128			* Look up the size in bytes of the open cache entry
129			* @param id The hash key
130			* @returns The size of the entry
131			* @retval -EBADF @p fd is not valid
132			*/
133			virtual int64_t GetSize(int fd);
134
135			/**
136			* Close the descriptor in the cache. Entries not marked as pinned will become
137			* subject to eviction once closed
138			* @param id The hash key
139			* @retval -EBADF @p fd is not valid
140			*/
141			virtual int Close(int fd);
142
143			/**
144			* Read a section from the cache entry. See pread(3) for a discussion of the
145			* arguments
146			* @param id The hash key
147			* @returns The number of bytes copied
148			* @retval -EBADF @p fd is not valid
149			*/
150			virtual int64_t Pread(int fd, void *buf, uint64_t size, uint64_t offset);
151
152			/**
153			* Duplicates the open file descriptor, allowing the original and the new one
154			* to be used independently
155			* @param id The hash key
156			* @returns A new, nonnegative integer fd
157			* @retval -EBADF @p fd is not valid
158			* @retval -ENFILE No handles are available
159			*/
160			virtual int Dup(int fd);
161
162			/**
163			* No effect for in-memory caches
164			* @param id The hash key
165			* @retval -EBADF @p fd is not valid
166			*/
167			virtual int Readahead(int fd);
168
169
170			/**
171			* Get the amount of space to be allocated for a call to @ref StartTxn
172			*/
173		43	virtual uint32_t SizeOfTxn() { return sizeof(Transaction); }
174
175
176			/**
177			* Start a new transaction, allocating the required memory and initializing
178			* the transaction parameters
179			* @param id The hash key
180			* @param size The total size of the new cache entry
181			* @param txn A pointer to space allocated for storing the transaction details
182			*/
183			virtual int StartTxn(const shash::Any &id, uint64_t size, void *txn);
184
185			/**
186			* Set the transaction parameters. At present, only @ref type is used for
187			* pinned, volatile, etc.
188			* @param description Unused
189			* @param type The type of the entry, e.g. pinned
190			* @param flags Unused
191			* @param txn A pointer to space allocated for storing the transaction details
192			*/
193			virtual void CtrlTxn(const ObjectInfo &object_info,
194			const int flags,
195			void *txn);
196
197			/**
198			* Copy the given memory region into the transaction buffer. Copying starts at
199			* the transaction's current offset
200			* @param buf The source address
201			* @param size The number of bytes to copy
202			* @param txn A pointer to space allocated for storing the transaction details
203			*/
204			virtual int64_t Write(const void buf, uint64_t size, void txn);
205
206			/**
207			* Seek to the beginning of the transaction buffer
208			* @param txn A pointer to space allocated for storing the transaction details
209			*/
210			virtual int Reset(void *txn);
211
212			/**
213			* Commit a transaction and open the resulting cache entry. This is necessary
214			* to avoid a race condition in which the cache entry is evicted between the
215			* calls to CommitTxn and Open.
216			* @param txn A pointer to space allocated for storing the transaction details
217			* @returns A file descriptor to the new cache entry, or a negative error from
218			* @ref CommitTxn
219			*/
220			virtual int OpenFromTxn(void *txn);
221
222			/**
223			* Free the resources allocated for the pending transaction
224			* @param txn A pointer to space allocated for storing the transaction details
225			*/
226			virtual int AbortTxn(void *txn);
227
228			/**
229			* Commit a transaction to the cache. If there is not enough free space in the
230			* cache, first try to make room by evicting volatile entries. If there is
231			* still not enough room, try evicting regular entries. If there is still
232			* not enough space, give up an return failure.
233			* @param txn A pointer to space allocated for storing the transaction details
234			* @retval -ENOSPC The transaction would exceed the size of the cache
235			* @retval -ENFILE No handles are available
236			* @retval -EEXIST An entry with nonzero reference count already exists
237			*/
238			virtual int CommitTxn(void *txn);
239
240			virtual void Spawn() { }
241
242			private:
243			// The null hash (hashed output is all null bytes) serves as a marker for
244			// an invalid handle
245			static const shash::Any kInvalidHandle;
246
247			struct ReadOnlyHandle {
248		48	ReadOnlyHandle()
249			: handle(kInvalidHandle)
250		48	, is_volatile(false)
251		48	{ }
252		18	ReadOnlyHandle(const shash::Any &h, bool v)
253			: handle(h)
254		18	, is_volatile(v)
255		18	{ }
256		22085	bool operator ==(const ReadOnlyHandle &other) const {
257		22085	return this->handle == other.handle;
258			}
259		44119	bool operator !=(const ReadOnlyHandle &other) const {
260		44119	return this->handle != other.handle;
261			}
262
263			shash::Any handle;
264			bool is_volatile;
265			};
266
267			struct Transaction {
268		37	Transaction()
269			: buffer()
270			, expected_size(0)
271		37	, pos(0) { }
272			MemoryBuffer buffer;
273			uint64_t expected_size;
274			uint64_t pos;
275			std::string description;
276			};
277
278		22088	inline MemoryKvStore *GetStore(const ReadOnlyHandle &fd) {
279	✓✓	22088	if (fd.is_volatile) {
280		6	return &volatile_entries_;
281			} else {
282		22082	return &regular_entries_;
283			}
284			}
285
286			inline MemoryKvStore GetTransactionStore(Transaction txn) {
287			if (txn->buffer.object_type == kTypeVolatile) {
288			return &volatile_entries_;
289			} else {
290			return &regular_entries_;
291			}
292			}
293
294			int AddFd(const ReadOnlyHandle &handle);
295			int64_t CommitToKvStore(Transaction *transaction);
296			virtual int DoOpen(const shash::Any &id);
297
298			uint64_t max_size_;
299			FdTable<ReadOnlyHandle> fd_table_;
300			pthread_rwlock_t rwlock_;
301			MemoryKvStore regular_entries_;
302			MemoryKvStore volatile_entries_;
303			Counters counters_;
304			}; // class RamCacheManager
305
306			#endif // CVMFS_CACHE_RAM_H_