CernVM-FS  2.10.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
quota_posix.h
Go to the documentation of this file.
1 
5 #ifndef CVMFS_QUOTA_POSIX_H_
6 #define CVMFS_QUOTA_POSIX_H_
7 
8 #include <pthread.h>
9 #include <stdint.h>
10 #include <sys/types.h>
11 #include <unistd.h>
12 
13 #include <map>
14 #include <string>
15 #include <vector>
16 
17 #include "duplex_sqlite3.h"
18 #include "gtest/gtest_prod.h"
19 #include "hash.h"
20 #include "quota.h"
21 #include "statistics.h"
22 #include "util/single_copy.h"
23 #include "util/string.h"
24 
25 namespace perf {
26 class Recorder;
27 }
28 
36  FRIEND_TEST(T_QuotaManager, BindReturnPipe);
37  FRIEND_TEST(T_QuotaManager, Cleanup);
38  FRIEND_TEST(T_QuotaManager, Contains);
39  FRIEND_TEST(T_QuotaManager, InitDatabase);
40  FRIEND_TEST(T_QuotaManager, MakeReturnPipe);
41 
42  public:
43  static PosixQuotaManager *Create(const std::string &cache_workspace,
44  const uint64_t limit, const uint64_t cleanup_threshold,
45  const bool rebuild_database);
47  const std::string &exe_path,
48  const std::string &cache_workspace,
49  const uint64_t limit,
50  const uint64_t cleanup_threshold,
51  bool foreground);
52  static int MainCacheManager(int argc, char **argv);
53 
54  virtual ~PosixQuotaManager();
55  virtual bool HasCapability(Capabilities capability) { return true; }
56 
57  virtual void Insert(const shash::Any &hash, const uint64_t size,
58  const std::string &description);
59  virtual void InsertVolatile(const shash::Any &hash, const uint64_t size,
60  const std::string &description);
61  virtual bool Pin(const shash::Any &hash, const uint64_t size,
62  const std::string &description, const bool is_catalog);
63  virtual void Unpin(const shash::Any &hash);
64  virtual void Touch(const shash::Any &hash);
65  virtual void Remove(const shash::Any &file);
66  virtual bool Cleanup(const uint64_t leave_size);
67 
68  virtual void RegisterBackChannel(int back_channel[2],
69  const std::string &channel_id);
70  virtual void UnregisterBackChannel(int back_channel[2],
71  const std::string &channel_id);
72 
73  virtual std::vector<std::string> List();
74  virtual std::vector<std::string> ListPinned();
75  virtual std::vector<std::string> ListCatalogs();
76  virtual std::vector<std::string> ListVolatile();
77  virtual uint64_t GetMaxFileSize();
78  virtual uint64_t GetCapacity();
79  virtual uint64_t GetSize();
80  virtual uint64_t GetSizePinned();
81  virtual uint64_t GetCleanupRate(uint64_t period_s);
82 
83  virtual void Spawn();
84  virtual pid_t GetPid();
85  virtual uint32_t GetProtocolRevision();
86 
87  private:
91  enum FileTypes {
94  };
95 
99  enum CommandType {
100  kTouch = 0,
118  // as of protocol revision 2
121  };
122 
128  struct LruCommand {
130  uint64_t size;
137  uint16_t desc_length;
138 
140  : command_type(static_cast<CommandType>(0))
141  , size(0)
142  , return_pipe(-1)
143  , desc_length(0)
144  {
145  memset(digest, 0, shash::kMaxDigestSize);
146  }
147 
148  void SetSize(const uint64_t new_size) {
149  uint64_t mask = 7;
150  mask = ~(mask << (64-3));
151  size = (new_size & mask) | size;
152  }
153 
154  uint64_t GetSize() const {
155  uint64_t mask = 7;
156  mask = ~(mask << (64-3));
157  return size & mask;
158  }
159 
160  void StoreHash(const shash::Any &hash) {
161  memcpy(digest, hash.digest, hash.GetDigestSize());
162  // Exclude MD5
163  uint64_t algo_flags = hash.algorithm - 1;
164  algo_flags = algo_flags << (64-3);
165  size |= algo_flags;
166  }
167 
169  uint64_t algo_flags = size >> (64-3);
170  shash::Any result(static_cast<shash::Algorithms>(algo_flags+1));
171  memcpy(result.digest, digest, result.GetDigestSize());
172  return result;
173  }
174  };
175 
179  static const unsigned kSqliteMemPerThread = 2*1024*1024;
180 
185  static const unsigned kCommandBufferSize = 32;
186 
191  static const unsigned kMaxDescription = 512-sizeof(LruCommand);
192 
197  static const unsigned kHighPinWatermark = 75;
198 
204  static const uint64_t kVolatileFlag = 1ULL << 63;
205 
206  bool InitDatabase(const bool rebuild_database);
207  bool RebuildDatabase();
208  void CloseDatabase();
209  bool Contains(const std::string &hash_str);
210  bool DoCleanup(const uint64_t leave_size);
211 
212  void MakeReturnPipe(int pipe[2]);
213  int BindReturnPipe(int pipe_wronly);
214  void UnbindReturnPipe(int pipe_wronly);
215  void UnlinkReturnPipe(int pipe_wronly);
216  void CloseReturnPipe(int pipe[2]);
217  void CleanupPipes();
218 
219  void CheckFreeSpace();
220  void CheckHighPinWatermark();
221  void ProcessCommandBunch(const unsigned num,
222  const LruCommand *commands,
223  const char *descriptions);
224  static void *MainCommandServer(void *data);
225 
226  void DoInsert(const shash::Any &hash, const uint64_t size,
227  const std::string &description, const CommandType command_type);
228  std::vector<std::string> DoList(const CommandType list_command);
229  void GetSharedStatus(uint64_t *gauge, uint64_t *pinned);
230  void GetLimits(uint64_t *limit, uint64_t *cleanup_threshold);
231 
232  static void ParseDirectories(const std::string cache_workspace,
233  std::string *cache_dir,
234  std::string *workspace_dir);
235  PosixQuotaManager(const uint64_t limit, const uint64_t cleanup_threshold,
236  const std::string &cache_workspace);
237 
242  bool shared_;
243 
248  bool spawned_;
249 
253  uint64_t limit_;
254 
259 
263  uint64_t gauge_;
264 
268  uint64_t pinned_;
269 
274  uint64_t seq_;
275 
279  std::string cache_dir_;
280 
286  std::string workspace_dir_;
287 
291  std::map<shash::Any, uint64_t> pinned_chunks_;
292 
296  int pipe_lru_[2];
297 
301  pthread_t thread_lru_;
302 
307 
313 
319 
320  sqlite3 *database_;
321  sqlite3_stmt *stmt_touch_;
322  sqlite3_stmt *stmt_unpin_;
323  sqlite3_stmt *stmt_block_;
324  sqlite3_stmt *stmt_unblock_;
325  sqlite3_stmt *stmt_new_;
326  sqlite3_stmt *stmt_lru_;
327  sqlite3_stmt *stmt_size_;
328  sqlite3_stmt *stmt_rm_;
329  sqlite3_stmt *stmt_list_;
330  sqlite3_stmt *stmt_list_pinned_;
331  sqlite3_stmt *stmt_list_catalogs_;
332  sqlite3_stmt *stmt_list_volatile_;
333 
338 }; // class PosixQuotaManager
339 
340 #endif // CVMFS_QUOTA_POSIX_H_
virtual uint32_t GetProtocolRevision()
Definition: quota_posix.cc:641
sqlite3_stmt * stmt_new_
Definition: quota_posix.h:325
sqlite3_stmt * stmt_block_
Definition: quota_posix.h:323
sqlite3_stmt * stmt_list_pinned_
Definition: quota_posix.h:330
virtual void UnregisterBackChannel(int back_channel[2], const std::string &channel_id)
std::vector< std::string > DoList(const CommandType list_command)
Definition: quota_posix.cc:556
virtual pid_t GetPid()
Definition: quota_posix.cc:622
virtual uint64_t GetCleanupRate(uint64_t period_s)
Definition: quota_posix.cc:690
static const unsigned kCommandBufferSize
Definition: quota_posix.h:185
sqlite3_stmt * stmt_list_
Definition: quota_posix.h:329
void GetLimits(uint64_t *limit, uint64_t *cleanup_threshold)
Definition: quota_posix.cc:598
virtual uint64_t GetSize()
Definition: quota_posix.cc:674
sqlite3_stmt * stmt_rm_
Definition: quota_posix.h:328
virtual bool Cleanup(const uint64_t leave_size)
Definition: quota_posix.cc:125
static const unsigned kSqliteMemPerThread
Definition: quota_posix.h:179
virtual uint64_t GetMaxFileSize()
Definition: quota_posix.cc:617
uint64_t cleanup_threshold_
Definition: quota_posix.h:258
Capabilities
Definition: quota.h:47
bool DoCleanup(const uint64_t leave_size)
Definition: quota_posix.cc:424
uint64_t GetSize() const
Definition: quota_posix.h:154
static void * MainCommandServer(void *data)
virtual void Insert(const shash::Any &hash, const uint64_t size, const std::string &description)
Definition: quota_posix.cc:894
bool InitDatabase(const bool rebuild_database)
Definition: quota_posix.cc:708
int BindReturnPipe(int pipe_wronly)
Definition: quota_posix.cc:64
std::string cache_dir_
Definition: quota_posix.h:279
bool Contains(const std::string &hash_str)
Definition: quota_posix.cc:189
Algorithms algorithm
Definition: hash.h:124
perf::MultiRecorder cleanup_recorder_
Definition: quota_posix.h:318
sqlite3 * database_
Definition: quota_posix.h:320
unsigned char digest[digest_size_]
Definition: hash.h:123
virtual std::vector< std::string > ListCatalogs()
Definition: quota_posix.cc:936
void CheckHighPinWatermark()
Definition: quota_posix.cc:82
sqlite3_stmt * stmt_lru_
Definition: quota_posix.h:326
void GetSharedStatus(uint64_t *gauge, uint64_t *pinned)
Definition: quota_posix.cc:660
unsigned GetDigestSize() const
Definition: hash.h:167
std::string workspace_dir_
Definition: quota_posix.h:286
void MakeReturnPipe(int pipe[2])
virtual std::vector< std::string > ListPinned()
Definition: quota_posix.cc:928
pthread_t thread_lru_
Definition: quota_posix.h:301
virtual uint64_t GetSizePinned()
Definition: quota_posix.cc:682
virtual bool HasCapability(Capabilities capability)
Definition: quota_posix.h:55
virtual bool Pin(const shash::Any &hash, const uint64_t size, const std::string &description, const bool is_catalog)
virtual std::vector< std::string > ListVolatile()
Definition: quota_posix.cc:944
virtual uint64_t GetCapacity()
Definition: quota_posix.cc:582
FRIEND_TEST(T_QuotaManager, BindReturnPipe)
void SetSize(const uint64_t new_size)
Definition: quota_posix.h:148
shash::Any RetrieveHash() const
Definition: quota_posix.h:168
static const unsigned kHighPinWatermark
Definition: quota_posix.h:197
virtual void RegisterBackChannel(int back_channel[2], const std::string &channel_id)
void UnlinkReturnPipe(int pipe_wronly)
virtual void Remove(const shash::Any &file)
static PosixQuotaManager * Create(const std::string &cache_workspace, const uint64_t limit, const uint64_t cleanup_threshold, const bool rebuild_database)
Definition: quota_posix.cc:230
sqlite3_stmt * stmt_touch_
Definition: quota_posix.h:321
void DoInsert(const shash::Any &hash, const uint64_t size, const std::string &description, const CommandType command_type)
Definition: quota_posix.cc:531
sqlite3_stmt * stmt_list_volatile_
Definition: quota_posix.h:332
void ProcessCommandBunch(const unsigned num, const LruCommand *commands, const char *descriptions)
sqlite3_stmt * stmt_unblock_
Definition: quota_posix.h:324
virtual void Unpin(const shash::Any &hash)
PosixQuotaManager(const uint64_t limit, const uint64_t cleanup_threshold, const std::string &cache_workspace)
virtual std::vector< std::string > List()
Definition: quota_posix.cc:920
static int MainCacheManager(int argc, char **argv)
Definition: quota_posix.cc:952
static const unsigned kMaxDescription
Definition: quota_posix.h:191
virtual void InsertVolatile(const shash::Any &hash, const uint64_t size, const std::string &description)
Definition: quota_posix.cc:908
unsigned char digest[shash::kMaxDigestSize]
Definition: quota_posix.h:132
sqlite3_stmt * stmt_list_catalogs_
Definition: quota_posix.h:331
static PosixQuotaManager * CreateShared(const std::string &exe_path, const std::string &cache_workspace, const uint64_t limit, const uint64_t cleanup_threshold, bool foreground)
Definition: quota_posix.cc:262
const unsigned kMaxDigestSize
Definition: hash.h:71
virtual ~PosixQuotaManager()
static void ParseDirectories(const std::string cache_workspace, std::string *cache_dir, std::string *workspace_dir)
std::map< shash::Any, uint64_t > pinned_chunks_
Definition: quota_posix.h:291
sqlite3_stmt * stmt_unpin_
Definition: quota_posix.h:322
void CloseReturnPipe(int pipe[2])
Definition: quota_posix.cc:179
virtual void Spawn()
static void size_t size
Definition: smalloc.h:47
void UnbindReturnPipe(int pipe_wronly)
virtual void Touch(const shash::Any &hash)
void StoreHash(const shash::Any &hash)
Definition: quota_posix.h:160
static const uint64_t kVolatileFlag
Definition: quota_posix.h:204
sqlite3_stmt * stmt_size_
Definition: quota_posix.h:327