![]() |
CernVM-FS
2.11.0
|
#include <quota_posix.h>
Classes | |
struct | LruCommand |
Public Member Functions | |
virtual | ~PosixQuotaManager () |
virtual bool | HasCapability (Capabilities capability) |
virtual void | Insert (const shash::Any &hash, const uint64_t size, const std::string &description) |
virtual void | InsertVolatile (const shash::Any &hash, const uint64_t size, const std::string &description) |
virtual bool | Pin (const shash::Any &hash, const uint64_t size, const std::string &description, const bool is_catalog) |
virtual void | Unpin (const shash::Any &hash) |
virtual void | Touch (const shash::Any &hash) |
virtual void | Remove (const shash::Any &file) |
virtual bool | Cleanup (const uint64_t leave_size) |
virtual void | RegisterBackChannel (int back_channel[2], const std::string &channel_id) |
virtual void | UnregisterBackChannel (int back_channel[2], const std::string &channel_id) |
virtual std::vector< std::string > | List () |
virtual std::vector< std::string > | ListPinned () |
virtual std::vector< std::string > | ListCatalogs () |
virtual std::vector< std::string > | ListVolatile () |
virtual uint64_t | GetMaxFileSize () |
virtual uint64_t | GetCapacity () |
virtual uint64_t | GetSize () |
virtual uint64_t | GetSizePinned () |
virtual uint64_t | GetCleanupRate (uint64_t period_s) |
virtual void | Spawn () |
virtual pid_t | GetPid () |
virtual uint32_t | GetProtocolRevision () |
![]() | |
QuotaManager () | |
virtual | ~QuotaManager () |
void | BroadcastBackchannels (const std::string &message) |
Static Public Member Functions | |
static PosixQuotaManager * | Create (const std::string &cache_workspace, const uint64_t limit, const uint64_t cleanup_threshold, const bool rebuild_database) |
static PosixQuotaManager * | CreateShared (const std::string &exe_path, const std::string &cache_workspace, const uint64_t limit, const uint64_t cleanup_threshold, bool foreground) |
static int | MainCacheManager (int argc, char **argv) |
Private Types | |
enum | FileTypes { kFileRegular = 0, kFileCatalog } |
enum | CommandType { kTouch = 0, kInsert, kReserve, kPin, kUnpin, kRemove, kCleanup, kList, kListPinned, kListCatalogs, kStatus, kLimits, kPid, kPinRegular, kRegisterBackChannel, kUnregisterBackChannel, kGetProtocolRevision, kInsertVolatile, kListVolatile, kCleanupRate } |
Private Member Functions | |
FRIEND_TEST (T_QuotaManager, BindReturnPipe) | |
FRIEND_TEST (T_QuotaManager, Cleanup) | |
FRIEND_TEST (T_QuotaManager, Contains) | |
FRIEND_TEST (T_QuotaManager, InitDatabase) | |
FRIEND_TEST (T_QuotaManager, MakeReturnPipe) | |
bool | InitDatabase (const bool rebuild_database) |
bool | RebuildDatabase () |
void | CloseDatabase () |
bool | Contains (const std::string &hash_str) |
bool | DoCleanup (const uint64_t leave_size) |
void | MakeReturnPipe (int pipe[2]) |
int | BindReturnPipe (int pipe_wronly) |
void | UnbindReturnPipe (int pipe_wronly) |
void | UnlinkReturnPipe (int pipe_wronly) |
void | CloseReturnPipe (int pipe[2]) |
void | CleanupPipes () |
void | CheckFreeSpace () |
void | CheckHighPinWatermark () |
void | ProcessCommandBunch (const unsigned num, const LruCommand *commands, const char *descriptions) |
void | DoInsert (const shash::Any &hash, const uint64_t size, const std::string &description, const CommandType command_type) |
std::vector< std::string > | DoList (const CommandType list_command) |
void | GetSharedStatus (uint64_t *gauge, uint64_t *pinned) |
void | GetLimits (uint64_t *limit, uint64_t *cleanup_threshold) |
PosixQuotaManager (const uint64_t limit, const uint64_t cleanup_threshold, const std::string &cache_workspace) | |
Static Private Member Functions | |
static void * | MainCommandServer (void *data) |
static void | ParseDirectories (const std::string cache_workspace, std::string *cache_dir, std::string *workspace_dir) |
Private Attributes | |
bool | shared_ |
bool | spawned_ |
uint64_t | limit_ |
uint64_t | cleanup_threshold_ |
uint64_t | gauge_ |
uint64_t | pinned_ |
uint64_t | seq_ |
std::string | cache_dir_ |
std::string | workspace_dir_ |
std::map< shash::Any, uint64_t > | pinned_chunks_ |
int | pipe_lru_ [2] |
pthread_t | thread_lru_ |
int | fd_lock_cachedb_ |
bool | async_delete_ |
perf::MultiRecorder | cleanup_recorder_ |
sqlite3 * | database_ |
sqlite3_stmt * | stmt_touch_ |
sqlite3_stmt * | stmt_unpin_ |
sqlite3_stmt * | stmt_block_ |
sqlite3_stmt * | stmt_unblock_ |
sqlite3_stmt * | stmt_new_ |
sqlite3_stmt * | stmt_lru_ |
sqlite3_stmt * | stmt_size_ |
sqlite3_stmt * | stmt_rm_ |
sqlite3_stmt * | stmt_list_ |
sqlite3_stmt * | stmt_list_pinned_ |
sqlite3_stmt * | stmt_list_catalogs_ |
sqlite3_stmt * | stmt_list_volatile_ |
bool | initialized_ |
Static Private Attributes | |
static const unsigned | kSqliteMemPerThread = 2*1024*1024 |
static const unsigned | kCommandBufferSize = 32 |
static const unsigned | kMaxDescription = 512-sizeof(LruCommand) |
static const unsigned | kHighPinWatermark = 75 |
static const uint64_t | kVolatileFlag = 1ULL << 63 |
Additional Inherited Members | |
![]() | |
enum | Capabilities { kCapIntrospectSize = 0, kCapIntrospectCleanupRate, kCapList, kCapShrink, kCapListeners } |
![]() | |
static const uint32_t | kProtocolRevision = 2 |
![]() | |
void | LockBackChannels () |
void | UnlockBackChannels () |
![]() | |
std::map< shash::Md5, int > | back_channels_ |
pthread_mutex_t * | lock_back_channels_ |
uint32_t | protocol_revision_ |
Works with the PosixCacheManager. Uses an SQlite database for cache contents tracking. Tracking is asynchronously.
TODO(jblomer): split into client, server, and protocol classes.
Definition at line 35 of file quota_posix.h.
|
private |
List of RPCs that can be sent to the cache manager.
Definition at line 99 of file quota_posix.h.
|
private |
Loaded catalogs are pinned in the LRU and have to be treated differently.
Enumerator | |
---|---|
kFileRegular | |
kFileCatalog |
Definition at line 91 of file quota_posix.h.
|
virtual |
|
private |
|
private |
Definition at line 64 of file quota_posix.cc.
Referenced by MainCommandServer().
|
private |
Definition at line 204 of file quota_posix.cc.
Referenced by Create(), and MainCacheManager().
|
private |
Definition at line 82 of file quota_posix.cc.
Referenced by MainCommandServer().
|
virtual |
Cleans up in data cache, until cache size is below leave_size. The actual unlinking is done in a separate process (fork).
Implements QuotaManager.
Definition at line 125 of file quota_posix.cc.
Referenced by FileSystem::SetupPosixQuotaMgr().
|
private |
Definition at line 93 of file quota_posix.cc.
Referenced by MainCacheManager().
|
private |
Definition at line 146 of file quota_posix.cc.
Referenced by MainCacheManager(), and ~PosixQuotaManager().
|
private |
Definition at line 179 of file quota_posix.cc.
Referenced by Remove().
|
private |
Definition at line 189 of file quota_posix.cc.
Referenced by ProcessCommandBunch().
|
static |
Definition at line 230 of file quota_posix.cc.
Referenced by FileSystem::SetupPosixQuotaMgr().
|
static |
Connects to a running shared local quota manager. Creates one if necessary.
Definition at line 262 of file quota_posix.cc.
Referenced by FileSystem::SetupPosixQuotaMgr().
|
private |
Definition at line 424 of file quota_posix.cc.
Referenced by MainCommandServer(), and ProcessCommandBunch().
|
private |
|
private |
|
private |
|
private |
|
private |
|
private |
|
private |
|
virtual |
Implements QuotaManager.
Definition at line 580 of file quota_posix.cc.
Referenced by FileSystem::SetupPosixQuotaMgr().
|
virtual |
Implements QuotaManager.
Definition at line 688 of file quota_posix.cc.
|
private |
Definition at line 596 of file quota_posix.cc.
Referenced by CreateShared().
|
virtual |
Since we only cleanup until cleanup_threshold, we can only add files smaller than limit-cleanup_threshold.
Implements QuotaManager.
Definition at line 615 of file quota_posix.cc.
|
virtual |
Implements QuotaManager.
Definition at line 620 of file quota_posix.cc.
|
virtual |
Implements QuotaManager.
Definition at line 639 of file quota_posix.cc.
Referenced by CreateShared().
|
private |
Queries the shared local hard disk quota manager.
Definition at line 658 of file quota_posix.cc.
|
virtual |
Implements QuotaManager.
Definition at line 672 of file quota_posix.cc.
Referenced by FileSystem::SetupPosixQuotaMgr().
|
virtual |
Implements QuotaManager.
Definition at line 680 of file quota_posix.cc.
|
inlinevirtual |
Implements QuotaManager.
Definition at line 55 of file quota_posix.h.
|
private |
Definition at line 706 of file quota_posix.cc.
Referenced by Create(), and MainCacheManager().
|
virtual |
Inserts a new file into cache catalog. This file gets a new, highest sequence number. Does cache cleanup if necessary.
Implements QuotaManager.
Definition at line 892 of file quota_posix.cc.
|
virtual |
Inserts a new file into cache catalog. This file is marked as volatile and gets a new highest sequence number with the first bit set. Cache cleanup treats these files with priority.
Implements QuotaManager.
Definition at line 906 of file quota_posix.cc.
|
virtual |
Lists all path names from the cache db.
Implements QuotaManager.
Definition at line 918 of file quota_posix.cc.
|
virtual |
Lists all sqlite catalog files from the cache db.
Implements QuotaManager.
Definition at line 934 of file quota_posix.cc.
|
virtual |
Lists all pinned files from the cache db.
Implements QuotaManager.
Definition at line 926 of file quota_posix.cc.
|
virtual |
Lists only files flagged as volatile (priority removal)
Implements QuotaManager.
Definition at line 942 of file quota_posix.cc.
|
static |
Entry point for the shared cache manager process
Definition at line 950 of file quota_posix.cc.
Referenced by AltProcessFlavor().
|
staticprivate |
Definition at line 1084 of file quota_posix.cc.
Referenced by MainCacheManager(), and Spawn().
|
private |
Definition at line 1400 of file quota_posix.cc.
Referenced by RegisterBackChannel(), and Remove().
|
staticprivate |
Definition at line 1424 of file quota_posix.cc.
Referenced by PosixQuotaManager().
|
virtual |
Immediately inserts a new pinned catalog. Does cache cleanup if necessary.
Implements QuotaManager.
Definition at line 1449 of file quota_posix.cc.
|
private |
Definition at line 1581 of file quota_posix.cc.
Referenced by MainCommandServer().
|
private |
|
virtual |
Register a channel that allows the cache manager to trigger action to its clients. Currently used for releasing pinned catalogs.
Implements QuotaManager.
Definition at line 1807 of file quota_posix.cc.
|
virtual |
Removes a chunk from cache, if it exists.
Implements QuotaManager.
Definition at line 1839 of file quota_posix.cc.
|
virtual |
Implements QuotaManager.
Definition at line 1859 of file quota_posix.cc.
|
virtual |
Updates the sequence number of the file specified by the hash.
Implements QuotaManager.
Definition at line 1876 of file quota_posix.cc.
|
private |
Definition at line 1884 of file quota_posix.cc.
Referenced by MainCommandServer().
|
private |
Definition at line 1890 of file quota_posix.cc.
Referenced by MainCommandServer().
|
virtual |
Implements QuotaManager.
Definition at line 1896 of file quota_posix.cc.
|
virtual |
Implements QuotaManager.
Definition at line 1906 of file quota_posix.cc.
|
private |
If this is true, the unlink operations that correspond to a cleanup run will be performed in a detached, asynchronous process.
Definition at line 312 of file quota_posix.h.
|
private |
Should match the directory given to the cache manager.
Definition at line 279 of file quota_posix.h.
Referenced by MainCacheManager(), MainCommandServer(), PosixQuotaManager(), RebuildDatabase(), and Remove().
|
private |
Keeps track of the number of cleanups over time. Use by cvmfs_talk cleanup rate
Definition at line 318 of file quota_posix.h.
Referenced by MainCommandServer(), and PosixQuotaManager().
|
private |
Cleanup until cleanup_threshold_ are left in the cache.
Definition at line 258 of file quota_posix.h.
Referenced by CreateShared(), MainCacheManager(), MainCommandServer(), and ProcessCommandBunch().
|
private |
Definition at line 320 of file quota_posix.h.
Referenced by ProcessCommandBunch(), and RebuildDatabase().
|
private |
Ensures exclusive cache database access through POSIX file lock.
Definition at line 306 of file quota_posix.h.
|
private |
Current size of cache.
Definition at line 263 of file quota_posix.h.
Referenced by MainCommandServer(), ProcessCommandBunch(), and RebuildDatabase().
|
private |
Used in the destructor to steer closing of the database and so on.
Definition at line 337 of file quota_posix.h.
Referenced by Create(), CreateShared(), and ~PosixQuotaManager().
|
staticprivate |
Collect a number of insert and touch operations before processing them as sqlite commands.
Definition at line 185 of file quota_posix.h.
|
staticprivate |
Alarm when more than 75% of the cache fraction allowed for pinned files (50%) is filled with pinned files
Definition at line 197 of file quota_posix.h.
|
staticprivate |
Make sure that the amount of data transferred through the RPC pipe is within the OS's guarantees for atomicity.
Definition at line 191 of file quota_posix.h.
Referenced by ProcessCommandBunch().
|
staticprivate |
Maximum page cache per thread (Bytes).
Definition at line 179 of file quota_posix.h.
Referenced by MainCommandServer().
|
staticprivate |
The last bit in the sequence number indicates if an entry is volatile. Such sequence numbers are negative and they are preferred during cleanup. Volatile entries are used for instance for ALICE conditions data.
Definition at line 204 of file quota_posix.h.
Referenced by ProcessCommandBunch().
|
private |
Soft limit in bytes, start cleanup when reached.
Definition at line 253 of file quota_posix.h.
Referenced by CreateShared(), MainCacheManager(), MainCommandServer(), and ProcessCommandBunch().
|
private |
Size of pinned files in bytes (usually file catalogs).
Definition at line 268 of file quota_posix.h.
Referenced by MainCacheManager(), and MainCommandServer().
|
private |
Pinned content hashes and their size.
Definition at line 291 of file quota_posix.h.
Referenced by MainCommandServer().
|
private |
Used to send RPCs to the quota manager thread or process.
Definition at line 296 of file quota_posix.h.
Referenced by Create(), CreateShared(), MainCacheManager(), MainCommandServer(), PosixQuotaManager(), RegisterBackChannel(), Remove(), Touch(), Unpin(), UnregisterBackChannel(), and ~PosixQuotaManager().
|
private |
Current access sequence number. Gets increased on every access/insert operation.
Definition at line 274 of file quota_posix.h.
Referenced by ProcessCommandBunch(), and RebuildDatabase().
|
private |
Indicates if the cache manager is a shared process or a thread within the same process (exclusive cache manager)
Definition at line 242 of file quota_posix.h.
Referenced by CreateShared(), MainCacheManager(), UnbindReturnPipe(), UnlinkReturnPipe(), and ~PosixQuotaManager().
|
private |
True once the program switches into multi-threaded mode or the quota manager process has been forked resp.
Definition at line 248 of file quota_posix.h.
Referenced by CreateShared(), MainCacheManager(), Spawn(), and ~PosixQuotaManager().
|
private |
Definition at line 323 of file quota_posix.h.
|
private |
Definition at line 329 of file quota_posix.h.
Referenced by MainCommandServer().
|
private |
Definition at line 331 of file quota_posix.h.
Referenced by MainCommandServer().
|
private |
Loaded catalogs are pinned.
Definition at line 330 of file quota_posix.h.
Referenced by MainCommandServer().
|
private |
Definition at line 332 of file quota_posix.h.
Referenced by MainCommandServer().
|
private |
Definition at line 326 of file quota_posix.h.
|
private |
Definition at line 325 of file quota_posix.h.
Referenced by ProcessCommandBunch().
|
private |
Definition at line 328 of file quota_posix.h.
Referenced by MainCommandServer().
|
private |
Definition at line 327 of file quota_posix.h.
Referenced by MainCommandServer().
|
private |
Definition at line 321 of file quota_posix.h.
Referenced by ProcessCommandBunch().
|
private |
Definition at line 324 of file quota_posix.h.
|
private |
Definition at line 322 of file quota_posix.h.
Referenced by ProcessCommandBunch().
|
private |
In exclusive mode, controls the quota manager thread.
Definition at line 301 of file quota_posix.h.
Referenced by Spawn(), and ~PosixQuotaManager().
|
private |
Directory for the database lock (shared manager) and the pipes (also shared manager). Usually the same as cache_dir_. Can be different if CVMFS_WORKSPACE or CVMFS_CACHE_WORKSPACE is set.
Definition at line 286 of file quota_posix.h.
Referenced by MainCacheManager(), PosixQuotaManager(), and UnlinkReturnPipe().