CernVM-FS
2.12.0
|
#include <quota_posix.h>
Classes | |
struct | EvictCandidate |
struct | LruCommand |
Public Member Functions | |
virtual | ~PosixQuotaManager () |
virtual bool | HasCapability (Capabilities capability) |
virtual void | Insert (const shash::Any &hash, const uint64_t size, const std::string &description) |
virtual void | InsertVolatile (const shash::Any &hash, const uint64_t size, const std::string &description) |
virtual bool | Pin (const shash::Any &hash, const uint64_t size, const std::string &description, const bool is_catalog) |
virtual void | Unpin (const shash::Any &hash) |
virtual void | Touch (const shash::Any &hash) |
virtual void | Remove (const shash::Any &file) |
virtual bool | Cleanup (const uint64_t leave_size) |
virtual void | RegisterBackChannel (int back_channel[2], const std::string &channel_id) |
virtual void | UnregisterBackChannel (int back_channel[2], const std::string &channel_id) |
virtual std::vector< std::string > | List () |
virtual std::vector< std::string > | ListPinned () |
virtual std::vector< std::string > | ListCatalogs () |
virtual std::vector< std::string > | ListVolatile () |
virtual uint64_t | GetMaxFileSize () |
virtual uint64_t | GetCapacity () |
virtual uint64_t | GetSize () |
virtual uint64_t | GetSizePinned () |
virtual bool | SetLimit (uint64_t limit) |
virtual uint64_t | GetCleanupRate (uint64_t period_s) |
virtual void | Spawn () |
virtual pid_t | GetPid () |
virtual uint32_t | GetProtocolRevision () |
void | ManagedReadHalfPipe (int fd, void *buf, size_t nbyte) |
void | SetCacheMgrPid (pid_t pid_) |
Public Member Functions inherited from QuotaManager | |
QuotaManager () | |
virtual | ~QuotaManager () |
void | BroadcastBackchannels (const std::string &message) |
Static Public Member Functions | |
static PosixQuotaManager * | Create (const std::string &cache_workspace, const uint64_t limit, const uint64_t cleanup_threshold, const bool rebuild_database) |
static PosixQuotaManager * | CreateShared (const std::string &exe_path, const std::string &cache_workspace, const uint64_t limit, const uint64_t cleanup_threshold, bool foreground) |
static int | MainCacheManager (int argc, char **argv) |
Private Types | |
enum | FileTypes { kFileRegular = 0, kFileCatalog } |
enum | CommandType { kTouch = 0, kInsert, kReserve, kPin, kUnpin, kRemove, kCleanup, kList, kListPinned, kListCatalogs, kStatus, kLimits, kPid, kPinRegular, kRegisterBackChannel, kUnregisterBackChannel, kGetProtocolRevision, kInsertVolatile, kListVolatile, kCleanupRate, kSetLimit } |
Private Member Functions | |
FRIEND_TEST (T_QuotaManager, BindReturnPipe) | |
FRIEND_TEST (T_QuotaManager, Cleanup) | |
FRIEND_TEST (T_QuotaManager, Contains) | |
FRIEND_TEST (T_QuotaManager, InitDatabase) | |
FRIEND_TEST (T_QuotaManager, MakeReturnPipe) | |
bool | InitDatabase (const bool rebuild_database) |
bool | RebuildDatabase () |
void | CloseDatabase () |
bool | Contains (const std::string &hash_str) |
bool | DoCleanup (const uint64_t leave_size) |
bool | EmptyTrash (const std::vector< std::string > &trash) |
void | MakeReturnPipe (int pipe[2]) |
int | BindReturnPipe (int pipe_wronly) |
void | UnbindReturnPipe (int pipe_wronly) |
void | UnlinkReturnPipe (int pipe_wronly) |
void | CloseReturnPipe (int pipe[2]) |
void | CleanupPipes () |
void | CheckFreeSpace () |
void | CheckHighPinWatermark () |
void | ProcessCommandBunch (const unsigned num, const LruCommand *commands, const char *descriptions) |
void | DoInsert (const shash::Any &hash, const uint64_t size, const std::string &description, const CommandType command_type) |
std::vector< std::string > | DoList (const CommandType list_command) |
void | GetSharedStatus (uint64_t *gauge, uint64_t *pinned) |
bool | SetSharedLimit (uint64_t limit) |
void | GetLimits (uint64_t *limit, uint64_t *cleanup_threshold) |
PosixQuotaManager (const uint64_t limit, const uint64_t cleanup_threshold, const std::string &cache_workspace) | |
Static Private Member Functions | |
static void * | MainCommandServer (void *data) |
static void | ParseDirectories (const std::string cache_workspace, std::string *cache_dir, std::string *workspace_dir) |
Private Attributes | |
bool | shared_ |
bool | spawned_ |
uint64_t | limit_ |
uint64_t | cleanup_threshold_ |
uint64_t | gauge_ |
uint64_t | pinned_ |
uint64_t | seq_ |
std::string | cache_dir_ |
std::string | workspace_dir_ |
std::map< shash::Any, uint64_t > | pinned_chunks_ |
int | pipe_lru_ [2] |
pthread_t | thread_lru_ |
int | fd_lock_cachedb_ |
bool | async_delete_ |
pid_t | cachemgr_pid_ |
perf::MultiRecorder | cleanup_recorder_ |
sqlite3 * | database_ |
sqlite3_stmt * | stmt_touch_ |
sqlite3_stmt * | stmt_unpin_ |
sqlite3_stmt * | stmt_block_ |
sqlite3_stmt * | stmt_unblock_ |
sqlite3_stmt * | stmt_new_ |
sqlite3_stmt * | stmt_lru_ |
sqlite3_stmt * | stmt_size_ |
sqlite3_stmt * | stmt_rm_ |
sqlite3_stmt * | stmt_rm_batch_ |
sqlite3_stmt * | stmt_list_ |
sqlite3_stmt * | stmt_list_pinned_ |
sqlite3_stmt * | stmt_list_catalogs_ |
sqlite3_stmt * | stmt_list_volatile_ |
bool | initialized_ |
Static Private Attributes | |
static const unsigned | kLockFileMagicNumber = 142857 |
static const unsigned | kSqliteMemPerThread = 2*1024*1024 |
static const unsigned | kCommandBufferSize = 32 |
static const unsigned | kEvictBatchSize = 1000 |
static const unsigned | kMaxDescription = 512-sizeof(LruCommand) |
static const unsigned | kHighPinWatermark = 75 |
static const uint64_t | kVolatileFlag = 1ULL << 63 |
Additional Inherited Members | |
Public Types inherited from QuotaManager | |
enum | Capabilities { kCapIntrospectSize = 0, kCapIntrospectCleanupRate, kCapList, kCapShrink, kCapListeners } |
Static Public Attributes inherited from QuotaManager | |
static const uint32_t | kProtocolRevision = 2 |
Protected Member Functions inherited from QuotaManager | |
void | LockBackChannels () |
void | UnlockBackChannels () |
Protected Attributes inherited from QuotaManager | |
std::map< shash::Md5, int > | back_channels_ |
pthread_mutex_t * | lock_back_channels_ |
uint32_t | protocol_revision_ |
Works with the PosixCacheManager. Uses an SQlite database for cache contents tracking. Tracking is asynchronously.
TODO(jblomer): split into client, server, and protocol classes.
Definition at line 35 of file quota_posix.h.
|
private |
List of RPCs that can be sent to the cache manager.
Definition at line 104 of file quota_posix.h.
|
private |
Loaded catalogs are pinned in the LRU and have to be treated differently.
Enumerator | |
---|---|
kFileRegular | |
kFileCatalog |
Definition at line 96 of file quota_posix.h.
|
virtual |
|
private |
|
private |
Definition at line 64 of file quota_posix.cc.
Referenced by MainCommandServer().
|
private |
Definition at line 206 of file quota_posix.cc.
Referenced by Create(), and MainCacheManager().
|
private |
Definition at line 82 of file quota_posix.cc.
Referenced by MainCommandServer().
|
virtual |
Cleans up in data cache, until cache size is below leave_size. The actual unlinking is done in a separate process (fork).
Implements QuotaManager.
Definition at line 125 of file quota_posix.cc.
Referenced by FileSystem::SetupPosixQuotaMgr().
|
private |
Definition at line 93 of file quota_posix.cc.
Referenced by MainCacheManager().
|
private |
Definition at line 146 of file quota_posix.cc.
Referenced by MainCacheManager(), and ~PosixQuotaManager().
|
private |
Definition at line 181 of file quota_posix.cc.
Referenced by Remove().
|
private |
Definition at line 191 of file quota_posix.cc.
Referenced by ProcessCommandBunch().
|
static |
Definition at line 232 of file quota_posix.cc.
Referenced by FileSystem::SetupPosixQuotaMgr().
|
static |
Connects to a running shared local quota manager. Creates one if necessary.
Definition at line 264 of file quota_posix.cc.
Referenced by FileSystem::SetupPosixQuotaMgr().
|
private |
Definition at line 472 of file quota_posix.cc.
Referenced by MainCommandServer(), and ProcessCommandBunch().
|
private |
|
private |
|
private |
|
private |
|
private |
|
private |
|
private |
|
private |
|
virtual |
Implements QuotaManager.
Definition at line 657 of file quota_posix.cc.
Referenced by FileSystem::SetupPosixQuotaMgr().
|
virtual |
Implements QuotaManager.
Definition at line 793 of file quota_posix.cc.
|
private |
Definition at line 673 of file quota_posix.cc.
Referenced by CreateShared().
|
virtual |
Since we only cleanup until cleanup_threshold, we can only add files smaller than limit-cleanup_threshold.
Implements QuotaManager.
Definition at line 692 of file quota_posix.cc.
|
virtual |
Implements QuotaManager.
Definition at line 697 of file quota_posix.cc.
Referenced by CreateShared().
|
virtual |
Implements QuotaManager.
Definition at line 719 of file quota_posix.cc.
Referenced by CreateShared().
|
private |
Queries the shared local hard disk quota manager.
Definition at line 738 of file quota_posix.cc.
|
virtual |
Implements QuotaManager.
Definition at line 777 of file quota_posix.cc.
Referenced by FileSystem::SetupPosixQuotaMgr().
|
virtual |
Implements QuotaManager.
Definition at line 785 of file quota_posix.cc.
|
inlinevirtual |
Implements QuotaManager.
Definition at line 55 of file quota_posix.h.
|
private |
Definition at line 811 of file quota_posix.cc.
Referenced by Create(), and MainCacheManager().
|
virtual |
Inserts a new file into cache catalog. This file gets a new, highest sequence number. Does cache cleanup if necessary.
Implements QuotaManager.
Definition at line 1001 of file quota_posix.cc.
|
virtual |
Inserts a new file into cache catalog. This file is marked as volatile and gets a new highest sequence number with the first bit set. Cache cleanup treats these files with priority.
Implements QuotaManager.
Definition at line 1015 of file quota_posix.cc.
|
virtual |
Lists all path names from the cache db.
Implements QuotaManager.
Definition at line 1027 of file quota_posix.cc.
|
virtual |
Lists all sqlite catalog files from the cache db.
Implements QuotaManager.
Definition at line 1043 of file quota_posix.cc.
|
virtual |
Lists all pinned files from the cache db.
Implements QuotaManager.
Definition at line 1035 of file quota_posix.cc.
|
virtual |
Lists only files flagged as volatile (priority removal)
Implements QuotaManager.
Definition at line 1051 of file quota_posix.cc.
|
static |
Entry point for the shared cache manager process
Definition at line 1059 of file quota_posix.cc.
Referenced by AltProcessFlavor().
|
staticprivate |
Definition at line 1194 of file quota_posix.cc.
Referenced by MainCacheManager(), and Spawn().
|
private |
Definition at line 1524 of file quota_posix.cc.
Referenced by RegisterBackChannel(), and Remove().
void PosixQuotaManager::ManagedReadHalfPipe | ( | int | fd, |
void * | buf, | ||
size_t | nbyte | ||
) |
Definition at line 2052 of file quota_posix.cc.
Referenced by RegisterBackChannel(), and Remove().
|
staticprivate |
Definition at line 1548 of file quota_posix.cc.
Referenced by PosixQuotaManager().
|
virtual |
Immediately inserts a new pinned catalog. Does cache cleanup if necessary.
Implements QuotaManager.
Definition at line 1573 of file quota_posix.cc.
|
private |
Definition at line 1707 of file quota_posix.cc.
Referenced by MainCommandServer().
|
private |
|
virtual |
Register a channel that allows the cache manager to trigger action to its clients. Currently used for releasing pinned catalogs.
Implements QuotaManager.
Definition at line 1933 of file quota_posix.cc.
|
virtual |
Removes a chunk from cache, if it exists.
Implements QuotaManager.
Definition at line 1965 of file quota_posix.cc.
|
inline |
Definition at line 89 of file quota_posix.h.
Referenced by CreateShared().
|
virtual |
Implements QuotaManager.
Definition at line 767 of file quota_posix.cc.
|
private |
|
virtual |
Implements QuotaManager.
Definition at line 1985 of file quota_posix.cc.
|
virtual |
Updates the sequence number of the file specified by the hash.
Implements QuotaManager.
Definition at line 2002 of file quota_posix.cc.
|
private |
Definition at line 2010 of file quota_posix.cc.
Referenced by MainCommandServer().
|
private |
Definition at line 2016 of file quota_posix.cc.
Referenced by MainCommandServer().
|
virtual |
Implements QuotaManager.
Definition at line 2022 of file quota_posix.cc.
|
virtual |
Implements QuotaManager.
Definition at line 2032 of file quota_posix.cc.
|
private |
If this is true, the unlink operations that correspond to a cleanup run will be performed in a detached, asynchronous process.
Definition at line 341 of file quota_posix.h.
|
private |
Should match the directory given to the cache manager.
Definition at line 308 of file quota_posix.h.
Referenced by MainCacheManager(), MainCommandServer(), PosixQuotaManager(), RebuildDatabase(), and Remove().
|
private |
Record pid of current cache manager in order to check if its process disappeared.
Definition at line 348 of file quota_posix.h.
Referenced by ManagedReadHalfPipe(), and SetCacheMgrPid().
|
private |
Keeps track of the number of cleanups over time. Use by cvmfs_talk cleanup rate
Definition at line 354 of file quota_posix.h.
Referenced by MainCommandServer(), and PosixQuotaManager().
|
private |
Cleanup until cleanup_threshold_ are left in the cache.
Definition at line 287 of file quota_posix.h.
Referenced by CreateShared(), MainCacheManager(), MainCommandServer(), and ProcessCommandBunch().
|
private |
Definition at line 356 of file quota_posix.h.
Referenced by ProcessCommandBunch(), and RebuildDatabase().
|
private |
Ensures exclusive cache database access through POSIX file lock.
Definition at line 335 of file quota_posix.h.
|
private |
Current size of cache.
Definition at line 292 of file quota_posix.h.
Referenced by MainCommandServer(), ProcessCommandBunch(), and RebuildDatabase().
|
private |
Used in the destructor to steer closing of the database and so on.
Definition at line 374 of file quota_posix.h.
Referenced by Create(), CreateShared(), and ~PosixQuotaManager().
|
staticprivate |
Collect a number of insert and touch operations before processing them as sqlite commands.
Definition at line 207 of file quota_posix.h.
|
staticprivate |
Batch size for database operations during DoCleanup()
Definition at line 212 of file quota_posix.h.
|
staticprivate |
Alarm when more than 75% of the cache fraction allowed for pinned files (50%) is filled with pinned files
Definition at line 224 of file quota_posix.h.
|
staticprivate |
Magic number to make reading PIDs from lockfiles more robust and versionable
Definition at line 196 of file quota_posix.h.
Referenced by CreateShared().
|
staticprivate |
Make sure that the amount of data transferred through the RPC pipe is within the OS's guarantees for atomicity.
Definition at line 218 of file quota_posix.h.
Referenced by ProcessCommandBunch().
|
staticprivate |
Maximum page cache per thread (Bytes).
Definition at line 201 of file quota_posix.h.
Referenced by MainCommandServer().
|
staticprivate |
The last bit in the sequence number indicates if an entry is volatile. Such sequence numbers are negative and they are preferred during cleanup. Volatile entries are used for instance for ALICE conditions data.
Definition at line 231 of file quota_posix.h.
Referenced by ProcessCommandBunch().
|
private |
Soft limit in bytes, start cleanup when reached.
Definition at line 282 of file quota_posix.h.
Referenced by CreateShared(), MainCacheManager(), MainCommandServer(), and ProcessCommandBunch().
|
private |
Size of pinned files in bytes (usually file catalogs).
Definition at line 297 of file quota_posix.h.
Referenced by MainCacheManager(), and MainCommandServer().
|
private |
Pinned content hashes and their size.
Definition at line 320 of file quota_posix.h.
Referenced by MainCommandServer().
|
private |
Used to send RPCs to the quota manager thread or process.
Definition at line 325 of file quota_posix.h.
Referenced by Create(), CreateShared(), MainCacheManager(), MainCommandServer(), PosixQuotaManager(), RegisterBackChannel(), Remove(), Touch(), Unpin(), UnregisterBackChannel(), and ~PosixQuotaManager().
|
private |
Current access sequence number. Gets increased on every access/insert operation.
Definition at line 303 of file quota_posix.h.
Referenced by ProcessCommandBunch(), and RebuildDatabase().
|
private |
Indicates if the cache manager is a shared process or a thread within the same process (exclusive cache manager)
Definition at line 271 of file quota_posix.h.
Referenced by CreateShared(), MainCacheManager(), UnbindReturnPipe(), UnlinkReturnPipe(), and ~PosixQuotaManager().
|
private |
True once the program switches into multi-threaded mode or the quota manager process has been forked resp.
Definition at line 277 of file quota_posix.h.
Referenced by CreateShared(), MainCacheManager(), Spawn(), and ~PosixQuotaManager().
|
private |
Definition at line 359 of file quota_posix.h.
|
private |
Definition at line 366 of file quota_posix.h.
Referenced by MainCommandServer().
|
private |
Definition at line 368 of file quota_posix.h.
Referenced by MainCommandServer().
|
private |
Loaded catalogs are pinned.
Definition at line 367 of file quota_posix.h.
Referenced by MainCommandServer().
|
private |
Definition at line 369 of file quota_posix.h.
Referenced by MainCommandServer().
|
private |
Definition at line 362 of file quota_posix.h.
|
private |
Definition at line 361 of file quota_posix.h.
Referenced by ProcessCommandBunch().
|
private |
Definition at line 364 of file quota_posix.h.
Referenced by MainCommandServer().
|
private |
Definition at line 365 of file quota_posix.h.
|
private |
Definition at line 363 of file quota_posix.h.
Referenced by MainCommandServer().
|
private |
Definition at line 357 of file quota_posix.h.
Referenced by ProcessCommandBunch().
|
private |
Definition at line 360 of file quota_posix.h.
|
private |
Definition at line 358 of file quota_posix.h.
Referenced by ProcessCommandBunch().
|
private |
In exclusive mode, controls the quota manager thread.
Definition at line 330 of file quota_posix.h.
Referenced by Spawn(), and ~PosixQuotaManager().
|
private |
Directory for the database lock (shared manager) and the pipes (also shared manager). Usually the same as cache_dir_. Can be different if CVMFS_WORKSPACE or CVMFS_CACHE_WORKSPACE is set.
Definition at line 315 of file quota_posix.h.
Referenced by MainCacheManager(), PosixQuotaManager(), and UnlinkReturnPipe().