61 : revision(statistics->Lookup(
"publish.revision")->ToString())
62 , files_added(statistics->Lookup(
"publish.n_files_added")->ToString())
63 , files_removed(statistics->Lookup(
"publish.n_files_removed")->ToString())
64 , files_changed(statistics->Lookup(
"publish.n_files_changed")->ToString())
65 , chunks_added(statistics->Lookup(
"publish.n_chunks_added")->ToString())
67 statistics->Lookup(
"publish.n_chunks_duplicated")->ToString())
69 statistics->Lookup(
"publish.n_catalogs_added")->ToString())
71 statistics->Lookup(
"publish.n_directories_added")->ToString())
73 statistics->Lookup(
"publish.n_directories_removed")->ToString())
75 statistics->Lookup(
"publish.n_directories_changed")->ToString())
77 statistics->Lookup(
"publish.n_symlinks_added")->ToString())
79 statistics->Lookup(
"publish.n_symlinks_removed")->ToString())
81 statistics->Lookup(
"publish.n_symlinks_changed")->ToString())
82 , bytes_added(statistics->Lookup(
"publish.sz_added_bytes")->ToString())
84 statistics->Lookup(
"publish.sz_removed_bytes")->ToString())
86 statistics->Lookup(
"publish.sz_uploaded_bytes")->ToString())
87 , catalog_bytes_uploaded(
88 statistics->Lookup(
"publish.sz_uploaded_catalog_bytes")
102 c = statistics->
Lookup(
"gc.n_preserved_catalogs");
103 n_preserved_catalogs = c ? c->
ToString() :
"0";
104 c = statistics->
Lookup(
"gc.n_condemned_catalogs");
105 n_condemned_catalogs = c ? c->
ToString() :
"0";
106 c = statistics->
Lookup(
"gc.n_condemned_objects");
107 n_condemned_objects = c ? c->
ToString() :
"0";
108 c = statistics->
Lookup(
"gc.sz_condemned_bytes");
109 sz_condemned_bytes = c ? c->
ToString() :
"0";
110 c = statistics->
Lookup(
"gc.n_duplicate_delete_requests");
111 n_duplicate_delete_requests = c ? c->
ToString() :
"0";
123 const std::string &start_time,
124 const std::string &finish_time,
125 const bool success) {
127 std::string insert_statement =
"INSERT INTO publish_statistics ("
138 "directories_removed,"
139 "directories_changed,"
146 "sz_catalog_bytes_uploaded,"
150 + start_time +
"'," +
"'" + finish_time +
"',"
165 + (success ?
"1" :
"0") +
");";
166 return insert_statement;
180 const std::string &start_time,
181 const std::string &finish_time,
182 const std::string &repo_name,
183 const bool success) {
185 std::string insert_statement =
"";
187 insert_statement =
"INSERT INTO gc_statistics ("
190 "n_preserved_catalogs,"
191 "n_condemned_catalogs,"
192 "n_condemned_objects,"
193 "sz_condemned_bytes,"
194 "n_duplicate_delete_requests,"
198 + start_time +
"'," +
"'" + finish_time +
"',"
204 + (success ?
"1" :
"0") +
");";
207 insert_statement =
"INSERT INTO gc_statistics ("
210 "n_preserved_catalogs,"
211 "n_condemned_catalogs,"
212 "n_condemned_objects,"
213 "n_duplicate_delete_requests,"
217 + start_time +
"'," +
"'" + finish_time +
"',"
222 + (success ?
"1" :
"0") +
");";
224 return insert_statement;
233 "CREATE TABLE publish_statistics ("
234 "publish_id INTEGER PRIMARY KEY,"
238 "files_added INTEGER,"
239 "files_removed INTEGER,"
240 "files_changed INTEGER,"
241 "chunks_added INTEGER,"
242 "chunks_duplicated INTEGER,"
243 "catalogs_added INTEGER,"
244 "directories_added INTEGER,"
245 "directories_removed INTEGER,"
246 "directories_changed INTEGER,"
247 "symlinks_added INTEGER,"
248 "symlinks_removed INTEGER,"
249 "symlinks_changed INTEGER,"
250 "sz_bytes_added INTEGER,"
251 "sz_bytes_removed INTEGER,"
252 "sz_bytes_uploaded INTEGER,"
253 "sz_catalog_bytes_uploaded INTEGER,"
257 "CREATE TABLE gc_statistics ("
258 "gc_id INTEGER PRIMARY KEY,"
261 "n_preserved_catalogs INTEGER,"
262 "n_condemned_catalogs INTEGER,"
263 "n_condemned_objects INTEGER,"
264 "sz_condemned_bytes INTEGER,"
265 "n_duplicate_delete_requests INTEGER,"
284 "upgrading schema revision (1 --> 2) of "
285 "statistics database");
290 "publish_statistics RENAME COLUMN finished_time TO finish_time;");
293 "publish_statistics ADD revision INTEGER;");
296 "publish_statistics RENAME COLUMN "
297 "duplicated_files TO chunks_duplicated;");
300 "publish_statistics ADD chunks_added INTEGER;");
303 "publish_statistics ADD symlinks_added INTEGER;");
306 "publish_statistics ADD symlinks_removed INTEGER;");
309 "publish_statistics ADD symlinks_changed INTEGER;");
312 "publish_statistics ADD catalogs_added INTEGER;");
316 "publish_statistics ADD sz_catalog_bytes_uploaded INTEGER;");
318 if (!publish_upgrade2_1.
Execute() || !publish_upgrade2_2.
Execute()
322 || !publish_upgrade2_9.
Execute()) {
324 "failed to upgrade publish_statistics"
325 " table of statistics database");
330 "ALTER TABLE gc_statistics"
331 " RENAME COLUMN finished_time TO finish_time;");
333 if (!gc_upgrade2_1.
Execute()) {
335 "failed to upgrade gc_statistics"
336 " table of statistics database");
343 "failed to upgrade schema revision"
344 " of statistics database");
352 "upgrading schema revision (2 --> 3) of "
353 "statistics database");
357 "publish_statistics ADD success INTEGER;");
359 if (!publish_upgrade3_1.
Execute()) {
361 "failed to upgrade publish_statistics"
362 " table of statistics database");
367 " ADD success INTEGER;");
369 if (!gc_upgrade3_1.
Execute()) {
371 "failed to upgrade gc_statistics"
372 " table of statistics database");
379 "failed to upgrade schema revision"
380 " of statistics database");
388 "upgrading schema revision (3 --> 4) of "
389 "statistics database");
392 "ALTER TABLE gc_statistics"
393 " ADD n_duplicate_delete_requests INTEGER;");
395 if (!gc_upgrade4_1.
Execute()) {
397 "failed to upgrade gc_statistics "
398 "table of statistics database");
405 "failed to upgrade schema revision "
406 "of statistics database");
424 const std::string repo_name) {
426 std::string db_file_path;
427 uint32_t days_to_keep;
428 GetDBParams(repo_name, &db_file_path, &days_to_keep);
433 }
else if (db->
GetProperty<std::string>(
"repo_name") != repo_name) {
435 "'repo_name' property of the statistics database %s "
436 "is incorrect. Please fix the database.",
437 db_file_path.c_str());
439 if (!db->
Prune(days_to_keep)) {
447 }
else if (!db->
SetProperty(
"repo_name", repo_name)) {
458 const std::string &start_time,
459 const bool success) {
462 finish_time, success);
468 const std::string &start_time,
469 const bool success) {
472 statistics, start_time, finish_time,
repo_name_, success);
482 "Couldn't store statistics in %s: insert.Execute failed!",
498 publish_stmt =
"DELETE FROM publish_statistics WHERE "
499 "julianday('now','start of day')-julianday(start_time) > "
503 gc_stmt =
"DELETE FROM gc_statistics WHERE "
504 "julianday('now','start of day')-julianday(start_time) > "
511 "Couldn't prune statistics DB %s: SQL Execute() failed!",
517 "Couldn't prune statistics DB %s: Vacuum() failed!",
527 std::string local_path) {
528 if (local_path ==
"") {
532 spooler->WaitForUpload();
533 unsigned errors_before = spooler->GetNumberOfErrors();
534 spooler->Mkdir(
"stats");
535 spooler->RemoveAsync(
"stats/stats.db");
536 spooler->WaitForUpload();
537 spooler->Upload(local_path,
"stats/stats.db");
538 spooler->WaitForUpload();
539 unsigned errors_after = spooler->GetNumberOfErrors();
541 if (errors_before != errors_after) {
543 "Could not upload statistics DB file into storage backend");
551 std::string local_path) {
552 if (local_path ==
"") {
561 uploader->
UploadFile(local_path,
"stats/stats.db");
564 return errors_before == errors_after;
570 uint32_t *days_to_keep) {
572 const std::string db_default_path =
"/var/spool/cvmfs/" + repo_name
574 const std::string repo_config_file =
"/etc/cvmfs/repositories.d/" + repo_name
580 "Could not parse repository configuration: %s.",
581 repo_config_file.c_str());
582 *path = db_default_path;
587 std::string statistics_db =
"";
588 if (!parser.
GetValue(
"CVMFS_STATISTICS_DB", &statistics_db)) {
590 "Parameter %s was not set in the repository configuration file. "
591 "Using default value: %s",
592 "CVMFS_STATISTICS_DB", db_default_path.c_str());
593 *path = db_default_path;
596 int mode = S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH
600 "Couldn't write statistics at the specified path %s.",
601 statistics_db.c_str());
602 *path = db_default_path;
604 *path = statistics_db;
608 std::string days_to_keep_str =
"";
609 if (!parser.
GetValue(
"CVMFS_STATS_DB_DAYS_TO_KEEP", &statistics_db)) {
611 "Parameter %s was not set in the repository configuration file. "
612 "Using default value: %s",
613 "CVMFS_STATS_DB_DAYS_TO_KEEP",
617 *days_to_keep =
static_cast<uint32_t
>(
String2Uint64(days_to_keep_str));
630 std::string param_value =
"";
631 const std::string repo_config_file =
"/etc/cvmfs/repositories.d/" + repo_name
636 "Could not parse repository configuration: %s.",
637 repo_config_file.c_str());
640 if (!parser.
GetValue(
"CVMFS_EXTENDED_GC_STATS", ¶m_value)) {
642 "Parameter %s was not set in the repository configuration file. "
643 "condemned_bytes were not counted.",
644 "CVMFS_EXTENDED_GC_STATS");
645 }
else if (parser.
IsOn(param_value)) {
655 , create_empty_db_calls(0)
656 , check_compatibility_calls(0)
657 , live_upgrade_calls(0)
virtual void WaitForUpload() const
std::string chunks_duplicated
StatisticsDatabase(const std::string &filename, const OpenMode open_mode)
std::string n_condemned_objects
std::string sz_condemned_bytes
std::string symlinks_changed
std::string GetGMTimestamp(const std::string &format)
std::string PrepareStatementIntoGc(const perf::Statistics *statistics, const std::string &start_time, const std::string &finish_time, const std::string &repo_name, const bool success)
const std::string & filename() const
bool UploadStatistics(upload::Spooler *spooler, std::string local_path="")
bool StoreGCStatistics(const perf::Statistics *statistics, const std::string &start_time, const bool success)
std::string catalog_bytes_uploaded
static unsigned int instances
bool IsOn(const std::string ¶m_value) const
std::string catalogs_added
bool CreateEmptyDatabase()
std::string symlinks_removed
unsigned int live_upgrade_calls
float schema_version() const
bool Prune(uint32_t days)
upload::Spooler * spooler
PublishStats(const perf::Statistics *statistics)
std::string StringifyUint(const uint64_t value)
void set_schema_revision(const unsigned rev)
std::string PrepareStatementIntoPublish(const perf::Statistics *statistics, const std::string &start_time, const std::string &finish_time, const bool success)
static StatisticsDatabase * Open(const std::string &filename, const OpenMode open_mode)
T GetProperty(const std::string &key) const
unsigned schema_revision() const
std::string n_condemned_catalogs
bool LiveSchemaUpgradeIfNecessary()
std::string bytes_uploaded
static bool compacting_fails
bool FileExists(const std::string &path)
Counter * Lookup(const std::string &name) const
bool TryParsePath(const std::string &config_file)
std::string symlinks_added
static const float kLatestCompatibleSchema
std::string files_changed
bool MkdirDeep(const std::string &path, const mode_t mode, bool verify_writable)
unsigned int create_empty_db_calls
static float kLatestSchema
std::string files_removed
sqlite3 * sqlite_db() const
unsigned int check_compatibility_calls
static unsigned kLatestSchemaRevision
std::string n_duplicate_delete_requests
bool GetValue(const std::string &key, std::string *value) const
virtual unsigned int GetNumberOfErrors() const =0
static StatisticsDatabase * OpenStandardDB(const std::string repo_name)
bool SetProperty(const std::string &key, const T value)
std::string bytes_removed
static bool GcExtendedStats(const std::string &repo_name)
bool StoreEntry(const std::string &insert_statement)
void UploadFile(const std::string &local_path, const std::string &remote_path, const CallbackTN *callback=NULL)
bool StorePublishStatistics(const perf::Statistics *statistics, const std::string &start_time, const bool success)
static const uint32_t kDefaultDaysToKeep
bool IsEqualSchema(const float value, const float compare) const
uint64_t String2Uint64(const string &value)
unsigned int compact_calls
void RemoveAsync(const std::string &file_to_delete)
PathString GetParentPath(const PathString &path)
std::string n_preserved_catalogs
GcStats(const perf::Statistics *statistics)
bool StoreSchemaRevision()
bool CheckSchemaCompatibility()
static void GetDBParams(const std::string &repo_name, std::string *path, uint32_t *days_to_keep)
static StatisticsDatabase * Create(const std::string &filename)
bool CompactDatabase() const
CVMFS_EXPORT void LogCvmfs(const LogSource source, const int mask, const char *format,...)