Head

GCC Code Coverage Report

Directory:	cvmfs/		Exec	Total	Coverage
File:	cvmfs/statistics_database.cc	Lines:	0	95	0.0 %
Date:	2019-02-03 02:48:13	Branches:	0	44	0.0 %


/**
 * This file is part of the CernVM File System.
 */

#include "statistics_database.h"


const float    StatisticsDatabase::kLatestCompatibleSchema = 1.0f;
float          StatisticsDatabase::kLatestSchema           = 1.0f;
unsigned       StatisticsDatabase::kLatestSchemaRevision   =
                                              RevisionFlags::kInitialRevision;
unsigned int   StatisticsDatabase::instances               = 0;
bool           StatisticsDatabase::compacting_fails        = false;


namespace {

struct PublishStats {
  std::string files_added;
  std::string files_removed;
  std::string files_changed;
  std::string duplicated_files;
  std::string dir_added;
  std::string dir_removed;
  std::string dir_changed;
  std::string bytes_added;
  std::string bytes_removed;
  std::string bytes_uploaded;

  explicit PublishStats(const perf::Statistics *statistics):
    files_added(statistics->
                    Lookup("Publish.n_files_added")->ToString()),
    files_removed(statistics->
                    Lookup("Publish.n_files_removed")->ToString()),
    files_changed(statistics->
                    Lookup("Publish.n_files_changed")->ToString()),
    duplicated_files(statistics->
                    Lookup("Publish.n_duplicated_files")->ToString()),
    dir_added(statistics->
                    Lookup("Publish.n_directories_added")->ToString()),
    dir_removed(statistics->
                    Lookup("Publish.n_directories_removed")->ToString()),
    dir_changed(statistics->
                    Lookup("Publish.n_directories_changed")->ToString()),
    bytes_added(statistics->
                    Lookup("Publish.sz_added_bytes")->ToString()),
    bytes_removed(statistics->
                    Lookup("Publish.sz_removed_bytes")->ToString()),
    bytes_uploaded(statistics->
                    Lookup("Publish.sz_uploaded_bytes")->ToString()) {
  }
};


struct GcStats {
  std::string n_preserved_catalogs;
  std::string n_condemned_catalogs;
  std::string n_condemned_objects;
  std::string sz_condemned_bytes;

  explicit GcStats(const perf::Statistics *statistics):
    n_preserved_catalogs(statistics->
                    Lookup("gc.n_preserved_catalogs")->ToString()),
    n_condemned_catalogs(statistics->
                    Lookup("gc.n_condemned_catalogs")->ToString()),
    n_condemned_objects(statistics->
                    Lookup("gc.n_condemned_objects")->ToString()),
    sz_condemned_bytes(statistics->
                    Lookup("gc.sz_condemned_bytes")->ToString()) {
  }
};


/**
  * Build the insert statement into publish_statistics table.
  *
  * @param stats a struct with all values stored in strings
  * @return the insert statement
  */
std::string PrepareStatementIntoPublish(const perf::Statistics *statistics,
                            const std::string &start_time,
                            const std::string &finished_time) {
  struct PublishStats stats = PublishStats(statistics);
  std::string insert_statement =
    "INSERT INTO publish_statistics ("
    "start_time,"
    "finished_time,"
    "files_added,"
    "files_removed,"
    "files_changed,"
    "duplicated_files,"
    "directories_added,"
    "directories_removed,"
    "directories_changed,"
    "sz_bytes_added,"
    "sz_bytes_removed,"
    "sz_bytes_uploaded)"
    " VALUES("
    "'"+start_time+"',"+
    "'"+finished_time+"',"+
    stats.files_added+"," +
    stats.files_removed +","+
    stats.files_changed + "," +
    stats.duplicated_files + "," +
    stats.dir_added + "," +
    stats.dir_removed + "," +
    stats.dir_changed + "," +
    stats.bytes_added + "," +
    stats.bytes_removed + "," +
    stats.bytes_uploaded + ");";
  return insert_statement;
}


/**
  * Build the insert statement into gc_statistics table.
  *
  * @param stats a struct with values stored in strings
  * @param start_time, finished_time to run Main() of the command
  * @param repo_name fully qualified name of the repository
  *
  * @return the insert statement
  */
std::string PrepareStatementIntoGc(const perf::Statistics *statistics,
                            const std::string &start_time,
                            const std::string &finished_time,
                            const std::string &repo_name) {
  struct GcStats stats = GcStats(statistics);
  std::string insert_statement = "";
  if (StatisticsDatabase::GcExtendedStats(repo_name)) {
    insert_statement =
      "INSERT INTO gc_statistics ("
      "start_time,"
      "finished_time,"
      "n_preserved_catalogs,"
      "n_condemned_catalogs,"
      "n_condemned_objects,"
      "sz_condemned_bytes)"
      " VALUES("
      "'" + start_time + "'," +
      "'" + finished_time + "'," +
      stats.n_preserved_catalogs + "," +
      stats.n_condemned_catalogs + ","+
      stats.n_condemned_objects + "," +
      stats.sz_condemned_bytes + ");";
  } else {
    // insert values except sz_condemned_bytes
    insert_statement =
      "INSERT INTO gc_statistics ("
      "start_time,"
      "finished_time,"
      "n_preserved_catalogs,"
      "n_condemned_catalogs,"
      "n_condemned_objects)"
      " VALUES("
      "'" + start_time + "'," +
      "'" + finished_time + "'," +
      stats.n_preserved_catalogs + "," +
      stats.n_condemned_catalogs + ","+
      stats.n_condemned_objects + ");";
  }
  return insert_statement;
}

}  // namespace


bool StatisticsDatabase::CreateEmptyDatabase() {
  ++create_empty_db_calls;
  bool ret1 = sqlite::Sql(sqlite_db(),
    "CREATE TABLE publish_statistics ("
    "publish_id INTEGER PRIMARY KEY,"
    "start_time TEXT,"
    "finished_time TEXT,"
    "files_added INTEGER,"
    "files_removed INTEGER,"
    "files_changed INTEGER,"
    "duplicated_files INTEGER,"
    "directories_added INTEGER,"
    "directories_removed INTEGER,"
    "directories_changed INTEGER,"
    "sz_bytes_added INTEGER,"
    "sz_bytes_removed INTEGER,"
    "sz_bytes_uploaded INTEGER);").Execute();
  bool ret2 = sqlite::Sql(sqlite_db(),
    "CREATE TABLE gc_statistics ("
    "gc_id INTEGER PRIMARY KEY,"
    "start_time TEXT,"
    "finished_time TEXT,"
    "n_preserved_catalogs INTEGER,"
    "n_condemned_catalogs INTEGER,"
    "n_condemned_objects INTEGER,"
    "sz_condemned_bytes INTEGER);").Execute();
  return ret1 & ret2;
}


bool StatisticsDatabase::CheckSchemaCompatibility() {
  ++check_compatibility_calls;
  return (schema_version() > kLatestCompatibleSchema - 0.1 &&
          schema_version() < kLatestCompatibleSchema + 0.1);
}


bool StatisticsDatabase::LiveSchemaUpgradeIfNecessary() {
  ++live_upgrade_calls;
  const unsigned int revision = schema_revision();

  if (revision == RevisionFlags::kInitialRevision) {
    return true;
  }

  if (revision == RevisionFlags::kUpdatableRevision) {
    set_schema_revision(RevisionFlags::kUpdatedRevision);
    StoreSchemaRevision();
    return true;
  }

  if (revision == RevisionFlags::kFailingRevision) {
    return false;
  }

  return false;
}


bool StatisticsDatabase::CompactDatabase() const {
  ++compact_calls;
  return !compacting_fails;
}


StatisticsDatabase::~StatisticsDatabase() {
  --StatisticsDatabase::instances;
}


int StatisticsDatabase::StoreStatistics(const perf::Statistics *statistics,
                                        const std::string &start_time,
                                        const std::string &finished_time,
                                        const std::string &command_name,
                                        const std::string &repo_name) {
  std::string insert_statement;
  if (command_name == "ingest" || command_name == "sync") {
    insert_statement = PrepareStatementIntoPublish(statistics, start_time,
                                                               finished_time);
  } else if (command_name == "gc") {
    insert_statement = PrepareStatementIntoGc(statistics, start_time,
                                              finished_time, repo_name);
  } else {
    return -5;
  }

  sqlite::Sql insert(this->sqlite_db(), insert_statement);

  if (!this->BeginTransaction()) {
    LogCvmfs(kLogCvmfs, kLogSyslogErr, "BeginTransaction failed!");
    return -1;
  }

  if (!insert.Execute()) {
    LogCvmfs(kLogCvmfs, kLogSyslogErr, "insert.Execute failed!");
    return -2;
  }

  if (!insert.Reset()) {
    LogCvmfs(kLogCvmfs, kLogSyslogErr, "insert.Reset() failed!");
    return -3;
  }

  if (!this->CommitTransaction()) {
    LogCvmfs(kLogCvmfs, kLogSyslogErr, "CommitTransaction failed!");
    return -4;
  }

  return 0;
}


std::string StatisticsDatabase::GetDBPath(const std::string &repo_name) {
  // default location
  const std::string db_default_path =
      "/var/spool/cvmfs/" + repo_name + "/stats.db";
  const std::string repo_config_file =
      "/etc/cvmfs/repositories.d/" + repo_name + "/server.conf";
  SimpleOptionsParser parser;

  if (!parser.TryParsePath(repo_config_file)) {
    LogCvmfs(kLogCvmfs, kLogSyslogErr,
             "Could not parse repository configuration: %s.",
             repo_config_file.c_str());
    return db_default_path;
  }

  std::string statistics_db = "";
  if (!parser.GetValue("CVMFS_STATISTICS_DB", &statistics_db)) {
    LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslog,
             "Parameter %s was not set in the repository configuration file. "
             "Using default value: %s",
             "CVMFS_STATISTICS_DB", db_default_path.c_str());
    return db_default_path;
  }

  std::string dirname = GetParentPath(statistics_db);
  int mode = S_IRUSR | S_IWUSR | S_IXUSR |
             S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH;  // 755
  if (!MkdirDeep(dirname, mode, true)) {
    LogCvmfs(kLogCvmfs, kLogSyslogErr,
      "Couldn't write statistics at the specified path %s.",
      statistics_db.c_str());
    return db_default_path;
  }

  return statistics_db;
}


/**
  * Check if the CVMFS_EXTENDED_GC_STATS is ON or not
  *
  * @param repo_name fully qualified name of the repository
  * @return true if CVMFS_EXTENDED_GC_STATS is ON
  */
bool StatisticsDatabase::GcExtendedStats(const std::string &repo_name) {
  SimpleOptionsParser parser;
  std::string param_value = "";
  const std::string repo_config_file =
      "/etc/cvmfs/repositories.d/" + repo_name + "/server.conf";

  if (!parser.TryParsePath(repo_config_file)) {
    LogCvmfs(kLogCvmfs, kLogSyslogErr,
             "Could not parse repository configuration: %s.",
             repo_config_file.c_str());
    return false;
  }
  if (!parser.GetValue("CVMFS_EXTENDED_GC_STATS", &param_value)) {
    LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslog,
             "Parameter %s was not set in the repository configuration file. "
             "condemned_bytes were not counted.",
             "CVMFS_EXTENDED_GC_STATS");
  } else if (parser.IsOn(param_value)) {
    return true;
  }
  return false;
}


StatisticsDatabase::StatisticsDatabase(const std::string  &filename,
              const OpenMode      open_mode) :
  sqlite::Database<StatisticsDatabase>(filename, open_mode),
  create_empty_db_calls(0),  check_compatibility_calls(0),
  live_upgrade_calls(0), compact_calls(0)
{
  ++StatisticsDatabase::instances;
}


Generated by: GCOVR (Version 4.1)

Line	Branch	Exec	Source
1			/**
2			* This file is part of the CernVM File System.
3			*/
4
5			#include "statistics_database.h"
6
7
8			const float StatisticsDatabase::kLatestCompatibleSchema = 1.0f;
9			float StatisticsDatabase::kLatestSchema = 1.0f;
10			unsigned StatisticsDatabase::kLatestSchemaRevision =
11			RevisionFlags::kInitialRevision;
12			unsigned int StatisticsDatabase::instances = 0;
13			bool StatisticsDatabase::compacting_fails = false;
14
15
16			namespace {
17
18			struct PublishStats {
19			std::string files_added;
20			std::string files_removed;
21			std::string files_changed;
22			std::string duplicated_files;
23			std::string dir_added;
24			std::string dir_removed;
25			std::string dir_changed;
26			std::string bytes_added;
27			std::string bytes_removed;
28			std::string bytes_uploaded;
29
30			explicit PublishStats(const perf::Statistics *statistics):
31			files_added(statistics->
32			Lookup("Publish.n_files_added")->ToString()),
33			files_removed(statistics->
34			Lookup("Publish.n_files_removed")->ToString()),
35			files_changed(statistics->
36			Lookup("Publish.n_files_changed")->ToString()),
37			duplicated_files(statistics->
38			Lookup("Publish.n_duplicated_files")->ToString()),
39			dir_added(statistics->
40			Lookup("Publish.n_directories_added")->ToString()),
41			dir_removed(statistics->
42			Lookup("Publish.n_directories_removed")->ToString()),
43			dir_changed(statistics->
44			Lookup("Publish.n_directories_changed")->ToString()),
45			bytes_added(statistics->
46			Lookup("Publish.sz_added_bytes")->ToString()),
47			bytes_removed(statistics->
48			Lookup("Publish.sz_removed_bytes")->ToString()),
49			bytes_uploaded(statistics->
50			Lookup("Publish.sz_uploaded_bytes")->ToString()) {
51			}
52			};
53
54
55			struct GcStats {
56			std::string n_preserved_catalogs;
57			std::string n_condemned_catalogs;
58			std::string n_condemned_objects;
59			std::string sz_condemned_bytes;
60
61			explicit GcStats(const perf::Statistics *statistics):
62			n_preserved_catalogs(statistics->
63			Lookup("gc.n_preserved_catalogs")->ToString()),
64			n_condemned_catalogs(statistics->
65			Lookup("gc.n_condemned_catalogs")->ToString()),
66			n_condemned_objects(statistics->
67			Lookup("gc.n_condemned_objects")->ToString()),
68			sz_condemned_bytes(statistics->
69			Lookup("gc.sz_condemned_bytes")->ToString()) {
70			}
71			};
72
73
74			/**
75			* Build the insert statement into publish_statistics table.
76			*
77			* @param stats a struct with all values stored in strings
78			* @return the insert statement
79			*/
80			std::string PrepareStatementIntoPublish(const perf::Statistics *statistics,
81			const std::string &start_time,
82			const std::string &finished_time) {
83			struct PublishStats stats = PublishStats(statistics);
84			std::string insert_statement =
85			"INSERT INTO publish_statistics ("
86			"start_time,"
87			"finished_time,"
88			"files_added,"
89			"files_removed,"
90			"files_changed,"
91			"duplicated_files,"
92			"directories_added,"
93			"directories_removed,"
94			"directories_changed,"
95			"sz_bytes_added,"
96			"sz_bytes_removed,"
97			"sz_bytes_uploaded)"
98			" VALUES("
99			"'"+start_time+"',"+
100			"'"+finished_time+"',"+
101			stats.files_added+"," +
102			stats.files_removed +","+
103			stats.files_changed + "," +
104			stats.duplicated_files + "," +
105			stats.dir_added + "," +
106			stats.dir_removed + "," +
107			stats.dir_changed + "," +
108			stats.bytes_added + "," +
109			stats.bytes_removed + "," +
110			stats.bytes_uploaded + ");";
111			return insert_statement;
112			}
113
114
115			/**
116			* Build the insert statement into gc_statistics table.
117			*
118			* @param stats a struct with values stored in strings
119			* @param start_time, finished_time to run Main() of the command
120			* @param repo_name fully qualified name of the repository
121			*
122			* @return the insert statement
123			*/
124			std::string PrepareStatementIntoGc(const perf::Statistics *statistics,
125			const std::string &start_time,
126			const std::string &finished_time,
127			const std::string &repo_name) {
128			struct GcStats stats = GcStats(statistics);
129			std::string insert_statement = "";
130			if (StatisticsDatabase::GcExtendedStats(repo_name)) {
131			insert_statement =
132			"INSERT INTO gc_statistics ("
133			"start_time,"
134			"finished_time,"
135			"n_preserved_catalogs,"
136			"n_condemned_catalogs,"
137			"n_condemned_objects,"
138			"sz_condemned_bytes)"
139			" VALUES("
140			"'" + start_time + "'," +
141			"'" + finished_time + "'," +
142			stats.n_preserved_catalogs + "," +
143			stats.n_condemned_catalogs + ","+
144			stats.n_condemned_objects + "," +
145			stats.sz_condemned_bytes + ");";
146			} else {
147			// insert values except sz_condemned_bytes
148			insert_statement =
149			"INSERT INTO gc_statistics ("
150			"start_time,"
151			"finished_time,"
152			"n_preserved_catalogs,"
153			"n_condemned_catalogs,"
154			"n_condemned_objects)"
155			" VALUES("
156			"'" + start_time + "'," +
157			"'" + finished_time + "'," +
158			stats.n_preserved_catalogs + "," +
159			stats.n_condemned_catalogs + ","+
160			stats.n_condemned_objects + ");";
161			}
162			return insert_statement;
163			}
164
165			} // namespace
166
167
168			bool StatisticsDatabase::CreateEmptyDatabase() {
169			++create_empty_db_calls;
170			bool ret1 = sqlite::Sql(sqlite_db(),
171			"CREATE TABLE publish_statistics ("
172			"publish_id INTEGER PRIMARY KEY,"
173			"start_time TEXT,"
174			"finished_time TEXT,"
175			"files_added INTEGER,"
176			"files_removed INTEGER,"
177			"files_changed INTEGER,"
178			"duplicated_files INTEGER,"
179			"directories_added INTEGER,"
180			"directories_removed INTEGER,"
181			"directories_changed INTEGER,"
182			"sz_bytes_added INTEGER,"
183			"sz_bytes_removed INTEGER,"
184			"sz_bytes_uploaded INTEGER);").Execute();
185			bool ret2 = sqlite::Sql(sqlite_db(),
186			"CREATE TABLE gc_statistics ("
187			"gc_id INTEGER PRIMARY KEY,"
188			"start_time TEXT,"
189			"finished_time TEXT,"
190			"n_preserved_catalogs INTEGER,"
191			"n_condemned_catalogs INTEGER,"
192			"n_condemned_objects INTEGER,"
193			"sz_condemned_bytes INTEGER);").Execute();
194			return ret1 & ret2;
195			}
196
197
198			bool StatisticsDatabase::CheckSchemaCompatibility() {
199			++check_compatibility_calls;
200			return (schema_version() > kLatestCompatibleSchema - 0.1 &&
201			schema_version() < kLatestCompatibleSchema + 0.1);
202			}
203
204
205			bool StatisticsDatabase::LiveSchemaUpgradeIfNecessary() {
206			++live_upgrade_calls;
207			const unsigned int revision = schema_revision();
208
209			if (revision == RevisionFlags::kInitialRevision) {
210			return true;
211			}
212
213			if (revision == RevisionFlags::kUpdatableRevision) {
214			set_schema_revision(RevisionFlags::kUpdatedRevision);
215			StoreSchemaRevision();
216			return true;
217			}
218
219			if (revision == RevisionFlags::kFailingRevision) {
220			return false;
221			}
222
223			return false;
224			}
225
226
227			bool StatisticsDatabase::CompactDatabase() const {
228			++compact_calls;
229			return !compacting_fails;
230			}
231
232
233			StatisticsDatabase::~StatisticsDatabase() {
234			--StatisticsDatabase::instances;
235			}
236
237
238			int StatisticsDatabase::StoreStatistics(const perf::Statistics *statistics,
239			const std::string &start_time,
240			const std::string &finished_time,
241			const std::string &command_name,
242			const std::string &repo_name) {
243			std::string insert_statement;
244			if (command_name == "ingest" \|\| command_name == "sync") {
245			insert_statement = PrepareStatementIntoPublish(statistics, start_time,
246			finished_time);
247			} else if (command_name == "gc") {
248			insert_statement = PrepareStatementIntoGc(statistics, start_time,
249			finished_time, repo_name);
250			} else {
251			return -5;
252			}
253
254			sqlite::Sql insert(this->sqlite_db(), insert_statement);
255
256			if (!this->BeginTransaction()) {
257			LogCvmfs(kLogCvmfs, kLogSyslogErr, "BeginTransaction failed!");
258			return -1;
259			}
260
261			if (!insert.Execute()) {
262			LogCvmfs(kLogCvmfs, kLogSyslogErr, "insert.Execute failed!");
263			return -2;
264			}
265
266			if (!insert.Reset()) {
267			LogCvmfs(kLogCvmfs, kLogSyslogErr, "insert.Reset() failed!");
268			return -3;
269			}
270
271			if (!this->CommitTransaction()) {
272			LogCvmfs(kLogCvmfs, kLogSyslogErr, "CommitTransaction failed!");
273			return -4;
274			}
275
276			return 0;
277			}
278
279
280			std::string StatisticsDatabase::GetDBPath(const std::string &repo_name) {
281			// default location
282			const std::string db_default_path =
283			"/var/spool/cvmfs/" + repo_name + "/stats.db";
284			const std::string repo_config_file =
285			"/etc/cvmfs/repositories.d/" + repo_name + "/server.conf";
286			SimpleOptionsParser parser;
287
288			if (!parser.TryParsePath(repo_config_file)) {
289			LogCvmfs(kLogCvmfs, kLogSyslogErr,
290			"Could not parse repository configuration: %s.",
291			repo_config_file.c_str());
292			return db_default_path;
293			}
294
295			std::string statistics_db = "";
296			if (!parser.GetValue("CVMFS_STATISTICS_DB", &statistics_db)) {
297			LogCvmfs(kLogCvmfs, kLogDebug \| kLogSyslog,
298			"Parameter %s was not set in the repository configuration file. "
299			"Using default value: %s",
300			"CVMFS_STATISTICS_DB", db_default_path.c_str());
301			return db_default_path;
302			}
303
304			std::string dirname = GetParentPath(statistics_db);
305			int mode = S_IRUSR \| S_IWUSR \| S_IXUSR \|
306			S_IRGRP \| S_IXGRP \| S_IROTH \| S_IXOTH; // 755
307			if (!MkdirDeep(dirname, mode, true)) {
308			LogCvmfs(kLogCvmfs, kLogSyslogErr,
309			"Couldn't write statistics at the specified path %s.",
310			statistics_db.c_str());
311			return db_default_path;
312			}
313
314			return statistics_db;
315			}
316
317
318			/**
319			* Check if the CVMFS_EXTENDED_GC_STATS is ON or not
320			*
321			* @param repo_name fully qualified name of the repository
322			* @return true if CVMFS_EXTENDED_GC_STATS is ON
323			*/
324			bool StatisticsDatabase::GcExtendedStats(const std::string &repo_name) {
325			SimpleOptionsParser parser;
326			std::string param_value = "";
327			const std::string repo_config_file =
328			"/etc/cvmfs/repositories.d/" + repo_name + "/server.conf";
329
330			if (!parser.TryParsePath(repo_config_file)) {
331			LogCvmfs(kLogCvmfs, kLogSyslogErr,
332			"Could not parse repository configuration: %s.",
333			repo_config_file.c_str());
334			return false;
335			}
336			if (!parser.GetValue("CVMFS_EXTENDED_GC_STATS", &param_value)) {
337			LogCvmfs(kLogCvmfs, kLogDebug \| kLogSyslog,
338			"Parameter %s was not set in the repository configuration file. "
339			"condemned_bytes were not counted.",
340			"CVMFS_EXTENDED_GC_STATS");
341			} else if (parser.IsOn(param_value)) {
342			return true;
343			}
344			return false;
345			}
346
347
348			StatisticsDatabase::StatisticsDatabase(const std::string &filename,
349			const OpenMode open_mode) :
350			sqlite::Database<StatisticsDatabase>(filename, open_mode),
351			create_empty_db_calls(0), check_compatibility_calls(0),
352			live_upgrade_calls(0), compact_calls(0)
353			{
354			++StatisticsDatabase::instances;
355			}