GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/statistics_database.cc
Date: 2025-11-09 02:35:23
Exec Total Coverage
Lines: 63 267 23.6%
Branches: 109 809 13.5%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 */
4
5 #include "statistics_database.h"
6
7 #include "util/exception.h"
8
9
10 const float StatisticsDatabase::kLatestCompatibleSchema = 1.0f;
11 float StatisticsDatabase::kLatestSchema = 1.0f;
12
13 // Changelog
14 // 1 --> 2: (Sep 4 2019)
15 // * change column name `finished_time` -> `finish_time`
16 // in publish_statistics table
17 // * add column `revision` to publish_statistics table
18 // * change column name `duplicated_files` -> `chunks_duplicated`
19 // in publish_statistics table
20 // * add column `chunks_added` to publish_statistics table
21 // * add column `symlinks_added` to publish_statistics table
22 // * add column `symlinks_removed` to publish_statistics table
23 // * add column `symlinks_changed` to publish_statistics table
24 // * change column name `finished_time` -> `finish_time`
25 // in gc_statistics table
26 // 2 --> 3: (Jan 14 2020)
27 // * add `success` column to publish_statistics table (1 for success
28 // 0 for fail)
29 // * add `success` column to gc_statistics table (1 for success
30 // 0 for fail)
31 // 3 --> 4: (Feb 1 2022)
32 // * add column `n_duplicate_delete_requests` to gc_statistics table
33
34 unsigned StatisticsDatabase::kLatestSchemaRevision = 4;
35 unsigned int StatisticsDatabase::instances = 0;
36 bool StatisticsDatabase::compacting_fails = false;
37
38
39 namespace {
40
41 struct PublishStats {
42 std::string revision;
43 std::string files_added;
44 std::string files_removed;
45 std::string files_changed;
46 std::string chunks_added;
47 std::string chunks_duplicated;
48 std::string catalogs_added;
49 std::string dirs_added;
50 std::string dirs_removed;
51 std::string dirs_changed;
52 std::string symlinks_added;
53 std::string symlinks_removed;
54 std::string symlinks_changed;
55 std::string bytes_added;
56 std::string bytes_removed;
57 std::string bytes_uploaded;
58 std::string catalog_bytes_uploaded;
59
60 ✗ explicit PublishStats(const perf::Statistics *statistics)
61 ✗ : revision(statistics->Lookup("publish.revision")->ToString())
62 ✗ , files_added(statistics->Lookup("publish.n_files_added")->ToString())
63 ✗ , files_removed(statistics->Lookup("publish.n_files_removed")->ToString())
64 ✗ , files_changed(statistics->Lookup("publish.n_files_changed")->ToString())
65 ✗ , chunks_added(statistics->Lookup("publish.n_chunks_added")->ToString())
66 ✗ , chunks_duplicated(
67 statistics->Lookup("publish.n_chunks_duplicated")->ToString())
68 ✗ , catalogs_added(
69 statistics->Lookup("publish.n_catalogs_added")->ToString())
70 ✗ , dirs_added(
71 statistics->Lookup("publish.n_directories_added")->ToString())
72 ✗ , dirs_removed(
73 statistics->Lookup("publish.n_directories_removed")->ToString())
74 ✗ , dirs_changed(
75 statistics->Lookup("publish.n_directories_changed")->ToString())
76 ✗ , symlinks_added(
77 statistics->Lookup("publish.n_symlinks_added")->ToString())
78 ✗ , symlinks_removed(
79 statistics->Lookup("publish.n_symlinks_removed")->ToString())
80 ✗ , symlinks_changed(
81 statistics->Lookup("publish.n_symlinks_changed")->ToString())
82 ✗ , bytes_added(statistics->Lookup("publish.sz_added_bytes")->ToString())
83 ✗ , bytes_removed(
84 statistics->Lookup("publish.sz_removed_bytes")->ToString())
85 ✗ , bytes_uploaded(
86 statistics->Lookup("publish.sz_uploaded_bytes")->ToString())
87 ✗ , catalog_bytes_uploaded(
88 statistics->Lookup("publish.sz_uploaded_catalog_bytes")
89 ✗ ->ToString()) { }
90 };
91
92
93 struct GcStats {
94 std::string n_preserved_catalogs;
95 std::string n_condemned_catalogs;
96 std::string n_condemned_objects;
97 std::string sz_condemned_bytes;
98 std::string n_duplicate_delete_requests;
99
100 ✗ explicit GcStats(const perf::Statistics *statistics) {
101 ✗ perf::Counter *c = NULL;
102 ✗ c = statistics->Lookup("gc.n_preserved_catalogs");
103 ✗ n_preserved_catalogs = c ? c->ToString() : "0";
104 ✗ c = statistics->Lookup("gc.n_condemned_catalogs");
105 ✗ n_condemned_catalogs = c ? c->ToString() : "0";
106 ✗ c = statistics->Lookup("gc.n_condemned_objects");
107 ✗ n_condemned_objects = c ? c->ToString() : "0";
108 ✗ c = statistics->Lookup("gc.sz_condemned_bytes");
109 ✗ sz_condemned_bytes = c ? c->ToString() : "0";
110 ✗ c = statistics->Lookup("gc.n_duplicate_delete_requests");
111 ✗ n_duplicate_delete_requests = c ? c->ToString() : "0";
112 }
113 };
114
115
116 /**
117 * Build the insert statement into publish_statistics table.
118 *
119 * @param stats a struct with all values stored in strings
120 * @return the insert statement
121 */
122 ✗ std::string PrepareStatementIntoPublish(const perf::Statistics *statistics,
123 const std::string &start_time,
124 const std::string &finish_time,
125 const bool success) {
126 ✗ struct PublishStats const stats = PublishStats(statistics);
127 std::string insert_statement = "INSERT INTO publish_statistics ("
128 "start_time,"
129 "finish_time,"
130 "revision,"
131 "files_added,"
132 "files_removed,"
133 "files_changed,"
134 "chunks_added,"
135 "chunks_duplicated,"
136 "catalogs_added,"
137 "directories_added,"
138 "directories_removed,"
139 "directories_changed,"
140 "symlinks_added,"
141 "symlinks_removed,"
142 "symlinks_changed,"
143 "sz_bytes_added,"
144 "sz_bytes_removed,"
145 "sz_bytes_uploaded,"
146 "sz_catalog_bytes_uploaded,"
147 "success)"
148 " VALUES("
149 "'"
150 ✗ + start_time + "'," + "'" + finish_time + "',"
151 ✗ + stats.revision + "," + stats.files_added
152 ✗ + "," + stats.files_removed + ","
153 ✗ + stats.files_changed + ","
154 ✗ + stats.chunks_added + ","
155 ✗ + stats.chunks_duplicated + ","
156 ✗ + stats.catalogs_added + "," + stats.dirs_added
157 ✗ + "," + stats.dirs_removed + ","
158 ✗ + stats.dirs_changed + ","
159 ✗ + stats.symlinks_added + ","
160 ✗ + stats.symlinks_removed + ","
161 ✗ + stats.symlinks_changed + ","
162 ✗ + stats.bytes_added + "," + stats.bytes_removed
163 ✗ + "," + stats.bytes_uploaded + ","
164 ✗ + stats.catalog_bytes_uploaded + ","
165 ✗ + (success ? "1" : "0") + ");";
166 ✗ return insert_statement;
167 }
168
169
170 /**
171 * Build the insert statement into gc_statistics table.
172 *
173 * @param stats a struct with values stored in strings
174 * @param start_time, finish_time to run Main() of the command
175 * @param repo_name fully qualified name of the repository
176 *
177 * @return the insert statement
178 */
179 ✗ std::string PrepareStatementIntoGc(const perf::Statistics *statistics,
180 const std::string &start_time,
181 const std::string &finish_time,
182 const std::string &repo_name,
183 const bool success) {
184 ✗ struct GcStats const stats = GcStats(statistics);
185 ✗ std::string insert_statement = "";
186 ✗ if (StatisticsDatabase::GcExtendedStats(repo_name)) {
187 insert_statement = "INSERT INTO gc_statistics ("
188 "start_time,"
189 "finish_time,"
190 "n_preserved_catalogs,"
191 "n_condemned_catalogs,"
192 "n_condemned_objects,"
193 "sz_condemned_bytes,"
194 "n_duplicate_delete_requests,"
195 "success)"
196 " VALUES("
197 "'"
198 ✗ + start_time + "'," + "'" + finish_time + "',"
199 ✗ + stats.n_preserved_catalogs + ","
200 ✗ + stats.n_condemned_catalogs + ","
201 ✗ + stats.n_condemned_objects + ","
202 ✗ + stats.sz_condemned_bytes + ","
203 ✗ + stats.n_duplicate_delete_requests + ","
204 ✗ + (success ? "1" : "0") + ");";
205 } else {
206 // insert values except sz_condemned_bytes
207 insert_statement = "INSERT INTO gc_statistics ("
208 "start_time,"
209 "finish_time,"
210 "n_preserved_catalogs,"
211 "n_condemned_catalogs,"
212 "n_condemned_objects,"
213 "n_duplicate_delete_requests,"
214 "success)"
215 " VALUES("
216 "'"
217 ✗ + start_time + "'," + "'" + finish_time + "',"
218 ✗ + stats.n_preserved_catalogs + ","
219 ✗ + stats.n_condemned_catalogs + ","
220 ✗ + stats.n_condemned_objects + ","
221 ✗ + stats.n_duplicate_delete_requests + ","
222 ✗ + (success ? "1" : "0") + ");";
223 }
224 ✗ return insert_statement;
225 }
226
227 } // namespace
228
229
230 18 bool StatisticsDatabase::CreateEmptyDatabase() {
231 18 ++create_empty_db_calls;
232
3/6
✓ Branch 2 taken 18 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 18 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 18 times.
✗ Branch 9 not taken.
 36 const bool ret1 = sqlite::Sql(sqlite_db(),
233 "CREATE TABLE publish_statistics ("
234 "publish_id INTEGER PRIMARY KEY,"
235 "start_time TEXT,"
236 "finish_time TEXT,"
237 "revision INTEGER,"
238 "files_added INTEGER,"
239 "files_removed INTEGER,"
240 "files_changed INTEGER,"
241 "chunks_added INTEGER,"
242 "chunks_duplicated INTEGER,"
243 "catalogs_added INTEGER,"
244 "directories_added INTEGER,"
245 "directories_removed INTEGER,"
246 "directories_changed INTEGER,"
247 "symlinks_added INTEGER,"
248 "symlinks_removed INTEGER,"
249 "symlinks_changed INTEGER,"
250 "sz_bytes_added INTEGER,"
251 "sz_bytes_removed INTEGER,"
252 "sz_bytes_uploaded INTEGER,"
253 "sz_catalog_bytes_uploaded INTEGER,"
254 "success INTEGER);")
255
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 18 .Execute();
256
3/6
✓ Branch 2 taken 18 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 18 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 18 times.
✗ Branch 9 not taken.
 36 const bool ret2 = sqlite::Sql(sqlite_db(),
257 "CREATE TABLE gc_statistics ("
258 "gc_id INTEGER PRIMARY KEY,"
259 "start_time TEXT,"
260 "finish_time TEXT,"
261 "n_preserved_catalogs INTEGER,"
262 "n_condemned_catalogs INTEGER,"
263 "n_condemned_objects INTEGER,"
264 "sz_condemned_bytes INTEGER,"
265 "n_duplicate_delete_requests INTEGER,"
266 "success INTEGER);")
267
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 18 .Execute();
268 18 return ret1 & ret2;
269 }
270
271
272 18 bool StatisticsDatabase::CheckSchemaCompatibility() {
273 18 ++check_compatibility_calls;
274 18 return (schema_version() > kLatestCompatibleSchema - 0.1
275
2/4
✓ Branch 0 taken 18 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 18 times.
✗ Branch 4 not taken.
 18 && schema_version() < kLatestCompatibleSchema + 0.1);
276 }
277
278
279 18 bool StatisticsDatabase::LiveSchemaUpgradeIfNecessary() {
280 18 ++live_upgrade_calls;
281 18 if (IsEqualSchema(schema_version(), kLatestSchema)
282
5/6
✓ Branch 0 taken 18 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 9 times.
✓ Branch 4 taken 9 times.
✓ Branch 5 taken 9 times.
✓ Branch 6 taken 9 times.
 18 && (schema_revision() == 1)) {
283
1/2
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
 9 LogCvmfs(kLogCvmfs, kLogDebug,
284 "upgrading schema revision (1 --> 2) of "
285 "statistics database");
286
287 sqlite::Sql publish_upgrade2_1(
288 this->sqlite_db(),
289 "ALTER TABLE "
290
3/6
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 9 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 9 times.
✗ Branch 9 not taken.
 18 "publish_statistics RENAME COLUMN finished_time TO finish_time;");
291 sqlite::Sql publish_upgrade2_2(this->sqlite_db(),
292 "ALTER TABLE "
293
3/6
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 9 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 9 times.
✗ Branch 9 not taken.
 18 "publish_statistics ADD revision INTEGER;");
294 sqlite::Sql publish_upgrade2_3(this->sqlite_db(),
295 "ALTER TABLE "
296 "publish_statistics RENAME COLUMN "
297
3/6
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 9 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 9 times.
✗ Branch 9 not taken.
 18 "duplicated_files TO chunks_duplicated;");
298 sqlite::Sql publish_upgrade2_4(
299 this->sqlite_db(), "ALTER TABLE "
300
3/6
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 9 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 9 times.
✗ Branch 9 not taken.
 18 "publish_statistics ADD chunks_added INTEGER;");
301 sqlite::Sql publish_upgrade2_5(
302 this->sqlite_db(), "ALTER TABLE "
303
3/6
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 9 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 9 times.
✗ Branch 9 not taken.
 18 "publish_statistics ADD symlinks_added INTEGER;");
304 sqlite::Sql publish_upgrade2_6(
305 this->sqlite_db(), "ALTER TABLE "
306
3/6
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 9 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 9 times.
✗ Branch 9 not taken.
 18 "publish_statistics ADD symlinks_removed INTEGER;");
307 sqlite::Sql publish_upgrade2_7(
308 this->sqlite_db(), "ALTER TABLE "
309
3/6
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 9 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 9 times.
✗ Branch 9 not taken.
 18 "publish_statistics ADD symlinks_changed INTEGER;");
310 sqlite::Sql publish_upgrade2_8(
311 this->sqlite_db(), "ALTER TABLE "
312
3/6
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 9 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 9 times.
✗ Branch 9 not taken.
 18 "publish_statistics ADD catalogs_added INTEGER;");
313 sqlite::Sql publish_upgrade2_9(
314 this->sqlite_db(),
315 "ALTER TABLE "
316
3/6
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 9 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 9 times.
✗ Branch 9 not taken.
 18 "publish_statistics ADD sz_catalog_bytes_uploaded INTEGER;");
317
318
3/7
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 9 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 9 times.
✗ Branch 7 not taken.
 18 if (!publish_upgrade2_1.Execute() || !publish_upgrade2_2.Execute()
319
4/9
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 9 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 9 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 9 times.
✗ Branch 9 not taken.
 9 || !publish_upgrade2_3.Execute() || !publish_upgrade2_4.Execute()
320
4/9
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 9 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 9 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 9 times.
✗ Branch 9 not taken.
 9 || !publish_upgrade2_5.Execute() || !publish_upgrade2_6.Execute()
321
4/9
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 9 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 9 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 9 times.
✗ Branch 9 not taken.
 9 || !publish_upgrade2_7.Execute() || !publish_upgrade2_8.Execute()
322
4/8
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 9 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 9 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 9 times.
 18 || !publish_upgrade2_9.Execute()) {
323 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr,
324 "failed to upgrade publish_statistics"
325 " table of statistics database");
326 ✗ return false;
327 }
328
329 sqlite::Sql gc_upgrade2_1(this->sqlite_db(),
330 "ALTER TABLE gc_statistics"
331
3/6
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 9 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 9 times.
✗ Branch 9 not taken.
 18 " RENAME COLUMN finished_time TO finish_time;");
332
333
2/4
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 9 times.
 9 if (!gc_upgrade2_1.Execute()) {
334 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr,
335 "failed to upgrade gc_statistics"
336 " table of statistics database");
337 ✗ return false;
338 }
339
340 9 set_schema_revision(2);
341
2/4
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 9 times.
 9 if (!StoreSchemaRevision()) {
342 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr,
343 "failed to upgrade schema revision"
344 " of statistics database");
345 ✗ return false;
346 }
347
10/20
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 9 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 9 times.
✗ Branch 8 not taken.
✓ Branch 10 taken 9 times.
✗ Branch 11 not taken.
✓ Branch 13 taken 9 times.
✗ Branch 14 not taken.
✓ Branch 16 taken 9 times.
✗ Branch 17 not taken.
✓ Branch 19 taken 9 times.
✗ Branch 20 not taken.
✓ Branch 22 taken 9 times.
✗ Branch 23 not taken.
✓ Branch 25 taken 9 times.
✗ Branch 26 not taken.
✓ Branch 28 taken 9 times.
✗ Branch 29 not taken.
 9 }
348
349 18 if (IsEqualSchema(schema_version(), kLatestSchema)
350
3/6
✓ Branch 0 taken 18 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 18 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 18 times.
✗ Branch 6 not taken.
 18 && (schema_revision() == 2)) {
351
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 18 LogCvmfs(kLogCvmfs, kLogDebug,
352 "upgrading schema revision (2 --> 3) of "
353 "statistics database");
354
355 sqlite::Sql publish_upgrade3_1(this->sqlite_db(),
356 "ALTER TABLE "
357
3/6
✓ Branch 2 taken 18 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 18 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 18 times.
✗ Branch 9 not taken.
 36 "publish_statistics ADD success INTEGER;");
358
359
2/4
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 18 times.
 18 if (!publish_upgrade3_1.Execute()) {
360 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr,
361 "failed to upgrade publish_statistics"
362 " table of statistics database");
363 ✗ return false;
364 }
365
366 sqlite::Sql gc_upgrade3_1(this->sqlite_db(), "ALTER TABLE gc_statistics"
367
3/6
✓ Branch 2 taken 18 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 18 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 18 times.
✗ Branch 9 not taken.
 36 " ADD success INTEGER;");
368
369
2/4
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 18 times.
 18 if (!gc_upgrade3_1.Execute()) {
370 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr,
371 "failed to upgrade gc_statistics"
372 " table of statistics database");
373 ✗ return false;
374 }
375
376 18 set_schema_revision(3);
377
2/4
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 18 times.
 18 if (!StoreSchemaRevision()) {
378 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr,
379 "failed to upgrade schema revision"
380 " of statistics database");
381 ✗ return false;
382 }
383
2/4
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 18 times.
✗ Branch 5 not taken.
 18 }
384
385 18 if (IsEqualSchema(schema_version(), kLatestSchema)
386
3/6
✓ Branch 0 taken 18 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 18 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 18 times.
✗ Branch 6 not taken.
 18 && (schema_revision() == 3)) {
387
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 18 LogCvmfs(kLogCvmfs, kLogDebug,
388 "upgrading schema revision (3 --> 4) of "
389 "statistics database");
390
391 sqlite::Sql gc_upgrade4_1(this->sqlite_db(),
392 "ALTER TABLE gc_statistics"
393
3/6
✓ Branch 2 taken 18 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 18 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 18 times.
✗ Branch 9 not taken.
 36 " ADD n_duplicate_delete_requests INTEGER;");
394
395
2/4
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 18 times.
 18 if (!gc_upgrade4_1.Execute()) {
396 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr,
397 "failed to upgrade gc_statistics "
398 "table of statistics database");
399 ✗ return false;
400 }
401
402 18 set_schema_revision(4);
403
2/4
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 18 times.
 18 if (!StoreSchemaRevision()) {
404 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr,
405 "failed to upgrade schema revision "
406 "of statistics database");
407 ✗ return false;
408 }
409
1/2
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
 18 }
410 18 return true;
411 }
412
413
414 ✗ bool StatisticsDatabase::CompactDatabase() const {
415 ✗ ++compact_calls;
416 ✗ return !compacting_fails;
417 }
418
419
420 36 StatisticsDatabase::~StatisticsDatabase() { --StatisticsDatabase::instances; }
421
422
423 ✗ StatisticsDatabase *StatisticsDatabase::OpenStandardDB(
424 const std::string repo_name) {
425 StatisticsDatabase *db;
426 ✗ std::string db_file_path;
427 uint32_t days_to_keep;
428 ✗ GetDBParams(repo_name, &db_file_path, &days_to_keep);
429 ✗ if (FileExists(db_file_path)) {
430 ✗ db = StatisticsDatabase::Open(db_file_path, kOpenReadWrite);
431 ✗ if (db == NULL) {
432 ✗ PANIC(kLogSyslogErr, "Couldn't create StatisticsDatabase object!");
433 ✗ } else if (db->GetProperty<std::string>("repo_name") != repo_name) {
434 ✗ PANIC(kLogSyslogErr,
435 "'repo_name' property of the statistics database %s "
436 "is incorrect. Please fix the database.",
437 db_file_path.c_str());
438 }
439 ✗ if (!db->Prune(days_to_keep)) {
440 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr, "Failed to prune statistics database");
441 }
442 } else {
443 ✗ db = StatisticsDatabase::Create(db_file_path);
444 ✗ if (db == NULL) {
445 ✗ PANIC(kLogSyslogErr, "Couldn't create StatisticsDatabase object!");
446 // insert repo_name into properties table
447 ✗ } else if (!db->SetProperty("repo_name", repo_name)) {
448 ✗ PANIC(kLogSyslogErr, "Couldn't insert repo_name into properties table!");
449 }
450 }
451 ✗ db->repo_name_ = repo_name;
452 ✗ return db;
453 }
454
455
456 ✗ bool StatisticsDatabase::StorePublishStatistics(
457 const perf::Statistics *statistics,
458 const std::string &start_time,
459 const bool success) {
460 ✗ const std::string finish_time = GetGMTimestamp();
461 const std::string statement = PrepareStatementIntoPublish(
462 ✗ statistics, start_time, finish_time, success);
463 ✗ return StoreEntry(statement);
464 }
465
466
467 ✗ bool StatisticsDatabase::StoreGCStatistics(const perf::Statistics *statistics,
468 const std::string &start_time,
469 const bool success) {
470 ✗ const std::string finish_time = GetGMTimestamp();
471 const std::string statement = PrepareStatementIntoGc(
472 ✗ statistics, start_time, finish_time, repo_name_, success);
473 ✗ return StoreEntry(statement);
474 }
475
476
477 ✗ bool StatisticsDatabase::StoreEntry(const std::string &insert_statement) {
478 ✗ sqlite::Sql insert(this->sqlite_db(), insert_statement);
479
480 ✗ if (!insert.Execute()) {
481 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr,
482 "Couldn't store statistics in %s: insert.Execute failed!",
483 ✗ this->filename().c_str());
484 ✗ return false;
485 }
486
487 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Statistics stored at: %s",
488 ✗ this->filename().c_str());
489 ✗ return true;
490 }
491
492
493 ✗ bool StatisticsDatabase::Prune(uint32_t days) {
494 ✗ if (days == 0)
495 ✗ return true;
496
497 const std::string
498 publish_stmt = "DELETE FROM publish_statistics WHERE "
499 "julianday('now','start of day')-julianday(start_time) > "
500 ✗ + StringifyUint(days) + ";";
501
502 const std::string
503 gc_stmt = "DELETE FROM gc_statistics WHERE "
504 "julianday('now','start of day')-julianday(start_time) > "
505 ✗ + StringifyUint(days) + ";";
506
507 ✗ sqlite::Sql publish_sql(this->sqlite_db(), publish_stmt);
508 ✗ sqlite::Sql gc_sql(this->sqlite_db(), gc_stmt);
509 ✗ if (!publish_sql.Execute() || !gc_sql.Execute()) {
510 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr,
511 "Couldn't prune statistics DB %s: SQL Execute() failed!",
512 ✗ this->filename().c_str());
513 ✗ return false;
514 }
515 ✗ if (!this->Vacuum()) {
516 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr,
517 "Couldn't prune statistics DB %s: Vacuum() failed!",
518 ✗ this->filename().c_str());
519 ✗ return false;
520 }
521
522 ✗ return true;
523 }
524
525
526 ✗ bool StatisticsDatabase::UploadStatistics(upload::Spooler *spooler,
527 std::string local_path) {
528 ✗ if (local_path == "") {
529 ✗ local_path = this->filename();
530 }
531
532 ✗ spooler->WaitForUpload();
533 ✗ const unsigned errors_before = spooler->GetNumberOfErrors();
534 ✗ spooler->Mkdir("stats");
535 ✗ spooler->RemoveAsync("stats/stats.db");
536 ✗ spooler->WaitForUpload();
537 ✗ spooler->Upload(local_path, "stats/stats.db");
538 ✗ spooler->WaitForUpload();
539 ✗ const unsigned errors_after = spooler->GetNumberOfErrors();
540
541 ✗ if (errors_before != errors_after) {
542 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr,
543 "Could not upload statistics DB file into storage backend");
544 ✗ return false;
545 }
546 ✗ return true;
547 }
548
549
550 ✗ bool StatisticsDatabase::UploadStatistics(upload::AbstractUploader *uploader,
551 std::string local_path) {
552 ✗ if (local_path == "") {
553 ✗ local_path = this->filename();
554 }
555
556 ✗ uploader->WaitForUpload();
557 ✗ const unsigned errors_before = uploader->GetNumberOfErrors();
558
559 ✗ uploader->RemoveAsync("stats/stats.db");
560 ✗ uploader->WaitForUpload();
561 ✗ uploader->UploadFile(local_path, "stats/stats.db");
562 ✗ uploader->WaitForUpload();
563 ✗ const unsigned errors_after = uploader->GetNumberOfErrors();
564 ✗ return errors_before == errors_after;
565 }
566
567
568 ✗ void StatisticsDatabase::GetDBParams(const std::string &repo_name,
569 std::string *path,
570 uint32_t *days_to_keep) {
571 // default location
572 ✗ const std::string db_default_path = "/var/spool/cvmfs/" + repo_name
573 ✗ + "/stats.db";
574 ✗ const std::string repo_config_file = "/etc/cvmfs/repositories.d/" + repo_name
575 ✗ + "/server.conf";
576 ✗ SimpleOptionsParser parser;
577
578 ✗ if (!parser.TryParsePath(repo_config_file)) {
579 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr,
580 "Could not parse repository configuration: %s.",
581 repo_config_file.c_str());
582 ✗ *path = db_default_path;
583 ✗ *days_to_keep = kDefaultDaysToKeep;
584 ✗ return;
585 }
586
587 ✗ std::string statistics_db = "";
588 ✗ if (!parser.GetValue("CVMFS_STATISTICS_DB", &statistics_db)) {
589 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslog,
590 "Parameter %s was not set in the repository configuration file. "
591 "Using default value: %s",
592 "CVMFS_STATISTICS_DB", db_default_path.c_str());
593 ✗ *path = db_default_path;
594 } else {
595 ✗ const std::string dirname = GetParentPath(statistics_db);
596 ✗ const int mode = S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH
597 | S_IXOTH; // 755
598 ✗ if (!MkdirDeep(dirname, mode, true)) {
599 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr,
600 "Couldn't write statistics at the specified path %s.",
601 statistics_db.c_str());
602 ✗ *path = db_default_path;
603 } else {
604 ✗ *path = statistics_db;
605 }
606 }
607
608 ✗ const std::string days_to_keep_str = "";
609 ✗ if (!parser.GetValue("CVMFS_STATS_DB_DAYS_TO_KEEP", &statistics_db)) {
610 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslog,
611 "Parameter %s was not set in the repository configuration file. "
612 "Using default value: %s",
613 "CVMFS_STATS_DB_DAYS_TO_KEEP",
614 ✗ StringifyUint(kDefaultDaysToKeep).c_str());
615 ✗ *days_to_keep = kDefaultDaysToKeep;
616 } else {
617 ✗ *days_to_keep = static_cast<uint32_t>(String2Uint64(days_to_keep_str));
618 }
619 }
620
621
622 /**
623 * Check if the CVMFS_EXTENDED_GC_STATS is ON or not
624 *
625 * @param repo_name fully qualified name of the repository
626 * @return true if CVMFS_EXTENDED_GC_STATS is ON
627 */
628 ✗ bool StatisticsDatabase::GcExtendedStats(const std::string &repo_name) {
629 ✗ SimpleOptionsParser parser;
630 ✗ std::string param_value = "";
631 ✗ const std::string repo_config_file = "/etc/cvmfs/repositories.d/" + repo_name
632 ✗ + "/server.conf";
633
634 ✗ if (!parser.TryParsePath(repo_config_file)) {
635 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr,
636 "Could not parse repository configuration: %s.",
637 repo_config_file.c_str());
638 ✗ return false;
639 }
640 ✗ if (!parser.GetValue("CVMFS_EXTENDED_GC_STATS", &param_value)) {
641 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslog,
642 "Parameter %s was not set in the repository configuration file. "
643 "condemned_bytes were not counted.",
644 "CVMFS_EXTENDED_GC_STATS");
645 ✗ } else if (parser.IsOn(param_value)) {
646 ✗ return true;
647 }
648 ✗ return false;
649 }
650
651
652 36 StatisticsDatabase::StatisticsDatabase(const std::string &filename,
653 36 const OpenMode open_mode)
654 : sqlite::Database<StatisticsDatabase>(filename, open_mode)
655 36 , create_empty_db_calls(0)
656 36 , check_compatibility_calls(0)
657 36 , live_upgrade_calls(0)
658 36 , compact_calls(0) {
659 36 ++StatisticsDatabase::instances;
660 36 }
661
662 const uint32_t StatisticsDatabase::kDefaultDaysToKeep = 365;
663