GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/statistics_database.cc
Date: 2025-02-09 02:34:19
Exec Total Coverage
Lines: 67 274 24.5%
Branches: 109 805 13.5%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 */
4
5 #include "statistics_database.h"
6
7 #include "util/exception.h"
8
9
10 const float StatisticsDatabase::kLatestCompatibleSchema = 1.0f;
11 float StatisticsDatabase::kLatestSchema = 1.0f;
12
13 // Changelog
14 // 1 --> 2: (Sep 4 2019)
15 // * change column name `finished_time` -> `finish_time`
16 // in publish_statistics table
17 // * add column `revision` to publish_statistics table
18 // * change column name `duplicated_files` -> `chunks_duplicated`
19 // in publish_statistics table
20 // * add column `chunks_added` to publish_statistics table
21 // * add column `symlinks_added` to publish_statistics table
22 // * add column `symlinks_removed` to publish_statistics table
23 // * add column `symlinks_changed` to publish_statistics table
24 // * change column name `finished_time` -> `finish_time`
25 // in gc_statistics table
26 // 2 --> 3: (Jan 14 2020)
27 // * add `success` column to publish_statistics table (1 for success
28 // 0 for fail)
29 // * add `success` column to gc_statistics table (1 for success
30 // 0 for fail)
31 // 3 --> 4: (Feb 1 2022)
32 // * add column `n_duplicate_delete_requests` to gc_statistics table
33
34 unsigned StatisticsDatabase::kLatestSchemaRevision = 4;
35 unsigned int StatisticsDatabase::instances = 0;
36 bool StatisticsDatabase::compacting_fails = false;
37
38
39 namespace {
40
41 struct PublishStats {
42 std::string revision;
43 std::string files_added;
44 std::string files_removed;
45 std::string files_changed;
46 std::string chunks_added;
47 std::string chunks_duplicated;
48 std::string catalogs_added;
49 std::string dirs_added;
50 std::string dirs_removed;
51 std::string dirs_changed;
52 std::string symlinks_added;
53 std::string symlinks_removed;
54 std::string symlinks_changed;
55 std::string bytes_added;
56 std::string bytes_removed;
57 std::string bytes_uploaded;
58 std::string catalog_bytes_uploaded;
59
60 ✗ explicit PublishStats(const perf::Statistics *statistics):
61 ✗ revision(statistics->
62 Lookup("publish.revision")->ToString()),
63 ✗ files_added(statistics->
64 Lookup("publish.n_files_added")->ToString()),
65 ✗ files_removed(statistics->
66 Lookup("publish.n_files_removed")->ToString()),
67 ✗ files_changed(statistics->
68 Lookup("publish.n_files_changed")->ToString()),
69 ✗ chunks_added(statistics->
70 Lookup("publish.n_chunks_added")->ToString()),
71 ✗ chunks_duplicated(statistics->
72 Lookup("publish.n_chunks_duplicated")->ToString()),
73 ✗ catalogs_added(statistics->
74 Lookup("publish.n_catalogs_added")->ToString()),
75 ✗ dirs_added(statistics->
76 Lookup("publish.n_directories_added")->ToString()),
77 ✗ dirs_removed(statistics->
78 Lookup("publish.n_directories_removed")->ToString()),
79 ✗ dirs_changed(statistics->
80 Lookup("publish.n_directories_changed")->ToString()),
81 ✗ symlinks_added(statistics->
82 Lookup("publish.n_symlinks_added")->ToString()),
83 ✗ symlinks_removed(statistics->
84 Lookup("publish.n_symlinks_removed")->ToString()),
85 ✗ symlinks_changed(statistics->
86 Lookup("publish.n_symlinks_changed")->ToString()),
87 ✗ bytes_added(statistics->
88 Lookup("publish.sz_added_bytes")->ToString()),
89 ✗ bytes_removed(statistics->
90 Lookup("publish.sz_removed_bytes")->ToString()),
91 ✗ bytes_uploaded(statistics->
92 Lookup("publish.sz_uploaded_bytes")->ToString()),
93 ✗ catalog_bytes_uploaded(statistics->
94 Lookup("publish.sz_uploaded_catalog_bytes")->ToString()) {
95 }
96 };
97
98
99 struct GcStats {
100 std::string n_preserved_catalogs;
101 std::string n_condemned_catalogs;
102 std::string n_condemned_objects;
103 std::string sz_condemned_bytes;
104 std::string n_duplicate_delete_requests;
105
106 ✗ explicit GcStats(const perf::Statistics *statistics) {
107 ✗ perf::Counter *c = NULL;
108 ✗ c = statistics->Lookup("gc.n_preserved_catalogs");
109 ✗ n_preserved_catalogs = c ? c->ToString() : "0";
110 ✗ c = statistics->Lookup("gc.n_condemned_catalogs");
111 ✗ n_condemned_catalogs = c ? c->ToString() : "0";
112 ✗ c = statistics->Lookup("gc.n_condemned_objects");
113 ✗ n_condemned_objects = c ? c->ToString() : "0";
114 ✗ c = statistics->Lookup("gc.sz_condemned_bytes");
115 ✗ sz_condemned_bytes = c ? c->ToString() : "0";
116 ✗ c = statistics->Lookup("gc.n_duplicate_delete_requests");
117 ✗ n_duplicate_delete_requests = c ? c->ToString() : "0";
118 }
119 };
120
121
122 /**
123 * Build the insert statement into publish_statistics table.
124 *
125 * @param stats a struct with all values stored in strings
126 * @return the insert statement
127 */
128 ✗ std::string PrepareStatementIntoPublish(const perf::Statistics *statistics,
129 const std::string &start_time,
130 const std::string &finish_time,
131 const bool success) {
132 ✗ struct PublishStats stats = PublishStats(statistics);
133 std::string insert_statement =
134 "INSERT INTO publish_statistics ("
135 "start_time,"
136 "finish_time,"
137 "revision,"
138 "files_added,"
139 "files_removed,"
140 "files_changed,"
141 "chunks_added,"
142 "chunks_duplicated,"
143 "catalogs_added,"
144 "directories_added,"
145 "directories_removed,"
146 "directories_changed,"
147 "symlinks_added,"
148 "symlinks_removed,"
149 "symlinks_changed,"
150 "sz_bytes_added,"
151 "sz_bytes_removed,"
152 "sz_bytes_uploaded,"
153 "sz_catalog_bytes_uploaded,"
154 "success)"
155 " VALUES("
156 ✗ "'"+start_time+"',"+
157 ✗ "'"+finish_time+"',"+
158 ✗ stats.revision+"," +
159 ✗ stats.files_added+"," +
160 ✗ stats.files_removed +","+
161 ✗ stats.files_changed + "," +
162 ✗ stats.chunks_added + "," +
163 ✗ stats.chunks_duplicated + "," +
164 ✗ stats.catalogs_added + "," +
165 ✗ stats.dirs_added + "," +
166 ✗ stats.dirs_removed + "," +
167 ✗ stats.dirs_changed + "," +
168 ✗ stats.symlinks_added + "," +
169 ✗ stats.symlinks_removed + "," +
170 ✗ stats.symlinks_changed + "," +
171 ✗ stats.bytes_added + "," +
172 ✗ stats.bytes_removed + "," +
173 ✗ stats.bytes_uploaded + "," +
174 ✗ stats.catalog_bytes_uploaded + "," +
175 ✗ (success ? "1" : "0") + ");";
176 ✗ return insert_statement;
177 }
178
179
180 /**
181 * Build the insert statement into gc_statistics table.
182 *
183 * @param stats a struct with values stored in strings
184 * @param start_time, finish_time to run Main() of the command
185 * @param repo_name fully qualified name of the repository
186 *
187 * @return the insert statement
188 */
189 ✗ std::string PrepareStatementIntoGc(const perf::Statistics *statistics,
190 const std::string &start_time,
191 const std::string &finish_time,
192 const std::string &repo_name,
193 const bool success) {
194 ✗ struct GcStats stats = GcStats(statistics);
195 ✗ std::string insert_statement = "";
196 ✗ if (StatisticsDatabase::GcExtendedStats(repo_name)) {
197 insert_statement =
198 "INSERT INTO gc_statistics ("
199 "start_time,"
200 "finish_time,"
201 "n_preserved_catalogs,"
202 "n_condemned_catalogs,"
203 "n_condemned_objects,"
204 "sz_condemned_bytes,"
205 "n_duplicate_delete_requests,"
206 "success)"
207 " VALUES("
208 ✗ "'" + start_time + "'," +
209 ✗ "'" + finish_time + "'," +
210 ✗ stats.n_preserved_catalogs + "," +
211 ✗ stats.n_condemned_catalogs + ","+
212 ✗ stats.n_condemned_objects + "," +
213 ✗ stats.sz_condemned_bytes + "," +
214 ✗ stats.n_duplicate_delete_requests + "," +
215 ✗ (success ? "1" : "0") + ");";
216 } else {
217 // insert values except sz_condemned_bytes
218 insert_statement =
219 "INSERT INTO gc_statistics ("
220 "start_time,"
221 "finish_time,"
222 "n_preserved_catalogs,"
223 "n_condemned_catalogs,"
224 "n_condemned_objects,"
225 "n_duplicate_delete_requests,"
226 "success)"
227 " VALUES("
228 ✗ "'" + start_time + "'," +
229 ✗ "'" + finish_time + "'," +
230 ✗ stats.n_preserved_catalogs + "," +
231 ✗ stats.n_condemned_catalogs + "," +
232 ✗ stats.n_condemned_objects + "," +
233 ✗ stats.n_duplicate_delete_requests + "," +
234 ✗ (success ? "1" : "0") + ");";
235 }
236 ✗ return insert_statement;
237 }
238
239 } // namespace
240
241
242 2 bool StatisticsDatabase::CreateEmptyDatabase() {
243 2 ++create_empty_db_calls;
244
3/6
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 2 times.
✗ Branch 9 not taken.
 4 bool ret1 = sqlite::Sql(sqlite_db(),
245 "CREATE TABLE publish_statistics ("
246 "publish_id INTEGER PRIMARY KEY,"
247 "start_time TEXT,"
248 "finish_time TEXT,"
249 "revision INTEGER,"
250 "files_added INTEGER,"
251 "files_removed INTEGER,"
252 "files_changed INTEGER,"
253 "chunks_added INTEGER,"
254 "chunks_duplicated INTEGER,"
255 "catalogs_added INTEGER,"
256 "directories_added INTEGER,"
257 "directories_removed INTEGER,"
258 "directories_changed INTEGER,"
259 "symlinks_added INTEGER,"
260 "symlinks_removed INTEGER,"
261 "symlinks_changed INTEGER,"
262 "sz_bytes_added INTEGER,"
263 "sz_bytes_removed INTEGER,"
264 "sz_bytes_uploaded INTEGER,"
265 "sz_catalog_bytes_uploaded INTEGER,"
266
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 "success INTEGER);").Execute();
267
3/6
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 2 times.
✗ Branch 9 not taken.
 4 bool ret2 = sqlite::Sql(sqlite_db(),
268 "CREATE TABLE gc_statistics ("
269 "gc_id INTEGER PRIMARY KEY,"
270 "start_time TEXT,"
271 "finish_time TEXT,"
272 "n_preserved_catalogs INTEGER,"
273 "n_condemned_catalogs INTEGER,"
274 "n_condemned_objects INTEGER,"
275 "sz_condemned_bytes INTEGER,"
276 "n_duplicate_delete_requests INTEGER,"
277
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 "success INTEGER);").Execute();
278 2 return ret1 & ret2;
279 }
280
281
282 2 bool StatisticsDatabase::CheckSchemaCompatibility() {
283 2 ++check_compatibility_calls;
284
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 4 return (schema_version() > kLatestCompatibleSchema - 0.1 &&
285
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 4 schema_version() < kLatestCompatibleSchema + 0.1);
286 }
287
288
289 2 bool StatisticsDatabase::LiveSchemaUpgradeIfNecessary() {
290 2 ++live_upgrade_calls;
291
5/6
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 times.
✓ Branch 5 taken 1 times.
✓ Branch 6 taken 1 times.
✓ Branch 7 taken 1 times.
 4 if (IsEqualSchema(schema_version(), kLatestSchema) &&
292 2 (schema_revision() == 1)) {
293
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 LogCvmfs(kLogCvmfs, kLogDebug, "upgrading schema revision (1 --> 2) of "
294 "statistics database");
295
296 sqlite::Sql publish_upgrade2_1(this->sqlite_db(), "ALTER TABLE "
297
3/6
✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1 times.
✗ Branch 9 not taken.
 2 "publish_statistics RENAME COLUMN finished_time TO finish_time;");
298 sqlite::Sql publish_upgrade2_2(this->sqlite_db(), "ALTER TABLE "
299
3/6
✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1 times.
✗ Branch 9 not taken.
 2 "publish_statistics ADD revision INTEGER;");
300 sqlite::Sql publish_upgrade2_3(this->sqlite_db(), "ALTER TABLE "
301
3/6
✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1 times.
✗ Branch 9 not taken.
 2 "publish_statistics RENAME COLUMN duplicated_files TO chunks_duplicated;");
302 sqlite::Sql publish_upgrade2_4(this->sqlite_db(), "ALTER TABLE "
303
3/6
✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1 times.
✗ Branch 9 not taken.
 2 "publish_statistics ADD chunks_added INTEGER;");
304 sqlite::Sql publish_upgrade2_5(this->sqlite_db(), "ALTER TABLE "
305
3/6
✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1 times.
✗ Branch 9 not taken.
 2 "publish_statistics ADD symlinks_added INTEGER;");
306 sqlite::Sql publish_upgrade2_6(this->sqlite_db(), "ALTER TABLE "
307
3/6
✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1 times.
✗ Branch 9 not taken.
 2 "publish_statistics ADD symlinks_removed INTEGER;");
308 sqlite::Sql publish_upgrade2_7(this->sqlite_db(), "ALTER TABLE "
309
3/6
✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1 times.
✗ Branch 9 not taken.
 2 "publish_statistics ADD symlinks_changed INTEGER;");
310 sqlite::Sql publish_upgrade2_8(this->sqlite_db(), "ALTER TABLE "
311
3/6
✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1 times.
✗ Branch 9 not taken.
 2 "publish_statistics ADD catalogs_added INTEGER;");
312 sqlite::Sql publish_upgrade2_9(this->sqlite_db(), "ALTER TABLE "
313
3/6
✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1 times.
✗ Branch 9 not taken.
 2 "publish_statistics ADD sz_catalog_bytes_uploaded INTEGER;");
314
315
1/2
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
 1 if (!publish_upgrade2_1.Execute() ||
316
2/4
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 times.
✗ Branch 4 not taken.
 1 !publish_upgrade2_2.Execute() ||
317
2/4
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 times.
✗ Branch 4 not taken.
 1 !publish_upgrade2_3.Execute() ||
318
2/4
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 times.
✗ Branch 4 not taken.
 1 !publish_upgrade2_4.Execute() ||
319
2/4
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 times.
✗ Branch 4 not taken.
 1 !publish_upgrade2_5.Execute() ||
320
2/4
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 times.
✗ Branch 4 not taken.
 1 !publish_upgrade2_6.Execute() ||
321
2/4
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 times.
✗ Branch 4 not taken.
 1 !publish_upgrade2_7.Execute() ||
322
4/8
✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 1 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 1 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 1 times.
 3 !publish_upgrade2_8.Execute() ||
323
2/4
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 times.
 1 !publish_upgrade2_9.Execute()) {
324 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr, "failed to upgrade publish_statistics"
325 " table of statistics database");
326 ✗ return false;
327 }
328
329 sqlite::Sql gc_upgrade2_1(this->sqlite_db(), "ALTER TABLE gc_statistics"
330
3/6
✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 1 times.
✗ Branch 9 not taken.
 2 " RENAME COLUMN finished_time TO finish_time;");
331
332
2/4
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 times.
 1 if (!gc_upgrade2_1.Execute()) {
333 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr, "failed to upgrade gc_statistics"
334 " table of statistics database");
335 ✗ return false;
336 }
337
338 1 set_schema_revision(2);
339
2/4
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 times.
 1 if (!StoreSchemaRevision()) {
340 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr, "failed to upgrade schema revision"
341 " of statistics database");
342 ✗ return false;
343 }
344
10/20
✓ Branch 1 taken 1 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 1 times.
✗ Branch 8 not taken.
✓ Branch 10 taken 1 times.
✗ Branch 11 not taken.
✓ Branch 13 taken 1 times.
✗ Branch 14 not taken.
✓ Branch 16 taken 1 times.
✗ Branch 17 not taken.
✓ Branch 19 taken 1 times.
✗ Branch 20 not taken.
✓ Branch 22 taken 1 times.
✗ Branch 23 not taken.
✓ Branch 25 taken 1 times.
✗ Branch 26 not taken.
✓ Branch 28 taken 1 times.
✗ Branch 29 not taken.
 1 }
345
346
3/6
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 2 times.
✗ Branch 7 not taken.
 4 if (IsEqualSchema(schema_version(), kLatestSchema) &&
347 2 (schema_revision() == 2)) {
348
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 LogCvmfs(kLogCvmfs, kLogDebug, "upgrading schema revision (2 --> 3) of "
349 "statistics database");
350
351 sqlite::Sql publish_upgrade3_1(this->sqlite_db(), "ALTER TABLE "
352
3/6
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 2 times.
✗ Branch 9 not taken.
 4 "publish_statistics ADD success INTEGER;");
353
354
2/4
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 2 times.
 2 if (!publish_upgrade3_1.Execute()) {
355 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr, "failed to upgrade publish_statistics"
356 " table of statistics database");
357 ✗ return false;
358 }
359
360 sqlite::Sql gc_upgrade3_1(this->sqlite_db(), "ALTER TABLE gc_statistics"
361
3/6
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 2 times.
✗ Branch 9 not taken.
 4 " ADD success INTEGER;");
362
363
2/4
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 2 times.
 2 if (!gc_upgrade3_1.Execute()) {
364 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr, "failed to upgrade gc_statistics"
365 " table of statistics database");
366 ✗ return false;
367 }
368
369 2 set_schema_revision(3);
370
2/4
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 2 times.
 2 if (!StoreSchemaRevision()) {
371 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr, "failed to upgrade schema revision"
372 " of statistics database");
373 ✗ return false;
374 }
375
2/4
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2 times.
✗ Branch 5 not taken.
 2 }
376
377
3/6
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 2 times.
✗ Branch 7 not taken.
 4 if (IsEqualSchema(schema_version(), kLatestSchema) &&
378 2 (schema_revision() == 3)) {
379
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 LogCvmfs(kLogCvmfs, kLogDebug, "upgrading schema revision (3 --> 4) of "
380 "statistics database");
381
382 sqlite::Sql gc_upgrade4_1(this->sqlite_db(), "ALTER TABLE gc_statistics"
383
3/6
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 2 times.
✗ Branch 6 not taken.
✓ Branch 8 taken 2 times.
✗ Branch 9 not taken.
 4 " ADD n_duplicate_delete_requests INTEGER;");
384
385
2/4
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 2 times.
 2 if (!gc_upgrade4_1.Execute()) {
386 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr, "failed to upgrade gc_statistics "
387 "table of statistics database");
388 ✗ return false;
389 }
390
391 2 set_schema_revision(4);
392
2/4
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 2 times.
 2 if (!StoreSchemaRevision()) {
393 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr, "failed to upgrade schema revision "
394 "of statistics database");
395 ✗ return false;
396 }
397
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 }
398 2 return true;
399 }
400
401
402 ✗ bool StatisticsDatabase::CompactDatabase() const {
403 ✗ ++compact_calls;
404 ✗ return !compacting_fails;
405 }
406
407
408 4 StatisticsDatabase::~StatisticsDatabase() {
409 4 --StatisticsDatabase::instances;
410 4 }
411
412
413 ✗ StatisticsDatabase *StatisticsDatabase::OpenStandardDB(
414 const std::string repo_name)
415 {
416 StatisticsDatabase *db;
417 ✗ std::string db_file_path;
418 uint32_t days_to_keep;
419 ✗ GetDBParams(repo_name, &db_file_path, &days_to_keep);
420 ✗ if (FileExists(db_file_path)) {
421 ✗ db = StatisticsDatabase::Open(db_file_path, kOpenReadWrite);
422 ✗ if (db == NULL) {
423 ✗ PANIC(kLogSyslogErr, "Couldn't create StatisticsDatabase object!");
424 ✗ } else if (db->GetProperty<std::string>("repo_name") != repo_name) {
425 ✗ PANIC(kLogSyslogErr, "'repo_name' property of the statistics database %s "
426 "is incorrect. Please fix the database.", db_file_path.c_str());
427 }
428 ✗ if (!db->Prune(days_to_keep)) {
429 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr, "Failed to prune statistics database");
430 }
431 } else {
432 ✗ db = StatisticsDatabase::Create(db_file_path);
433 ✗ if (db == NULL) {
434 ✗ PANIC(kLogSyslogErr, "Couldn't create StatisticsDatabase object!");
435 // insert repo_name into properties table
436 ✗ } else if (!db->SetProperty("repo_name", repo_name)) {
437 ✗ PANIC(kLogSyslogErr,
438 "Couldn't insert repo_name into properties table!");
439 }
440 }
441 ✗ db->repo_name_ = repo_name;
442 ✗ return db;
443 }
444
445
446 ✗ bool StatisticsDatabase::StorePublishStatistics(
447 const perf::Statistics *statistics,
448 const std::string &start_time,
449 const bool success)
450 {
451 ✗ std::string finish_time = GetGMTimestamp();
452 std::string statement = PrepareStatementIntoPublish(statistics, start_time,
453 ✗ finish_time, success);
454 ✗ return StoreEntry(statement);
455 }
456
457
458 ✗ bool StatisticsDatabase::StoreGCStatistics(
459 const perf::Statistics *statistics,
460 const std::string &start_time,
461 const bool success)
462 {
463 ✗ std::string finish_time = GetGMTimestamp();
464 std::string statement =
465 PrepareStatementIntoGc(statistics, start_time, finish_time,
466 ✗ repo_name_, success);
467 ✗ return StoreEntry(statement);
468 }
469
470
471 ✗ bool StatisticsDatabase::StoreEntry(const std::string &insert_statement) {
472 ✗ sqlite::Sql insert(this->sqlite_db(), insert_statement);
473
474 ✗ if (!insert.Execute()) {
475 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr,
476 "Couldn't store statistics in %s: insert.Execute failed!",
477 ✗ this->filename().c_str());
478 ✗ return false;
479 }
480
481 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Statistics stored at: %s",
482 ✗ this->filename().c_str());
483 ✗ return true;
484 }
485
486
487 ✗ bool StatisticsDatabase::Prune(uint32_t days) {
488 ✗ if (days == 0) return true;
489
490 std::string publish_stmt =
491 "DELETE FROM publish_statistics WHERE "
492 ✗ "julianday('now','start of day')-julianday(start_time) > " +
493 ✗ StringifyUint(days) + ";";
494
495 std::string gc_stmt =
496 "DELETE FROM gc_statistics WHERE "
497 ✗ "julianday('now','start of day')-julianday(start_time) > " +
498 ✗ StringifyUint(days) + ";";
499
500 ✗ sqlite::Sql publish_sql(this->sqlite_db(), publish_stmt);
501 ✗ sqlite::Sql gc_sql(this->sqlite_db(), gc_stmt);
502 ✗ if (!publish_sql.Execute() || !gc_sql.Execute()) {
503 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr,
504 "Couldn't prune statistics DB %s: SQL Execute() failed!",
505 ✗ this->filename().c_str());
506 ✗ return false;
507 }
508 ✗ if (!this->Vacuum()) {
509 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr,
510 "Couldn't prune statistics DB %s: Vacuum() failed!",
511 ✗ this->filename().c_str());
512 ✗ return false;
513 }
514
515 ✗ return true;
516 }
517
518
519 ✗ bool StatisticsDatabase::UploadStatistics(upload::Spooler *spooler,
520 std::string local_path)
521 {
522 ✗ if (local_path == "") {
523 ✗ local_path = this->filename();
524 }
525
526 ✗ spooler->WaitForUpload();
527 ✗ unsigned errors_before = spooler->GetNumberOfErrors();
528 ✗ spooler->Mkdir("stats");
529 ✗ spooler->RemoveAsync("stats/stats.db");
530 ✗ spooler->WaitForUpload();
531 ✗ spooler->Upload(local_path, "stats/stats.db");
532 ✗ spooler->WaitForUpload();
533 ✗ unsigned errors_after = spooler->GetNumberOfErrors();
534
535 ✗ if (errors_before != errors_after) {
536 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr,
537 "Could not upload statistics DB file into storage backend");
538 ✗ return false;
539 }
540 ✗ return true;
541 }
542
543
544 ✗ bool StatisticsDatabase::UploadStatistics(
545 upload::AbstractUploader *uploader,
546 std::string local_path)
547 {
548 ✗ if (local_path == "") {
549 ✗ local_path = this->filename();
550 }
551
552 ✗ uploader->WaitForUpload();
553 ✗ unsigned errors_before = uploader->GetNumberOfErrors();
554
555 ✗ uploader->RemoveAsync("stats/stats.db");
556 ✗ uploader->WaitForUpload();
557 ✗ uploader->UploadFile(local_path, "stats/stats.db");
558 ✗ uploader->WaitForUpload();
559 ✗ unsigned errors_after = uploader->GetNumberOfErrors();
560 ✗ return errors_before == errors_after;
561 }
562
563
564 ✗ void StatisticsDatabase::GetDBParams(const std::string &repo_name,
565 std::string *path,
566 uint32_t *days_to_keep)
567 {
568 // default location
569 const std::string db_default_path =
570 ✗ "/var/spool/cvmfs/" + repo_name + "/stats.db";
571 const std::string repo_config_file =
572 ✗ "/etc/cvmfs/repositories.d/" + repo_name + "/server.conf";
573 ✗ SimpleOptionsParser parser;
574
575 ✗ if (!parser.TryParsePath(repo_config_file)) {
576 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr,
577 "Could not parse repository configuration: %s.",
578 repo_config_file.c_str());
579 ✗ *path = db_default_path;
580 ✗ *days_to_keep = kDefaultDaysToKeep;
581 ✗ return;
582 }
583
584 ✗ std::string statistics_db = "";
585 ✗ if (!parser.GetValue("CVMFS_STATISTICS_DB", &statistics_db)) {
586 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslog,
587 "Parameter %s was not set in the repository configuration file. "
588 "Using default value: %s",
589 "CVMFS_STATISTICS_DB", db_default_path.c_str());
590 ✗ *path = db_default_path;
591 } else {
592 ✗ std::string dirname = GetParentPath(statistics_db);
593 ✗ int mode = S_IRUSR | S_IWUSR | S_IXUSR |
594 S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH; // 755
595 ✗ if (!MkdirDeep(dirname, mode, true)) {
596 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr,
597 "Couldn't write statistics at the specified path %s.",
598 statistics_db.c_str());
599 ✗ *path = db_default_path;
600 } else {
601 ✗ *path = statistics_db;
602 }
603 }
604
605 ✗ std::string days_to_keep_str = "";
606 ✗ if (!parser.GetValue("CVMFS_STATS_DB_DAYS_TO_KEEP", &statistics_db)) {
607 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslog,
608 "Parameter %s was not set in the repository configuration file. "
609 "Using default value: %s",
610 "CVMFS_STATS_DB_DAYS_TO_KEEP",
611 ✗ StringifyUint(kDefaultDaysToKeep).c_str());
612 ✗ *days_to_keep = kDefaultDaysToKeep;
613 } else {
614 ✗ *days_to_keep = static_cast<uint32_t> (String2Uint64(days_to_keep_str));
615 }
616 }
617
618
619 /**
620 * Check if the CVMFS_EXTENDED_GC_STATS is ON or not
621 *
622 * @param repo_name fully qualified name of the repository
623 * @return true if CVMFS_EXTENDED_GC_STATS is ON
624 */
625 ✗ bool StatisticsDatabase::GcExtendedStats(const std::string &repo_name) {
626 ✗ SimpleOptionsParser parser;
627 ✗ std::string param_value = "";
628 const std::string repo_config_file =
629 ✗ "/etc/cvmfs/repositories.d/" + repo_name + "/server.conf";
630
631 ✗ if (!parser.TryParsePath(repo_config_file)) {
632 ✗ LogCvmfs(kLogCvmfs, kLogSyslogErr,
633 "Could not parse repository configuration: %s.",
634 repo_config_file.c_str());
635 ✗ return false;
636 }
637 ✗ if (!parser.GetValue("CVMFS_EXTENDED_GC_STATS", &param_value)) {
638 ✗ LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslog,
639 "Parameter %s was not set in the repository configuration file. "
640 "condemned_bytes were not counted.",
641 "CVMFS_EXTENDED_GC_STATS");
642 ✗ } else if (parser.IsOn(param_value)) {
643 ✗ return true;
644 }
645 ✗ return false;
646 }
647
648
649 4 StatisticsDatabase::StatisticsDatabase(const std::string &filename,
650 4 const OpenMode open_mode) :
651 sqlite::Database<StatisticsDatabase>(filename, open_mode),
652 4 create_empty_db_calls(0), check_compatibility_calls(0),
653 4 live_upgrade_calls(0), compact_calls(0)
654 {
655 4 ++StatisticsDatabase::instances;
656 4 }
657
658 const uint32_t StatisticsDatabase::kDefaultDaysToKeep = 365;
659