GCC Code Coverage Report
Directory: cvmfs/ Exec Total Coverage
File: cvmfs/swissknife_gc.cc Lines: 0 125 0.0 %
Date: 2019-02-03 02:48:13 Branches: 0 74 0.0 %

Line Branch Exec Source
1
/**
2
 * This file is part of the CernVM File System.
3
 *
4
 * This command processes a repository's catalog structure to detect and remove
5
 * outdated and/or unneeded data objects.
6
 */
7
8
#include "cvmfs_config.h"
9
#include "swissknife_gc.h"
10
11
#include <string>
12
13
#include "garbage_collection/garbage_collector.h"
14
#include "garbage_collection/gc_aux.h"
15
#include "garbage_collection/hash_filter.h"
16
#include "manifest.h"
17
#include "reflog.h"
18
#include "statistics_database.h"
19
#include "upload_facility.h"
20
#include "util/posix.h"
21
#include "util/string.h"
22
23
namespace swissknife {
24
25
typedef HttpObjectFetcher<> ObjectFetcher;
26
typedef CatalogTraversal<ObjectFetcher> ReadonlyCatalogTraversal;
27
typedef SmallhashFilter HashFilter;
28
typedef GarbageCollector<ReadonlyCatalogTraversal, HashFilter> GC;
29
typedef GarbageCollectorAux<ReadonlyCatalogTraversal, HashFilter> GCAux;
30
typedef GC::Configuration GcConfig;
31
32
33
ParameterList CommandGc::GetParams() const {
34
  ParameterList r;
35
  r.push_back(Parameter::Mandatory('r', "repository url"));
36
  r.push_back(Parameter::Mandatory('u', "spooler definition string"));
37
  r.push_back(Parameter::Mandatory('n', "fully qualified repository name"));
38
  r.push_back(Parameter::Mandatory('R', "path to reflog.chksum file"));
39
  r.push_back(Parameter::Optional('h', "conserve <h> revisions"));
40
  r.push_back(Parameter::Optional('z', "conserve revisions younger than <z>"));
41
  r.push_back(Parameter::Optional('k', "repository master key(s) / dir"));
42
  r.push_back(Parameter::Optional('t', "temporary directory"));
43
  r.push_back(Parameter::Optional('L', "path to deletion log file"));
44
  r.push_back(Parameter::Switch('d', "dry run"));
45
  r.push_back(Parameter::Switch('l', "list objects to be removed"));
46
  return r;
47
}
48
49
50
int CommandGc::Main(const ArgumentList &args) {
51
  const std::string &repo_url = *args.find('r')->second;
52
  const std::string &spooler = *args.find('u')->second;
53
  const std::string &repo_name = *args.find('n')->second;
54
  const std::string &reflog_chksum_path = *args.find('R')->second;
55
  shash::Any reflog_hash;
56
  if (!manifest::Reflog::ReadChecksum(reflog_chksum_path, &reflog_hash)) {
57
    LogCvmfs(kLogCvmfs, kLogStderr, "Could not read reflog checksum");
58
    return 1;
59
  }
60
61
  const int64_t revisions = (args.count('h') > 0) ?
62
    String2Int64(*args.find('h')->second) : GcConfig::kFullHistory;
63
  const time_t timestamp  = (args.count('z') > 0)
64
    ? static_cast<time_t>(String2Int64(*args.find('z')->second))
65
    : GcConfig::kNoTimestamp;
66
  std::string repo_keys = (args.count('k') > 0) ?
67
    *args.find('k')->second : "";
68
  if (DirectoryExists(repo_keys))
69
    repo_keys = JoinStrings(FindFilesBySuffix(repo_keys, ".pub"), ":");
70
  const bool dry_run = (args.count('d') > 0);
71
  const bool list_condemned_objects = (args.count('l') > 0);
72
  const std::string temp_directory = (args.count('t') > 0) ?
73
    *args.find('t')->second : "/tmp";
74
  const std::string deletion_log_path = (args.count('L') > 0) ?
75
    *args.find('L')->second : "";
76
77
  if (revisions < 0) {
78
    LogCvmfs(kLogCvmfs, kLogStderr,
79
             "at least one revision needs to be preserved");
80
    return 1;
81
  }
82
83
  if (timestamp == GcConfig::kNoTimestamp &&
84
      revisions == GcConfig::kFullHistory) {
85
    LogCvmfs(kLogCvmfs, kLogStderr,
86
             "neither a timestamp nor history threshold given");
87
    return 1;
88
  }
89
90
  const bool follow_redirects = false;
91
  if (!this->InitDownloadManager(follow_redirects) ||
92
      !this->InitVerifyingSignatureManager(repo_keys)) {
93
    LogCvmfs(kLogCatalog, kLogStderr, "failed to init repo connection");
94
    return 1;
95
  }
96
97
  ObjectFetcher object_fetcher(repo_name,
98
                               repo_url,
99
                               temp_directory,
100
                               download_manager(),
101
                               signature_manager());
102
103
  UniquePtr<manifest::Manifest> manifest;
104
  ObjectFetcher::Failures retval = object_fetcher.FetchManifest(&manifest);
105
  if (retval != ObjectFetcher::kFailOk) {
106
    LogCvmfs(kLogCvmfs, kLogStderr, "failed to load repository manifest "
107
                                    "(%d - %s)",
108
                                    retval, Code2Ascii(retval));
109
    return 1;
110
  }
111
112
  if (!manifest->garbage_collectable()) {
113
    LogCvmfs(kLogCvmfs, kLogStderr,
114
             "repository does not allow garbage collection");
115
    return 1;
116
  }
117
118
  UniquePtr<manifest::Reflog> reflog;
119
  reflog = FetchReflog(&object_fetcher, repo_name, reflog_hash);
120
  assert(reflog.IsValid());
121
122
  const upload::SpoolerDefinition spooler_definition(spooler, shash::kAny);
123
  UniquePtr<upload::AbstractUploader> uploader(
124
                       upload::AbstractUploader::Construct(spooler_definition));
125
126
  if (!uploader.IsValid()) {
127
    LogCvmfs(kLogCvmfs, kLogStderr, "failed to initialize spooler for '%s'",
128
             spooler.c_str());
129
    return 1;
130
  }
131
132
  FILE *deletion_log_file = NULL;
133
  if (!deletion_log_path.empty()) {
134
    deletion_log_file = fopen(deletion_log_path.c_str(), "a+");
135
    if (NULL == deletion_log_file) {
136
      LogCvmfs(kLogCvmfs, kLogStderr, "failed to open deletion log file "
137
                                      "(errno: %d)", errno);
138
      uploader->TearDown();
139
      return 1;
140
    }
141
  }
142
143
  bool extended_stats = StatisticsDatabase::GcExtendedStats(repo_name);
144
145
  reflog->BeginTransaction();
146
147
  GcConfig config;
148
  config.uploader                = uploader.weak_ref();
149
  config.keep_history_depth      = revisions;
150
  config.keep_history_timestamp  = timestamp;
151
  config.dry_run                 = dry_run;
152
  config.verbose                 = list_condemned_objects;
153
  config.object_fetcher          = &object_fetcher;
154
  config.reflog                  = reflog.weak_ref();
155
  config.deleted_objects_logfile = deletion_log_file;
156
  config.statistics              = statistics();
157
  config.extended_stats          = extended_stats;
158
159
  if (deletion_log_file != NULL) {
160
    const int bytes_written = fprintf(deletion_log_file,
161
                                      "# Garbage Collection started at %s\n",
162
                                      StringifyTime(time(NULL), true).c_str());
163
    if (bytes_written < 0) {
164
      LogCvmfs(kLogCvmfs, kLogStderr, "failed to write to deletion log '%s' "
165
                                      "(errno: %d)",
166
                                      deletion_log_path.c_str(), errno);
167
      uploader->TearDown();
168
      return 1;
169
    }
170
  }
171
172
  // File catalogs
173
  GC collector(config);
174
  collector.UseReflogTimestamps();
175
  bool success = collector.Collect();
176
177
  if (!success) {
178
    LogCvmfs(kLogCvmfs, kLogStderr, "garbage collection failed");
179
    uploader->TearDown();
180
    return 1;
181
  }
182
183
  // Tag databases, meta infos, certificates
184
  HashFilter preserved_objects;
185
  preserved_objects.Fill(manifest->certificate());
186
  preserved_objects.Fill(manifest->history());
187
  preserved_objects.Fill(manifest->meta_info());
188
  GCAux collector_aux(config);
189
  success = collector_aux.CollectOlderThan(
190
    collector.oldest_trunk_catalog(), preserved_objects);
191
  if (!success) {
192
    LogCvmfs(kLogCvmfs, kLogStderr,
193
             "garbage collection of auxiliary files failed");
194
    uploader->TearDown();
195
    return 1;
196
  }
197
198
  // As of here: garbage collection succeeded, cleanup & commit
199
200
  if (deletion_log_file != NULL) {
201
    const int bytes_written = fprintf(deletion_log_file,
202
                                      "# Garbage Collection finished at %s\n\n",
203
                                      StringifyTime(time(NULL), true).c_str());
204
    assert(bytes_written >= 0);
205
    fclose(deletion_log_file);
206
  }
207
208
  reflog->CommitTransaction();
209
  // Has to be outside the transaction
210
  success = reflog->Vacuum();
211
  assert(success);
212
  reflog->DropDatabaseFileOwnership();
213
  const std::string reflog_db = reflog->database_file();
214
  reflog.Destroy();
215
216
  if (!dry_run) {
217
    uploader->Upload(reflog_db, ".cvmfsreflog");
218
    manifest::Reflog::HashDatabase(reflog_db, &reflog_hash);
219
    uploader->WaitForUpload();
220
    manifest::Reflog::WriteChecksum(reflog_chksum_path, reflog_hash);
221
  }
222
223
  unlink(reflog_db.c_str());
224
225
  if (uploader->GetNumberOfErrors() > 0 && !dry_run) {
226
    LogCvmfs(kLogCvmfs, kLogStderr, "failed to upload updated Reflog");
227
    uploader->TearDown();
228
    return 1;
229
  }
230
231
  uploader->TearDown();
232
  return 0;
233
}
234
235
}  // namespace swissknife