1 |
|
|
/** |
2 |
|
|
* This file is part of the CernVM File System. |
3 |
|
|
* |
4 |
|
|
* This command processes a repository's catalog structure to detect and remove |
5 |
|
|
* outdated and/or unneeded data objects. |
6 |
|
|
*/ |
7 |
|
|
|
8 |
|
|
#include "cvmfs_config.h" |
9 |
|
|
#include "swissknife_gc.h" |
10 |
|
|
|
11 |
|
|
#include <string> |
12 |
|
|
|
13 |
|
|
#include "garbage_collection/garbage_collector.h" |
14 |
|
|
#include "garbage_collection/gc_aux.h" |
15 |
|
|
#include "garbage_collection/hash_filter.h" |
16 |
|
|
#include "manifest.h" |
17 |
|
|
#include "reflog.h" |
18 |
|
|
#include "statistics_database.h" |
19 |
|
|
#include "upload_facility.h" |
20 |
|
|
#include "util/posix.h" |
21 |
|
|
#include "util/string.h" |
22 |
|
|
|
23 |
|
|
namespace swissknife { |
24 |
|
|
|
25 |
|
|
typedef HttpObjectFetcher<> ObjectFetcher; |
26 |
|
|
typedef CatalogTraversal<ObjectFetcher> ReadonlyCatalogTraversal; |
27 |
|
|
typedef SmallhashFilter HashFilter; |
28 |
|
|
typedef GarbageCollector<ReadonlyCatalogTraversal, HashFilter> GC; |
29 |
|
|
typedef GarbageCollectorAux<ReadonlyCatalogTraversal, HashFilter> GCAux; |
30 |
|
|
typedef GC::Configuration GcConfig; |
31 |
|
|
|
32 |
|
|
|
33 |
|
|
ParameterList CommandGc::GetParams() const { |
34 |
|
|
ParameterList r; |
35 |
|
|
r.push_back(Parameter::Mandatory('r', "repository url")); |
36 |
|
|
r.push_back(Parameter::Mandatory('u', "spooler definition string")); |
37 |
|
|
r.push_back(Parameter::Mandatory('n', "fully qualified repository name")); |
38 |
|
|
r.push_back(Parameter::Mandatory('R', "path to reflog.chksum file")); |
39 |
|
|
r.push_back(Parameter::Optional('h', "conserve <h> revisions")); |
40 |
|
|
r.push_back(Parameter::Optional('z', "conserve revisions younger than <z>")); |
41 |
|
|
r.push_back(Parameter::Optional('k', "repository master key(s) / dir")); |
42 |
|
|
r.push_back(Parameter::Optional('t', "temporary directory")); |
43 |
|
|
r.push_back(Parameter::Optional('L', "path to deletion log file")); |
44 |
|
|
r.push_back(Parameter::Switch('d', "dry run")); |
45 |
|
|
r.push_back(Parameter::Switch('l', "list objects to be removed")); |
46 |
|
|
return r; |
47 |
|
|
} |
48 |
|
|
|
49 |
|
|
|
50 |
|
|
int CommandGc::Main(const ArgumentList &args) { |
51 |
|
|
const std::string &repo_url = *args.find('r')->second; |
52 |
|
|
const std::string &spooler = *args.find('u')->second; |
53 |
|
|
const std::string &repo_name = *args.find('n')->second; |
54 |
|
|
const std::string &reflog_chksum_path = *args.find('R')->second; |
55 |
|
|
shash::Any reflog_hash; |
56 |
|
|
if (!manifest::Reflog::ReadChecksum(reflog_chksum_path, &reflog_hash)) { |
57 |
|
|
LogCvmfs(kLogCvmfs, kLogStderr, "Could not read reflog checksum"); |
58 |
|
|
return 1; |
59 |
|
|
} |
60 |
|
|
|
61 |
|
|
const int64_t revisions = (args.count('h') > 0) ? |
62 |
|
|
String2Int64(*args.find('h')->second) : GcConfig::kFullHistory; |
63 |
|
|
const time_t timestamp = (args.count('z') > 0) |
64 |
|
|
? static_cast<time_t>(String2Int64(*args.find('z')->second)) |
65 |
|
|
: GcConfig::kNoTimestamp; |
66 |
|
|
std::string repo_keys = (args.count('k') > 0) ? |
67 |
|
|
*args.find('k')->second : ""; |
68 |
|
|
if (DirectoryExists(repo_keys)) |
69 |
|
|
repo_keys = JoinStrings(FindFilesBySuffix(repo_keys, ".pub"), ":"); |
70 |
|
|
const bool dry_run = (args.count('d') > 0); |
71 |
|
|
const bool list_condemned_objects = (args.count('l') > 0); |
72 |
|
|
const std::string temp_directory = (args.count('t') > 0) ? |
73 |
|
|
*args.find('t')->second : "/tmp"; |
74 |
|
|
const std::string deletion_log_path = (args.count('L') > 0) ? |
75 |
|
|
*args.find('L')->second : ""; |
76 |
|
|
|
77 |
|
|
if (revisions < 0) { |
78 |
|
|
LogCvmfs(kLogCvmfs, kLogStderr, |
79 |
|
|
"at least one revision needs to be preserved"); |
80 |
|
|
return 1; |
81 |
|
|
} |
82 |
|
|
|
83 |
|
|
if (timestamp == GcConfig::kNoTimestamp && |
84 |
|
|
revisions == GcConfig::kFullHistory) { |
85 |
|
|
LogCvmfs(kLogCvmfs, kLogStderr, |
86 |
|
|
"neither a timestamp nor history threshold given"); |
87 |
|
|
return 1; |
88 |
|
|
} |
89 |
|
|
|
90 |
|
|
const bool follow_redirects = false; |
91 |
|
|
if (!this->InitDownloadManager(follow_redirects) || |
92 |
|
|
!this->InitVerifyingSignatureManager(repo_keys)) { |
93 |
|
|
LogCvmfs(kLogCatalog, kLogStderr, "failed to init repo connection"); |
94 |
|
|
return 1; |
95 |
|
|
} |
96 |
|
|
|
97 |
|
|
ObjectFetcher object_fetcher(repo_name, |
98 |
|
|
repo_url, |
99 |
|
|
temp_directory, |
100 |
|
|
download_manager(), |
101 |
|
|
signature_manager()); |
102 |
|
|
|
103 |
|
|
UniquePtr<manifest::Manifest> manifest; |
104 |
|
|
ObjectFetcher::Failures retval = object_fetcher.FetchManifest(&manifest); |
105 |
|
|
if (retval != ObjectFetcher::kFailOk) { |
106 |
|
|
LogCvmfs(kLogCvmfs, kLogStderr, "failed to load repository manifest " |
107 |
|
|
"(%d - %s)", |
108 |
|
|
retval, Code2Ascii(retval)); |
109 |
|
|
return 1; |
110 |
|
|
} |
111 |
|
|
|
112 |
|
|
if (!manifest->garbage_collectable()) { |
113 |
|
|
LogCvmfs(kLogCvmfs, kLogStderr, |
114 |
|
|
"repository does not allow garbage collection"); |
115 |
|
|
return 1; |
116 |
|
|
} |
117 |
|
|
|
118 |
|
|
UniquePtr<manifest::Reflog> reflog; |
119 |
|
|
reflog = FetchReflog(&object_fetcher, repo_name, reflog_hash); |
120 |
|
|
assert(reflog.IsValid()); |
121 |
|
|
|
122 |
|
|
const upload::SpoolerDefinition spooler_definition(spooler, shash::kAny); |
123 |
|
|
UniquePtr<upload::AbstractUploader> uploader( |
124 |
|
|
upload::AbstractUploader::Construct(spooler_definition)); |
125 |
|
|
|
126 |
|
|
if (!uploader.IsValid()) { |
127 |
|
|
LogCvmfs(kLogCvmfs, kLogStderr, "failed to initialize spooler for '%s'", |
128 |
|
|
spooler.c_str()); |
129 |
|
|
return 1; |
130 |
|
|
} |
131 |
|
|
|
132 |
|
|
FILE *deletion_log_file = NULL; |
133 |
|
|
if (!deletion_log_path.empty()) { |
134 |
|
|
deletion_log_file = fopen(deletion_log_path.c_str(), "a+"); |
135 |
|
|
if (NULL == deletion_log_file) { |
136 |
|
|
LogCvmfs(kLogCvmfs, kLogStderr, "failed to open deletion log file " |
137 |
|
|
"(errno: %d)", errno); |
138 |
|
|
uploader->TearDown(); |
139 |
|
|
return 1; |
140 |
|
|
} |
141 |
|
|
} |
142 |
|
|
|
143 |
|
|
bool extended_stats = StatisticsDatabase::GcExtendedStats(repo_name); |
144 |
|
|
|
145 |
|
|
reflog->BeginTransaction(); |
146 |
|
|
|
147 |
|
|
GcConfig config; |
148 |
|
|
config.uploader = uploader.weak_ref(); |
149 |
|
|
config.keep_history_depth = revisions; |
150 |
|
|
config.keep_history_timestamp = timestamp; |
151 |
|
|
config.dry_run = dry_run; |
152 |
|
|
config.verbose = list_condemned_objects; |
153 |
|
|
config.object_fetcher = &object_fetcher; |
154 |
|
|
config.reflog = reflog.weak_ref(); |
155 |
|
|
config.deleted_objects_logfile = deletion_log_file; |
156 |
|
|
config.statistics = statistics(); |
157 |
|
|
config.extended_stats = extended_stats; |
158 |
|
|
|
159 |
|
|
if (deletion_log_file != NULL) { |
160 |
|
|
const int bytes_written = fprintf(deletion_log_file, |
161 |
|
|
"# Garbage Collection started at %s\n", |
162 |
|
|
StringifyTime(time(NULL), true).c_str()); |
163 |
|
|
if (bytes_written < 0) { |
164 |
|
|
LogCvmfs(kLogCvmfs, kLogStderr, "failed to write to deletion log '%s' " |
165 |
|
|
"(errno: %d)", |
166 |
|
|
deletion_log_path.c_str(), errno); |
167 |
|
|
uploader->TearDown(); |
168 |
|
|
return 1; |
169 |
|
|
} |
170 |
|
|
} |
171 |
|
|
|
172 |
|
|
// File catalogs |
173 |
|
|
GC collector(config); |
174 |
|
|
collector.UseReflogTimestamps(); |
175 |
|
|
bool success = collector.Collect(); |
176 |
|
|
|
177 |
|
|
if (!success) { |
178 |
|
|
LogCvmfs(kLogCvmfs, kLogStderr, "garbage collection failed"); |
179 |
|
|
uploader->TearDown(); |
180 |
|
|
return 1; |
181 |
|
|
} |
182 |
|
|
|
183 |
|
|
// Tag databases, meta infos, certificates |
184 |
|
|
HashFilter preserved_objects; |
185 |
|
|
preserved_objects.Fill(manifest->certificate()); |
186 |
|
|
preserved_objects.Fill(manifest->history()); |
187 |
|
|
preserved_objects.Fill(manifest->meta_info()); |
188 |
|
|
GCAux collector_aux(config); |
189 |
|
|
success = collector_aux.CollectOlderThan( |
190 |
|
|
collector.oldest_trunk_catalog(), preserved_objects); |
191 |
|
|
if (!success) { |
192 |
|
|
LogCvmfs(kLogCvmfs, kLogStderr, |
193 |
|
|
"garbage collection of auxiliary files failed"); |
194 |
|
|
uploader->TearDown(); |
195 |
|
|
return 1; |
196 |
|
|
} |
197 |
|
|
|
198 |
|
|
// As of here: garbage collection succeeded, cleanup & commit |
199 |
|
|
|
200 |
|
|
if (deletion_log_file != NULL) { |
201 |
|
|
const int bytes_written = fprintf(deletion_log_file, |
202 |
|
|
"# Garbage Collection finished at %s\n\n", |
203 |
|
|
StringifyTime(time(NULL), true).c_str()); |
204 |
|
|
assert(bytes_written >= 0); |
205 |
|
|
fclose(deletion_log_file); |
206 |
|
|
} |
207 |
|
|
|
208 |
|
|
reflog->CommitTransaction(); |
209 |
|
|
// Has to be outside the transaction |
210 |
|
|
success = reflog->Vacuum(); |
211 |
|
|
assert(success); |
212 |
|
|
reflog->DropDatabaseFileOwnership(); |
213 |
|
|
const std::string reflog_db = reflog->database_file(); |
214 |
|
|
reflog.Destroy(); |
215 |
|
|
|
216 |
|
|
if (!dry_run) { |
217 |
|
|
uploader->Upload(reflog_db, ".cvmfsreflog"); |
218 |
|
|
manifest::Reflog::HashDatabase(reflog_db, &reflog_hash); |
219 |
|
|
uploader->WaitForUpload(); |
220 |
|
|
manifest::Reflog::WriteChecksum(reflog_chksum_path, reflog_hash); |
221 |
|
|
} |
222 |
|
|
|
223 |
|
|
unlink(reflog_db.c_str()); |
224 |
|
|
|
225 |
|
|
if (uploader->GetNumberOfErrors() > 0 && !dry_run) { |
226 |
|
|
LogCvmfs(kLogCvmfs, kLogStderr, "failed to upload updated Reflog"); |
227 |
|
|
uploader->TearDown(); |
228 |
|
|
return 1; |
229 |
|
|
} |
230 |
|
|
|
231 |
|
|
uploader->TearDown(); |
232 |
|
|
return 0; |
233 |
|
|
} |
234 |
|
|
|
235 |
|
|
} // namespace swissknife |