5 #ifndef CVMFS_GARBAGE_COLLECTION_GARBAGE_COLLECTOR_IMPL_H_
6 #define CVMFS_GARBAGE_COLLECTION_GARBAGE_COLLECTOR_IMPL_H_
16 template<
class CatalogTraversalT,
class HashFilterT>
18 HashFilterT>::Configuration::kFullHistory =
19 std::numeric_limits<unsigned int>::max();
21 template<
class CatalogTraversalT,
class HashFilterT>
23 HashFilterT>::Configuration::kNoHistory = 0;
25 template<
class CatalogTraversalT,
class HashFilterT>
27 HashFilterT>::Configuration::kNoTimestamp = 0;
30 template <
class CatalogTraversalT,
class HashFilterT>
33 : configuration_(configuration)
34 , catalog_info_shim_(configuration.
reflog)
39 , use_reflog_timestamps_(false)
40 , oldest_trunk_catalog_(static_cast<uint64_t>(-1))
41 , oldest_trunk_catalog_found_(false)
42 , preserved_catalogs_(0)
43 , unreferenced_trees_(0)
45 , condemned_catalogs_(0)
46 , last_reported_status_(0.0)
47 , condemned_objects_(0)
54 template <
class CatalogTraversalT,
class HashFilterT>
56 traversal_.SetCatalogInfoShim(&catalog_info_shim_);
57 use_reflog_timestamps_ =
true;
61 template <
class CatalogTraversalT,
class HashFilterT>
70 params.no_repeat_history =
true;
71 params.ignore_load_failure =
true;
78 template <
class CatalogTraversalT,
class HashFilterT>
83 ++preserved_catalogs_;
85 if (data.catalog->IsRoot()) {
86 const uint64_t mtime = use_reflog_timestamps_
87 ? catalog_info_shim_.GetLastModified(data.catalog)
88 : data.catalog->GetLastModified();
89 if (!oldest_trunk_catalog_found_)
90 oldest_trunk_catalog_ = std::min(oldest_trunk_catalog_, mtime);
91 if (configuration_.verbose) {
92 const int rev = data.catalog->revision();
94 "Preserving Revision %d (%s / added @ %s)",
97 StringifyTime(catalog_info_shim_.GetLastModified(data.catalog),
99 PrintCatalogTreeEntry(data.tree_level, data.catalog);
101 if (data.catalog->schema() < 0.99) {
103 "legacy catalog does not provide access to nested catalog hierarchy.\n"
104 " Some unreferenced objects may remain in the repository.");
109 hash_filter_.Fill(data.catalog->hash());
112 const HashVector &referenced_hashes = data.catalog->GetReferencedObjects();
113 typename HashVector::const_iterator i = referenced_hashes.begin();
114 const typename HashVector::const_iterator iend = referenced_hashes.end();
115 for (; i != iend; ++i) {
116 hash_filter_.Fill(*i);
121 template <
class CatalogTraversalT,
class HashFilterT>
126 ++condemned_catalogs_;
127 if (data.catalog->IsRoot())
130 if (configuration_.verbose) {
131 if (data.catalog->IsRoot()) {
132 const int rev = data.catalog->revision();
133 const time_t mtime =
static_cast<time_t
>(data.catalog->GetLastModified());
137 PrintCatalogTreeEntry(data.tree_level, data.catalog);
142 const HashVector &referenced_hashes = data.catalog->GetReferencedObjects();
143 typename HashVector::const_iterator i = referenced_hashes.begin();
144 const typename HashVector::const_iterator iend = referenced_hashes.end();
145 for (; i != iend; ++i) {
150 CheckAndSweep(data.catalog->hash());
153 static_cast<float>(condemned_trees_) /
154 static_cast<float>(unreferenced_trees_);
155 if (threshold > last_reported_status_ + 0.1) {
157 " - %02.0f%% %u / %u unreferenced revisions removed [%s]",
158 100.0 * threshold, condemned_trees_, unreferenced_trees_,
160 last_reported_status_ = threshold;
165 template <
class CatalogTraversalT,
class HashFilterT>
169 if (!hash_filter_.Contains(hash))
174 template <
class CatalogTraversalT,
class HashFilterT>
177 ++condemned_objects_;
178 if (configuration_.extended_stats) {
180 int64_t condemned_bytes = configuration_.uploader->GetObjectSize(hash);
181 if (condemned_bytes > 0) {
182 condemned_bytes_ += condemned_bytes;
188 if (configuration_.dry_run) {
192 configuration_.uploader->RemoveAsync(hash);
196 template <
class CatalogTraversalT,
class HashFilterT>
201 return (configuration_.dry_run)
203 : configuration_.reflog->Remove(catalog);
207 template <
class CatalogTraversalT,
class HashFilterT>
209 return AnalyzePreservedCatalogTree() &&
210 CheckPreservedRevisions() &&
215 template <
class CatalogTraversalT,
class HashFilterT>
221 if (configuration_.verbose) {
223 "Preserving data objects in latest revision");
226 typename CatalogTraversalT::CallbackTN *callback =
227 traversal_.RegisterListener(
231 bool success = traversal_.Traverse();
232 oldest_trunk_catalog_found_ =
true;
233 success = success && traversal_.TraverseNamedSnapshots();
234 traversal_.UnregisterListener(callback);
240 template <
class CatalogTraversalT,
class HashFilterT>
243 const bool keeps_revisions = (preserved_catalog_count() > 0);
244 if (!keeps_revisions && configuration_.verbose) {
246 "This would delete everything! Abort.");
249 return keeps_revisions;
253 template <
class CatalogTraversalT,
class HashFilterT>
259 std::vector<shash::Any> catalogs;
265 typename CatalogTraversalT::CallbackTN *callback =
266 traversal_.RegisterListener(
270 std::vector<shash::Any> to_sweep;
271 std::vector<shash::Any>::const_iterator i = catalogs.begin();
272 std::vector<shash::Any>::const_iterator iend = catalogs.end();
273 for (; i != iend; ++i) {
274 if (!hash_filter_.Contains(*i)) {
275 to_sweep.push_back(*i);
278 unreferenced_trees_ = to_sweep.size();
279 bool success = traversal_.TraverseList(to_sweep,
280 CatalogTraversalT::kDepthFirst);
281 traversal_.UnregisterListener(callback);
283 i = to_sweep.begin();
284 iend = to_sweep.end();
285 for (; i != iend; ++i) {
286 success = success && RemoveCatalogFromReflog(*i);
290 if (configuration_.statistics) {
292 configuration_.statistics->Register(
293 "gc.n_preserved_catalogs",
"number of live catalogs");
295 configuration_.statistics->Register(
296 "gc.n_condemned_catalogs",
"number of dead catalogs");
298 configuration_.statistics->Register(
299 "gc.n_condemned_objects",
"number of deleted objects");
301 configuration_.statistics->Register(
302 "gc.sz_condemned_bytes",
"number of deleted bytes");
303 ctr_preserved_catalogs->
Set(preserved_catalog_count());
304 ctr_condemned_catalogs->
Set(condemned_catalog_count());
305 ctr_condemned_objects->
Set(condemned_objects_count());
306 ctr_condemned_bytes->
Set(condemned_bytes_count());
309 configuration_.uploader->WaitForUpload();
312 return success && (configuration_.uploader->GetNumberOfErrors() == 0);
316 template <
class CatalogTraversalT,
class HashFilterT>
318 const unsigned int tree_level,
321 std::string tree_indent;
322 for (
unsigned int i = 0; i < tree_level; ++i) {
323 tree_indent +=
"\u2502 ";
325 tree_indent +=
"\u251C\u2500 ";
327 const std::string hash_string = catalog->hash().ToString();
328 const std::string path =
329 (catalog->mountpoint().IsEmpty()) ?
"/" : catalog->mountpoint().ToString();
336 hash_string.c_str(), path.c_str());
340 template <
class CatalogTraversalT,
class HashFilterT>
343 if (configuration_.verbose) {
348 if (configuration_.has_deletion_log()) {
349 const int written = fprintf(configuration_.deleted_objects_logfile,
357 #endif // CVMFS_GARBAGE_COLLECTION_GARBAGE_COLLECTOR_IMPL_H_
#define LogCvmfs(source, mask,...)
bool AnalyzePreservedCatalogTree()
void UseReflogTimestamps()
std::string ToStringWithSuffix() const
void LogDeletion(const shash::Any &hash) const
TraversalParameters GetTraversalParams(const Configuration &configuration)
ObjectFetcherTN * object_fetcher
assert((mem||(size==0))&&"Out Of Memory")
void Sweep(const shash::Any &hash)
string StringifyTime(const time_t seconds, const bool utc)
CatalogTraversalT::Parameters TraversalParameters
CatalogTraversalT::CallbackDataTN TraversalCallbackDataTN
void PrintCatalogTreeEntry(const unsigned int tree_level, const CatalogTN *catalog) const
std::string RfcTimestamp()
void Set(const int64_t val)
ObjectFetcherTN::ReflogTN ReflogTN
const char kSuffixPartial
const char kSuffixCatalog
CatalogTraversalT::CatalogTN CatalogTN
manifest::Reflog * reflog
upload::AbstractUploader * uploader
unsigned int keep_history_depth
bool RemoveCatalogFromReflog(const shash::Any &catalog)
void PreserveDataObjects(const TraversalCallbackDataTN &data)
void SweepDataObjects(const TraversalCallbackDataTN &data)
const Configuration configuration_
void CheckAndSweep(const shash::Any &hash)
time_t keep_history_timestamp
std::vector< shash::Any > HashVector
bool CheckPreservedRevisions()
GarbageCollector(const Configuration &configuration)