garbage_collector.h

Go to the documentation of this file.

#ifndef CVMFS_GARBAGE_COLLECTION_GARBAGE_COLLECTOR_H_
#define CVMFS_GARBAGE_COLLECTION_GARBAGE_COLLECTOR_H_

#include <inttypes.h>

#include <vector>

#include "catalog_traversal_parallel.h"
#include "garbage_collection/hash_filter.h"
#include "statistics.h"
#include "upload_facility.h"

template<class CatalogTraversalT, class HashFilterT>
class GarbageCollector {
 protected:
  typedef typename CatalogTraversalT::ObjectFetcherTN ObjectFetcherTN;
  typedef typename ObjectFetcherTN::HistoryTN HistoryTN;
  typedef typename ObjectFetcherTN::ReflogTN ReflogTN;
  typedef typename CatalogTraversalT::CatalogTN CatalogTN;
  typedef typename CatalogTraversalT::CallbackDataTN TraversalCallbackDataTN;
  typedef typename CatalogTraversalT::Parameters TraversalParameters;
  typedef std::vector<shash::Any> HashVector;

 public:
  struct Configuration {
    static const uint64_t kFullHistory;
    static const uint64_t kNoHistory;
    static const time_t kNoTimestamp;
    static const shash::Any kLatestHistoryDatabase;

    Configuration()
        : uploader(NULL)
        , object_fetcher(NULL)
        , reflog(NULL)
        , keep_history_depth(kFullHistory)
        , keep_history_timestamp(kNoTimestamp)
        , dry_run(false)
        , verbose(false)
        , deleted_objects_logfile(NULL)
        , statistics(NULL)
        , extended_stats(false)
        , num_threads(8) { }

    bool has_deletion_log() const { return deleted_objects_logfile != NULL; }

    upload::AbstractUploader *uploader;
    ObjectFetcherTN *object_fetcher;
    ReflogTN *reflog;
    uint64_t keep_history_depth;
    time_t keep_history_timestamp;
    bool dry_run;
    bool verbose;
    FILE *deleted_objects_logfile;
    perf::Statistics *statistics;
    bool extended_stats;
    unsigned int num_threads;
  };

 public:
  explicit GarbageCollector(const Configuration &configuration);

  void UseReflogTimestamps();
  bool Collect();

  uint64_t preserved_catalog_count() const { return preserved_catalogs_; }
  uint64_t condemned_catalog_count() const { return condemned_catalogs_; }
  uint64_t condemned_objects_count() const { return condemned_objects_; }
  uint64_t duplicate_delete_requests() const {
    return duplicate_delete_requests_;
  }
  uint64_t condemned_bytes_count() const { return condemned_bytes_; }
  uint64_t oldest_trunk_catalog() const { return oldest_trunk_catalog_; }

 protected:
  TraversalParameters GetTraversalParams(const Configuration &configuration);

  void PreserveDataObjects(const TraversalCallbackDataTN &data);
  void SweepDataObjects(const TraversalCallbackDataTN &data);

  bool AnalyzePreservedCatalogTree();
  bool CheckPreservedRevisions();
  bool SweepReflog();

  void CheckAndSweep(const shash::Any &hash);
  void Sweep(const shash::Any &hash);
  bool RemoveCatalogFromReflog(const shash::Any &catalog);

  void PrintCatalogTreeEntry(const unsigned int tree_level,
                             const CatalogTN *catalog) const;
  void LogDeletion(const shash::Any &hash) const;

 private:
  class ReflogBasedInfoShim
      : public swissknife::CatalogTraversalInfoShim<CatalogTN> {
   public:
    explicit ReflogBasedInfoShim(ReflogTN *reflog) : reflog_(reflog) {
      pthread_mutex_init(&reflog_mutex_, NULL);
    }
    virtual ~ReflogBasedInfoShim() { pthread_mutex_destroy(&reflog_mutex_); }
    virtual uint64_t GetLastModified(const CatalogTN *catalog) {
      uint64_t timestamp;
      MutexLockGuard m(&reflog_mutex_);
      bool retval = reflog_->GetCatalogTimestamp(catalog->hash(), &timestamp);
      return retval ? timestamp : catalog->GetLastModified();
    }

   private:
    ReflogTN *reflog_;
    pthread_mutex_t reflog_mutex_;
  };

  const Configuration configuration_;
  ReflogBasedInfoShim catalog_info_shim_;
  CatalogTraversalT traversal_;
  HashFilterT hash_filter_;
  HashFilterT hash_map_delete_requests_;


  bool use_reflog_timestamps_;
  uint64_t oldest_trunk_catalog_;
  bool oldest_trunk_catalog_found_;
  uint64_t preserved_catalogs_;
  uint64_t unreferenced_trees_;
  uint64_t condemned_trees_;
  uint64_t condemned_catalogs_;
  float last_reported_status_;

  uint64_t condemned_objects_;
  uint64_t condemned_bytes_;
  uint64_t duplicate_delete_requests_;
};

#include "garbage_collector_impl.h"

#endif  // CVMFS_GARBAGE_COLLECTION_GARBAGE_COLLECTOR_H_