swissknife_migrate.h

Go to the documentation of this file.

#ifndef CVMFS_SWISSKNIFE_MIGRATE_H_
#define CVMFS_SWISSKNIFE_MIGRATE_H_

#include <map>
#include <string>
#include <vector>

#include "catalog.h"
#include "catalog_traversal.h"
#include "crypto/hash.h"
#include "history_sqlite.h"
#include "manifest.h"
#include "swissknife.h"
#include "uid_map.h"
#include "upload.h"
#include "util/algorithm.h"
#include "util/atomic.h"
#include "util/concurrency.h"
#include "util/future.h"
#include "util/logging.h"
#include "util/pointer.h"

namespace catalog {
class WritableCatalog;
}

namespace swissknife {

class CommandMigrate : public Command {
 protected:
  struct CatalogStatistics {
    CatalogStatistics()
        : max_row_id(0)
        , entry_count(0)
        , hardlink_group_count(0)
        , aggregated_linkcounts(0)
        , migration_time(0.0) { }
    unsigned int max_row_id;
    unsigned int entry_count;

    unsigned int hardlink_group_count;
    unsigned int aggregated_linkcounts;

    double migration_time;

    std::string root_path;
  };

  class CatalogStatisticsList : protected std::vector<CatalogStatistics>,
                                public Lockable {
    friend class CommandMigrate;

   public:
    inline void Insert(const CatalogStatistics &statistics) {
      LockGuard<CatalogStatisticsList> lock(this);
      this->push_back(statistics);
    }
  };

 public:
  struct PendingCatalog;
  typedef std::vector<PendingCatalog *> PendingCatalogList;
  struct PendingCatalog {
    explicit PendingCatalog(const catalog::Catalog *old_catalog = NULL)
        : success(false)
        , old_catalog(old_catalog)
        , new_catalog(NULL)
        , new_catalog_size(0) { }
    virtual ~PendingCatalog();

    inline const std::string root_path() const {
      return old_catalog->mountpoint().ToString();
    }
    inline bool IsRoot() const { return old_catalog->IsRoot(); }
    inline bool HasNew() const { return new_catalog != NULL; }

    inline bool HasChanges() const {
      return (new_catalog != NULL
              || old_catalog->database().GetModifiedRowCount() > 0);
    }

    inline shash::Any GetOldContentHash() const { return old_catalog->hash(); }

    bool success;

    const catalog::Catalog *old_catalog;
    catalog::WritableCatalog *new_catalog;

    PendingCatalogList nested_catalogs;
    Future<catalog::DirectoryEntry> root_entry;
    Future<catalog::DeltaCounters> nested_statistics;

    CatalogStatistics statistics;

    // Note: As soon as the `was_updated` future is set to 'true', both
    //       `new_catalog_hash` and `new_catalog_size` are assumed to be set
    //       accordingly. If it is set to 'false' they will be ignored.
    Future<bool> was_updated;
    shash::Any new_catalog_hash;
    size_t new_catalog_size;
  };

  class PendingCatalogMap
      : public std::map<std::string, const PendingCatalog *>,
        public Lockable { };

  template<class DerivedT>
  class AbstractMigrationWorker : public ConcurrentWorker<DerivedT> {
   public:
    typedef CommandMigrate::PendingCatalog *expected_data;
    typedef CommandMigrate::PendingCatalog *returned_data;

    struct worker_context {
      worker_context(const std::string &temporary_directory,
                     const bool collect_catalog_statistics)
          : temporary_directory(temporary_directory)
          , collect_catalog_statistics(collect_catalog_statistics) { }
      const std::string temporary_directory;
      const bool collect_catalog_statistics;
    };

   public:
    explicit AbstractMigrationWorker(const worker_context *context);
    virtual ~AbstractMigrationWorker();

    void operator()(const expected_data &data);

   protected:
    bool RunMigration(PendingCatalog *data) const { return false; }

    bool UpdateNestedCatalogReferences(PendingCatalog *data) const;
    bool UpdateCatalogMetadata(PendingCatalog *data) const;
    bool CleanupNestedCatalogs(PendingCatalog *data) const;
    bool CollectAndAggregateStatistics(PendingCatalog *data) const;

    catalog::WritableCatalog *GetWritable(
        const catalog::Catalog *catalog) const;

   protected:
    const std::string temporary_directory_;
    const bool collect_catalog_statistics_;

    StopWatch migration_stopwatch_;
  };

  class MigrationWorker_20x
      : public AbstractMigrationWorker<MigrationWorker_20x> {
    friend class AbstractMigrationWorker<MigrationWorker_20x>;

   protected:
    static const float kSchema;
    static const unsigned kSchemaRevision;

   public:
    struct worker_context
        : AbstractMigrationWorker<MigrationWorker_20x>::worker_context {
      worker_context(const std::string &temporary_directory,
                     const bool collect_catalog_statistics,
                     const bool fix_nested_catalog_transitions,
                     const bool analyze_file_linkcounts,
                     const uid_t uid,
                     const gid_t gid)
          : AbstractMigrationWorker<MigrationWorker_20x>::worker_context(
                temporary_directory, collect_catalog_statistics)
          , fix_nested_catalog_transitions(fix_nested_catalog_transitions)
          , analyze_file_linkcounts(analyze_file_linkcounts)
          , uid(uid)
          , gid(gid) { }
      const bool fix_nested_catalog_transitions;
      const bool analyze_file_linkcounts;
      const uid_t uid;
      const gid_t gid;
    };

   public:
    explicit MigrationWorker_20x(const worker_context *context);

   protected:
    bool RunMigration(PendingCatalog *data) const;

    bool CreateNewEmptyCatalog(PendingCatalog *data) const;
    bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const;
    bool AttachOldCatalogDatabase(PendingCatalog *data) const;
    bool StartDatabaseTransaction(PendingCatalog *data) const;
    bool MigrateFileMetadata(PendingCatalog *data) const;
    bool AnalyzeFileLinkcounts(PendingCatalog *data) const;
    bool RemoveDanglingNestedMountpoints(PendingCatalog *data) const;
    bool MigrateNestedCatalogMountPoints(PendingCatalog *data) const;
    bool FixNestedCatalogTransitionPoints(PendingCatalog *data) const;
    bool GenerateCatalogStatistics(PendingCatalog *data) const;
    bool FindRootEntryInformation(PendingCatalog *data) const;
    bool CommitDatabaseTransaction(PendingCatalog *data) const;
    bool DetachOldCatalogDatabase(PendingCatalog *data) const;

   private:
    const bool fix_nested_catalog_transitions_;
    const bool analyze_file_linkcounts_;
    const uid_t uid_;
    const gid_t gid_;
  };

  class MigrationWorker_217
      : public AbstractMigrationWorker<MigrationWorker_217> {
    friend class AbstractMigrationWorker<MigrationWorker_217>;

   public:
    explicit MigrationWorker_217(const worker_context *context);

   protected:
    bool RunMigration(PendingCatalog *data) const;

    bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const;
    bool StartDatabaseTransaction(PendingCatalog *data) const;
    bool GenerateNewStatisticsCounters(PendingCatalog *data) const;
    bool UpdateCatalogSchema(PendingCatalog *data) const;
    bool CommitDatabaseTransaction(PendingCatalog *data) const;
  };

  class ChownMigrationWorker
      : public AbstractMigrationWorker<ChownMigrationWorker> {
    friend class AbstractMigrationWorker<ChownMigrationWorker>;

   public:
    struct worker_context
        : AbstractMigrationWorker<ChownMigrationWorker>::worker_context {
      worker_context(const std::string &temporary_directory,
                     const bool collect_catalog_statistics,
                     const UidMap &uid_map,
                     const GidMap &gid_map)
          : AbstractMigrationWorker<ChownMigrationWorker>::worker_context(
                temporary_directory, collect_catalog_statistics)
          , uid_map(uid_map)
          , gid_map(gid_map) { }
      const UidMap &uid_map;
      const GidMap &gid_map;
    };

   public:
    explicit ChownMigrationWorker(const worker_context *context);

   protected:
    bool RunMigration(PendingCatalog *data) const;
    bool ApplyPersonaMappings(PendingCatalog *data) const;

   private:
    template<class MapT>
    std::string GenerateMappingStatement(const MapT &map,
                                         const std::string &column) const;

   private:
    const std::string uid_map_statement_;
    const std::string gid_map_statement_;
  };

  class HardlinkRemovalMigrationWorker
      : public AbstractMigrationWorker<HardlinkRemovalMigrationWorker> {
    friend class AbstractMigrationWorker<HardlinkRemovalMigrationWorker>;

   public:
    explicit HardlinkRemovalMigrationWorker(const worker_context *context)
        : AbstractMigrationWorker<HardlinkRemovalMigrationWorker>(context) { }

   protected:
    bool RunMigration(PendingCatalog *data) const;

    bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const;
    bool BreakUpHardlinks(PendingCatalog *data) const;
  };

  class BulkhashRemovalMigrationWorker
      : public AbstractMigrationWorker<BulkhashRemovalMigrationWorker> {
    friend class AbstractMigrationWorker<BulkhashRemovalMigrationWorker>;

   public:
    explicit BulkhashRemovalMigrationWorker(const worker_context *context)
        : AbstractMigrationWorker<BulkhashRemovalMigrationWorker>(context) { }

   protected:
    bool RunMigration(PendingCatalog *data) const;

    bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const;
    bool RemoveRedundantBulkHashes(PendingCatalog *data) const;
  };

  // Regenerate / repair statistics counters
  class StatsMigrationWorker
      : public AbstractMigrationWorker<StatsMigrationWorker> {
    friend class AbstractMigrationWorker<StatsMigrationWorker>;

   public:
    explicit StatsMigrationWorker(const worker_context *context);

   protected:
    bool RunMigration(PendingCatalog *data) const;

    bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const;
    bool StartDatabaseTransaction(PendingCatalog *data) const;
    bool RepairStatisticsCounters(PendingCatalog *data) const;
    bool CommitDatabaseTransaction(PendingCatalog *data) const;
  };

 public:
  CommandMigrate();
  ~CommandMigrate() { }
  virtual std::string GetName() const { return "migrate"; }
  virtual std::string GetDescription() const {
    return "CernVM-FS catalog repository migration \n"
           "This command migrates the whole catalog structure of a given "
           "repository";
  }
  virtual ParameterList GetParams() const;

  int Main(const ArgumentList &args);

  static void FixNestedCatalogTransitionPoint(
      const catalog::DirectoryEntry &nested_root,
      catalog::DirectoryEntry *mountpoint);
  static const catalog::DirectoryEntry &GetNestedCatalogMarkerDirent();

 protected:
  template<class ObjectFetcherT>
  bool LoadCatalogs(const shash::Any &manual_root_hash,
                    ObjectFetcherT *object_fetcher) {
    ObjectFetcherFailures::Failures retval;
    retval = object_fetcher->FetchManifest(&manifest_upstream_);
    if (retval != ObjectFetcherFailures::kFailOk) {
      LogCvmfs(kLogCvmfs, kLogStdout, "could not get manifest (%d)", retval);
      return false;
    }

    if (!manifest_upstream_->history().IsNull()) {
      retval = object_fetcher->FetchHistory(&history_upstream_,
                                            manifest_upstream_->history());
      if (retval != ObjectFetcherFailures::kFailOk) {
        LogCvmfs(kLogCvmfs, kLogStdout, "could not get history (%d)", retval);
        return false;
      }
    }

    typename CatalogTraversal<ObjectFetcherT>::Parameters params;
    const bool generate_full_catalog_tree = true;
    params.no_close = generate_full_catalog_tree;
    params.object_fetcher = object_fetcher;
    CatalogTraversal<ObjectFetcherT> traversal(params);
    traversal.RegisterListener(&CommandMigrate::CatalogCallback, this);

    if (manual_root_hash.IsNull())
      return traversal.Traverse();
    return traversal.Traverse(manual_root_hash);
  }

  void CatalogCallback(
      const CatalogTraversalData<catalog::WritableCatalog> &data);
  void MigrationCallback(PendingCatalog * const &data);
  void UploadCallback(const upload::SpoolerResult &result);

  void PrintStatusMessage(const PendingCatalog *catalog,
                          const shash::Any &content_hash,
                          const std::string &message);

  template<class MigratorT>
  bool DoMigrationAndCommit(const std::string &manifest_path,
                            typename MigratorT::worker_context *context);

  template<class MigratorT>
  void ConvertCatalogsRecursively(PendingCatalog *catalog, MigratorT *migrator);
  bool RaiseFileDescriptorLimit() const;
  bool ConfigureSQLite() const;
  void AnalyzeCatalogStatistics() const;
  bool ReadPersona(const std::string &uid, const std::string &gid);
  bool ReadPersonaMaps(const std::string &uid_map_path,
                       const std::string &gid_map_path,
                       UidMap *uid_map,
                       GidMap *gid_map) const;

  bool GenerateNestedCatalogMarkerChunk();
  void CreateNestedCatalogMarkerDirent(const shash::Any &content_hash);

  void UploadHistoryClosure(const upload::SpoolerResult &result,
                            Future<shash::Any> *hash);
  bool UpdateUndoTags(PendingCatalog *root_catalog,
                      uint64_t revision,
                      time_t timestamp,
                      shash::Any *history_hash);

 private:
  unsigned int file_descriptor_limit_;
  CatalogStatisticsList catalog_statistics_list_;
  unsigned int catalog_count_;
  atomic_int32 catalogs_processed_;
  bool has_committed_new_revision_;

  uid_t uid_;
  gid_t gid_;

  std::string temporary_directory_;
  std::string nested_catalog_marker_tmp_path_;
  static catalog::DirectoryEntry nested_catalog_marker_;

  UniquePtr<manifest::Manifest> manifest_upstream_;
  UniquePtr<history::SqliteHistory> history_upstream_;
  catalog::Catalog const *root_catalog_;
  UniquePtr<upload::Spooler> spooler_;
  PendingCatalogMap pending_catalogs_;

  StopWatch catalog_loading_stopwatch_;
  StopWatch migration_stopwatch_;
};

}  // namespace swissknife

#endif  // CVMFS_SWISSKNIFE_MIGRATE_H_