GCC Code Coverage Report
Directory: cvmfs/ Exec Total Coverage
File: cvmfs/swissknife_migrate.h Lines: 0 60 0.0 %
Date: 2019-02-03 02:48:13 Branches: 0 34 0.0 %

Line Branch Exec Source
1
/**
2
 * This file is part of the CernVM File System.
3
 */
4
5
#ifndef CVMFS_SWISSKNIFE_MIGRATE_H_
6
#define CVMFS_SWISSKNIFE_MIGRATE_H_
7
8
#include "swissknife.h"
9
10
#include <map>
11
#include <string>
12
#include <vector>
13
14
#include "atomic.h"
15
#include "catalog.h"
16
#include "catalog_traversal.h"
17
#include "hash.h"
18
#include "history_sqlite.h"
19
#include "logging.h"
20
#include "manifest.h"
21
#include "uid_map.h"
22
#include "upload.h"
23
#include "util/algorithm.h"
24
#include "util/pointer.h"
25
#include "util_concurrency.h"
26
27
namespace catalog {
28
class WritableCatalog;
29
}
30
31
namespace swissknife {
32
33
class CommandMigrate : public Command {
34
 protected:
35
  struct CatalogStatistics {
36
    CatalogStatistics()
37
      : max_row_id(0)
38
      , entry_count(0)
39
      , hardlink_group_count(0)
40
      , aggregated_linkcounts(0)
41
      , migration_time(0.0) { }
42
    unsigned int max_row_id;
43
    unsigned int entry_count;
44
45
    unsigned int hardlink_group_count;
46
    unsigned int aggregated_linkcounts;
47
48
    double       migration_time;
49
50
    std::string root_path;
51
  };
52
53
  class CatalogStatisticsList : protected std::vector<CatalogStatistics>,
54
                                public    Lockable {
55
    friend class CommandMigrate;
56
57
   public:
58
    inline void Insert(const CatalogStatistics &statistics) {
59
      LockGuard<CatalogStatisticsList> lock(this);
60
      this->push_back(statistics);
61
    }
62
  };
63
64
 public:
65
  struct PendingCatalog;
66
  typedef std::vector<PendingCatalog *> PendingCatalogList;
67
  struct PendingCatalog {
68
    explicit PendingCatalog(const catalog::Catalog *old_catalog = NULL)
69
      : success(false)
70
      , old_catalog(old_catalog)
71
      , new_catalog(NULL)
72
      , new_catalog_size(0) { }
73
    virtual ~PendingCatalog();
74
75
    inline const std::string root_path() const {
76
      return old_catalog->mountpoint().ToString();
77
    }
78
    inline bool IsRoot() const { return old_catalog->IsRoot(); }
79
    inline bool HasNew() const { return new_catalog != NULL;   }
80
81
    inline bool HasChanges() const {
82
      return (new_catalog != NULL ||
83
              old_catalog->database().GetModifiedRowCount() > 0);
84
    }
85
86
    inline shash::Any GetOldContentHash() const {
87
      return old_catalog->hash();
88
    }
89
90
    bool                              success;
91
92
    const catalog::Catalog           *old_catalog;
93
    catalog::WritableCatalog         *new_catalog;
94
95
    PendingCatalogList                nested_catalogs;
96
    Future<catalog::DirectoryEntry>   root_entry;
97
    Future<catalog::DeltaCounters>    nested_statistics;
98
99
    CatalogStatistics                 statistics;
100
101
    // Note: As soon as the `was_updated` future is set to 'true', both
102
    //       `new_catalog_hash` and `new_catalog_size` are assumed to be set
103
    //       accordingly. If it is set to 'false' they will be ignored.
104
    Future<bool>                      was_updated;
105
    shash::Any                        new_catalog_hash;
106
    size_t                            new_catalog_size;
107
  };
108
109
  class PendingCatalogMap : public std::map<std::string, const PendingCatalog*>,
110
                            public Lockable {};
111
112
  template<class DerivedT>
113
  class AbstractMigrationWorker : public ConcurrentWorker<DerivedT> {
114
   public:
115
    typedef CommandMigrate::PendingCatalog* expected_data;
116
    typedef CommandMigrate::PendingCatalog* returned_data;
117
118
    struct worker_context {
119
      worker_context(const std::string  &temporary_directory,
120
                     const bool          collect_catalog_statistics) :
121
        temporary_directory(temporary_directory),
122
        collect_catalog_statistics(collect_catalog_statistics) {}
123
      const std::string  temporary_directory;
124
      const bool         collect_catalog_statistics;
125
    };
126
127
   public:
128
    explicit AbstractMigrationWorker(const worker_context *context);
129
    virtual ~AbstractMigrationWorker();
130
131
    void operator()(const expected_data &data);
132
133
   protected:
134
    bool RunMigration(PendingCatalog *data) const { return false; }
135
136
    bool UpdateNestedCatalogReferences(PendingCatalog *data) const;
137
    bool UpdateCatalogMetadata(PendingCatalog *data) const;
138
    bool CleanupNestedCatalogs(PendingCatalog *data) const;
139
    bool CollectAndAggregateStatistics(PendingCatalog *data) const;
140
141
    catalog::WritableCatalog*
142
      GetWritable(const catalog::Catalog *catalog) const;
143
144
   protected:
145
    const std::string  temporary_directory_;
146
    const bool         collect_catalog_statistics_;
147
148
    StopWatch          migration_stopwatch_;
149
  };
150
151
  class MigrationWorker_20x :
152
    public AbstractMigrationWorker<MigrationWorker_20x>
153
  {
154
    friend class AbstractMigrationWorker<MigrationWorker_20x>;
155
   protected:
156
    static const float    kSchema;
157
    static const unsigned kSchemaRevision;
158
159
   public:
160
    struct worker_context :
161
      AbstractMigrationWorker<MigrationWorker_20x>::worker_context
162
    {
163
      worker_context(const std::string  &temporary_directory,
164
                     const bool          collect_catalog_statistics,
165
                     const bool          fix_nested_catalog_transitions,
166
                     const bool          analyze_file_linkcounts,
167
                     const uid_t         uid,
168
                     const gid_t         gid)
169
        : AbstractMigrationWorker<MigrationWorker_20x>::worker_context(
170
            temporary_directory, collect_catalog_statistics)
171
        , fix_nested_catalog_transitions(fix_nested_catalog_transitions)
172
        , analyze_file_linkcounts(analyze_file_linkcounts)
173
        , uid(uid)
174
        , gid(gid) { }
175
      const bool  fix_nested_catalog_transitions;
176
      const bool  analyze_file_linkcounts;
177
      const uid_t uid;
178
      const gid_t gid;
179
    };
180
181
   public:
182
    explicit MigrationWorker_20x(const worker_context *context);
183
184
   protected:
185
    bool RunMigration(PendingCatalog *data) const;
186
187
    bool CreateNewEmptyCatalog(PendingCatalog *data) const;
188
    bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const;
189
    bool AttachOldCatalogDatabase(PendingCatalog *data) const;
190
    bool StartDatabaseTransaction(PendingCatalog *data) const;
191
    bool MigrateFileMetadata(PendingCatalog *data) const;
192
    bool AnalyzeFileLinkcounts(PendingCatalog *data) const;
193
    bool RemoveDanglingNestedMountpoints(PendingCatalog *data) const;
194
    bool MigrateNestedCatalogMountPoints(PendingCatalog *data) const;
195
    bool FixNestedCatalogTransitionPoints(PendingCatalog *data) const;
196
    bool GenerateCatalogStatistics(PendingCatalog *data) const;
197
    bool FindRootEntryInformation(PendingCatalog *data) const;
198
    bool CommitDatabaseTransaction(PendingCatalog *data) const;
199
    bool DetachOldCatalogDatabase(PendingCatalog *data) const;
200
201
   private:
202
    const bool  fix_nested_catalog_transitions_;
203
    const bool  analyze_file_linkcounts_;
204
    const uid_t uid_;
205
    const gid_t gid_;
206
  };
207
208
  class MigrationWorker_217 :
209
    public AbstractMigrationWorker<MigrationWorker_217>
210
  {
211
    friend class AbstractMigrationWorker<MigrationWorker_217>;
212
213
   public:
214
    explicit MigrationWorker_217(const worker_context *context);
215
216
   protected:
217
    bool RunMigration(PendingCatalog *data) const;
218
219
    bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const;
220
    bool StartDatabaseTransaction(PendingCatalog *data) const;
221
    bool GenerateNewStatisticsCounters(PendingCatalog *data) const;
222
    bool UpdateCatalogSchema(PendingCatalog *data) const;
223
    bool CommitDatabaseTransaction(PendingCatalog *data) const;
224
  };
225
226
  class ChownMigrationWorker :
227
    public AbstractMigrationWorker<ChownMigrationWorker>
228
  {
229
    friend class AbstractMigrationWorker<ChownMigrationWorker>;
230
   public:
231
    struct worker_context :
232
      AbstractMigrationWorker<ChownMigrationWorker>::worker_context
233
    {
234
      worker_context(const std::string  &temporary_directory,
235
                     const bool          collect_catalog_statistics,
236
                     const UidMap       &uid_map,
237
                     const GidMap       &gid_map)
238
        : AbstractMigrationWorker<ChownMigrationWorker>::worker_context(
239
            temporary_directory, collect_catalog_statistics)
240
        , uid_map(uid_map)
241
        , gid_map(gid_map) { }
242
      const UidMap &uid_map;
243
      const GidMap &gid_map;
244
    };
245
246
   public:
247
    explicit ChownMigrationWorker(const worker_context *context);
248
249
   protected:
250
    bool RunMigration(PendingCatalog *data) const;
251
    bool ApplyPersonaMappings(PendingCatalog *data) const;
252
253
   private:
254
    template <class MapT>
255
    std::string GenerateMappingStatement(const MapT         &map,
256
                                         const std::string  &column) const;
257
258
   private:
259
    const std::string uid_map_statement_;
260
    const std::string gid_map_statement_;
261
  };
262
263
  class HardlinkRemovalMigrationWorker :
264
    public AbstractMigrationWorker<HardlinkRemovalMigrationWorker>
265
  {
266
    friend class AbstractMigrationWorker<HardlinkRemovalMigrationWorker>;
267
268
   public:
269
    explicit HardlinkRemovalMigrationWorker(const worker_context *context) :
270
      AbstractMigrationWorker<HardlinkRemovalMigrationWorker>(context) {}
271
272
   protected:
273
    bool RunMigration(PendingCatalog *data) const;
274
275
    bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const;
276
    bool BreakUpHardlinks(PendingCatalog *data) const;
277
  };
278
279
 public:
280
  CommandMigrate();
281
  ~CommandMigrate() { }
282
  virtual std::string GetName() const { return "migrate"; }
283
  virtual std::string GetDescription() const {
284
    return "CernVM-FS catalog repository migration \n"
285
      "This command migrates the whole catalog structure of a given repository";
286
  }
287
  virtual ParameterList GetParams() const;
288
289
  int Main(const ArgumentList &args);
290
291
  static void FixNestedCatalogTransitionPoint(
292
    const catalog::DirectoryEntry &nested_root,
293
    catalog::DirectoryEntry *mountpoint);
294
  static const catalog::DirectoryEntry& GetNestedCatalogMarkerDirent();
295
296
 protected:
297
  template <class ObjectFetcherT>
298
  bool LoadCatalogs(const shash::Any &manual_root_hash,
299
                    ObjectFetcherT *object_fetcher)
300
  {
301
    ObjectFetcherFailures::Failures retval;
302
    retval = object_fetcher->FetchManifest(&manifest_upstream_);
303
    if (retval != ObjectFetcherFailures::kFailOk) {
304
      LogCvmfs(kLogCvmfs, kLogStdout, "could not get manifest (%d)", retval);
305
      return false;
306
    }
307
308
    if (!manifest_upstream_->history().IsNull()) {
309
      retval = object_fetcher->FetchHistory(
310
        &history_upstream_, manifest_upstream_->history());
311
      if (retval != ObjectFetcherFailures::kFailOk) {
312
        LogCvmfs(kLogCvmfs, kLogStdout, "could not get history (%d)", retval);
313
        return false;
314
      }
315
    }
316
317
    typename CatalogTraversal<ObjectFetcherT>::Parameters params;
318
    const bool generate_full_catalog_tree = true;
319
    params.no_close       = generate_full_catalog_tree;
320
    params.object_fetcher = object_fetcher;
321
    CatalogTraversal<ObjectFetcherT> traversal(params);
322
    traversal.RegisterListener(&CommandMigrate::CatalogCallback, this);
323
324
    if (manual_root_hash.IsNull())
325
      return traversal.Traverse();
326
    return traversal.Traverse(manual_root_hash);
327
  }
328
329
  void CatalogCallback(
330
    const CatalogTraversalData<catalog::WritableCatalog> &data);
331
  void MigrationCallback(PendingCatalog *const &data);
332
  void UploadCallback(const upload::SpoolerResult &result);
333
334
  void PrintStatusMessage(const PendingCatalog *catalog,
335
                          const shash::Any     &content_hash,
336
                          const std::string    &message);
337
338
  template <class MigratorT>
339
  bool DoMigrationAndCommit(const std::string                   &manifest_path,
340
                            typename MigratorT::worker_context  *context);
341
342
  template <class MigratorT>
343
  void ConvertCatalogsRecursively(PendingCatalog *catalog, MigratorT *migrator);
344
  bool RaiseFileDescriptorLimit() const;
345
  bool ConfigureSQLite() const;
346
  void AnalyzeCatalogStatistics() const;
347
  bool ReadPersona(const std::string &uid, const std::string &gid);
348
  bool ReadPersonaMaps(const std::string &uid_map_path,
349
                       const std::string &gid_map_path,
350
                             UidMap      *uid_map,
351
                             GidMap      *gid_map) const;
352
353
  bool GenerateNestedCatalogMarkerChunk();
354
  void CreateNestedCatalogMarkerDirent(const shash::Any &content_hash);
355
356
  void UploadHistoryClosure(const upload::SpoolerResult &result,
357
                            Future<shash::Any> *hash);
358
  bool UpdateUndoTags(PendingCatalog *root_catalog,
359
                      unsigned revision,
360
                      time_t timestamp,
361
                      shash::Any *history_hash);
362
363
 private:
364
  unsigned int           file_descriptor_limit_;
365
  CatalogStatisticsList  catalog_statistics_list_;
366
  unsigned int           catalog_count_;
367
  atomic_int32           catalogs_processed_;
368
  bool                   has_committed_new_revision_;
369
370
  uid_t                  uid_;
371
  gid_t                  gid_;
372
373
  std::string                     temporary_directory_;
374
  std::string                     nested_catalog_marker_tmp_path_;
375
  static catalog::DirectoryEntry  nested_catalog_marker_;
376
377
  UniquePtr<manifest::Manifest> manifest_upstream_;
378
  UniquePtr<history::SqliteHistory> history_upstream_;
379
  catalog::Catalog const*     root_catalog_;
380
  UniquePtr<upload::Spooler>  spooler_;
381
  PendingCatalogMap           pending_catalogs_;
382
383
  StopWatch  catalog_loading_stopwatch_;
384
  StopWatch  migration_stopwatch_;
385
};
386
387
}  // namespace swissknife
388
389
#endif  // CVMFS_SWISSKNIFE_MIGRATE_H_