GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/swissknife_migrate.h
Date: 2024-04-28 02:33:07
Exec Total Coverage
Lines: 0 64 0.0%
Branches: 0 12 0.0%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 */
4
5 #ifndef CVMFS_SWISSKNIFE_MIGRATE_H_
6 #define CVMFS_SWISSKNIFE_MIGRATE_H_
7
8 #include "swissknife.h"
9
10 #include <map>
11 #include <string>
12 #include <vector>
13
14 #include "catalog.h"
15 #include "catalog_traversal.h"
16 #include "crypto/hash.h"
17 #include "history_sqlite.h"
18 #include "manifest.h"
19 #include "uid_map.h"
20 #include "upload.h"
21 #include "util/algorithm.h"
22 #include "util/atomic.h"
23 #include "util/concurrency.h"
24 #include "util/future.h"
25 #include "util/logging.h"
26 #include "util/pointer.h"
27
28 namespace catalog {
29 class WritableCatalog;
30 }
31
32 namespace swissknife {
33
34 class CommandMigrate : public Command {
35 protected:
36 struct CatalogStatistics {
37 ✗ CatalogStatistics()
38 ✗ : max_row_id(0)
39 ✗ , entry_count(0)
40 ✗ , hardlink_group_count(0)
41 ✗ , aggregated_linkcounts(0)
42 ✗ , migration_time(0.0) { }
43 unsigned int max_row_id;
44 unsigned int entry_count;
45
46 unsigned int hardlink_group_count;
47 unsigned int aggregated_linkcounts;
48
49 double migration_time;
50
51 std::string root_path;
52 };
53
54 class CatalogStatisticsList : protected std::vector<CatalogStatistics>,
55 public Lockable {
56 friend class CommandMigrate;
57
58 public:
59 ✗ inline void Insert(const CatalogStatistics &statistics) {
60 ✗ LockGuard<CatalogStatisticsList> lock(this);
61 ✗ this->push_back(statistics);
62 }
63 };
64
65 public:
66 struct PendingCatalog;
67 typedef std::vector<PendingCatalog *> PendingCatalogList;
68 struct PendingCatalog {
69 ✗ explicit PendingCatalog(const catalog::Catalog *old_catalog = NULL)
70 ✗ : success(false)
71 ✗ , old_catalog(old_catalog)
72 ✗ , new_catalog(NULL)
73 ✗ , new_catalog_size(0) { }
74 virtual ~PendingCatalog();
75
76 ✗ inline const std::string root_path() const {
77 ✗ return old_catalog->mountpoint().ToString();
78 }
79 ✗ inline bool IsRoot() const { return old_catalog->IsRoot(); }
80 ✗ inline bool HasNew() const { return new_catalog != NULL; }
81
82 ✗ inline bool HasChanges() const {
83 ✗ return (new_catalog != NULL ||
84 ✗ old_catalog->database().GetModifiedRowCount() > 0);
85 }
86
87 ✗ inline shash::Any GetOldContentHash() const {
88 ✗ return old_catalog->hash();
89 }
90
91 bool success;
92
93 const catalog::Catalog *old_catalog;
94 catalog::WritableCatalog *new_catalog;
95
96 PendingCatalogList nested_catalogs;
97 Future<catalog::DirectoryEntry> root_entry;
98 Future<catalog::DeltaCounters> nested_statistics;
99
100 CatalogStatistics statistics;
101
102 // Note: As soon as the `was_updated` future is set to 'true', both
103 // `new_catalog_hash` and `new_catalog_size` are assumed to be set
104 // accordingly. If it is set to 'false' they will be ignored.
105 Future<bool> was_updated;
106 shash::Any new_catalog_hash;
107 size_t new_catalog_size;
108 };
109
110 class PendingCatalogMap : public std::map<std::string, const PendingCatalog*>,
111 public Lockable {};
112
113 template<class DerivedT>
114 class AbstractMigrationWorker : public ConcurrentWorker<DerivedT> {
115 public:
116 typedef CommandMigrate::PendingCatalog* expected_data;
117 typedef CommandMigrate::PendingCatalog* returned_data;
118
119 struct worker_context {
120 ✗ worker_context(const std::string &temporary_directory,
121 const bool collect_catalog_statistics) :
122 ✗ temporary_directory(temporary_directory),
123 ✗ collect_catalog_statistics(collect_catalog_statistics) {}
124 const std::string temporary_directory;
125 const bool collect_catalog_statistics;
126 };
127
128 public:
129 explicit AbstractMigrationWorker(const worker_context *context);
130 virtual ~AbstractMigrationWorker();
131
132 void operator()(const expected_data &data);
133
134 protected:
135 bool RunMigration(PendingCatalog *data) const { return false; }
136
137 bool UpdateNestedCatalogReferences(PendingCatalog *data) const;
138 bool UpdateCatalogMetadata(PendingCatalog *data) const;
139 bool CleanupNestedCatalogs(PendingCatalog *data) const;
140 bool CollectAndAggregateStatistics(PendingCatalog *data) const;
141
142 catalog::WritableCatalog*
143 GetWritable(const catalog::Catalog *catalog) const;
144
145 protected:
146 const std::string temporary_directory_;
147 const bool collect_catalog_statistics_;
148
149 StopWatch migration_stopwatch_;
150 };
151
152 class MigrationWorker_20x :
153 public AbstractMigrationWorker<MigrationWorker_20x>
154 {
155 friend class AbstractMigrationWorker<MigrationWorker_20x>;
156 protected:
157 static const float kSchema;
158 static const unsigned kSchemaRevision;
159
160 public:
161 struct worker_context :
162 AbstractMigrationWorker<MigrationWorker_20x>::worker_context
163 {
164 ✗ worker_context(const std::string &temporary_directory,
165 const bool collect_catalog_statistics,
166 const bool fix_nested_catalog_transitions,
167 const bool analyze_file_linkcounts,
168 const uid_t uid,
169 const gid_t gid)
170 ✗ : AbstractMigrationWorker<MigrationWorker_20x>::worker_context(
171 temporary_directory, collect_catalog_statistics)
172 ✗ , fix_nested_catalog_transitions(fix_nested_catalog_transitions)
173 ✗ , analyze_file_linkcounts(analyze_file_linkcounts)
174 ✗ , uid(uid)
175 ✗ , gid(gid) { }
176 const bool fix_nested_catalog_transitions;
177 const bool analyze_file_linkcounts;
178 const uid_t uid;
179 const gid_t gid;
180 };
181
182 public:
183 explicit MigrationWorker_20x(const worker_context *context);
184
185 protected:
186 bool RunMigration(PendingCatalog *data) const;
187
188 bool CreateNewEmptyCatalog(PendingCatalog *data) const;
189 bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const;
190 bool AttachOldCatalogDatabase(PendingCatalog *data) const;
191 bool StartDatabaseTransaction(PendingCatalog *data) const;
192 bool MigrateFileMetadata(PendingCatalog *data) const;
193 bool AnalyzeFileLinkcounts(PendingCatalog *data) const;
194 bool RemoveDanglingNestedMountpoints(PendingCatalog *data) const;
195 bool MigrateNestedCatalogMountPoints(PendingCatalog *data) const;
196 bool FixNestedCatalogTransitionPoints(PendingCatalog *data) const;
197 bool GenerateCatalogStatistics(PendingCatalog *data) const;
198 bool FindRootEntryInformation(PendingCatalog *data) const;
199 bool CommitDatabaseTransaction(PendingCatalog *data) const;
200 bool DetachOldCatalogDatabase(PendingCatalog *data) const;
201
202 private:
203 const bool fix_nested_catalog_transitions_;
204 const bool analyze_file_linkcounts_;
205 const uid_t uid_;
206 const gid_t gid_;
207 };
208
209 class MigrationWorker_217 :
210 public AbstractMigrationWorker<MigrationWorker_217>
211 {
212 friend class AbstractMigrationWorker<MigrationWorker_217>;
213
214 public:
215 explicit MigrationWorker_217(const worker_context *context);
216
217 protected:
218 bool RunMigration(PendingCatalog *data) const;
219
220 bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const;
221 bool StartDatabaseTransaction(PendingCatalog *data) const;
222 bool GenerateNewStatisticsCounters(PendingCatalog *data) const;
223 bool UpdateCatalogSchema(PendingCatalog *data) const;
224 bool CommitDatabaseTransaction(PendingCatalog *data) const;
225 };
226
227 class ChownMigrationWorker :
228 public AbstractMigrationWorker<ChownMigrationWorker>
229 {
230 friend class AbstractMigrationWorker<ChownMigrationWorker>;
231 public:
232 struct worker_context :
233 AbstractMigrationWorker<ChownMigrationWorker>::worker_context
234 {
235 ✗ worker_context(const std::string &temporary_directory,
236 const bool collect_catalog_statistics,
237 const UidMap &uid_map,
238 const GidMap &gid_map)
239 ✗ : AbstractMigrationWorker<ChownMigrationWorker>::worker_context(
240 temporary_directory, collect_catalog_statistics)
241 ✗ , uid_map(uid_map)
242 ✗ , gid_map(gid_map) { }
243 const UidMap &uid_map;
244 const GidMap &gid_map;
245 };
246
247 public:
248 explicit ChownMigrationWorker(const worker_context *context);
249
250 protected:
251 bool RunMigration(PendingCatalog *data) const;
252 bool ApplyPersonaMappings(PendingCatalog *data) const;
253
254 private:
255 template <class MapT>
256 std::string GenerateMappingStatement(const MapT &map,
257 const std::string &column) const;
258
259 private:
260 const std::string uid_map_statement_;
261 const std::string gid_map_statement_;
262 };
263
264 class HardlinkRemovalMigrationWorker :
265 public AbstractMigrationWorker<HardlinkRemovalMigrationWorker>
266 {
267 friend class AbstractMigrationWorker<HardlinkRemovalMigrationWorker>;
268
269 public:
270 ✗ explicit HardlinkRemovalMigrationWorker(const worker_context *context) :
271 ✗ AbstractMigrationWorker<HardlinkRemovalMigrationWorker>(context) {}
272
273 protected:
274 bool RunMigration(PendingCatalog *data) const;
275
276 bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const;
277 bool BreakUpHardlinks(PendingCatalog *data) const;
278 };
279
280 class BulkhashRemovalMigrationWorker :
281 public AbstractMigrationWorker<BulkhashRemovalMigrationWorker>
282 {
283 friend class AbstractMigrationWorker<BulkhashRemovalMigrationWorker>;
284
285 public:
286 ✗ explicit BulkhashRemovalMigrationWorker(const worker_context *context) :
287 ✗ AbstractMigrationWorker<BulkhashRemovalMigrationWorker>(context) {}
288
289 protected:
290 bool RunMigration(PendingCatalog *data) const;
291
292 bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const;
293 bool RemoveRedundantBulkHashes(PendingCatalog *data) const;
294 };
295
296 // Regenerate / repair statistics counters
297 class StatsMigrationWorker :
298 public AbstractMigrationWorker<StatsMigrationWorker>
299 {
300 friend class AbstractMigrationWorker<StatsMigrationWorker>;
301
302 public:
303 explicit StatsMigrationWorker(const worker_context *context);
304
305 protected:
306 bool RunMigration(PendingCatalog *data) const;
307
308 bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const;
309 bool StartDatabaseTransaction(PendingCatalog *data) const;
310 bool RepairStatisticsCounters(PendingCatalog *data) const;
311 bool CommitDatabaseTransaction(PendingCatalog *data) const;
312 };
313
314 public:
315 CommandMigrate();
316 ✗ ~CommandMigrate() { }
317 ✗ virtual std::string GetName() const { return "migrate"; }
318 ✗ virtual std::string GetDescription() const {
319 return "CernVM-FS catalog repository migration \n"
320 ✗ "This command migrates the whole catalog structure of a given repository";
321 }
322 virtual ParameterList GetParams() const;
323
324 int Main(const ArgumentList &args);
325
326 static void FixNestedCatalogTransitionPoint(
327 const catalog::DirectoryEntry &nested_root,
328 catalog::DirectoryEntry *mountpoint);
329 static const catalog::DirectoryEntry& GetNestedCatalogMarkerDirent();
330
331 protected:
332 template <class ObjectFetcherT>
333 ✗ bool LoadCatalogs(const shash::Any &manual_root_hash,
334 ObjectFetcherT *object_fetcher)
335 {
336 ObjectFetcherFailures::Failures retval;
337 ✗ retval = object_fetcher->FetchManifest(&manifest_upstream_);
338 ✗ if (retval != ObjectFetcherFailures::kFailOk) {
339 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "could not get manifest (%d)", retval);
340 ✗ return false;
341 }
342
343 ✗ if (!manifest_upstream_->history().IsNull()) {
344 ✗ retval = object_fetcher->FetchHistory(
345 ✗ &history_upstream_, manifest_upstream_->history());
346 ✗ if (retval != ObjectFetcherFailures::kFailOk) {
347 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "could not get history (%d)", retval);
348 ✗ return false;
349 }
350 }
351
352 ✗ typename CatalogTraversal<ObjectFetcherT>::Parameters params;
353 ✗ const bool generate_full_catalog_tree = true;
354 ✗ params.no_close = generate_full_catalog_tree;
355 ✗ params.object_fetcher = object_fetcher;
356 ✗ CatalogTraversal<ObjectFetcherT> traversal(params);
357 ✗ traversal.RegisterListener(&CommandMigrate::CatalogCallback, this);
358
359 ✗ if (manual_root_hash.IsNull())
360 ✗ return traversal.Traverse();
361 ✗ return traversal.Traverse(manual_root_hash);
362 }
363
364 void CatalogCallback(
365 const CatalogTraversalData<catalog::WritableCatalog> &data);
366 void MigrationCallback(PendingCatalog *const &data);
367 void UploadCallback(const upload::SpoolerResult &result);
368
369 void PrintStatusMessage(const PendingCatalog *catalog,
370 const shash::Any &content_hash,
371 const std::string &message);
372
373 template <class MigratorT>
374 bool DoMigrationAndCommit(const std::string &manifest_path,
375 typename MigratorT::worker_context *context);
376
377 template <class MigratorT>
378 void ConvertCatalogsRecursively(PendingCatalog *catalog, MigratorT *migrator);
379 bool RaiseFileDescriptorLimit() const;
380 bool ConfigureSQLite() const;
381 void AnalyzeCatalogStatistics() const;
382 bool ReadPersona(const std::string &uid, const std::string &gid);
383 bool ReadPersonaMaps(const std::string &uid_map_path,
384 const std::string &gid_map_path,
385 UidMap *uid_map,
386 GidMap *gid_map) const;
387
388 bool GenerateNestedCatalogMarkerChunk();
389 void CreateNestedCatalogMarkerDirent(const shash::Any &content_hash);
390
391 void UploadHistoryClosure(const upload::SpoolerResult &result,
392 Future<shash::Any> *hash);
393 bool UpdateUndoTags(PendingCatalog *root_catalog,
394 uint64_t revision,
395 time_t timestamp,
396 shash::Any *history_hash);
397
398 private:
399 unsigned int file_descriptor_limit_;
400 CatalogStatisticsList catalog_statistics_list_;
401 unsigned int catalog_count_;
402 atomic_int32 catalogs_processed_;
403 bool has_committed_new_revision_;
404
405 uid_t uid_;
406 gid_t gid_;
407
408 std::string temporary_directory_;
409 std::string nested_catalog_marker_tmp_path_;
410 static catalog::DirectoryEntry nested_catalog_marker_;
411
412 UniquePtr<manifest::Manifest> manifest_upstream_;
413 UniquePtr<history::SqliteHistory> history_upstream_;
414 catalog::Catalog const* root_catalog_;
415 UniquePtr<upload::Spooler> spooler_;
416 PendingCatalogMap pending_catalogs_;
417
418 StopWatch catalog_loading_stopwatch_;
419 StopWatch migration_stopwatch_;
420 };
421
422 } // namespace swissknife
423
424 #endif // CVMFS_SWISSKNIFE_MIGRATE_H_
425