GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/swissknife_migrate.h
Date: 2025-06-22 02:36:02
Exec Total Coverage
Lines: 0 63 0.0%
Branches: 0 12 0.0%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 */
4
5 #ifndef CVMFS_SWISSKNIFE_MIGRATE_H_
6 #define CVMFS_SWISSKNIFE_MIGRATE_H_
7
8 #include <map>
9 #include <string>
10 #include <vector>
11
12 #include "catalog.h"
13 #include "catalog_traversal.h"
14 #include "crypto/hash.h"
15 #include "history_sqlite.h"
16 #include "manifest.h"
17 #include "swissknife.h"
18 #include "uid_map.h"
19 #include "upload.h"
20 #include "util/algorithm.h"
21 #include "util/atomic.h"
22 #include "util/concurrency.h"
23 #include "util/future.h"
24 #include "util/logging.h"
25 #include "util/pointer.h"
26
27 namespace catalog {
28 class WritableCatalog;
29 }
30
31 namespace swissknife {
32
33 class CommandMigrate : public Command {
34 protected:
35 struct CatalogStatistics {
36 ✗ CatalogStatistics()
37 ✗ : max_row_id(0)
38 ✗ , entry_count(0)
39 ✗ , hardlink_group_count(0)
40 ✗ , aggregated_linkcounts(0)
41 ✗ , migration_time(0.0) { }
42 unsigned int max_row_id;
43 unsigned int entry_count;
44
45 unsigned int hardlink_group_count;
46 unsigned int aggregated_linkcounts;
47
48 double migration_time;
49
50 std::string root_path;
51 };
52
53 class CatalogStatisticsList : protected std::vector<CatalogStatistics>,
54 public Lockable {
55 friend class CommandMigrate;
56
57 public:
58 ✗ inline void Insert(const CatalogStatistics &statistics) {
59 ✗ const LockGuard<CatalogStatisticsList> lock(this);
60 ✗ this->push_back(statistics);
61 }
62 };
63
64 public:
65 struct PendingCatalog;
66 typedef std::vector<PendingCatalog *> PendingCatalogList;
67 struct PendingCatalog {
68 ✗ explicit PendingCatalog(const catalog::Catalog *old_catalog = NULL)
69 ✗ : success(false)
70 ✗ , old_catalog(old_catalog)
71 ✗ , new_catalog(NULL)
72 ✗ , new_catalog_size(0) { }
73 virtual ~PendingCatalog();
74
75 ✗ inline const std::string root_path() const {
76 ✗ return old_catalog->mountpoint().ToString();
77 }
78 ✗ inline bool IsRoot() const { return old_catalog->IsRoot(); }
79 ✗ inline bool HasNew() const { return new_catalog != NULL; }
80
81 ✗ inline bool HasChanges() const {
82 ✗ return (new_catalog != NULL
83 ✗ || old_catalog->database().GetModifiedRowCount() > 0);
84 }
85
86 ✗ inline shash::Any GetOldContentHash() const { return old_catalog->hash(); }
87
88 bool success;
89
90 const catalog::Catalog *old_catalog;
91 catalog::WritableCatalog *new_catalog;
92
93 PendingCatalogList nested_catalogs;
94 Future<catalog::DirectoryEntry> root_entry;
95 Future<catalog::DeltaCounters> nested_statistics;
96
97 CatalogStatistics statistics;
98
99 // Note: As soon as the `was_updated` future is set to 'true', both
100 // `new_catalog_hash` and `new_catalog_size` are assumed to be set
101 // accordingly. If it is set to 'false' they will be ignored.
102 Future<bool> was_updated;
103 shash::Any new_catalog_hash;
104 size_t new_catalog_size;
105 };
106
107 class PendingCatalogMap
108 : public std::map<std::string, const PendingCatalog *>,
109 public Lockable { };
110
111 template<class DerivedT>
112 class AbstractMigrationWorker : public ConcurrentWorker<DerivedT> {
113 public:
114 typedef CommandMigrate::PendingCatalog *expected_data;
115 typedef CommandMigrate::PendingCatalog *returned_data;
116
117 struct worker_context {
118 ✗ worker_context(const std::string &temporary_directory,
119 const bool collect_catalog_statistics)
120 ✗ : temporary_directory(temporary_directory)
121 ✗ , collect_catalog_statistics(collect_catalog_statistics) { }
122 const std::string temporary_directory;
123 const bool collect_catalog_statistics;
124 };
125
126 public:
127 explicit AbstractMigrationWorker(const worker_context *context);
128 virtual ~AbstractMigrationWorker();
129
130 void operator()(const expected_data &data);
131
132 protected:
133 bool RunMigration(PendingCatalog *data) const { return false; }
134
135 bool UpdateNestedCatalogReferences(PendingCatalog *data) const;
136 bool UpdateCatalogMetadata(PendingCatalog *data) const;
137 bool CleanupNestedCatalogs(PendingCatalog *data) const;
138 bool CollectAndAggregateStatistics(PendingCatalog *data) const;
139
140 catalog::WritableCatalog *GetWritable(
141 const catalog::Catalog *catalog) const;
142
143 protected:
144 const std::string temporary_directory_;
145 const bool collect_catalog_statistics_;
146
147 StopWatch migration_stopwatch_;
148 };
149
150 class MigrationWorker_20x
151 : public AbstractMigrationWorker<MigrationWorker_20x> {
152 friend class AbstractMigrationWorker<MigrationWorker_20x>;
153
154 protected:
155 static const float kSchema;
156 static const unsigned kSchemaRevision;
157
158 public:
159 struct worker_context
160 : AbstractMigrationWorker<MigrationWorker_20x>::worker_context {
161 ✗ worker_context(const std::string &temporary_directory,
162 const bool collect_catalog_statistics,
163 const bool fix_nested_catalog_transitions,
164 const bool analyze_file_linkcounts,
165 const uid_t uid,
166 const gid_t gid)
167 ✗ : AbstractMigrationWorker<MigrationWorker_20x>::worker_context(
168 temporary_directory, collect_catalog_statistics)
169 ✗ , fix_nested_catalog_transitions(fix_nested_catalog_transitions)
170 ✗ , analyze_file_linkcounts(analyze_file_linkcounts)
171 ✗ , uid(uid)
172 ✗ , gid(gid) { }
173 const bool fix_nested_catalog_transitions;
174 const bool analyze_file_linkcounts;
175 const uid_t uid;
176 const gid_t gid;
177 };
178
179 public:
180 explicit MigrationWorker_20x(const worker_context *context);
181
182 protected:
183 bool RunMigration(PendingCatalog *data) const;
184
185 bool CreateNewEmptyCatalog(PendingCatalog *data) const;
186 bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const;
187 bool AttachOldCatalogDatabase(PendingCatalog *data) const;
188 bool StartDatabaseTransaction(PendingCatalog *data) const;
189 bool MigrateFileMetadata(PendingCatalog *data) const;
190 bool AnalyzeFileLinkcounts(PendingCatalog *data) const;
191 bool RemoveDanglingNestedMountpoints(PendingCatalog *data) const;
192 bool MigrateNestedCatalogMountPoints(PendingCatalog *data) const;
193 bool FixNestedCatalogTransitionPoints(PendingCatalog *data) const;
194 bool GenerateCatalogStatistics(PendingCatalog *data) const;
195 bool FindRootEntryInformation(PendingCatalog *data) const;
196 bool CommitDatabaseTransaction(PendingCatalog *data) const;
197 bool DetachOldCatalogDatabase(PendingCatalog *data) const;
198
199 private:
200 const bool fix_nested_catalog_transitions_;
201 const bool analyze_file_linkcounts_;
202 const uid_t uid_;
203 const gid_t gid_;
204 };
205
206 class MigrationWorker_217
207 : public AbstractMigrationWorker<MigrationWorker_217> {
208 friend class AbstractMigrationWorker<MigrationWorker_217>;
209
210 public:
211 explicit MigrationWorker_217(const worker_context *context);
212
213 protected:
214 bool RunMigration(PendingCatalog *data) const;
215
216 bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const;
217 bool StartDatabaseTransaction(PendingCatalog *data) const;
218 bool GenerateNewStatisticsCounters(PendingCatalog *data) const;
219 bool UpdateCatalogSchema(PendingCatalog *data) const;
220 bool CommitDatabaseTransaction(PendingCatalog *data) const;
221 };
222
223 class ChownMigrationWorker
224 : public AbstractMigrationWorker<ChownMigrationWorker> {
225 friend class AbstractMigrationWorker<ChownMigrationWorker>;
226
227 public:
228 struct worker_context
229 : AbstractMigrationWorker<ChownMigrationWorker>::worker_context {
230 ✗ worker_context(const std::string &temporary_directory,
231 const bool collect_catalog_statistics,
232 const UidMap &uid_map,
233 const GidMap &gid_map)
234 ✗ : AbstractMigrationWorker<ChownMigrationWorker>::worker_context(
235 temporary_directory, collect_catalog_statistics)
236 ✗ , uid_map(uid_map)
237 ✗ , gid_map(gid_map) { }
238 const UidMap &uid_map;
239 const GidMap &gid_map;
240 };
241
242 public:
243 explicit ChownMigrationWorker(const worker_context *context);
244
245 protected:
246 bool RunMigration(PendingCatalog *data) const;
247 bool ApplyPersonaMappings(PendingCatalog *data) const;
248
249 private:
250 template<class MapT>
251 std::string GenerateMappingStatement(const MapT &map,
252 const std::string &column) const;
253
254 private:
255 const std::string uid_map_statement_;
256 const std::string gid_map_statement_;
257 };
258
259 class HardlinkRemovalMigrationWorker
260 : public AbstractMigrationWorker<HardlinkRemovalMigrationWorker> {
261 friend class AbstractMigrationWorker<HardlinkRemovalMigrationWorker>;
262
263 public:
264 ✗ explicit HardlinkRemovalMigrationWorker(const worker_context *context)
265 ✗ : AbstractMigrationWorker<HardlinkRemovalMigrationWorker>(context) { }
266
267 protected:
268 bool RunMigration(PendingCatalog *data) const;
269
270 bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const;
271 bool BreakUpHardlinks(PendingCatalog *data) const;
272 };
273
274 class BulkhashRemovalMigrationWorker
275 : public AbstractMigrationWorker<BulkhashRemovalMigrationWorker> {
276 friend class AbstractMigrationWorker<BulkhashRemovalMigrationWorker>;
277
278 public:
279 ✗ explicit BulkhashRemovalMigrationWorker(const worker_context *context)
280 ✗ : AbstractMigrationWorker<BulkhashRemovalMigrationWorker>(context) { }
281
282 protected:
283 bool RunMigration(PendingCatalog *data) const;
284
285 bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const;
286 bool RemoveRedundantBulkHashes(PendingCatalog *data) const;
287 };
288
289 // Regenerate / repair statistics counters
290 class StatsMigrationWorker
291 : public AbstractMigrationWorker<StatsMigrationWorker> {
292 friend class AbstractMigrationWorker<StatsMigrationWorker>;
293
294 public:
295 explicit StatsMigrationWorker(const worker_context *context);
296
297 protected:
298 bool RunMigration(PendingCatalog *data) const;
299
300 bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const;
301 bool StartDatabaseTransaction(PendingCatalog *data) const;
302 bool RepairStatisticsCounters(PendingCatalog *data) const;
303 bool CommitDatabaseTransaction(PendingCatalog *data) const;
304 };
305
306 public:
307 CommandMigrate();
308 ✗ ~CommandMigrate() { }
309 ✗ virtual std::string GetName() const { return "migrate"; }
310 ✗ virtual std::string GetDescription() const {
311 return "CernVM-FS catalog repository migration \n"
312 "This command migrates the whole catalog structure of a given "
313 ✗ "repository";
314 }
315 virtual ParameterList GetParams() const;
316
317 int Main(const ArgumentList &args);
318
319 static void FixNestedCatalogTransitionPoint(
320 const catalog::DirectoryEntry &nested_root,
321 catalog::DirectoryEntry *mountpoint);
322 static const catalog::DirectoryEntry &GetNestedCatalogMarkerDirent();
323
324 protected:
325 template<class ObjectFetcherT>
326 ✗ bool LoadCatalogs(const shash::Any &manual_root_hash,
327 ObjectFetcherT *object_fetcher) {
328 ObjectFetcherFailures::Failures retval;
329 ✗ retval = object_fetcher->FetchManifest(&manifest_upstream_);
330 ✗ if (retval != ObjectFetcherFailures::kFailOk) {
331 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "could not get manifest (%d)", retval);
332 ✗ return false;
333 }
334
335 ✗ if (!manifest_upstream_->history().IsNull()) {
336 ✗ retval = object_fetcher->FetchHistory(&history_upstream_,
337 ✗ manifest_upstream_->history());
338 ✗ if (retval != ObjectFetcherFailures::kFailOk) {
339 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "could not get history (%d)", retval);
340 ✗ return false;
341 }
342 }
343
344 ✗ typename CatalogTraversal<ObjectFetcherT>::Parameters params;
345 ✗ const bool generate_full_catalog_tree = true;
346 ✗ params.no_close = generate_full_catalog_tree;
347 ✗ params.object_fetcher = object_fetcher;
348 ✗ CatalogTraversal<ObjectFetcherT> traversal(params);
349 ✗ traversal.RegisterListener(&CommandMigrate::CatalogCallback, this);
350
351 ✗ if (manual_root_hash.IsNull())
352 ✗ return traversal.Traverse();
353 ✗ return traversal.Traverse(manual_root_hash);
354 }
355
356 void CatalogCallback(
357 const CatalogTraversalData<catalog::WritableCatalog> &data);
358 void MigrationCallback(PendingCatalog * const &data);
359 void UploadCallback(const upload::SpoolerResult &result);
360
361 void PrintStatusMessage(const PendingCatalog *catalog,
362 const shash::Any &content_hash,
363 const std::string &message);
364
365 template<class MigratorT>
366 bool DoMigrationAndCommit(const std::string &manifest_path,
367 typename MigratorT::worker_context *context);
368
369 template<class MigratorT>
370 void ConvertCatalogsRecursively(PendingCatalog *catalog, MigratorT *migrator);
371 bool RaiseFileDescriptorLimit() const;
372 bool ConfigureSQLite() const;
373 void AnalyzeCatalogStatistics() const;
374 bool ReadPersona(const std::string &uid, const std::string &gid);
375 bool ReadPersonaMaps(const std::string &uid_map_path,
376 const std::string &gid_map_path,
377 UidMap *uid_map,
378 GidMap *gid_map) const;
379
380 bool GenerateNestedCatalogMarkerChunk();
381 void CreateNestedCatalogMarkerDirent(const shash::Any &content_hash);
382
383 void UploadHistoryClosure(const upload::SpoolerResult &result,
384 Future<shash::Any> *hash);
385 bool UpdateUndoTags(PendingCatalog *root_catalog,
386 uint64_t revision,
387 time_t timestamp,
388 shash::Any *history_hash);
389
390 private:
391 unsigned int file_descriptor_limit_;
392 CatalogStatisticsList catalog_statistics_list_;
393 unsigned int catalog_count_;
394 atomic_int32 catalogs_processed_;
395 bool has_committed_new_revision_;
396
397 uid_t uid_;
398 gid_t gid_;
399
400 std::string temporary_directory_;
401 std::string nested_catalog_marker_tmp_path_;
402 static catalog::DirectoryEntry nested_catalog_marker_;
403
404 UniquePtr<manifest::Manifest> manifest_upstream_;
405 UniquePtr<history::SqliteHistory> history_upstream_;
406 catalog::Catalog const *root_catalog_;
407 UniquePtr<upload::Spooler> spooler_;
408 PendingCatalogMap pending_catalogs_;
409
410 StopWatch catalog_loading_stopwatch_;
411 StopWatch migration_stopwatch_;
412 };
413
414 } // namespace swissknife
415
416 #endif // CVMFS_SWISSKNIFE_MIGRATE_H_
417