1 |
|
|
/** |
2 |
|
|
* This file is part of the CernVM File System. |
3 |
|
|
*/ |
4 |
|
|
|
5 |
|
|
#ifndef CVMFS_SWISSKNIFE_MIGRATE_H_ |
6 |
|
|
#define CVMFS_SWISSKNIFE_MIGRATE_H_ |
7 |
|
|
|
8 |
|
|
#include "swissknife.h" |
9 |
|
|
|
10 |
|
|
#include <map> |
11 |
|
|
#include <string> |
12 |
|
|
#include <vector> |
13 |
|
|
|
14 |
|
|
#include "atomic.h" |
15 |
|
|
#include "catalog.h" |
16 |
|
|
#include "catalog_traversal.h" |
17 |
|
|
#include "hash.h" |
18 |
|
|
#include "history_sqlite.h" |
19 |
|
|
#include "logging.h" |
20 |
|
|
#include "manifest.h" |
21 |
|
|
#include "uid_map.h" |
22 |
|
|
#include "upload.h" |
23 |
|
|
#include "util/algorithm.h" |
24 |
|
|
#include "util/pointer.h" |
25 |
|
|
#include "util_concurrency.h" |
26 |
|
|
|
27 |
|
|
namespace catalog { |
28 |
|
|
class WritableCatalog; |
29 |
|
|
} |
30 |
|
|
|
31 |
|
|
namespace swissknife { |
32 |
|
|
|
33 |
|
|
class CommandMigrate : public Command { |
34 |
|
|
protected: |
35 |
|
|
struct CatalogStatistics { |
36 |
|
|
CatalogStatistics() |
37 |
|
|
: max_row_id(0) |
38 |
|
|
, entry_count(0) |
39 |
|
|
, hardlink_group_count(0) |
40 |
|
|
, aggregated_linkcounts(0) |
41 |
|
|
, migration_time(0.0) { } |
42 |
|
|
unsigned int max_row_id; |
43 |
|
|
unsigned int entry_count; |
44 |
|
|
|
45 |
|
|
unsigned int hardlink_group_count; |
46 |
|
|
unsigned int aggregated_linkcounts; |
47 |
|
|
|
48 |
|
|
double migration_time; |
49 |
|
|
|
50 |
|
|
std::string root_path; |
51 |
|
|
}; |
52 |
|
|
|
53 |
|
|
class CatalogStatisticsList : protected std::vector<CatalogStatistics>, |
54 |
|
|
public Lockable { |
55 |
|
|
friend class CommandMigrate; |
56 |
|
|
|
57 |
|
|
public: |
58 |
|
|
inline void Insert(const CatalogStatistics &statistics) { |
59 |
|
|
LockGuard<CatalogStatisticsList> lock(this); |
60 |
|
|
this->push_back(statistics); |
61 |
|
|
} |
62 |
|
|
}; |
63 |
|
|
|
64 |
|
|
public: |
65 |
|
|
struct PendingCatalog; |
66 |
|
|
typedef std::vector<PendingCatalog *> PendingCatalogList; |
67 |
|
|
struct PendingCatalog { |
68 |
|
|
explicit PendingCatalog(const catalog::Catalog *old_catalog = NULL) |
69 |
|
|
: success(false) |
70 |
|
|
, old_catalog(old_catalog) |
71 |
|
|
, new_catalog(NULL) |
72 |
|
|
, new_catalog_size(0) { } |
73 |
|
|
virtual ~PendingCatalog(); |
74 |
|
|
|
75 |
|
|
inline const std::string root_path() const { |
76 |
|
|
return old_catalog->mountpoint().ToString(); |
77 |
|
|
} |
78 |
|
|
inline bool IsRoot() const { return old_catalog->IsRoot(); } |
79 |
|
|
inline bool HasNew() const { return new_catalog != NULL; } |
80 |
|
|
|
81 |
|
|
inline bool HasChanges() const { |
82 |
|
|
return (new_catalog != NULL || |
83 |
|
|
old_catalog->database().GetModifiedRowCount() > 0); |
84 |
|
|
} |
85 |
|
|
|
86 |
|
|
inline shash::Any GetOldContentHash() const { |
87 |
|
|
return old_catalog->hash(); |
88 |
|
|
} |
89 |
|
|
|
90 |
|
|
bool success; |
91 |
|
|
|
92 |
|
|
const catalog::Catalog *old_catalog; |
93 |
|
|
catalog::WritableCatalog *new_catalog; |
94 |
|
|
|
95 |
|
|
PendingCatalogList nested_catalogs; |
96 |
|
|
Future<catalog::DirectoryEntry> root_entry; |
97 |
|
|
Future<catalog::DeltaCounters> nested_statistics; |
98 |
|
|
|
99 |
|
|
CatalogStatistics statistics; |
100 |
|
|
|
101 |
|
|
// Note: As soon as the `was_updated` future is set to 'true', both |
102 |
|
|
// `new_catalog_hash` and `new_catalog_size` are assumed to be set |
103 |
|
|
// accordingly. If it is set to 'false' they will be ignored. |
104 |
|
|
Future<bool> was_updated; |
105 |
|
|
shash::Any new_catalog_hash; |
106 |
|
|
size_t new_catalog_size; |
107 |
|
|
}; |
108 |
|
|
|
109 |
|
|
class PendingCatalogMap : public std::map<std::string, const PendingCatalog*>, |
110 |
|
|
public Lockable {}; |
111 |
|
|
|
112 |
|
|
template<class DerivedT> |
113 |
|
|
class AbstractMigrationWorker : public ConcurrentWorker<DerivedT> { |
114 |
|
|
public: |
115 |
|
|
typedef CommandMigrate::PendingCatalog* expected_data; |
116 |
|
|
typedef CommandMigrate::PendingCatalog* returned_data; |
117 |
|
|
|
118 |
|
|
struct worker_context { |
119 |
|
|
worker_context(const std::string &temporary_directory, |
120 |
|
|
const bool collect_catalog_statistics) : |
121 |
|
|
temporary_directory(temporary_directory), |
122 |
|
|
collect_catalog_statistics(collect_catalog_statistics) {} |
123 |
|
|
const std::string temporary_directory; |
124 |
|
|
const bool collect_catalog_statistics; |
125 |
|
|
}; |
126 |
|
|
|
127 |
|
|
public: |
128 |
|
|
explicit AbstractMigrationWorker(const worker_context *context); |
129 |
|
|
virtual ~AbstractMigrationWorker(); |
130 |
|
|
|
131 |
|
|
void operator()(const expected_data &data); |
132 |
|
|
|
133 |
|
|
protected: |
134 |
|
|
bool RunMigration(PendingCatalog *data) const { return false; } |
135 |
|
|
|
136 |
|
|
bool UpdateNestedCatalogReferences(PendingCatalog *data) const; |
137 |
|
|
bool UpdateCatalogMetadata(PendingCatalog *data) const; |
138 |
|
|
bool CleanupNestedCatalogs(PendingCatalog *data) const; |
139 |
|
|
bool CollectAndAggregateStatistics(PendingCatalog *data) const; |
140 |
|
|
|
141 |
|
|
catalog::WritableCatalog* |
142 |
|
|
GetWritable(const catalog::Catalog *catalog) const; |
143 |
|
|
|
144 |
|
|
protected: |
145 |
|
|
const std::string temporary_directory_; |
146 |
|
|
const bool collect_catalog_statistics_; |
147 |
|
|
|
148 |
|
|
StopWatch migration_stopwatch_; |
149 |
|
|
}; |
150 |
|
|
|
151 |
|
|
class MigrationWorker_20x : |
152 |
|
|
public AbstractMigrationWorker<MigrationWorker_20x> |
153 |
|
|
{ |
154 |
|
|
friend class AbstractMigrationWorker<MigrationWorker_20x>; |
155 |
|
|
protected: |
156 |
|
|
static const float kSchema; |
157 |
|
|
static const unsigned kSchemaRevision; |
158 |
|
|
|
159 |
|
|
public: |
160 |
|
|
struct worker_context : |
161 |
|
|
AbstractMigrationWorker<MigrationWorker_20x>::worker_context |
162 |
|
|
{ |
163 |
|
|
worker_context(const std::string &temporary_directory, |
164 |
|
|
const bool collect_catalog_statistics, |
165 |
|
|
const bool fix_nested_catalog_transitions, |
166 |
|
|
const bool analyze_file_linkcounts, |
167 |
|
|
const uid_t uid, |
168 |
|
|
const gid_t gid) |
169 |
|
|
: AbstractMigrationWorker<MigrationWorker_20x>::worker_context( |
170 |
|
|
temporary_directory, collect_catalog_statistics) |
171 |
|
|
, fix_nested_catalog_transitions(fix_nested_catalog_transitions) |
172 |
|
|
, analyze_file_linkcounts(analyze_file_linkcounts) |
173 |
|
|
, uid(uid) |
174 |
|
|
, gid(gid) { } |
175 |
|
|
const bool fix_nested_catalog_transitions; |
176 |
|
|
const bool analyze_file_linkcounts; |
177 |
|
|
const uid_t uid; |
178 |
|
|
const gid_t gid; |
179 |
|
|
}; |
180 |
|
|
|
181 |
|
|
public: |
182 |
|
|
explicit MigrationWorker_20x(const worker_context *context); |
183 |
|
|
|
184 |
|
|
protected: |
185 |
|
|
bool RunMigration(PendingCatalog *data) const; |
186 |
|
|
|
187 |
|
|
bool CreateNewEmptyCatalog(PendingCatalog *data) const; |
188 |
|
|
bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const; |
189 |
|
|
bool AttachOldCatalogDatabase(PendingCatalog *data) const; |
190 |
|
|
bool StartDatabaseTransaction(PendingCatalog *data) const; |
191 |
|
|
bool MigrateFileMetadata(PendingCatalog *data) const; |
192 |
|
|
bool AnalyzeFileLinkcounts(PendingCatalog *data) const; |
193 |
|
|
bool RemoveDanglingNestedMountpoints(PendingCatalog *data) const; |
194 |
|
|
bool MigrateNestedCatalogMountPoints(PendingCatalog *data) const; |
195 |
|
|
bool FixNestedCatalogTransitionPoints(PendingCatalog *data) const; |
196 |
|
|
bool GenerateCatalogStatistics(PendingCatalog *data) const; |
197 |
|
|
bool FindRootEntryInformation(PendingCatalog *data) const; |
198 |
|
|
bool CommitDatabaseTransaction(PendingCatalog *data) const; |
199 |
|
|
bool DetachOldCatalogDatabase(PendingCatalog *data) const; |
200 |
|
|
|
201 |
|
|
private: |
202 |
|
|
const bool fix_nested_catalog_transitions_; |
203 |
|
|
const bool analyze_file_linkcounts_; |
204 |
|
|
const uid_t uid_; |
205 |
|
|
const gid_t gid_; |
206 |
|
|
}; |
207 |
|
|
|
208 |
|
|
class MigrationWorker_217 : |
209 |
|
|
public AbstractMigrationWorker<MigrationWorker_217> |
210 |
|
|
{ |
211 |
|
|
friend class AbstractMigrationWorker<MigrationWorker_217>; |
212 |
|
|
|
213 |
|
|
public: |
214 |
|
|
explicit MigrationWorker_217(const worker_context *context); |
215 |
|
|
|
216 |
|
|
protected: |
217 |
|
|
bool RunMigration(PendingCatalog *data) const; |
218 |
|
|
|
219 |
|
|
bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const; |
220 |
|
|
bool StartDatabaseTransaction(PendingCatalog *data) const; |
221 |
|
|
bool GenerateNewStatisticsCounters(PendingCatalog *data) const; |
222 |
|
|
bool UpdateCatalogSchema(PendingCatalog *data) const; |
223 |
|
|
bool CommitDatabaseTransaction(PendingCatalog *data) const; |
224 |
|
|
}; |
225 |
|
|
|
226 |
|
|
class ChownMigrationWorker : |
227 |
|
|
public AbstractMigrationWorker<ChownMigrationWorker> |
228 |
|
|
{ |
229 |
|
|
friend class AbstractMigrationWorker<ChownMigrationWorker>; |
230 |
|
|
public: |
231 |
|
|
struct worker_context : |
232 |
|
|
AbstractMigrationWorker<ChownMigrationWorker>::worker_context |
233 |
|
|
{ |
234 |
|
|
worker_context(const std::string &temporary_directory, |
235 |
|
|
const bool collect_catalog_statistics, |
236 |
|
|
const UidMap &uid_map, |
237 |
|
|
const GidMap &gid_map) |
238 |
|
|
: AbstractMigrationWorker<ChownMigrationWorker>::worker_context( |
239 |
|
|
temporary_directory, collect_catalog_statistics) |
240 |
|
|
, uid_map(uid_map) |
241 |
|
|
, gid_map(gid_map) { } |
242 |
|
|
const UidMap &uid_map; |
243 |
|
|
const GidMap &gid_map; |
244 |
|
|
}; |
245 |
|
|
|
246 |
|
|
public: |
247 |
|
|
explicit ChownMigrationWorker(const worker_context *context); |
248 |
|
|
|
249 |
|
|
protected: |
250 |
|
|
bool RunMigration(PendingCatalog *data) const; |
251 |
|
|
bool ApplyPersonaMappings(PendingCatalog *data) const; |
252 |
|
|
|
253 |
|
|
private: |
254 |
|
|
template <class MapT> |
255 |
|
|
std::string GenerateMappingStatement(const MapT &map, |
256 |
|
|
const std::string &column) const; |
257 |
|
|
|
258 |
|
|
private: |
259 |
|
|
const std::string uid_map_statement_; |
260 |
|
|
const std::string gid_map_statement_; |
261 |
|
|
}; |
262 |
|
|
|
263 |
|
|
class HardlinkRemovalMigrationWorker : |
264 |
|
|
public AbstractMigrationWorker<HardlinkRemovalMigrationWorker> |
265 |
|
|
{ |
266 |
|
|
friend class AbstractMigrationWorker<HardlinkRemovalMigrationWorker>; |
267 |
|
|
|
268 |
|
|
public: |
269 |
|
|
explicit HardlinkRemovalMigrationWorker(const worker_context *context) : |
270 |
|
|
AbstractMigrationWorker<HardlinkRemovalMigrationWorker>(context) {} |
271 |
|
|
|
272 |
|
|
protected: |
273 |
|
|
bool RunMigration(PendingCatalog *data) const; |
274 |
|
|
|
275 |
|
|
bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const; |
276 |
|
|
bool BreakUpHardlinks(PendingCatalog *data) const; |
277 |
|
|
}; |
278 |
|
|
|
279 |
|
|
public: |
280 |
|
|
CommandMigrate(); |
281 |
|
|
~CommandMigrate() { } |
282 |
|
|
virtual std::string GetName() const { return "migrate"; } |
283 |
|
|
virtual std::string GetDescription() const { |
284 |
|
|
return "CernVM-FS catalog repository migration \n" |
285 |
|
|
"This command migrates the whole catalog structure of a given repository"; |
286 |
|
|
} |
287 |
|
|
virtual ParameterList GetParams() const; |
288 |
|
|
|
289 |
|
|
int Main(const ArgumentList &args); |
290 |
|
|
|
291 |
|
|
static void FixNestedCatalogTransitionPoint( |
292 |
|
|
const catalog::DirectoryEntry &nested_root, |
293 |
|
|
catalog::DirectoryEntry *mountpoint); |
294 |
|
|
static const catalog::DirectoryEntry& GetNestedCatalogMarkerDirent(); |
295 |
|
|
|
296 |
|
|
protected: |
297 |
|
|
template <class ObjectFetcherT> |
298 |
|
|
bool LoadCatalogs(const shash::Any &manual_root_hash, |
299 |
|
|
ObjectFetcherT *object_fetcher) |
300 |
|
|
{ |
301 |
|
|
ObjectFetcherFailures::Failures retval; |
302 |
|
|
retval = object_fetcher->FetchManifest(&manifest_upstream_); |
303 |
|
|
if (retval != ObjectFetcherFailures::kFailOk) { |
304 |
|
|
LogCvmfs(kLogCvmfs, kLogStdout, "could not get manifest (%d)", retval); |
305 |
|
|
return false; |
306 |
|
|
} |
307 |
|
|
|
308 |
|
|
if (!manifest_upstream_->history().IsNull()) { |
309 |
|
|
retval = object_fetcher->FetchHistory( |
310 |
|
|
&history_upstream_, manifest_upstream_->history()); |
311 |
|
|
if (retval != ObjectFetcherFailures::kFailOk) { |
312 |
|
|
LogCvmfs(kLogCvmfs, kLogStdout, "could not get history (%d)", retval); |
313 |
|
|
return false; |
314 |
|
|
} |
315 |
|
|
} |
316 |
|
|
|
317 |
|
|
typename CatalogTraversal<ObjectFetcherT>::Parameters params; |
318 |
|
|
const bool generate_full_catalog_tree = true; |
319 |
|
|
params.no_close = generate_full_catalog_tree; |
320 |
|
|
params.object_fetcher = object_fetcher; |
321 |
|
|
CatalogTraversal<ObjectFetcherT> traversal(params); |
322 |
|
|
traversal.RegisterListener(&CommandMigrate::CatalogCallback, this); |
323 |
|
|
|
324 |
|
|
if (manual_root_hash.IsNull()) |
325 |
|
|
return traversal.Traverse(); |
326 |
|
|
return traversal.Traverse(manual_root_hash); |
327 |
|
|
} |
328 |
|
|
|
329 |
|
|
void CatalogCallback( |
330 |
|
|
const CatalogTraversalData<catalog::WritableCatalog> &data); |
331 |
|
|
void MigrationCallback(PendingCatalog *const &data); |
332 |
|
|
void UploadCallback(const upload::SpoolerResult &result); |
333 |
|
|
|
334 |
|
|
void PrintStatusMessage(const PendingCatalog *catalog, |
335 |
|
|
const shash::Any &content_hash, |
336 |
|
|
const std::string &message); |
337 |
|
|
|
338 |
|
|
template <class MigratorT> |
339 |
|
|
bool DoMigrationAndCommit(const std::string &manifest_path, |
340 |
|
|
typename MigratorT::worker_context *context); |
341 |
|
|
|
342 |
|
|
template <class MigratorT> |
343 |
|
|
void ConvertCatalogsRecursively(PendingCatalog *catalog, MigratorT *migrator); |
344 |
|
|
bool RaiseFileDescriptorLimit() const; |
345 |
|
|
bool ConfigureSQLite() const; |
346 |
|
|
void AnalyzeCatalogStatistics() const; |
347 |
|
|
bool ReadPersona(const std::string &uid, const std::string &gid); |
348 |
|
|
bool ReadPersonaMaps(const std::string &uid_map_path, |
349 |
|
|
const std::string &gid_map_path, |
350 |
|
|
UidMap *uid_map, |
351 |
|
|
GidMap *gid_map) const; |
352 |
|
|
|
353 |
|
|
bool GenerateNestedCatalogMarkerChunk(); |
354 |
|
|
void CreateNestedCatalogMarkerDirent(const shash::Any &content_hash); |
355 |
|
|
|
356 |
|
|
void UploadHistoryClosure(const upload::SpoolerResult &result, |
357 |
|
|
Future<shash::Any> *hash); |
358 |
|
|
bool UpdateUndoTags(PendingCatalog *root_catalog, |
359 |
|
|
unsigned revision, |
360 |
|
|
time_t timestamp, |
361 |
|
|
shash::Any *history_hash); |
362 |
|
|
|
363 |
|
|
private: |
364 |
|
|
unsigned int file_descriptor_limit_; |
365 |
|
|
CatalogStatisticsList catalog_statistics_list_; |
366 |
|
|
unsigned int catalog_count_; |
367 |
|
|
atomic_int32 catalogs_processed_; |
368 |
|
|
bool has_committed_new_revision_; |
369 |
|
|
|
370 |
|
|
uid_t uid_; |
371 |
|
|
gid_t gid_; |
372 |
|
|
|
373 |
|
|
std::string temporary_directory_; |
374 |
|
|
std::string nested_catalog_marker_tmp_path_; |
375 |
|
|
static catalog::DirectoryEntry nested_catalog_marker_; |
376 |
|
|
|
377 |
|
|
UniquePtr<manifest::Manifest> manifest_upstream_; |
378 |
|
|
UniquePtr<history::SqliteHistory> history_upstream_; |
379 |
|
|
catalog::Catalog const* root_catalog_; |
380 |
|
|
UniquePtr<upload::Spooler> spooler_; |
381 |
|
|
PendingCatalogMap pending_catalogs_; |
382 |
|
|
|
383 |
|
|
StopWatch catalog_loading_stopwatch_; |
384 |
|
|
StopWatch migration_stopwatch_; |
385 |
|
|
}; |
386 |
|
|
|
387 |
|
|
} // namespace swissknife |
388 |
|
|
|
389 |
|
|
#endif // CVMFS_SWISSKNIFE_MIGRATE_H_ |