10 #include <sys/resource.h>
24 namespace swissknife {
28 CommandMigrate::CommandMigrate() :
29 file_descriptor_limit_(8192),
31 has_committed_new_revision_(false),
43 "migration base version ( 2.0.x | 2.1.7 | chown | hardlink | bulkhash | "
46 "repository URL (absolute local path or remote URL)"));
50 "temporary directory for catalog decompress"));
52 "user id to be used for this repository"));
54 "group id to be used for this repository"));
64 "enable collection of catalog statistics"));
69 static void Error(
const std::string &message) {
74 static void Error(
const std::string &message,
76 const std::string err_msg = message +
"\n"
82 static void Error(
const std::string &message,
85 const std::string err_msg =
89 Error(err_msg, catalog);
95 const std::string &migration_base = *args.find(
'v')->second;
96 const std::string &repo_url = *args.find(
'r')->second;
97 const std::string &
spooler = *args.find(
'u')->second;
98 const std::string &manifest_path = *args.find(
'o')->second;
99 const std::string &tmp_dir = *args.find(
't')->second;
100 const std::string &uid = (args.count(
'p') > 0) ?
101 *args.find(
'p')->second :
103 const std::string &gid = (args.count(
'g') > 0) ?
104 *args.find(
'g')->second :
106 const std::string &repo_name = (args.count(
'n') > 0) ?
107 *args.find(
'n')->second :
109 const std::string &repo_keys = (args.count(
'k') > 0) ?
110 *args.find(
'k')->second :
112 const std::string &uid_map_path = (args.count(
'i') > 0) ?
113 *args.find(
'i')->second :
115 const std::string &gid_map_path = (args.count(
'j') > 0) ?
116 *args.find(
'j')->second :
118 const bool fix_transition_points = (args.count(
'f') > 0);
119 const bool analyze_file_linkcounts = (args.count(
'l') == 0);
120 const bool collect_catalog_statistics = (args.count(
's') > 0);
121 if (args.count(
'h') > 0) {
128 Error(
"Failed to raise file descriptor limits");
134 Error(
"Failed to preconfigure SQLite library");
141 spooler_ = upload::Spooler::Construct(spooler_definition);
143 Error(
"Failed to create upstream Spooler.");
152 bool loading_successful =
false;
156 const bool follow_redirects =
false;
157 const string proxy = (args.count(
'@') > 0) ? *args.find(
'@')->second :
"";
164 ObjectFetcher fetcher(repo_name,
170 loading_successful =
LoadCatalogs(manual_root_hash, &fetcher);
173 ObjectFetcher fetcher(repo_url, tmp_dir);
174 loading_successful =
LoadCatalogs(manual_root_hash, &fetcher);
178 if (!loading_successful) {
179 Error(
"Failed to load catalog tree");
187 bool migration_succeeded =
false;
188 if (migration_base ==
"2.0.x") {
195 Error(
"Failed to create a nested catalog marker.");
202 collect_catalog_statistics,
203 fix_transition_points,
204 analyze_file_linkcounts,
207 migration_succeeded =
208 DoMigrationAndCommit<MigrationWorker_20x>(manifest_path, &context);
209 }
else if (migration_base ==
"2.1.7") {
211 collect_catalog_statistics);
212 migration_succeeded =
213 DoMigrationAndCommit<MigrationWorker_217>(manifest_path, &context);
214 }
else if (migration_base ==
"chown") {
217 if (!
ReadPersonaMaps(uid_map_path, gid_map_path, &uid_map, &gid_map)) {
218 Error(
"Failed to read UID and/or GID map");
222 collect_catalog_statistics,
225 migration_succeeded =
226 DoMigrationAndCommit<ChownMigrationWorker>(manifest_path, &context);
227 }
else if (migration_base ==
"hardlink") {
228 HardlinkRemovalMigrationWorker::worker_context
230 migration_succeeded =
231 DoMigrationAndCommit<HardlinkRemovalMigrationWorker>(manifest_path,
233 }
else if (migration_base ==
"bulkhash") {
234 BulkhashRemovalMigrationWorker::worker_context
236 migration_succeeded =
237 DoMigrationAndCommit<BulkhashRemovalMigrationWorker>(manifest_path,
239 }
else if (migration_base ==
"stats") {
240 StatsMigrationWorker::worker_context context(
242 migration_succeeded =
243 DoMigrationAndCommit<StatsMigrationWorker>(manifest_path, &context);
245 const std::string err_msg =
"Unknown migration base: " + migration_base;
251 if (!migration_succeeded) {
252 Error(
"Migration failed!");
268 const std::string &gid) {
270 Error(
"Please provide a user ID");
274 Error(
"Please provide a group ID");
286 const std::string &gid_map_path,
288 GidMap *gid_map)
const {
289 if (!uid_map->Read(uid_map_path) || !uid_map->IsValid()) {
290 Error(
"Failed to read UID map");
294 if (!gid_map->Read(gid_map_path) || !gid_map->IsValid()) {
295 Error(
"Failed to read GID map");
299 if (uid_map->RuleCount() == 0 && !uid_map->HasDefault()) {
300 Error(
"UID map appears to be empty");
304 if (gid_map->RuleCount() == 0 && !gid_map->HasDefault()) {
305 Error(
"GID map appears to be empty");
335 string filename_new = filename_old +
".new";
337 if (!retval)
return false;
340 history->TakeDatabaseFileOwnership();
346 if (!retval)
return false;
359 retval = history->Insert(tag_trunk_previous);
360 if (!retval)
return false;
361 retval = history->Insert(tag_trunk);
362 if (!retval)
return false;
366 history->DropDatabaseFileOwnership();
370 upload::Spooler::CallbackPtr callback =
spooler_->RegisterListener(
372 spooler_->ProcessHistory(filename_new);
374 spooler_->UnregisterListener(callback);
375 unlink(filename_new.c_str());
376 *history_hash = history_hash_new.
Get();
377 if (history_hash->IsNull()) {
378 Error(
"failed to upload tag database");
386 template <
class MigratorT>
388 const std::string &manifest_path,
389 typename MigratorT::worker_context *context
396 Error(
"Failed to initialize worker migration system.");
416 "Catalog Migration finished with %d errors.", errors);
419 "\nCatalog Migration produced errors\nAborting...");
425 "\nCommitting migrated repository revision...");
440 "Updating repository tag database... ");
446 Error(
"Updating tag database failed.\nAborting...");
453 if (!manifest.
Export(manifest_path)) {
454 Error(
"Manifest export failed.\nAborting...");
460 "\nNo catalogs migrated, skipping the commit...");
472 std::string tree_indent;
473 std::string hash_string;
476 for (
unsigned int i = 1; i < data.
tree_level; ++i) {
477 tree_indent +=
"\u2502 ";
481 tree_indent +=
"\u251C\u2500 ";
504 Error(
"Catalog migration failed! Aborting...");
528 if (new_catalog_size <= 0) {
529 Error(
"Failed to get uncompressed file size of catalog!", data);
545 Error(
"Failed to upload file " + path +
"\nAborting...");
552 unlink(path.c_str());
583 const std::string &message) {
595 template <
class MigratorT>
597 MigratorT *migrator) {
601 catalog::CatalogList::const_iterator i = nested_catalogs.begin();
602 catalog::CatalogList::const_iterator iend = nested_catalogs.end();
604 for (; i != iend; ++i) {
611 migrator->Schedule(catalog);
617 memset(&rpl, 0,
sizeof(rpl));
618 getrlimit(RLIMIT_NOFILE, &rpl);
623 const bool retval = setrlimit(RLIMIT_NOFILE, &rpl);
633 int retval = sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
634 return (retval == SQLITE_OK);
640 unsigned int aggregated_entry_count = 0;
641 unsigned int aggregated_max_row_id = 0;
642 unsigned int aggregated_hardlink_count = 0;
643 unsigned int aggregated_linkcounts = 0;
644 double aggregated_migration_time = 0.0;
648 for (; i != iend; ++i) {
649 aggregated_entry_count += i->entry_count;
650 aggregated_max_row_id += i->max_row_id;
651 aggregated_hardlink_count += i->hardlink_group_count;
652 aggregated_linkcounts += i->aggregated_linkcounts;
653 aggregated_migration_time += i->migration_time;
657 assert(aggregated_max_row_id > 0);
658 const unsigned int unused_inodes =
659 aggregated_max_row_id - aggregated_entry_count;
661 (
static_cast<float>(unused_inodes) /
662 static_cast<float>(aggregated_max_row_id)) * 100.0f;
664 "Allocated Inodes: %d\n"
665 " Unused Inodes: %d\n"
666 " Percentage of wasted Inodes: %.1f%%\n",
667 aggregated_entry_count, aggregated_max_row_id, unused_inodes, ratio);
670 const float average_linkcount = (aggregated_hardlink_count > 0)
671 ? aggregated_linkcounts /
672 aggregated_hardlink_count
675 "Average Linkcount per Group: %.1f\n",
676 aggregated_hardlink_count, average_linkcount);
679 const double average_migration_time =
680 aggregated_migration_time /
static_cast<double>(number_of_catalogs);
682 "Average Migration Time: %.2fs\n"
683 "Overall Migration Time: %.2fs\n"
684 "Aggregated Migration Time: %.2fs\n",
686 average_migration_time,
688 aggregated_migration_time);
703 template<
class DerivedT>
707 , collect_catalog_statistics_(context->collect_catalog_statistics)
711 template<
class DerivedT>
715 template<
class DerivedT>
719 const bool success =
static_cast<DerivedT*
>(
this)->RunMigration(data) &&
720 UpdateNestedCatalogReferences(data) &&
721 UpdateCatalogMetadata(data) &&
722 CollectAndAggregateStatistics(data) &&
723 CleanupNestedCatalogs(data);
739 template<
class DerivedT>
748 "INSERT OR REPLACE INTO nested_catalogs (path, sha1, size) "
749 " VALUES (:path, :sha1, :size);");
756 for (; i != iend; ++i) {
763 const std::string &root_path = nested_catalog->
root_path();
769 add_nested_catalog.
BindText(1, root_path) &&
771 add_nested_catalog.
BindInt64(3, catalog_size) &&
774 Error(
"Failed to add nested catalog link", add_nested_catalog, data);
777 add_nested_catalog.
Reset();
784 template<
class DerivedT>
806 template<
class DerivedT>
810 if (!collect_catalog_statistics_) {
821 "SELECT COUNT(*), MAX(rowid) FROM catalog;");
824 Error(
"Failed to count entries in catalog", wasted_inodes, data);
827 const unsigned int entry_count = wasted_inodes.
RetrieveInt64(0);
828 const unsigned int max_row_id = wasted_inodes.
RetrieveInt64(1);
839 template<
class DerivedT>
847 for (; i != iend; ++i) {
866 template<
class DerivedT>
881 , fix_nested_catalog_transitions_(context->fix_nested_catalog_transitions)
882 , analyze_file_linkcounts_(context->analyze_file_linkcounts)
884 ,
gid_(context->gid) { }
895 return CreateNewEmptyCatalog(data) &&
896 CheckDatabaseSchemaCompatibility(data) &&
897 AttachOldCatalogDatabase(data) &&
898 StartDatabaseTransaction(data) &&
899 MigrateFileMetadata(data) &&
900 MigrateNestedCatalogMountPoints(data) &&
901 FixNestedCatalogTransitionPoints(data) &&
902 RemoveDanglingNestedMountpoints(data) &&
903 GenerateCatalogStatistics(data) &&
904 FindRootEntryInformation(data) &&
905 CommitDatabaseTransaction(data) &&
906 DetachOldCatalogDatabase(data);
912 const string root_path = data->
root_path();
915 const string clg_db_path =
917 if (clg_db_path.empty()) {
918 Error(
"Failed to create temporary file for the new catalog database.");
921 const bool volatile_content =
false;
930 Error(
"Failed to create database for new catalog");
931 unlink(clg_db_path.c_str());
940 if (writable_catalog == NULL) {
941 Error(
"Failed to open database for new catalog");
942 unlink(clg_db_path.c_str());
968 Error(
"Failed to meet database requirements for migration.", data);
982 "ATTACH '" + old_catalog.
filename() +
"' AS old;");
983 bool retval = sql_attach_new.
Execute();
990 Error(
"Failed to attach database of old catalog", sql_attach_new, data);
1025 "CREATE TEMPORARY TABLE hardlinks "
1026 " ( hardlink_group_id INTEGER PRIMARY KEY AUTOINCREMENT, "
1028 " linkcount INTEGER, "
1029 " CONSTRAINT unique_inode UNIQUE (inode) );");
1030 retval = sql_create_hardlinks_table.
Execute();
1032 Error(
"Failed to create temporary hardlink analysis table",
1033 sql_create_hardlinks_table, data);
1043 "CREATE TEMPORARY TABLE dir_linkcounts "
1044 " ( inode INTEGER PRIMARY KEY, "
1045 " linkcount INTEGER );");
1046 retval = sql_create_linkcounts_table.
Execute();
1048 Error(
"Failed to create tmeporary directory linkcount analysis table",
1049 sql_create_linkcounts_table, data);
1055 if (analyze_file_linkcounts_) {
1056 retval = AnalyzeFileLinkcounts(data);
1073 "INSERT INTO dir_linkcounts "
1074 " SELECT c1.inode as inode, "
1075 " SUM(IFNULL(MIN(c2.inode,1),0)) + 2 as linkcount "
1076 " FROM old.catalog as c1 "
1077 " LEFT JOIN old.catalog as c2 "
1078 " ON c2.parent_1 = c1.md5path_1 AND "
1079 " c2.parent_2 = c1.md5path_2 AND "
1080 " c2.flags & :flag_dir_1 "
1081 " WHERE c1.flags & :flag_dir_2 "
1082 " GROUP BY c1.inode;");
1088 Error(
"Failed to analyze directory specific linkcounts",
1089 sql_dir_linkcounts, data);
1090 if (sql_dir_linkcounts.
GetLastError() == SQLITE_CONSTRAINT) {
1091 Error(
"Obviously your catalogs are corrupted, since we found a directory"
1092 "inode that is a file inode at the same time!");
1104 "INSERT INTO catalog "
1105 " SELECT md5path_1, md5path_2, "
1106 " parent_1, parent_2, "
1107 " IFNULL(hardlink_group_id, 0) << 32 | "
1108 " COALESCE(hardlinks.linkcount, dir_linkcounts.linkcount, 1) "
1110 " hash, size, mode, mtime, "
1111 " flags, name, symlink, "
1115 " FROM old.catalog "
1116 " LEFT JOIN hardlinks "
1117 " ON catalog.inode = hardlinks.inode "
1118 " LEFT JOIN dir_linkcounts "
1119 " ON catalog.inode = dir_linkcounts.inode;");
1122 migrate_file_meta_data.
Execute();
1124 Error(
"Failed to migrate the file system meta data",
1125 migrate_file_meta_data, data);
1136 const std::string root_path = data->
root_path();
1137 const std::string file_path = root_path +
1143 retval = insert_nested_marker.
BindPathHash(path_hash) &&
1145 insert_nested_marker.
BindDirent(nested_marker) &&
1147 insert_nested_marker.
Execute();
1149 Error(
"Failed to insert nested catalog marker into new nested catalog.",
1150 insert_nested_marker, data);
1161 "INSERT OR REPLACE INTO properties "
1162 " SELECT key, value "
1163 " FROM old.properties "
1164 " WHERE key != 'schema';");
1165 retval = copy_properties.
Execute();
1167 Error(
"Failed to migrate the properties table.", copy_properties, data);
1196 "CREATE TEMPORARY TABLE hl_scratch AS "
1197 " SELECT c1.inode AS inode, c1.md5path_1, c1.md5path_2, "
1198 " c1.parent_1 as c1p1, c1.parent_2 as c1p2, "
1199 " c2.parent_1 as c2p1, c2.parent_2 as c2p2 "
1200 " FROM old.catalog AS c1 "
1201 " INNER JOIN old.catalog AS c2 "
1202 " ON c1.inode == c2.inode AND "
1203 " (c1.md5path_1 != c2.md5path_1 OR "
1204 " c1.md5path_2 != c2.md5path_2);");
1205 retval = sql_create_hardlinks_scratch_table.
Execute();
1207 Error(
"Failed to create temporary scratch table for hardlink analysis",
1208 sql_create_hardlinks_scratch_table, data);
1217 "INSERT INTO hardlinks (inode, linkcount)"
1218 " SELECT inode, count(*) as linkcount "
1224 " SELECT DISTINCT hl.inode, hl.md5path_1, hl.md5path_2 "
1231 " SELECT inode, count(*) AS cnt "
1239 " SELECT DISTINCT inode,c1p1,c1p1,c2p1,c2p2 "
1240 " FROM hl_scratch AS hl "
1245 " ) AS supported_hardlinks "
1246 " LEFT JOIN hl_scratch AS hl "
1247 " ON supported_hardlinks.inode = hl.inode "
1249 " GROUP BY inode;");
1250 retval = fill_linkcount_table_for_files.
Execute();
1252 Error(
"Failed to analyze hardlink relationships for files.",
1253 fill_linkcount_table_for_files, data);
1260 "DROP TABLE hl_scratch;");
1261 retval = drop_hardlink_scratch_space.
Execute();
1263 Error(
"Failed to remove file linkcount analysis scratch table",
1264 drop_hardlink_scratch_space, data);
1269 if (collect_catalog_statistics_) {
1271 "SELECT count(*), sum(linkcount) FROM hardlinks;");
1272 retval = count_hardlinks.
FetchRow();
1274 Error(
"Failed to count the generated file hardlinks for statistics",
1275 count_hardlinks, data);
1297 "SET hardlinks = :linkcount "
1298 "WHERE md5path_1 = :md5_1 AND md5path_2 = :md5_2;");
1303 PendingCatalogList::const_iterator iend = data->
nested_catalogs.end();
1304 for (; i != iend; ++i) {
1308 const string &root_path = nested_catalog->
root_path();
1316 update_mntpnt_linkcount.
BindMd5(2, 3, mountpoint_hash) &&
1317 update_mntpnt_linkcount.
Execute();
1319 Error(
"Failed to update linkcount of nested catalog mountpoint",
1320 update_mntpnt_linkcount, data);
1323 update_mntpnt_linkcount.
Reset();
1334 if (!fix_nested_catalog_transitions_) {
1349 PendingCatalogList::const_iterator iend = data->
nested_catalogs.end();
1350 for (; i != iend; ++i) {
1355 const string &nested_root_path = nested_catalog->
root_path();
1357 shash::Md5(nested_root_path.data(), nested_root_path.size());
1360 retval = lookup_mountpoint.
BindPathHash(mountpoint_path_hash) &&
1363 Error(
"Failed to fetch nested catalog mountpoint to check for compatible"
1364 "transition points", lookup_mountpoint, data);
1370 lookup_mountpoint.
Reset();
1374 mountpoint_entry.
CompareTo(nested_root_entry);
1379 assert(diffs & Difference::kNestedCatalogTransitionFlags);
1380 assert((diffs & Difference::kName) == 0);
1384 if ((diffs ^ Difference::kNestedCatalogTransitionFlags) != 0) {
1387 if ((diffs & Difference::kChecksum) ||
1388 (diffs & Difference::kLinkcount) ||
1389 (diffs & Difference::kSymlink) ||
1390 (diffs & Difference::kChunkedFileFlag) )
1392 Error(
"Found an irreparable mismatch in a nested catalog transition "
1393 "point at '" + nested_root_path +
"'\nAborting...\n");
1399 nested_root_entry, &mountpoint_entry);
1402 retval = update_directory_entry.
BindPathHash(mountpoint_path_hash) &&
1403 update_directory_entry.
BindDirent(mountpoint_entry) &&
1404 update_directory_entry.
Execute();
1406 Error(
"Failed to save resynchronized nested catalog mountpoint into "
1407 "catalog database", update_directory_entry, data);
1410 update_directory_entry.
Reset();
1415 "NOTE: fixed incompatible nested catalog transition point at: "
1416 "'%s' ", nested_root_path.c_str());
1433 mountpoint->
uid_ = nested_root.
uid_;
1434 mountpoint->
gid_ = nested_root.
gid_;
1445 bool retval =
false;
1449 typedef std::map<shash::Md5, catalog::Catalog::NestedCatalog>
1451 const NestedCatalogList& nested_clgs =
1453 NestedCatalogList::const_iterator i = nested_clgs.begin();
1454 const NestedCatalogList::const_iterator iend = nested_clgs.end();
1455 NestedCatalogMap nested_catalog_path_hashes;
1456 for (; i != iend; ++i) {
1459 nested_catalog_path_hashes[hash] = *i;
1467 std::vector<catalog::DirectoryEntry> todo_dirent;
1468 std::vector<shash::Md5> todo_hash;
1472 while (sql_dangling_mountpoints.
FetchRow()) {
1480 const NestedCatalogMap::const_iterator nested_catalog =
1481 nested_catalog_path_hashes.find(path_hash);
1482 if (nested_catalog != nested_catalog_path_hashes.end()) {
1484 "WARNING: found a non-empty nested catalog mountpoint under "
1485 "'%s'", nested_catalog->second.mountpoint.c_str());
1491 todo_dirent.push_back(dangling_mountpoint);
1492 todo_hash.push_back(path_hash);
1495 for (
unsigned i = 0; i < todo_dirent.size(); ++i) {
1496 retval = save_updated_mountpoint.
BindPathHash(todo_hash[i]) &&
1497 save_updated_mountpoint.
BindDirent(todo_dirent[i]) &&
1498 save_updated_mountpoint.
Execute() &&
1499 save_updated_mountpoint.
Reset();
1501 Error(
"Failed to remove dangling nested catalog mountpoint entry in "
1502 "catalog", save_updated_mountpoint, data);
1508 "mountpoint entry called: '%s' ",
1509 todo_dirent[i].name().c_str());
1527 Error(
"Failed to create temp file for nested catalog marker dummy.");
1560 bool retval =
false;
1568 PendingCatalogList::const_iterator iend = data->
nested_catalogs.end();
1569 for (; i != iend; ++i) {
1578 "SELECT count(*) FROM catalog "
1579 " WHERE flags & :flag_file "
1580 " AND NOT flags & :flag_link;");
1582 "SELECT count(*) FROM catalog WHERE flags & :flag_link;");
1584 "SELECT count(*) FROM catalog WHERE flags & :flag_dir;");
1586 "SELECT sum(size) FROM catalog WHERE flags & :flag_file "
1587 " AND NOT flags & :flag_link");
1595 Error(
"Failed to count regular files.", count_regular_files, data);
1602 Error(
"Failed to count symlinks.", count_symlinks, data);
1609 Error(
"Failed to count directories.", count_directories, data);
1617 Error(
"Failed to aggregate the file sizes.", aggregate_file_size, data);
1644 std::string root_path = data->
root_path();
1648 retval = lookup_root_entry.BindPathHash(root_path_hash) &&
1649 lookup_root_entry.FetchRow();
1651 Error(
"Failed to retrieve root directory entry of migrated catalog",
1652 lookup_root_entry, data);
1659 Error(
"Retrieved linkcount of catalog root entry is not sane.", data);
1683 const bool retval = detach_old_catalog.
Execute();
1685 Error(
"Failed to detach old catalog database.", detach_old_catalog, data);
1696 const worker_context *context)
1704 return CheckDatabaseSchemaCompatibility(data) &&
1705 StartDatabaseTransaction(data) &&
1706 GenerateNewStatisticsCounters(data) &&
1707 UpdateCatalogSchema(data) &&
1708 CommitDatabaseTransaction(data);
1718 if ((old_catalog.schema_version() < 2.4 -
1721 (old_catalog.schema_version() > 2.4 +
1724 Error(
"Given Catalog is not Schema 2.4.", data);
1744 bool retval =
false;
1753 PendingCatalogList::const_iterator iend = data->
nested_catalogs.end();
1754 for (; i != iend; ++i) {
1763 "SELECT count(*), sum(size) FROM catalog "
1764 " WHERE flags & :flag_chunked_file;");
1766 "SELECT count(*) FROM chunks;");
1768 "SELECT sum(size) FROM catalog WHERE flags & :flag_file "
1769 " AND NOT flags & :flag_link;");
1776 Error(
"Failed to count chunked files.", count_chunked_files, data);
1779 retval = count_file_chunks.
FetchRow();
1781 Error(
"Failed to count file chunks", count_file_chunks, data);
1789 Error(
"Failed to aggregate the file sizes.", aggregate_file_size, data);
1803 Error(
"Failed to read old catalog statistics counters", data);
1809 Error(
"Failed to write new statistics counters to database", data);
1826 "UPDATE properties SET value = :schema_version WHERE key = 'schema';");
1830 update_schema_version.
Execute();
1832 Error(
"Failed to update catalog schema version",
1833 update_schema_version,
1856 , uid_map_statement_(GenerateMappingStatement(context->uid_map,
"uid"))
1857 , gid_map_statement_(GenerateMappingStatement(context->gid_map,
"gid"))
1862 return ApplyPersonaMappings(data);
1887 Error(
"Failed to update UIDs", uid_sql, data);
1893 Error(
"Failed to update GIDs", gid_sql, data);
1901 template <
class MapT>
1904 const std::string &column)
const {
1905 assert(map.RuleCount() > 0 || map.HasDefault());
1907 std::string stmt =
"UPDATE OR ABORT catalog SET " + column +
" = ";
1909 if (map.RuleCount() == 0) {
1914 stmt +=
"CASE " + column +
" ";
1915 typedef typename MapT::map_type::const_iterator map_iterator;
1916 map_iterator i = map.GetRuleMap().begin();
1917 const map_iterator iend = map.GetRuleMap().end();
1918 for (; i != iend; ++i) {
1924 stmt += (map.HasDefault())
1926 :
"ELSE " + column +
" ";
1940 return CheckDatabaseSchemaCompatibility(data) &&
1941 BreakUpHardlinks(data);
1990 const std::string stmt =
"UPDATE OR ABORT catalog "
1991 "SET hardlinks = 1 "
1992 "WHERE flags & :file_flag "
1993 " AND hardlinks > 1;";
1996 hardlink_removal_sql.
Execute();
2006 return CheckDatabaseSchemaCompatibility(data) &&
2007 RemoveRedundantBulkHashes(data);
2036 const std::string stmt =
"UPDATE OR ABORT catalog "
2038 "WHERE flags & :file_chunked_flag;";
2041 bulkhash_removal_sql.
Execute();
2051 const worker_context *context)
2059 return CheckDatabaseSchemaCompatibility(data) &&
2060 StartDatabaseTransaction(data) &&
2061 RepairStatisticsCounters(data) &&
2062 CommitDatabaseTransaction(data);
2074 Error(
"Given catalog schema is < 2.5.", data);
2079 Error(
"Given catalog revision is < 5", data);
2100 bool retval =
false;
2109 PendingCatalogList::const_iterator iend = data->
nested_catalogs.end();
2110 for (; i != iend; ++i) {
2119 std::string(
"SELECT count(*), sum(size) FROM catalog ") +
2125 std::string(
"SELECT count(*), sum(size) FROM catalog ") +
2129 std::string(
"SELECT count(*) FROM catalog ") +
2132 std::string(
"SELECT count(*) FROM catalog ") +
2136 std::string(
"SELECT count(*) FROM catalog ") +
2137 "WHERE xattr IS NOT NULL;");
2139 std::string(
"SELECT count(*), sum(size) FROM catalog ") +
2142 std::string(
"SELECT count(*) FROM catalog ") +
2145 "SELECT count(*) FROM chunks;");
2147 retval = count_regular.
FetchRow() &&
2156 Error(
"Failed to collect catalog statistics", data);
2178 Error(
"Failed to write new statistics counters to database", data);
bool InsertIntoDatabase(const CatalogDatabase &database) const
uint32_t linkcount() const
catalog::Catalog const * root_catalog_
StopWatch migration_stopwatch_
int return_code
the return value of the spooler operation
void ConvertCatalogsRecursively(PendingCatalog *catalog, MigratorT *migrator)
void operator()(const expected_data &data)
static Parameter Optional(const char key, const std::string &desc)
#define LogCvmfs(source, mask,...)
bool UpdateCatalogMetadata(PendingCatalog *data) const
std::string GetLastErrorMsg() const
bool AnalyzeFileLinkcounts(PendingCatalog *data) const
UniquePtr< upload::Spooler > spooler_
CallbackPtr RegisterListener(typename BoundClosure< WorkerT::returned_data, DelegateT, ClosureDataT >::CallbackMethod method, DelegateT *delegate, ClosureDataT data)
PendingCatalogMap pending_catalogs_
static const unsigned kSchemaRevision
Differences CompareTo(const DirectoryEntry &other) const
bool ReadPersonaMaps(const std::string &uid_map_path, const std::string &gid_map_path, UidMap *uid_map, GidMap *gid_map) const
const manifest::Manifest * manifest() const
bool Export(const std::string &path) const
std::string nested_catalog_marker_tmp_path_
shash::Md5 GetPathHash() const
static Parameter Switch(const char key, const std::string &desc)
catalog::WritableCatalog * new_catalog
bool MigrateFileMetadata(PendingCatalog *data) const
bool UpdateCatalogSchema(PendingCatalog *data) const
FileChunkList file_chunks
the file chunks generated during processing
bool BindText(const int index, const std::string &value)
void Insert(const CatalogStatistics &statistics)
std::string database_path() const
void set_catalog_hash(const shash::Any &catalog_hash)
bool FindRootEntryInformation(PendingCatalog *data) const
const std::string & filename() const
const catalog::Catalog * old_catalog
ConcurrentWorkers< DerivedWorkerT > * master() const
void WaitForEmptyQueue() const
bool StartDatabaseTransaction(PendingCatalog *data) const
bool LoadCatalogs(const shash::Any &manual_root_hash, ObjectFetcherT *object_fetcher)
bool BindDirent(const DirectoryEntry &entry)
void AnalyzeCatalogStatistics() const
ChownMigrationWorker(const worker_context *context)
bool FixNestedCatalogTransitionPoints(PendingCatalog *data) const
static const std::string kPreviousHeadTag
MigrationWorker_217(const worker_context *context)
std::vector< Parameter > ParameterList
void CreateNestedCatalogMarkerDirent(const shash::Any &content_hash)
atomic_int32 catalogs_processed_
std::string ToString(const bool with_suffix=false) const
void Assign(const char *chars, const unsigned length)
void UploadCallback(const upload::SpoolerResult &result)
const history::History * history() const
void UpdateLastModified()
void ApplyDelta(const DeltaCounters &delta)
bool IsHttpUrl(const std::string &path)
DirectoryEntry GetDirent(const Catalog *catalog, const bool expand_symlink=true) const
bool BeginTransaction() const
std::string CreateTempPath(const std::string &path_prefix, const int mode)
unsigned int GetNumberOfCpuCores()
void set_revision(const uint64_t revision)
static void FixNestedCatalogTransitionPoint(const catalog::DirectoryEntry &nested_root, catalog::DirectoryEntry *mountpoint)
virtual ParameterList GetParams() const
bool RunMigration(PendingCatalog *data) const
shash::Any new_catalog_hash
assert((mem||(size==0))&&"Out Of Memory")
static const int kFlagFileChunk
PendingCatalogList nested_catalogs
float schema_version() const
CatalogList GetChildren() const
void PrintStatusMessage(const PendingCatalog *catalog, const shash::Any &content_hash, const std::string &message)
upload::Spooler * spooler
void SetPreviousRevision(const shash::Any &hash)
bool RemoveDanglingNestedMountpoints(PendingCatalog *data) const
static const int kFlagFile
static catalog::DirectoryEntry nested_catalog_marker_
bool AttachOldCatalogDatabase(PendingCatalog *data) const
bool CopyPath2Path(const string &src, const string &dest)
bool ReadPersona(const std::string &uid, const std::string &gid)
const NestedCatalogList & ListNestedCatalogs() const
void Set(const T &object)
const std::string root_path() const
void JobSuccessful(const returned_data_t &data)
bool MigrateNestedCatalogMountPoints(PendingCatalog *data) const
shash::Any GetOldContentHash() const
unsigned schema_revision() const
void CatalogCallback(const CatalogTraversalData< catalog::WritableCatalog > &data)
UniquePtr< history::SqliteHistory > history_upstream_
bool ConfigureSQLite() const
static const int kFlagLink
static void Error(const std::string &message)
bool WriteToDatabase(const CatalogDatabase &database) const
unsigned int hardlink_group_count
bool IsNestedCatalogMountpoint() const
bool BindPathHash(const shash::Md5 &hash)
virtual ~AbstractMigrationWorker()
bool CommitDatabaseTransaction(PendingCatalog *data) const
static SqliteHistory * OpenWritable(const std::string &file_name)
static WritableCatalog * AttachFreely(const std::string &root_path, const std::string &file, const shash::Any &catalog_hash, Catalog *parent=NULL, const bool is_not_root=false)
uint64_t GetRevision() const
bool GenerateCatalogStatistics(PendingCatalog *data) const
StopWatch catalog_loading_stopwatch_
unsigned int file_descriptor_limit_
uint64_t GetLastModified() const
int64_t String2Int64(const string &value)
bool BindDouble(const int index, const double value)
bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const
void UploadHistoryClosure(const upload::SpoolerResult &result, Future< shash::Any > *hash)
std::string local_path
the local_path previously given as input
Future< catalog::DirectoryEntry > root_entry
std::string GenerateMappingStatement(const MapT &map, const std::string &column) const
static const std::string kPreviousHeadTagDescription
static const float kSchemaEpsilon
bool RepairStatisticsCounters(PendingCatalog *data) const
bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const
bool RunMigration(PendingCatalog *data) const
unsigned int catalog_count_
std::string temporary_directory_
static Parameter Mandatory(const char key, const std::string &desc)
bool StartDatabaseTransaction(PendingCatalog *data) const
bool DetachOldCatalogDatabase(PendingCatalog *data) const
const char kSuffixCatalog
bool ApplyPersonaMappings(PendingCatalog *data) const
static const int kFlagFileSpecial
AbstractMigrationWorker(const worker_context *context)
static const int kFlagFileExternal
bool CommitDatabaseTransaction(PendingCatalog *data) const
static const std::string kHeadTag
static const catalog::DirectoryEntry & GetNestedCatalogMarkerDirent()
void PopulateToParent(DeltaCounters *parent) const
const shash::Any catalog_hash
bool InsertInitialValues(const std::string &root_path, const bool volatile_content, const std::string &voms_authz, const DirectoryEntry &root_entry=DirectoryEntry(kDirentNegative))
sqlite3_int64 RetrieveInt64(const int idx_column) const
StatsMigrationWorker(const worker_context *context)
bool RunMigration(PendingCatalog *data) const
PathString mountpoint() const
bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const
bool BindPathHash(const struct shash::Md5 &hash)
string StringifyInt(const int64_t value)
bool GenerateNestedCatalogMarkerChunk()
bool CommitTransaction() const
bool ReadFromDatabase(const CatalogDatabase &database, const LegacyMode::Type legacy=LegacyMode::kNoLegacy)
bool BreakUpHardlinks(PendingCatalog *data) const
bool BindInt64(const int index, const sqlite3_int64 value)
Future< bool > was_updated
void set_history(const shash::Any &history_db)
Future< catalog::DeltaCounters > nested_statistics
bool CollectAndAggregateStatistics(PendingCatalog *data) const
std::vector< Catalog * > CatalogList
bool GenerateNewStatisticsCounters(PendingCatalog *data) const
void set_ttl(const uint32_t ttl)
bool CommitDatabaseTransaction(PendingCatalog *data) const
bool RunMigration(PendingCatalog *data) const
bool has_committed_new_revision_
std::string ToString() const
std::vector< NestedCatalog > NestedCatalogList
bool RemoveRedundantBulkHashes(PendingCatalog *data) const
int Main(const ArgumentList &args)
std::map< char, SharedPtr< std::string > > ArgumentList
ShortString< kDefaultMaxPath, 0 > PathString
virtual ~PendingCatalog()
MigrationWorker_20x(const worker_context *context)
UniquePtr< manifest::Manifest > manifest_upstream_
bool UpdateUndoTags(PendingCatalog *root_catalog, unsigned revision, time_t timestamp, shash::Any *history_hash)
bool CleanupNestedCatalogs(PendingCatalog *data) const
bool UpdateNestedCatalogReferences(PendingCatalog *data) const
bool CreateNewEmptyCatalog(PendingCatalog *data) const
void set_catalog_size(const uint64_t catalog_size)
CatalogStatisticsList catalog_statistics_list_
const CatalogDatabase & database() const
unsigned int GetNumberOfFailedJobs() const
Any MkFromHexPtr(const HexPtr hex, const char suffix)
int64_t GetFileSize(const std::string &path)
HttpObjectFetcher ObjectFetcher
unsigned int aggregated_linkcounts
bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const
CatalogStatistics statistics
static const float kLatestSupportedSchema
bool StartDatabaseTransaction(PendingCatalog *data) const
void JobFailed(const returned_data_t &data)
unsigned GetLength() const
static const char * kVirtualPath
bool CheckDatabaseSchemaCompatibility(PendingCatalog *data) const
bool BindMd5(const int idx_high, const int idx_low, const shash::Md5 &hash)
const char * GetChars() const
uint32_t hardlink_group() const
bool BindParentPathHash(const shash::Md5 &hash)
static const unsigned kLatestSchemaRevision
static const int kFlagDir
void MigrationCallback(PendingCatalog *const &data)
bool RunMigration(PendingCatalog *data) const
bool RaiseFileDescriptorLimit() const
void set_is_nested_catalog_mountpoint(const bool val)
bool BindPathHash(const shash::Md5 &hash)
catalog::WritableCatalog * GetWritable(const catalog::Catalog *catalog) const
const unsigned int tree_level
static DerivedT * Create(const std::string &filename)
bool DoMigrationAndCommit(const std::string &manifest_path, typename MigratorT::worker_context *context)
bool RunMigration(PendingCatalog *data) const
bool BindDirent(const DirectoryEntry &entry)
void set_root_path(const std::string &root_path)
static const float kSchema