GCC Code Coverage Report


Directory: cvmfs/
File: cvmfs/swissknife_ingestsql.cc
Date: 2026-11-08 02:41:18
Exec Total Coverage
Lines: 0 919 0.0%
Branches: 0 608 0.0%

Line Branch Exec Source
1 #include "swissknife_ingestsql.h"
2
3 #include <fcntl.h>
4 #include <grp.h>
5 #include <pwd.h>
6 #include <stdio.h>
7 #include <sys/resource.h>
8 #include <sys/time.h>
9 #include <sys/types.h>
10 #include <unistd.h>
11
12 #include <csignal>
13 #include <cstdlib>
14 #include <fstream>
15 #include <sstream>
16 #include <stack>
17 #include <unordered_map>
18 #include <unordered_set>
19
20 #include "acl.h"
21 #include "catalog_downloader.h"
22 #include "catalog_mgr_rw.h"
23 #include "curl/curl.h"
24 #include "gateway_util.h"
25 #include "shortstring.h"
26 #include "swissknife_lease_curl.h"
27 #include "swissknife_lease_json.h"
28 #include "swissknife_sync.h"
29 #include "upload.h"
30 #include "util/logging.h"
31
32 #define CHECK_SQLITE_ERROR(ret, expected) \
33 do { \
34 const int sqlite_result = (ret); \
35 if (sqlite_result != (expected)) { \
36 LogCvmfs(kLogCvmfs, kLogStderr, "SQLite error: %d", \
37 sqlite_result); \
38 abort(); \
39 } \
40 } while (0)
41
42 #define CUSTOM_ASSERT(check, msg, ...) \
43 do { \
44 if (!(check)) { \
45 LogCvmfs(kLogCvmfs, kLogStderr, msg, ##__VA_ARGS__); \
46 abort(); \
47 } \
48 } while (0)
49
50 #define SHOW_PROGRESS(item, freq, curr, total) \
51 do { \
52 if ((curr) % freq == 0 || (curr) == total) { \
53 LogCvmfs(kLogCvmfs, kLogStdout, "Processed %d/%d %s", (curr), total, \
54 item); \
55 } \
56 } while (0)
57
58
59 static const unsigned kExternalChunkSize = 24 * 1024 * 1024;
60 static const unsigned kInternalChunkSize = 6 * 1024 * 1024;
61 static const unsigned kDefaultLeaseBusyRetryInterval = 10;
62 static const unsigned kLeaseRefreshInterval = 90; // seconds
63
64
65 static bool g_lease_acquired = false;
66 static string g_gateway_url;
67 static string g_gateway_key_id;
68 static string g_gateway_secret;
69 static string g_session_token;
70 static string g_session_token_file;
71 static string g_s3_file;
72 static time_t g_last_lease_refresh = 0;
73 static bool g_stop_refresh = false;
74 static string g_wait_for_update;
75 static int g_wait_for_update_timeout_sec = -1;
76 static int64_t g_priority = 0;
77 static bool g_add_missing_catalogs = false;
78 static string get_lease_from_paths(vector<string> paths);
79 static vector<string> get_all_dirs_from_sqlite(vector<string> &sqlite_db_vec,
80 bool include_additions,
81 bool include_deletions);
82 static string get_parent(const string &path);
83 static string get_basename(const string &path);
84
85 static XattrList marshal_xattrs(const char *acl);
86 static string sanitise_name(const char *name_cstr, bool allow_leading_slash);
87 static void on_signal(int sig);
88 static string acquire_lease(const string &key_id, const string &secret,
89 const string &lease_path,
90 const string &repo_service_url,
91 bool force_cancel_lease, uint64_t *current_revision,
92 string &current_root_hash,
93 unsigned int refresh_interval);
94 static void cancel_lease();
95 static void refresh_lease();
96 static vector<string> get_file_list(string &path);
97 static int check_hash(const char *hash);
98 static void recursively_delete_directory(
99 PathString &path, catalog::WritableCatalogManager &catalog_manager);
100 static void create_empty_database(string &filename);
101 static void relax_db_locking(sqlite3 *db);
102 static bool check_prefix(const std::string &path, const std::string &prefix);
103
104 static bool isDatabaseMarkedComplete(const char *dbfile);
105 static void setDatabaseMarkedComplete(const char *dbfile);
106
107 static void invalidate_manifest(std::string proxy_list, std::string url);
108 static int wait_for_update(std::string path, long revision, int timeout_sec);
109
110 extern "C" void *lease_refresh_thread(void *payload);
111
112 extern long g_final_revision;
113
114
115 ✗ static string sanitise_name(const char *name_cstr,
116 bool allow_leading_slash = false) {
117 ✗ int reason = 0;
118 ✗ const char *c = name_cstr;
119 ✗ while (*c == '/') {
120 ✗ c++;
121 } // strip any leading slashes
122 ✗ string const name = string(c);
123 ✗ bool ok = true;
124
125 ✗ if (!allow_leading_slash && HasPrefix(name, "/", true)) {
126 ✗ reason = 1;
127 ✗ ok = false;
128 }
129 ✗ if (HasSuffix(name, "/", true)) {
130 ✗ if (!(allow_leading_slash
131 ✗ && name.size() == 1)) { // account for the case where name=="/"
132 ✗ reason = 2;
133 ✗ ok = false;
134 }
135 }
136 ✗ if (name.find("//") != string::npos) {
137 ✗ reason = 3;
138 ✗ ok = false;
139 }
140 ✗ if (HasPrefix(name, "./", true) || HasPrefix(name, "../", true)) {
141 ✗ reason = 4;
142 ✗ ok = false;
143 }
144 ✗ if (HasSuffix(name, "/.", true) || HasSuffix(name, "/..", true)) {
145 ✗ reason = 5;
146 ✗ ok = false;
147 }
148 ✗ if (name.find("/./") != string::npos || name.find("/../") != string::npos) {
149 ✗ reason = 6;
150 ✗ ok = false;
151 }
152 ✗ if (name == "") {
153 ✗ reason = 7;
154 ✗ ok = false;
155 }
156 ✗ CUSTOM_ASSERT(ok, "Name [%s] is invalid (reason %d)", name.c_str(), reason);
157 ✗ return string(name);
158 }
159
160 ✗ static string get_parent(const string &path) {
161 ✗ size_t const found = path.find_last_of('/');
162 ✗ if (found == string::npos) {
163 ✗ return string("");
164 }
165 ✗ return path.substr(0, found);
166 }
167
168 ✗ static string get_basename(const string &path) {
169 ✗ const size_t found = path.find_last_of('/');
170 ✗ if (found == string::npos) {
171 ✗ return path;
172 }
173 ✗ return path.substr(found + 1);
174 }
175
176 // this is copied from MakeRelativePath
177 ✗ static string MakeCatalogPath(const std::string &relative_path) {
178 ✗ return (relative_path == "") ? "" : "/" + relative_path;
179 }
180
181 ✗ static string acquire_lease(const string &key_id, const string &secret,
182 const string &lease_path,
183 const string &repo_service_url,
184 bool force_cancel_lease, uint64_t *current_revision,
185 string &current_root_hash,
186 unsigned int refresh_interval) {
187 ✗ const CURLcode ret = curl_global_init(CURL_GLOBAL_ALL);
188 ✗ CUSTOM_ASSERT(ret == CURLE_OK, "failed to init curl");
189
190 ✗ string gateway_metadata_str;
191 ✗ char *gateway_metadata = getenv("CVMFS_GATEWAY_METADATA");
192 ✗ if (gateway_metadata != NULL)
193 ✗ gateway_metadata_str = gateway_metadata;
194
195 while (true) {
196 ✗ CurlBuffer buffer;
197 ✗ if (MakeAcquireRequest(key_id, secret, lease_path, repo_service_url,
198 &buffer, gateway_metadata_str)) {
199 ✗ string session_token;
200
201 ✗ const LeaseReply rep = ParseAcquireReplyWithRevision(
202 buffer, &session_token, current_revision, current_root_hash);
203 ✗ switch (rep) {
204 ✗ case kLeaseReplySuccess:
205 ✗ g_lease_acquired = true;
206 ✗ g_last_lease_refresh = time(NULL);
207 ✗ return session_token;
208 break;
209 ✗ case kLeaseReplyBusy:
210 ✗ if (force_cancel_lease) {
211 ✗ LogCvmfs(kLogCvmfs, kLogStderr,
212 "Lease busy, forcing cancellation (TODO");
213 }
214 ✗ LogCvmfs(kLogCvmfs, kLogStderr, "Lease busy, retrying in %d sec",
215 refresh_interval);
216 ✗ sleep(refresh_interval);
217 ✗ break;
218 ✗ default:
219 ✗ LogCvmfs(kLogCvmfs, kLogStderr,
220 "Error acquiring lease: %s. Retrying in %d sec",
221 buffer.data.c_str(), refresh_interval);
222 ✗ sleep(refresh_interval);
223 }
224 ✗ } else {
225 ✗ LogCvmfs(kLogCvmfs, kLogStderr,
226 "Error making lease acquisition request. Retrying in %d sec",
227 refresh_interval);
228 ✗ sleep(refresh_interval);
229 }
230 }
231 assert(false);
232 return "";
233 }
234
235 ✗ static uint64_t make_commit_on_gateway(const std::string &old_root_hash,
236 const std::string &new_root_hash,
237 int64_t priority) {
238 ✗ CurlBuffer buffer;
239 char priorityStr[100];
240 ✗ (void)sprintf(priorityStr, "%" PRId64,
241 priority); // skipping return value check; no way such large
242 // buffer will overflow
243 ✗ buffer.data = "";
244
245 ✗ const std::string payload = "{\n\"old_root_hash\": \"" + old_root_hash
246 ✗ + "\",\n\"new_root_hash\": \"" + new_root_hash
247 ✗ + "\",\n\"priority\": " + priorityStr + "}";
248
249 ✗ return MakeEndRequest("POST", g_gateway_key_id, g_gateway_secret,
250 g_session_token, g_gateway_url, payload, &buffer,
251 ✗ true /*expect_final_revision*/);
252 }
253
254 ✗ static void refresh_lease() {
255 ✗ CurlBuffer buffer;
256 ✗ buffer.data = "";
257 ✗ if ((time(NULL) - g_last_lease_refresh) < kLeaseRefreshInterval) {
258 ✗ return;
259 }
260
261 ✗ if (MakeEndRequest("PATCH", g_gateway_key_id, g_gateway_secret,
262 g_session_token, g_gateway_url, "", &buffer,
263 false /*expect_final_revision*/)) {
264 ✗ const int ret = ParseDropReply(buffer);
265 ✗ if (kLeaseReplySuccess == ret) {
266 ✗ LogCvmfs(kLogCvmfs, kLogVerboseMsg, "Lease refreshed");
267 ✗ g_last_lease_refresh = time(NULL);
268 } else {
269 ✗ LogCvmfs(kLogCvmfs, kLogStderr, "Lease refresh failed: %d", ret);
270 }
271 } else {
272 ✗ LogCvmfs(kLogCvmfs, kLogStderr, "Lease refresh request failed");
273 ✗ if (buffer.data == "Method Not Allowed\n") {
274 ✗ g_last_lease_refresh = time(NULL);
275 ✗ LogCvmfs(kLogCvmfs, kLogStderr,
276 "This gateway does not support lease refresh");
277 }
278 }
279 }
280
281
282 ✗ static void cancel_lease() {
283 ✗ CurlBuffer buffer;
284 ✗ if (MakeEndRequest("DELETE", g_gateway_key_id, g_gateway_secret,
285 g_session_token, g_gateway_url, "", &buffer,
286 false /*expect_final_revision*/)) {
287 ✗ const int ret = ParseDropReply(buffer);
288 ✗ if (kLeaseReplySuccess == ret) {
289 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Lease cancelled");
290 } else {
291 ✗ LogCvmfs(kLogCvmfs, kLogStderr, "Lease cancellation failed: %d", ret);
292 }
293 } else {
294 ✗ LogCvmfs(kLogCvmfs, kLogStderr, "Lease cancellation request failed");
295 }
296 ✗ g_stop_refresh = true;
297 }
298
299 ✗ static void on_signal(int sig) {
300 ✗ (void)signal(sig, SIG_DFL);
301 ✗ if (g_lease_acquired) {
302 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Cancelling lease");
303 ✗ cancel_lease();
304 ✗ unlink(g_session_token_file.c_str());
305 }
306 ✗ if (sig == SIGINT || sig == SIGTERM)
307 ✗ exit(1);
308 }
309
310 ✗ static vector<string> get_all_dirs_from_sqlite(vector<string> &sqlite_db_vec,
311 bool include_additions,
312 bool include_deletions) {
313 int ret;
314 ✗ vector<string> paths;
315
316 ✗ for (vector<string>::iterator it = sqlite_db_vec.begin();
317 ✗ it != sqlite_db_vec.end();
318 ✗ it++) {
319 sqlite3 *db;
320 ✗ ret = sqlite3_open_v2((*it).c_str(), &db, SQLITE_OPEN_READONLY, NULL);
321 ✗ CHECK_SQLITE_ERROR(ret, SQLITE_OK);
322 ✗ relax_db_locking(db);
323
324 ✗ vector<string> tables;
325 ✗ if (include_additions) {
326 ✗ tables.push_back("dirs");
327 ✗ tables.push_back("links");
328 ✗ tables.push_back("files");
329 }
330 ✗ if (include_deletions) {
331 ✗ tables.push_back("deletions");
332 }
333
334 // get all the paths from the DB
335 ✗ for (vector<string>::iterator it = tables.begin(); it != tables.end();
336 ✗ it++) {
337 sqlite3_stmt *stmt;
338 ✗ const string query = "SELECT name FROM " + *it;
339 ✗ ret = sqlite3_prepare_v2(db, query.c_str(), -1, &stmt, NULL);
340 ✗ CHECK_SQLITE_ERROR(ret, SQLITE_OK);
341 ✗ while (sqlite3_step(stmt) == SQLITE_ROW) {
342 const char *name = reinterpret_cast<const char *>(
343 ✗ sqlite3_column_text(stmt, 0));
344 ✗ const string names = sanitise_name(name);
345 ✗ if (*it == "dirs") {
346 ✗ paths.push_back(names);
347 } else {
348 ✗ paths.push_back(get_parent(names));
349 }
350 }
351 ✗ ret = sqlite3_finalize(stmt);
352 ✗ CHECK_SQLITE_ERROR(ret, SQLITE_OK);
353 }
354 ✗ ret = sqlite3_close_v2(db);
355 ✗ CHECK_SQLITE_ERROR(ret, SQLITE_OK);
356 }
357 ✗ return paths;
358 }
359
360 ✗ static int get_db_schema_revision(sqlite3 *db,
361 const std::string &db_name = "") {
362 sqlite3_stmt *stmt;
363 ✗ std::ostringstream stmt_str;
364 stmt_str << "SELECT value FROM " << db_name
365 ✗ << "properties WHERE key = 'schema_revision'";
366 ✗ int ret = sqlite3_prepare_v2(db, stmt_str.str().c_str(), -1, &stmt, NULL);
367 ✗ CHECK_SQLITE_ERROR(ret, SQLITE_OK);
368
369 ✗ ret = sqlite3_step(stmt);
370 // if table exists, we require that it must have a schema_revision row
371 ✗ CHECK_SQLITE_ERROR(ret, SQLITE_ROW);
372 const std::string schema_revision_str(
373 ✗ reinterpret_cast<const char *>(sqlite3_column_text(stmt, 0)));
374 ✗ CHECK_SQLITE_ERROR(sqlite3_finalize(stmt), SQLITE_OK);
375 ✗ return std::stoi(schema_revision_str);
376 }
377
378 ✗ static int get_row_count(sqlite3 *db, const std::string &table_name) {
379 sqlite3_stmt *stmt;
380 ✗ std::ostringstream stmt_str;
381 ✗ stmt_str << "SELECT COUNT(*) FROM " << table_name;
382 ✗ int ret = sqlite3_prepare_v2(db, stmt_str.str().c_str(), -1, &stmt, NULL);
383 ✗ CHECK_SQLITE_ERROR(ret, SQLITE_OK);
384
385 ✗ ret = sqlite3_step(stmt);
386 ✗ CHECK_SQLITE_ERROR(ret, SQLITE_ROW);
387 const std::string count_str(
388 ✗ reinterpret_cast<const char *>(sqlite3_column_text(stmt, 0)));
389 ✗ CHECK_SQLITE_ERROR(sqlite3_finalize(stmt), SQLITE_OK);
390 ✗ return std::stoi(count_str);
391 }
392
393 ✗ static int calculate_print_frequency(int total) {
394 ✗ int base = 1000;
395 ✗ while (base * 50 < total)
396 ✗ base *= 10;
397 ✗ return base;
398 }
399
400 // compute a common path among for paths for use as the lease path
401 ✗ static string get_lease_from_paths(vector<string> paths) {
402 ✗ CUSTOM_ASSERT(!paths.empty(), "no paths are provided");
403
404 // we'd have to ensure path is relative
405 // (probably best to check this elsewhere as it is not just a requirement for
406 // this function)
407 ✗ auto lease = PathString(paths.at(0));
408 ✗ for (auto it = paths.begin() + 1; it != paths.end(); ++it) {
409 ✗ auto path = PathString(*it);
410 // shrink the lease path until it is a parent of "path"
411 ✗ while (!IsSubPath(lease, path)) {
412 ✗ auto i = lease.GetLength() - 1;
413 ✗ for (; i >= 0; --i) {
414 ✗ if (lease.GetChars()[i] == '/' || i == 0) {
415 ✗ lease.Truncate(i);
416 ✗ break;
417 }
418 }
419 }
420 ✗ if (lease.IsEmpty())
421 ✗ break; // early stop if lease is already at the root
422 }
423
424 ✗ auto prefix = "/" + lease.ToString();
425
426 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Longest prefix is %s", prefix.c_str());
427 ✗ return prefix;
428 }
429
430 ✗ static XattrList marshal_xattrs(const char *acl_string) {
431 ✗ XattrList aclobj;
432
433 ✗ if (acl_string == NULL || acl_string[0] == '\0') {
434 ✗ return aclobj;
435 }
436
437 bool equiv_mode;
438 size_t binary_size;
439 char *binary_acl;
440 ✗ const int ret = acl_from_text_to_xattr_value(string(acl_string), binary_acl,
441 binary_size, equiv_mode);
442 ✗ if (ret) {
443 ✗ LogCvmfs(kLogCvmfs, kLogStderr,
444 "failure of acl_from_text_to_xattr_value(%s)", acl_string);
445 ✗ abort();
446 }
447 ✗ if (!equiv_mode) {
448 ✗ CUSTOM_ASSERT(
449 aclobj.Set("system.posix_acl_access", string(binary_acl, binary_size)),
450 "failed to set system.posix_acl_access (ACL size %ld)", binary_size);
451 ✗ free(binary_acl);
452 }
453
454 ✗ return aclobj;
455 }
456
457 ✗ std::unordered_map<string, string> load_config(const string &config_file) {
458 ✗ std::unordered_map<string, string> config_map;
459 ✗ ifstream input(config_file);
460 ✗ if (!input) {
461 ✗ LogCvmfs(kLogCvmfs, kLogStderr, "could not open config file %s",
462 config_file.c_str());
463 ✗ return config_map;
464 }
465 ✗ vector<string> lines;
466 ✗ for (string line; getline(input, line);) {
467 ✗ lines.push_back(line);
468 }
469
470 ✗ for (auto it = lines.begin(); it != lines.end(); it++) {
471 ✗ const string l = *it;
472 ✗ const size_t p = l.find('=', 0);
473 ✗ if (p != string::npos) {
474 ✗ const string key = l.substr(0, p);
475 ✗ string val = l.substr(p + 1);
476 // trim any double quotes
477 ✗ if (val.front() == '"') {
478 ✗ val = val.substr(1, val.length() - 2);
479 }
480 ✗ config_map[key] = val;
481 }
482 }
483
484 ✗ return config_map;
485 }
486
487 ✗ string retrieve_config(std::unordered_map<string, string> &config_map,
488 const string &key) {
489 ✗ auto kv = config_map.find(key);
490 ✗ CUSTOM_ASSERT(kv != config_map.end(), "Parameter %s not found in config",
491 key.c_str());
492 ✗ return kv->second;
493 }
494
495 ✗ static vector<string> get_file_list(string &path) {
496 ✗ vector<string> paths;
497 ✗ const char *cpath = path.c_str();
498 struct stat st;
499 ✗ const int ret = stat(cpath, &st);
500 ✗ CUSTOM_ASSERT(ret == 0, "failed to stat file %s", cpath);
501
502 ✗ if (S_ISDIR(st.st_mode)) {
503 DIR *d;
504 struct dirent *dir;
505 ✗ d = opendir(cpath);
506 ✗ if (d) {
507 ✗ while ((dir = readdir(d)) != NULL) {
508 ✗ const char *t = strrchr(dir->d_name, '.');
509 ✗ if (t && !strcmp(t, ".db")) {
510 ✗ paths.push_back(path + "/" + dir->d_name);
511 }
512 }
513 ✗ closedir(d);
514 }
515 } else {
516 ✗ paths.push_back(path);
517 }
518 ✗ return paths;
519 }
520
521 extern bool g_log_with_time;
522
523 ✗ int swissknife::IngestSQL::Main(const swissknife::ArgumentList &args) {
524 // the catalog code uses assert() liberally.
525 // install ABRT signal handler to catch an abort and cancel lease
526 ✗ if (signal(SIGABRT, &on_signal) == SIG_ERR
527 ✗ || signal(SIGINT, &on_signal) == SIG_ERR
528 ✗ || signal(SIGTERM, &on_signal) == SIG_ERR) {
529 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Setting signal handlers failed");
530 ✗ exit(1);
531 }
532
533 ✗ const bool enable_corefiles = (args.find('c') != args.end());
534 ✗ if (!enable_corefiles) {
535 struct rlimit rlim;
536 ✗ rlim.rlim_cur = rlim.rlim_max = 0;
537 ✗ setrlimit(RLIMIT_CORE, &rlim);
538 }
539
540
541 ✗ if (args.find('n') != args.end()) {
542 ✗ create_empty_database(*args.find('n')->second);
543 ✗ exit(0);
544 }
545
546 ✗ if (args.find('B') != args.end()) {
547 ✗ g_wait_for_update = *args.find('B')->second;
548
549 ✗ if (args.find('W') != args.end()) {
550 ✗ const char *arg = (*args.find('W')->second).c_str();
551 char *at_null_terminator_if_number;
552 ✗ g_wait_for_update_timeout_sec = strtoll(arg, &at_null_terminator_if_number, 10);
553 ✗ if (*at_null_terminator_if_number != '\0') {
554 ✗ LogCvmfs(kLogCvmfs, kLogStderr,
555 "'W' parameter value '%s' parsing failed", arg);
556 ✗ return 1;
557 }
558 }
559 }
560
561 ✗ if (args.find('P') != args.end()) {
562 ✗ const char *arg = (*args.find('P')->second).c_str();
563 char *at_null_terminator_if_number;
564 ✗ g_priority = strtoll(arg, &at_null_terminator_if_number, 10);
565 ✗ if (*at_null_terminator_if_number != '\0') {
566 ✗ LogCvmfs(kLogCvmfs, kLogStderr,
567 "Priority parameter value '%s' parsing failed", arg);
568 ✗ return 1;
569 }
570 } else {
571 ✗ g_priority = -time(NULL);
572 }
573
574
575 ✗ unsigned int lease_busy_retry_interval = kDefaultLeaseBusyRetryInterval;
576 ✗ if (args.find('r') != args.end()) {
577 ✗ lease_busy_retry_interval = atoi((*args.find('r')->second).c_str());
578 }
579
580 ✗ string dir_temp = "";
581 const char *env_tmpdir;
582 ✗ if (args.find('t') != args.end()) {
583 ✗ dir_temp = MakeCanonicalPath(*args.find('t')->second);
584 ✗ } else if ((env_tmpdir = getenv("TMPDIR"))) {
585 ✗ dir_temp = MakeCanonicalPath(env_tmpdir);
586 } else {
587 ✗ LogCvmfs(kLogCvmfs, kLogStderr, "-t or TMPDIR required");
588 ✗ return 1;
589 }
590
591 ✗ string kConfigDir("/etc/cvmfs/gateway-client/");
592 ✗ if (args.find('C') != args.end()) {
593 ✗ kConfigDir = MakeCanonicalPath(*args.find('C')->second);
594 ✗ kConfigDir += "/";
595 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Overriding configuration dir prefix to %s",
596 kConfigDir.c_str());
597 }
598
599 // mandatory arguments
600 ✗ string const repo_name = *args.find('N')->second;
601 ✗ string sqlite_db_path = *args.find('D')->second;
602
603 ✗ vector<string> sqlite_db_vec = get_file_list(sqlite_db_path);
604
605 // optional arguments
606 ✗ bool const allow_deletions = (args.find('d') != args.end());
607 ✗ bool const force_cancel_lease = (args.find('x') != args.end());
608 ✗ bool const allow_additions = !allow_deletions
609 ✗ || (args.find('a') != args.end());
610 ✗ g_add_missing_catalogs = (args.find('z') != args.end());
611 ✗ bool const check_completed_graft_property = (args.find('Z') != args.end());
612 ✗ if (args.find('v') != args.end()) {
613 ✗ SetLogVerbosity(kLogVerbose);
614 }
615
616 ✗ if (check_completed_graft_property) {
617 ✗ if (sqlite_db_vec.size() != 1) {
618 ✗ LogCvmfs(kLogCvmfs, kLogStderr, "-Z requires a single DB file");
619 ✗ exit(1);
620 }
621 ✗ if (isDatabaseMarkedComplete(sqlite_db_vec[0].c_str())) {
622 ✗ LogCvmfs(kLogCvmfs, kLogStderr,
623 "DB file is already marked as completed_graft");
624 ✗ exit(0);
625 } else {
626 ✗ LogCvmfs(kLogCvmfs, kLogStderr,
627 "DB file is not marked as completed_graft");
628 }
629 }
630
631 ✗ string const config_file = kConfigDir + repo_name + "/config";
632 ✗ string stratum0;
633 ✗ string proxy;
634
635 ✗ string additional_prefix = "";
636 ✗ bool has_additional_prefix = false;
637 ✗ if (args.find('p') != args.end()) {
638 ✗ additional_prefix = *args.find('p')->second;
639 ✗ additional_prefix = sanitise_name(additional_prefix.c_str(), true);
640 ✗ if (additional_prefix.back() != '/') {
641 ✗ additional_prefix += "/";
642 }
643 ✗ has_additional_prefix = true;
644 ✗ LogCvmfs(kLogCvmfs, kLogStdout,
645 "Adding additional prefix %s to lease and all paths",
646 additional_prefix.c_str());
647 // now we are confident that any additional prefix has no leading / and does
648 // have a tailing /
649 }
650 ✗ auto config_map = load_config(config_file);
651
652 ✗ if (args.find('g') != args.end()) {
653 ✗ g_gateway_url = *args.find('g')->second;
654 } else {
655 ✗ g_gateway_url = retrieve_config(config_map, "CVMFS_GATEWAY");
656 }
657 ✗ if (args.find('w') != args.end()) {
658 ✗ stratum0 = *args.find('w')->second;
659 } else {
660 ✗ stratum0 = retrieve_config(config_map, "CVMFS_STRATUM0");
661 }
662
663 ✗ if (args.find('@') != args.end()) {
664 ✗ proxy = *args.find('@')->second;
665 } else {
666 ✗ proxy = retrieve_config(config_map, "CVMFS_HTTP_PROXY");
667 }
668
669 ✗ string lease_path = "";
670 // bool lease_autodetected = false;
671 ✗ if (args.find('l') != args.end()) {
672 ✗ lease_path = *args.find('l')->second;
673 } else {
674 // lease path wasn't specified, so try to autodetect it
675 vector<string> const paths = get_all_dirs_from_sqlite(
676 ✗ sqlite_db_vec, allow_additions, allow_deletions);
677 ✗ if (paths.size() == 0) {
678 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Database is empty, nothing to do");
679 ✗ return 0; // treat it as a success
680 }
681 ✗ lease_path = get_lease_from_paths(paths);
682 // lease_autodetected = true;
683 }
684
685 ✗ if (has_additional_prefix) {
686 ✗ if (lease_path == "/") {
687 ✗ lease_path = "/" + additional_prefix;
688 } else {
689 ✗ if (lease_path.substr(0, 1) == "/") {
690 ✗ lease_path = "/" + additional_prefix
691 ✗ + lease_path.substr(1, lease_path.size() - 1);
692 } else {
693 ✗ lease_path = "/" + additional_prefix
694 ✗ + lease_path; // prefix is certain to have a trailing /
695 }
696 }
697 }
698 ✗ if (lease_path.substr(0, 1) != "/") {
699 ✗ lease_path = "/" + lease_path;
700 }
701 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Lease path is %s", lease_path.c_str());
702
703
704 ✗ string public_keys = kConfigDir + repo_name + "/pubkey";
705 ✗ string key_file = kConfigDir + repo_name + "/gatewaykey";
706 ✗ string s3_file = kConfigDir + repo_name + "/s3.conf";
707
708 ✗ if (args.find('k') != args.end()) {
709 ✗ public_keys = *args.find('k')->second;
710 }
711 ✗ if (args.find('s') != args.end()) {
712 ✗ key_file = *args.find('s')->second;
713 }
714 ✗ if (args.find('3') != args.end()) {
715 ✗ s3_file = *args.find('3')->second;
716 }
717
718 ✗ CUSTOM_ASSERT(access(public_keys.c_str(), R_OK) == 0, "%s is not readable",
719 public_keys.c_str());
720 ✗ CUSTOM_ASSERT(access(key_file.c_str(), R_OK) == 0, "%s is not readable",
721 key_file.c_str());
722
723 // string spooler_definition_string = string("gw,,") + g_gateway_url;
724 // create a spooler that will upload to S3
725 ✗ string const spooler_definition_string = string("S3,") + dir_temp + ","
726 ✗ + repo_name + "@" + s3_file;
727
728 // load gateway lease
729 ✗ if (!gateway::ReadKeys(key_file, &g_gateway_key_id, &g_gateway_secret)) {
730 ✗ LogCvmfs(kLogCvmfs, kLogStderr, "gateway::ReadKeys failed");
731 ✗ return 1;
732 }
733
734 ✗ uint64_t current_revision = 0;
735 ✗ std::string current_root_hash = "";
736
737 // acquire lease and save token to a file in the tmpdir
738 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Acquiring gateway lease on %s",
739 lease_path.c_str());
740 ✗ g_session_token = acquire_lease(g_gateway_key_id, g_gateway_secret,
741 ✗ repo_name + lease_path, g_gateway_url,
742 force_cancel_lease, &current_revision,
743 ✗ current_root_hash, lease_busy_retry_interval);
744
745
746 ✗ char *_tmpfile = strdup((dir_temp + "/gateway_session_token_XXXXXX").c_str());
747 ✗ int const temp_fd = mkstemp(_tmpfile);
748 ✗ g_session_token_file = string(_tmpfile);
749 ✗ free(_tmpfile);
750
751 ✗ FILE *fout = fdopen(temp_fd, "wb");
752 ✗ CUSTOM_ASSERT(fout != NULL,
753 "failed to open session token file %s for writing",
754 g_session_token_file.c_str());
755 ✗ fputs(g_session_token.c_str(), fout);
756 ✗ fclose(fout);
757
758 // now start the lease refresh thread
759 pthread_t lease_thread;
760 ✗ if (0 != pthread_create(&lease_thread, NULL, lease_refresh_thread, NULL)) {
761 ✗ LogCvmfs(kLogCvmfs, kLogStderr, "Unable to start lease refresh thread");
762 ✗ cancel_lease();
763 ✗ return 1;
764 }
765
766 // now initialise the various bits we need
767
768 upload::SpoolerDefinition spooler_definition(
769 spooler_definition_string, shash::kSha1, zlib::kZlibDefault, false, true,
770 SyncParameters::kDefaultMinFileChunkSize,
771 SyncParameters::kDefaultAvgFileChunkSize,
772 ✗ SyncParameters::kDefaultMaxFileChunkSize, g_session_token_file, key_file);
773
774 ✗ if (args.find('q') != args.end()) {
775 ✗ spooler_definition.number_of_concurrent_uploads = String2Uint64(
776 ✗ *args.find('q')->second);
777 }
778
779 upload::SpoolerDefinition const spooler_definition_catalogs(
780 ✗ spooler_definition.Dup2DefaultCompression());
781
782 std::unique_ptr<upload::Spooler> const spooler_catalogs(
783 ✗ upload::Spooler::Construct(spooler_definition_catalogs, nullptr));
784
785 ✗ if (spooler_catalogs.get() == nullptr) {
786 ✗ LogCvmfs(kLogCvmfs, kLogStderr, "spooler_catalogs invalid");
787 ✗ cancel_lease();
788 ✗ return 1;
789 }
790 ✗ if (!InitDownloadManager(true, proxy, kCatalogDownloadMultiplier)) {
791 ✗ LogCvmfs(kLogCvmfs, kLogStderr, "download manager init failed");
792 ✗ cancel_lease();
793 ✗ return 1;
794 }
795 ✗ if (!InitSignatureManager(public_keys, "")) {
796 ✗ LogCvmfs(kLogCvmfs, kLogStderr, "signature manager init failed");
797 ✗ cancel_lease();
798 ✗ return 1;
799 }
800
801 ✗ std::unique_ptr<manifest::Manifest> manifest;
802
803 ✗ manifest.reset(FetchRemoteManifest(stratum0, repo_name, shash::Any()));
804
805 ✗ if (manifest.get() == nullptr) {
806 ✗ LogCvmfs(kLogCvmfs, kLogStderr, "manifest invalid");
807 ✗ cancel_lease();
808 ✗ return 1;
809 }
810
811 ✗ if (current_revision > 0) {
812 ✗ if (current_revision == manifest->revision()) {
813 ✗ if (current_root_hash != manifest->catalog_hash().ToString()) {
814 ✗ LogCvmfs(kLogCvmfs, kLogStderr,
815 "Mismatch between cvmfspublished and gateway hash for "
816 "revision %lu (%s!=%s)",
817 current_revision, current_root_hash.c_str(),
818 manifest->catalog_hash().ToString().c_str());
819 ✗ cancel_lease();
820 ✗ return 1;
821 } else {
822 ✗ LogCvmfs(kLogCvmfs, kLogStdout,
823 "Gateway and .cvmfspublished agree on repo version %lu",
824 current_revision);
825 }
826 }
827 ✗ if (current_revision > manifest->revision()) {
828 ✗ LogCvmfs(kLogCvmfs, kLogStdout,
829 "Gateway has supplied a newer revision than the current "
830 ".cvmfspublished %lu > %lu",
831 current_revision, manifest->revision());
832 ✗ manifest->set_revision(current_revision);
833 ✗ manifest->set_catalog_hash(shash::MkFromHexPtr(
834 shash::HexPtr(current_root_hash), shash::kSuffixCatalog));
835 ✗ } else if (current_revision < manifest->revision()) {
836 ✗ LogCvmfs(kLogCvmfs, kLogStdout,
837 "Gateway has supplied an older revision than the current "
838 ".cvmfspublished %lu < %lu",
839 current_revision, manifest->revision());
840 }
841 } else {
842 ✗ LogCvmfs(kLogCvmfs, kLogStdout,
843 "Gateway has not supplied a revision. Using .cvmfspublished");
844 }
845
846
847 // get hash of current root catalog, remove terminal "C", encode it
848 ✗ string const old_root_hash = manifest->catalog_hash().ToString(true);
849 ✗ string const hash = old_root_hash.substr(0, old_root_hash.length() - 1);
850 ✗ shash::Any const base_hash = shash::MkFromHexPtr(shash::HexPtr(hash),
851 shash::kSuffixCatalog);
852 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "old_root_hash: %s", old_root_hash.c_str());
853
854 ✗ bool const is_balanced = false;
855
856 catalog::WritableCatalogManager catalog_manager(
857 base_hash, stratum0, dir_temp, spooler_catalogs.get(), download_manager(),
858 false, SyncParameters::kDefaultNestedKcatalogLimit,
859 SyncParameters::kDefaultRootKcatalogLimit,
860 SyncParameters::kDefaultFileMbyteLimit, statistics(), is_balanced,
861 SyncParameters::kDefaultMaxWeight, SyncParameters::kDefaultMinWeight,
862 ✗ dir_temp /* dir_cache */);
863
864 ✗ catalog_manager.Init();
865
866
867 // now graft the contents of the DB
868 ✗ vector<sqlite3 *> open_dbs;
869 ✗ for (auto &&db_file : sqlite_db_vec) {
870 sqlite3 *db;
871 ✗ CHECK_SQLITE_ERROR(
872 sqlite3_open_v2(db_file.c_str(), &db, SQLITE_OPEN_READONLY, NULL),
873 SQLITE_OK);
874 ✗ relax_db_locking(db);
875 ✗ open_dbs.push_back(db);
876 }
877 ✗ process_sqlite(open_dbs, catalog_manager, allow_additions, allow_deletions,
878 ✗ lease_path.substr(1), additional_prefix);
879 ✗ for (auto &&db : open_dbs) {
880 ✗ CHECK_SQLITE_ERROR(sqlite3_close_v2(db), SQLITE_OK);
881 }
882
883 // commit changes
884 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Committing changes...");
885 ✗ if (!catalog_manager.Commit(false, false, manifest.get())) {
886 ✗ LogCvmfs(kLogCvmfs, kLogStderr, "something went wrong during sync");
887 ✗ cancel_lease();
888 ✗ return 1;
889 }
890
891 // finalize the spooler
892 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Waiting for all uploads to finish...");
893 ✗ spooler_catalogs->WaitForUpload();
894
895 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Exporting repository manifest");
896
897 // We call FinalizeSession(true) this time, to also trigger the commit
898 // operation on the gateway machine (if the upstream is of type "gw").
899
900 // Get the path of the new root catalog
901 ✗ const string new_root_hash = manifest->catalog_hash().ToString(true);
902
903 // if (!spooler_catalogs->FinalizeSession(true, old_root_hash, new_root_hash,
904 // RepositoryTag())) {
905 // LogCvmfs(kLogCvmfs, kLogStderr, "Failed to commit the transaction");
906 // // lease is only released on success
907 // cancel_lease();
908 // return 1;
909 // }
910
911 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Committing with priority %" PRId64,
912 g_priority);
913
914 ✗ bool const ok = make_commit_on_gateway(old_root_hash, new_root_hash,
915 ✗ g_priority);
916 ✗ if (!ok) {
917 ✗ LogCvmfs(kLogCvmfs, kLogStderr,
918 "something went wrong during commit on gateway");
919 ✗ cancel_lease();
920 ✗ exit(1);
921 }
922
923
924 ✗ unlink(g_session_token_file.c_str());
925
926 ✗ g_stop_refresh = true;
927
928 ✗ if (g_wait_for_update != "") {
929 ✗ invalidate_manifest(proxy, stratum0 + "/.cvmfspublished");
930 ✗ const int ret = wait_for_update(g_wait_for_update, g_final_revision, g_wait_for_update_timeout_sec);
931 ✗ if (ret) {
932 ✗ return ret;
933 }
934 }
935
936 ✗ if (check_completed_graft_property) {
937 ✗ setDatabaseMarkedComplete(sqlite_db_vec[0].c_str());
938 }
939
940
941 ✗ return 0;
942 }
943
944 ✗ size_t writeFunction(void *ptr, size_t size, size_t nmemb, std::string *data) {
945 ✗ return size * nmemb;
946 }
947
948 ✗ void replaceAllSubstrings(std::string &str, const std::string &from,
949 const std::string &to) {
950 ✗ if (from.empty()) {
951 ✗ return; // Avoid infinite loop if 'from' is an empty string.
952 }
953 ✗ size_t startPos = 0;
954 ✗ while ((startPos = str.find(from, startPos)) != std::string::npos) {
955 ✗ str.replace(startPos, from.length(), to);
956 ✗ startPos += to.length(); // Advance startPos to avoid replacing the
957 // substring just inserted.
958 }
959 }
960
961
962 ✗ void swissknife::IngestSQL::process_sqlite(
963 const std::vector<sqlite3 *> &dbs,
964 catalog::WritableCatalogManager &catalog_manager, bool allow_additions,
965 bool allow_deletions, const std::string &lease_path,
966 const std::string &additional_prefix) {
967 ✗ std::map<std::string, Directory> all_dirs;
968 ✗ std::map<std::string, std::vector<File> > all_files;
969 ✗ std::map<std::string, std::vector<Symlink> > all_symlinks;
970
971 ✗ for (auto &&db : dbs) {
972 ✗ load_dirs(db, lease_path, additional_prefix, all_dirs);
973 }
974
975 // put in a nested scope so we can free up memory of `dir_names`
976 {
977 ✗ LogCvmfs(kLogCvmfs, kLogStdout,
978 "Precaching existing directories (starting from %s)",
979 lease_path.c_str());
980 ✗ std::unordered_set<std::string> dir_names;
981 ✗ std::transform(all_dirs.begin(), all_dirs.end(),
982 std::inserter(dir_names, dir_names.end()),
983 ✗ [](const std::pair<std::string, Directory> &pair) {
984 ✗ return MakeCatalogPath(pair.first);
985 });
986 ✗ catalog_manager.LoadCatalogs(MakeCatalogPath(lease_path), dir_names);
987 }
988
989 ✗ for (auto &&db : dbs) {
990 ✗ load_files(db, lease_path, additional_prefix, all_files);
991 ✗ load_symlinks(db, lease_path, additional_prefix, all_symlinks);
992 }
993
994 // perform all deletions first
995 ✗ if (allow_deletions) {
996 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Processing deletions...");
997 ✗ for (auto &&db : dbs) {
998 ✗ CHECK_SQLITE_ERROR(
999 do_deletions(db, catalog_manager, lease_path, additional_prefix),
1000 SQLITE_OK);
1001 }
1002 }
1003
1004 ✗ if (allow_additions) {
1005 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Processing additions...");
1006 // first ensure all directories are present and create missing ones
1007 ✗ do_additions(all_dirs, all_files, all_symlinks, lease_path,
1008 catalog_manager);
1009 }
1010 }
1011
1012 ✗ void add_dir_to_tree(
1013 std::string path,
1014 std::unordered_map<std::string, std::set<std::string> > &tree,
1015 const std::string &lease_path) {
1016 ✗ tree[path];
1017 ✗ std::string parent_path = get_parent(path);
1018 // recursively create any missing parents in the tree
1019 // avoid creating a loop when we insert the root path
1020 ✗ while (path != parent_path && path != lease_path
1021 ✗ && !tree[parent_path].count(path)) {
1022 ✗ tree[parent_path].insert(path);
1023 ✗ path = parent_path;
1024 ✗ parent_path = get_parent(path);
1025 }
1026 }
1027
1028 ✗ int swissknife::IngestSQL::do_additions(
1029 const DirMap &all_dirs, const FileMap &all_files,
1030 const SymlinkMap &all_symlinks, const std::string &lease_path,
1031 catalog::WritableCatalogManager &catalog_manager) {
1032 // STEP 1:
1033 // - collect all the dirs/symlinks/files we need to process from the DB
1034 // - build a tree of paths for DFS traversal
1035 // - note the tree will contain all parent dirs of symlinks/files even if
1036 // those are not
1037 // explicitly added to the dirs table
1038 ✗ std::unordered_map<std::string, std::set<std::string> > tree;
1039 ✗ for (auto &&p : all_dirs) {
1040 ✗ add_dir_to_tree(p.first, tree, lease_path);
1041 }
1042 ✗ for (auto &&p : all_files) {
1043 ✗ add_dir_to_tree(p.first, tree, lease_path);
1044 }
1045 ✗ for (auto &&p : all_symlinks) {
1046 ✗ add_dir_to_tree(p.first, tree, lease_path);
1047 }
1048 ✗ int const row_count = static_cast<int>(tree.size());
1049 ✗ int const print_every = calculate_print_frequency(row_count);
1050 ✗ int curr_row = 0;
1051 ✗ LogCvmfs(kLogCvmfs, kLogStdout,
1052 "Changeset: %ld dirs, %ld files, %ld symlinks", tree.size(),
1053 all_files.size(), all_symlinks.size());
1054
1055 // STEP 2:
1056 // - process all the changes with DFS traversal
1057 // - make directories in pre-order
1058 // - add files/symlinks and schedule upload in post-order
1059 ✗ catalog_manager.SetupSingleCatalogUploadCallback();
1060 ✗ std::stack<string> dfs_stack;
1061 ✗ for (auto &&p : tree) {
1062 // figure out the starting point by checking whose parent is missing from
1063 // the tree
1064 ✗ if (p.first == "" || !tree.count(get_parent(p.first))) {
1065 ✗ CUSTOM_ASSERT(dfs_stack.empty(),
1066 "provided DB input forms more than one path trees");
1067 ✗ dfs_stack.push(p.first);
1068 }
1069 }
1070 ✗ std::set<string> visited;
1071 ✗ while (!dfs_stack.empty()) {
1072 ✗ string const curr_dir = dfs_stack.top();
1073 // add content for the dir in post-order traversal
1074 ✗ if (visited.count(curr_dir)) {
1075 ✗ curr_row++;
1076 ✗ if (all_symlinks.count(curr_dir)) {
1077 ✗ add_symlinks(catalog_manager, all_symlinks.at(curr_dir));
1078 }
1079 ✗ if (all_files.count(curr_dir)) {
1080 ✗ add_files(catalog_manager, all_files.at(curr_dir));
1081 }
1082 // snapshot the dir (if it's a nested catalog mountpoint)
1083 ✗ catalog::DirectoryEntry dir_entry;
1084 ✗ bool exists = false;
1085 ✗ exists = catalog_manager.LookupDirEntry(
1086 ✗ MakeCatalogPath(curr_dir), catalog::kLookupDefault, &dir_entry);
1087 ✗ CUSTOM_ASSERT(exists, "Directory %s is missing from the catalog",
1088 curr_dir.c_str());
1089 ✗ if (dir_entry.IsNestedCatalogMountpoint()
1090 ✗ || dir_entry.IsNestedCatalogRoot()) {
1091 ✗ catalog_manager.AddCatalogToQueue(curr_dir);
1092 ✗ catalog_manager.ScheduleReadyCatalogs();
1093 }
1094 ✗ dfs_stack.pop();
1095 ✗ SHOW_PROGRESS("directories", print_every, curr_row, row_count);
1096 ✗ } else {
1097 ✗ visited.insert(curr_dir);
1098 // push children to the stack
1099 ✗ auto it = tree.find(curr_dir);
1100 ✗ if (it != tree.end()) {
1101 ✗ for (auto &&child : it->second) {
1102 ✗ dfs_stack.push(child);
1103 }
1104 ✗ tree.erase(it);
1105 }
1106 ✗ if (!all_dirs.count(curr_dir))
1107 ✗ continue;
1108
1109 // create the dir first in pre-order traversal
1110 ✗ const IngestSQL::Directory &dir = all_dirs.at(curr_dir);
1111 ✗ catalog::DirectoryEntry dir_entry;
1112
1113 ✗ bool exists = false;
1114 ✗ exists = catalog_manager.LookupDirEntry(
1115 ✗ MakeCatalogPath(curr_dir), catalog::kLookupDefault, &dir_entry);
1116 ✗ CUSTOM_ASSERT(
1117 !(exists && !S_ISDIR(dir_entry.mode_)),
1118 "Refusing to replace existing file/symlink at %s with a directory",
1119 dir.name.c_str());
1120
1121 ✗ dir_entry.name_ = NameString(get_basename(dir.name));
1122 ✗ dir_entry.mtime_ = dir.mtime / 1000000000;
1123 ✗ dir_entry.mode_ = dir.mode | S_IFDIR;
1124 ✗ dir_entry.mode_ &= (S_IFDIR | 0777);
1125 ✗ dir_entry.uid_ = dir.owner;
1126 ✗ dir_entry.gid_ = dir.grp;
1127 ✗ dir_entry.has_xattrs_ = !dir.xattr.IsEmpty();
1128
1129 ✗ bool add_nested_catalog = false;
1130
1131 ✗ if (exists) {
1132 ✗ catalog_manager.TouchDirectory(dir_entry, dir.xattr, dir.name);
1133 ✗ if ((!dir_entry.IsNestedCatalogMountpoint()
1134 ✗ && !dir_entry.IsNestedCatalogRoot())
1135 ✗ && (g_add_missing_catalogs || dir.nested)) {
1136 ✗ add_nested_catalog = true;
1137 ✗ LogCvmfs(kLogCvmfs, kLogVerboseMsg,
1138 "Touching existing directory %s and adding nested catalog",
1139 dir.name.c_str());
1140 } else {
1141 ✗ LogCvmfs(kLogCvmfs, kLogVerboseMsg, "Touching existing directory %s",
1142 dir.name.c_str());
1143 }
1144 } else {
1145 ✗ LogCvmfs(kLogCvmfs, kLogVerboseMsg, "Adding directory [%s]",
1146 dir.name.c_str());
1147 ✗ catalog_manager.AddDirectory(dir_entry, dir.xattr,
1148 ✗ get_parent(dir.name));
1149 ✗ if (dir.nested) {
1150 ✗ add_nested_catalog = true;
1151 }
1152 }
1153 ✗ if (add_nested_catalog) {
1154 // now add a .cvmfscatalog file
1155 // so that manual changes won't remove the nested catalog
1156 ✗ LogCvmfs(kLogCvmfs, kLogVerboseMsg,
1157 "Placing .cvmfscatalog file in [%s]", dir.name.c_str());
1158 ✗ catalog::DirectoryEntryBase dir2;
1159 ✗ dir2.name_ = NameString(".cvmfscatalog");
1160 ✗ dir2.mtime_ = dir.mtime / 1000000000;
1161 ✗ dir2.mode_ = (S_IFREG | 0666);
1162 ✗ dir2.uid_ = 0;
1163 ✗ dir2.gid_ = 0;
1164 ✗ dir2.has_xattrs_ = 0;
1165 ✗ dir2.checksum_ = shash::MkFromHexPtr(
1166 shash::HexPtr("da39a3ee5e6b4b0d3255bfef95601890afd80709"),
1167 shash::kSuffixNone); // hash of ""
1168 ✗ XattrList const xattr2;
1169 ✗ catalog_manager.AddFile(dir2, xattr2, dir.name);
1170
1171 ✗ LogCvmfs(kLogCvmfs, kLogVerboseMsg, "Creating Nested Catalog [%s]",
1172 dir.name.c_str());
1173 ✗ catalog_manager.CreateNestedCatalog(dir.name);
1174 }
1175 }
1176 }
1177
1178 // sanity check that we have processed all the input
1179 ✗ CUSTOM_ASSERT(tree.empty(),
1180 "not all directories are processed, malformed input DB?");
1181 ✗ catalog_manager.RemoveSingleCatalogUploadCallback();
1182 ✗ return 0;
1183 }
1184
1185 ✗ int swissknife::IngestSQL::add_symlinks(
1186 catalog::WritableCatalogManager &catalog_manager,
1187 const std::vector<Symlink> &symlinks) {
1188 ✗ for (auto &&symlink : symlinks) {
1189 ✗ catalog::DirectoryEntry dir;
1190 ✗ catalog::DirectoryEntryBase dir2;
1191 ✗ XattrList const xattr;
1192 ✗ bool exists = false;
1193 ✗ exists = catalog_manager.LookupDirEntry(MakeCatalogPath(symlink.name),
1194 catalog::kLookupDefault, &dir);
1195
1196 ✗ dir2.name_ = NameString(get_basename(symlink.name));
1197 ✗ dir2.mtime_ = symlink.mtime / 1000000000;
1198 ✗ dir2.uid_ = symlink.owner;
1199 ✗ dir2.gid_ = symlink.grp;
1200 ✗ dir2.has_xattrs_ = false;
1201 ✗ dir2.symlink_ = LinkString(symlink.target);
1202 ✗ dir2.mode_ = S_IFLNK | 0777;
1203
1204 ✗ int noop = false;
1205
1206 ✗ if (exists) {
1207 ✗ if (symlink.skip_if_file_or_dir) {
1208 ✗ if (S_ISDIR(dir.mode_) || S_ISREG(dir.mode_)) {
1209 ✗ LogCvmfs(kLogCvmfs, kLogVerboseMsg,
1210 "File or directory for symlink [%s] exists, skipping "
1211 "symlink creation",
1212 symlink.name.c_str());
1213 ✗ noop = true;
1214 ✗ } else if (S_ISLNK(dir.mode_)) {
1215 ✗ LogCvmfs(kLogCvmfs, kLogVerboseMsg, "Removing existing symlink [%s]",
1216 symlink.name.c_str());
1217 ✗ catalog_manager.RemoveFile(symlink.name);
1218 } else {
1219 ✗ CUSTOM_ASSERT(0, "unknown mode for dirent: %d", dir.mode_);
1220 }
1221 } else {
1222 ✗ CUSTOM_ASSERT(!S_ISDIR(dir.mode_),
1223 "Not removing directory [%s] to create symlink",
1224 symlink.name.c_str());
1225 ✗ LogCvmfs(kLogCvmfs, kLogVerboseMsg,
1226 "Removing existing file/symlink [%s]", symlink.name.c_str());
1227 ✗ catalog_manager.RemoveFile(symlink.name);
1228 }
1229 }
1230 ✗ if (!noop) {
1231 ✗ string const parent = get_parent(symlink.name);
1232 ✗ LogCvmfs(kLogCvmfs, kLogVerboseMsg, "Adding symlink [%s] -> [%s]",
1233 symlink.name.c_str(), symlink.target.c_str());
1234 ✗ catalog_manager.AddFile(dir2, xattr, parent);
1235 }
1236 }
1237 ✗ return 0;
1238 }
1239
1240 ✗ static int check_hash(const char *hash) {
1241 ✗ if (strlen(hash) != 40) {
1242 ✗ return 1;
1243 }
1244 ✗ for (int i = 0; i < 40; i++) {
1245 // < '0' || > 'f' || ( > '9' && < 'a' )
1246 ✗ if (hash[i] < 0x30 || hash[i] > 0x66
1247 ✗ || (hash[i] > 0x39 && hash[i] < 0x61)) {
1248 ✗ return 1;
1249 }
1250 }
1251 ✗ return 0;
1252 }
1253
1254 ✗ bool check_prefix(const std::string &path, const std::string &prefix) {
1255 ✗ if (prefix == "" || prefix == "/") {
1256 ✗ return true;
1257 }
1258 ✗ if ("/" + path == prefix) {
1259 ✗ return true;
1260 }
1261 ✗ if (!HasPrefix(path, prefix, false)) {
1262 ✗ LogCvmfs(kLogCvmfs, kLogVerboseMsg, "Entry %s is outside lease path: %s",
1263 path.c_str(), prefix.c_str());
1264 ✗ return false;
1265 }
1266 ✗ return true;
1267 }
1268
1269 ✗ void swissknife::IngestSQL::load_dirs(
1270 sqlite3 *db, const std::string &lease_path,
1271 const std::string &additional_prefix,
1272 std::map<std::string, Directory> &all_dirs) {
1273 sqlite3_stmt *stmt;
1274 ✗ int const schema_revision = get_db_schema_revision(db);
1275 string select_stmt = "SELECT name, mode, mtime, owner, grp, acl, nested FROM "
1276 ✗ "dirs";
1277 ✗ if (schema_revision <= 3) {
1278 ✗ select_stmt = "SELECT name, mode, mtime, owner, grp, acl FROM dirs";
1279 }
1280 ✗ int const ret = sqlite3_prepare_v2(db, select_stmt.c_str(), -1, &stmt, NULL);
1281 ✗ CHECK_SQLITE_ERROR(ret, SQLITE_OK);
1282 ✗ while (sqlite3_step(stmt) == SQLITE_ROW) {
1283 ✗ char *name_cstr = (char *)sqlite3_column_text(stmt, 0);
1284 ✗ mode_t const mode = sqlite3_column_int(stmt, 1);
1285 ✗ time_t const mtime = sqlite3_column_int64(stmt, 2);
1286 ✗ uid_t const owner = sqlite3_column_int(stmt, 3);
1287 ✗ gid_t const grp = sqlite3_column_int(stmt, 4);
1288 ✗ int const nested = schema_revision <= 3 ? 1 : sqlite3_column_int(stmt, 6);
1289
1290 ✗ string const name = additional_prefix + sanitise_name(name_cstr);
1291 ✗ CUSTOM_ASSERT(check_prefix(name, lease_path),
1292 "%s is not below lease path %s", name.c_str(),
1293 lease_path.c_str());
1294
1295 ✗ Directory dir(name, mtime, mode, owner, grp, nested);
1296 ✗ char *acl = (char *)sqlite3_column_text(stmt, 5);
1297 ✗ dir.xattr = marshal_xattrs(acl);
1298 ✗ all_dirs.insert(std::make_pair(name, dir));
1299 }
1300 ✗ CHECK_SQLITE_ERROR(sqlite3_finalize(stmt), SQLITE_OK);
1301 }
1302
1303 ✗ void swissknife::IngestSQL::load_files(
1304 sqlite3 *db, const std::string &lease_path,
1305 const std::string &additional_prefix,
1306 std::map<std::string, std::vector<File> > &all_files) {
1307 sqlite3_stmt *stmt;
1308 ✗ int const schema_revision = get_db_schema_revision(db);
1309 string select_stmt = "SELECT name, mode, mtime, owner, grp, size, hashes, "
1310 ✗ "internal, compressed FROM files";
1311 ✗ if (schema_revision <= 2) {
1312 select_stmt = "SELECT name, mode, mtime, owner, grp, size, hashes, "
1313 ✗ "internal FROM files";
1314 }
1315 ✗ int const ret = sqlite3_prepare_v2(db, select_stmt.c_str(), -1, &stmt, NULL);
1316 ✗ CHECK_SQLITE_ERROR(ret, SQLITE_OK);
1317 ✗ while (sqlite3_step(stmt) == SQLITE_ROW) {
1318 ✗ char *name = (char *)sqlite3_column_text(stmt, 0);
1319 ✗ mode_t const mode = sqlite3_column_int(stmt, 1);
1320 ✗ time_t const mtime = sqlite3_column_int64(stmt, 2);
1321 ✗ uid_t const owner = sqlite3_column_int(stmt, 3);
1322 ✗ gid_t const grp = sqlite3_column_int(stmt, 4);
1323 ✗ size_t const size = sqlite3_column_int64(stmt, 5);
1324 ✗ char *hashes_cstr = (char *)sqlite3_column_text(stmt, 6);
1325 ✗ int const internal = sqlite3_column_int(stmt, 7);
1326 ✗ int const compressed = schema_revision <= 2 ? 0
1327 ✗ : sqlite3_column_int(stmt, 8);
1328
1329 ✗ string names = additional_prefix + sanitise_name(name);
1330 ✗ CUSTOM_ASSERT(check_prefix(names, lease_path),
1331 "%s is not below lease path %s", names.c_str(),
1332 lease_path.c_str());
1333 ✗ string const parent_dir = get_parent(names);
1334
1335 ✗ if (!all_files.count(parent_dir)) {
1336 ✗ all_files[parent_dir] = vector<swissknife::IngestSQL::File>();
1337 }
1338 ✗ all_files[parent_dir].emplace_back(std::move(names), mtime, size, owner,
1339 grp, mode, internal, compressed);
1340
1341 // tokenize hashes
1342 char *ref;
1343 char *tok;
1344 ✗ tok = strtok_r(hashes_cstr, ",", &ref);
1345 ✗ vector<off_t> offsets;
1346 ✗ vector<size_t> sizes;
1347 ✗ vector<shash::Any> hashes;
1348 ✗ off_t offset = 0;
1349
1350 CUSTOM_ASSERT(size >= 0, "file size cannot be negative [%s]",
1351 names.c_str());
1352 ✗ size_t const kChunkSize = internal ? kInternalChunkSize
1353 : kExternalChunkSize;
1354
1355 ✗ while (tok) {
1356 ✗ offsets.push_back(offset);
1357 // TODO: check the hash format
1358 ✗ CUSTOM_ASSERT(check_hash(tok) == 0,
1359 "provided hash for [%s] is invalid: %s", names.c_str(),
1360 tok);
1361 ✗ hashes.push_back(
1362 ✗ shash::MkFromHexPtr(shash::HexPtr(tok), shash::kSuffixNone));
1363 ✗ tok = strtok_r(NULL, ",", &ref);
1364 ✗ offset += kChunkSize; // in the future we might want variable chunk
1365 // sizes specified in the DB
1366 }
1367 ✗ size_t expected_num_chunks = size / kChunkSize;
1368 ✗ if (expected_num_chunks * (size_t)kChunkSize < (size_t)size || size == 0) {
1369 ✗ expected_num_chunks++;
1370 }
1371 ✗ CUSTOM_ASSERT(
1372 offsets.size() == expected_num_chunks,
1373 "offsets size %ld does not match expected number of chunks %ld",
1374 offsets.size(), expected_num_chunks);
1375 ✗ for (size_t i = 0; i < offsets.size() - 1; i++) {
1376 ✗ sizes.push_back(size_t(offsets[i + 1] - offsets[i]));
1377 }
1378
1379 ✗ sizes.push_back(size_t(size - offsets[offsets.size() - 1]));
1380 ✗ for (size_t i = 0; i < offsets.size(); i++) {
1381 ✗ FileChunk const chunk = FileChunk(hashes[i], offsets[i], sizes[i]);
1382 ✗ all_files[parent_dir].back().chunks.PushBack(chunk);
1383 }
1384 }
1385 ✗ CHECK_SQLITE_ERROR(sqlite3_finalize(stmt), SQLITE_OK);
1386 }
1387
1388 ✗ void swissknife::IngestSQL::load_symlinks(
1389 sqlite3 *db, const std::string &lease_path,
1390 const std::string &additional_prefix,
1391 std::map<std::string, std::vector<Symlink> > &all_symlinks) {
1392 sqlite3_stmt *stmt;
1393 string const select_stmt = "SELECT name, target, mtime, owner, grp, "
1394 ✗ "skip_if_file_or_dir FROM links";
1395 ✗ int const ret = sqlite3_prepare_v2(db, select_stmt.c_str(), -1, &stmt, NULL);
1396 ✗ CHECK_SQLITE_ERROR(ret, SQLITE_OK);
1397 ✗ while (sqlite3_step(stmt) == SQLITE_ROW) {
1398 ✗ char *name_cstr = (char *)sqlite3_column_text(stmt, 0);
1399 ✗ char *target_cstr = (char *)sqlite3_column_text(stmt, 1);
1400 ✗ time_t const mtime = sqlite3_column_int64(stmt, 2);
1401 ✗ uid_t const owner = sqlite3_column_int(stmt, 3);
1402 ✗ gid_t const grp = sqlite3_column_int(stmt, 4);
1403 ✗ int const skip_if_file_or_dir = sqlite3_column_int(stmt, 5);
1404
1405 ✗ string names = additional_prefix + sanitise_name(name_cstr);
1406 ✗ CUSTOM_ASSERT(check_prefix(names, lease_path),
1407 "%s is not below lease path %s", names.c_str(),
1408 lease_path.c_str());
1409 ✗ string target = target_cstr;
1410 ✗ string const parent_dir = get_parent(names);
1411
1412 ✗ if (!all_symlinks.count(parent_dir)) {
1413 ✗ all_symlinks[parent_dir] = vector<swissknife::IngestSQL::Symlink>();
1414 }
1415 ✗ all_symlinks[parent_dir].emplace_back(std::move(names), std::move(target),
1416 mtime, owner, grp,
1417 skip_if_file_or_dir);
1418 }
1419 ✗ CHECK_SQLITE_ERROR(sqlite3_finalize(stmt), SQLITE_OK);
1420 }
1421
1422 ✗ int swissknife::IngestSQL::add_files(
1423 catalog::WritableCatalogManager &catalog_manager,
1424 const std::vector<File> &files) {
1425 ✗ for (auto &&file : files) {
1426 ✗ catalog::DirectoryEntry dir;
1427 ✗ XattrList const xattr;
1428 ✗ bool exists = false;
1429 ✗ exists = catalog_manager.LookupDirEntry(MakeCatalogPath(file.name),
1430 catalog::kLookupDefault, &dir);
1431
1432 ✗ dir.name_ = NameString(get_basename(file.name));
1433 ✗ dir.mtime_ = file.mtime / 1000000000;
1434 ✗ dir.mode_ = file.mode | S_IFREG;
1435 ✗ dir.mode_ &= (S_IFREG | 0777);
1436 ✗ dir.uid_ = file.owner;
1437 ✗ dir.gid_ = file.grp;
1438 ✗ dir.size_ = file.size;
1439 ✗ dir.has_xattrs_ = false;
1440 ✗ dir.is_external_file_ = !file.internal;
1441 ✗ dir.set_is_chunked_file(true);
1442 ✗ dir.checksum_ = shash::MkFromHexPtr(
1443 shash::HexPtr("0000000000000000000000000000000000000000"),
1444 shash::kSuffixNone);
1445
1446 // compression is permitted only for internal data
1447 ✗ CUSTOM_ASSERT(file.internal || (!file.internal && file.compressed < 2),
1448 "compression is only allowed for internal data [%s]",
1449 file.name.c_str());
1450
1451 ✗ switch (file.compressed) {
1452 ✗ case 1: // Uncompressed
1453 ✗ dir.compression_algorithm_ = zlib::kNoCompression;
1454 ✗ break;
1455 ✗ case 2: // Compressed with Zlib
1456 ✗ dir.compression_algorithm_ = zlib::kZlibDefault;
1457 ✗ break;
1458 // future cases: different compression schemes
1459 ✗ default: // default behaviour: compressed if internal, content-addressed.
1460 // Uncompressed if external
1461 ✗ dir.compression_algorithm_ = file.internal ? zlib::kZlibDefault
1462 : zlib::kNoCompression;
1463 }
1464
1465 ✗ if (exists) {
1466 ✗ CUSTOM_ASSERT(
1467 !S_ISDIR(dir.mode()) && !S_ISLNK(dir.mode()),
1468 "Refusing to replace existing dir/symlink at %s with a file",
1469 file.name.c_str());
1470 ✗ LogCvmfs(kLogCvmfs, kLogVerboseMsg, "Removing existing file [%s]",
1471 file.name.c_str());
1472 ✗ catalog_manager.RemoveFile(file.name);
1473 }
1474 ✗ string const parent = get_parent(file.name);
1475 ✗ LogCvmfs(kLogCvmfs, kLogVerboseMsg, "Adding chunked file [%s]",
1476 file.name.c_str());
1477 ✗ catalog_manager.AddChunkedFile(dir, xattr, parent, file.chunks);
1478 }
1479
1480 ✗ return 0;
1481 }
1482
1483 ✗ int swissknife::IngestSQL::do_deletions(
1484 sqlite3 *db, catalog::WritableCatalogManager &catalog_manager,
1485 const std::string &lease_path, const std::string &additional_prefix) {
1486 sqlite3_stmt *stmt;
1487 ✗ int const row_count = get_row_count(db, "deletions");
1488 ✗ int const print_every = calculate_print_frequency(row_count);
1489 ✗ int curr_row = 0;
1490 ✗ int ret = sqlite3_prepare_v2(db,
1491 "SELECT name, directory, file, link FROM "
1492 "deletions ORDER BY length(name) DESC",
1493 -1, &stmt, NULL);
1494 ✗ CHECK_SQLITE_ERROR(ret, SQLITE_OK);
1495 ✗ while (sqlite3_step(stmt) == SQLITE_ROW) {
1496 ✗ curr_row++;
1497
1498 ✗ char *name = (char *)sqlite3_column_text(stmt, 0);
1499 ✗ int64_t const isdir = sqlite3_column_int64(stmt, 1);
1500 ✗ int64_t const isfile = sqlite3_column_int64(stmt, 2);
1501 ✗ int64_t const islink = sqlite3_column_int64(stmt, 3);
1502
1503 ✗ string const names = additional_prefix + sanitise_name(name);
1504 ✗ CUSTOM_ASSERT(check_prefix(names, lease_path),
1505 "%s is not below lease path %s", names.c_str(),
1506 lease_path.c_str());
1507
1508 ✗ catalog::DirectoryEntry dirent;
1509 ✗ bool exists = false;
1510 ✗ exists = catalog_manager.LookupDirEntry(MakeCatalogPath(names),
1511 catalog::kLookupDefault, &dirent);
1512 ✗ if (exists) {
1513 ✗ if ((isdir && S_ISDIR(dirent.mode()))
1514 ✗ || (islink && S_ISLNK(dirent.mode()))
1515 ✗ || (isfile && S_ISREG(dirent.mode()))) {
1516 ✗ if (S_ISDIR(dirent.mode())) {
1517 ✗ PathString names_path(names);
1518 ✗ recursively_delete_directory(names_path, catalog_manager);
1519 ✗ } else {
1520 ✗ LogCvmfs(kLogCvmfs, kLogVerboseMsg, "Removing link/file [%s]",
1521 names.c_str());
1522 ✗ catalog_manager.RemoveFile(names);
1523 }
1524 } else {
1525 ✗ LogCvmfs(kLogCvmfs, kLogVerboseMsg,
1526 "Mismatch in deletion type, not deleting: [%s] (dir %ld/%d , "
1527 "link %ld/%d, file %ld/%d)",
1528 names.c_str(), isdir, S_ISDIR(dirent.mode()), islink,
1529 S_ISLNK(dirent.mode()), isfile, S_ISREG(dirent.mode()));
1530 }
1531 } else {
1532 ✗ LogCvmfs(kLogCvmfs, kLogVerboseMsg, "Not Removing non-existent [%s]",
1533 names.c_str());
1534 }
1535
1536 ✗ SHOW_PROGRESS("deletions", print_every, curr_row, row_count);
1537 }
1538 ✗ ret = sqlite3_finalize(stmt);
1539 ✗ return ret;
1540 }
1541
1542 const char *schema[] = {"PRAGMA journal_mode=WAL;",
1543
1544 "CREATE TABLE IF NOT EXISTS dirs ( \
1545 name TEXT PRIMARY KEY, \
1546 mode INTEGER NOT NULL DEFAULT 493,\
1547 mtime INTEGER NOT NULL DEFAULT 0,\
1548 owner INTEGER NOT NULL DEFAULT 0, \
1549 grp INTEGER NOT NULL DEFAULT 0, \
1550 acl TEXT NOT NULL DEFAULT '', \
1551 nested INTEGER DEFAULT 1);",
1552
1553 "CREATE TABLE IF NOT EXISTS files ( \
1554 name TEXT PRIMARY KEY, \
1555 mode INTEGER NOT NULL DEFAULT 420, \
1556 mtime INTEGER NOT NULL DEFAULT 0,\
1557 owner INTEGER NOT NULL DEFAULT 0,\
1558 grp INTEGER NOT NULL DEFAULT 0,\
1559 size INTEGER NOT NULL DEFAULT 0,\
1560 hashes TEXT NOT NULL DEFAULT '',\
1561 internal INTEGER NOT NULL DEFAULT 0,\
1562 compressed INTEGER NOT NULL DEFAULT 0\
1563 );",
1564
1565 "CREATE TABLE IF NOT EXISTS links (\
1566 name TEXT PRIMARY KEY,\
1567 target TEXT NOT NULL DEFAULT '',\
1568 mtime INTEGER NOT NULL DEFAULT 0,\
1569 owner INTEGER NOT NULL DEFAULT 0,\
1570 grp INTEGER NOT NULL DEFAULT 0,\
1571 skip_if_file_or_dir INTEGER NOT NULL DEFAULT 0\
1572 );",
1573
1574 "CREATE TABLE IF NOT EXISTS deletions (\
1575 name TEXT PRIMARY KEY,\
1576 directory INTEGER NOT NULL DEFAULT 0,\
1577 file INTEGER NOT NULL DEFAULT 0,\
1578 link INTEGER NOT NULL DEFAULT 0\
1579 );",
1580
1581 "CREATE TABLE IF NOT EXISTS properties (\
1582 key TEXT PRIMARY KEY,\
1583 value TEXT NOT NULL\
1584 );",
1585
1586 "INSERT INTO properties VALUES ('schema_revision', "
1587 "'4') ON CONFLICT DO NOTHING;",
1588 NULL};
1589
1590 ✗ static void create_empty_database(string &filename) {
1591 sqlite3 *db_out;
1592 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Creating empty database file %s",
1593 filename.c_str());
1594 ✗ int ret = sqlite3_open_v2(filename.c_str(), &db_out,
1595 SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL);
1596 ✗ CHECK_SQLITE_ERROR(ret, SQLITE_OK);
1597 ✗ relax_db_locking(db_out);
1598
1599 ✗ const char **ptr = schema;
1600 ✗ while (*ptr != NULL) {
1601 ✗ ret = sqlite3_exec(db_out, *ptr, NULL, NULL, NULL);
1602 ✗ CHECK_SQLITE_ERROR(ret, SQLITE_OK);
1603 ✗ ptr++;
1604 }
1605 ✗ sqlite3_close(db_out);
1606 }
1607
1608 ✗ static void recursively_delete_directory(
1609 PathString &path, catalog::WritableCatalogManager &catalog_manager) {
1610 ✗ catalog::DirectoryEntryList const listing;
1611
1612 // Add all names
1613 ✗ catalog::StatEntryList listing_from_catalog;
1614 ✗ bool const retval = catalog_manager.ListingStat(
1615 ✗ PathString("/" + path.ToString()), &listing_from_catalog);
1616
1617 ✗ CUSTOM_ASSERT(retval, "failed to call ListingStat for %s", path.c_str());
1618
1619 ✗ if (!catalog_manager.IsTransitionPoint(path.ToString())) {
1620 ✗ for (unsigned i = 0; i < listing_from_catalog.size(); ++i) {
1621 ✗ PathString entry_path;
1622 ✗ entry_path.Assign(path);
1623 ✗ entry_path.Append("/", 1);
1624 ✗ entry_path.Append(listing_from_catalog.AtPtr(i)->name.GetChars(),
1625 ✗ listing_from_catalog.AtPtr(i)->name.GetLength());
1626
1627 ✗ if (S_ISDIR(listing_from_catalog.AtPtr(i)->info.st_mode)) {
1628 ✗ LogCvmfs(kLogCvmfs, kLogVerboseMsg, "Recursing into %s/",
1629 entry_path.ToString().c_str());
1630 ✗ recursively_delete_directory(entry_path, catalog_manager);
1631
1632
1633 } else {
1634 ✗ LogCvmfs(kLogCvmfs, kLogVerboseMsg, " Recursively removing %s",
1635 entry_path.ToString().c_str());
1636 ✗ catalog_manager.RemoveFile(entry_path.ToString());
1637 }
1638 }
1639 } else {
1640 ✗ LogCvmfs(kLogCvmfs, kLogVerboseMsg, "Removing nested catalog %s",
1641 path.ToString().c_str());
1642 ✗ catalog_manager.RemoveNestedCatalog(path.ToString(), false);
1643 }
1644 ✗ LogCvmfs(kLogCvmfs, kLogVerboseMsg, "Removing directory %s",
1645 path.ToString().c_str());
1646 ✗ catalog_manager.RemoveDirectory(path.ToString());
1647 }
1648
1649
1650 ✗ static void relax_db_locking(sqlite3 *db) {
1651 ✗ int ret = 0;
1652 ✗ ret = sqlite3_exec(db, "PRAGMA temp_store=2", NULL, NULL, NULL);
1653 ✗ CHECK_SQLITE_ERROR(ret, SQLITE_OK);
1654 ✗ ret = sqlite3_exec(db, "PRAGMA synchronous=OFF", NULL, NULL, NULL);
1655 ✗ CHECK_SQLITE_ERROR(ret, SQLITE_OK);
1656 }
1657
1658
1659 ✗ extern "C" void *lease_refresh_thread(void *payload) {
1660 ✗ while (!g_stop_refresh) {
1661 ✗ sleep(2);
1662 ✗ refresh_lease();
1663 }
1664 ✗ return NULL;
1665 }
1666
1667 ✗ static bool isDatabaseMarkedComplete(const char *dbfile) {
1668 int ret;
1669 sqlite3 *db;
1670 sqlite3_stmt *stmt;
1671 ✗ bool retval = false;
1672
1673 ✗ ret = sqlite3_open(dbfile, &db);
1674 ✗ if (ret != SQLITE_OK) {
1675 ✗ return false;
1676 }
1677
1678 ✗ const char *req = "SELECT value FROM properties WHERE key='completed_graft'";
1679 ✗ ret = sqlite3_prepare_v2(db, req, -1, &stmt, NULL);
1680 ✗ if (ret != SQLITE_OK) {
1681 ✗ return false;
1682 }
1683 ✗ if (sqlite3_step(stmt) == SQLITE_ROW) {
1684 ✗ int const id = sqlite3_column_int(stmt, 0);
1685 ✗ if (id > 0) {
1686 ✗ retval = true;
1687 }
1688 }
1689 ✗ sqlite3_close(db);
1690 ✗ return retval;
1691 }
1692
1693 ✗ static void setDatabaseMarkedComplete(const char *dbfile) {
1694 int ret;
1695 sqlite3 *db;
1696 char *err;
1697
1698 ✗ ret = sqlite3_open(dbfile, &db);
1699 ✗ if (ret != SQLITE_OK) {
1700 ✗ return;
1701 }
1702
1703 ✗ const char *req = "INSERT INTO properties (key, value) VALUES "
1704 "('completed_graft',1) ON CONFLICT(key) DO UPDATE SET "
1705 "value=1 WHERE key='completed_graft'";
1706
1707 ✗ ret = sqlite3_exec(db, req, 0, 0, &err);
1708 ✗ if (ret != SQLITE_OK) {
1709 ✗ return;
1710 }
1711 ✗ sqlite3_close(db);
1712 }
1713
1714 ✗ static void invalidate_manifest(std::string proxy_list, std::string url) {
1715 // split the proxy string -- remove any '"' and split on '|' or ';'
1716 ✗ size_t pos = 0;
1717 // replace any ';' with '|' to simplify subsequent split
1718 ✗ while ((pos = proxy_list.find(';', pos)) != std::string::npos) {
1719 ✗ proxy_list.replace(pos, 1, 1, '|');
1720 ✗ ++pos;
1721 }
1722 // remove any leading or trailing '"'
1723 ✗ if (HasPrefix(proxy_list, "\"", true)) {
1724 ✗ proxy_list = proxy_list.substr(1);
1725 }
1726 ✗ if (HasSuffix(proxy_list, "\"", true)) {
1727 ✗ proxy_list = proxy_list.substr(0, proxy_list.size() - 1);
1728 }
1729 // rewrite the port from 6086 to 6081 to ensure the invalidation works
1730 ✗ replaceAllSubstrings(proxy_list, ":6086", ":6081");
1731
1732 ✗ std::vector<std::string> proxies = SplitString(proxy_list, '|');
1733
1734 // now iterate over all the proxies
1735 // for the first, force a no-cache GET back to google
1736 // for the remainder just PURGE
1737 ✗ bool first = true;
1738 ✗ for (auto p = proxies.begin(); p != proxies.end(); p++) {
1739 ✗ bool ok = true;
1740 ✗ const string proxy = *p;
1741 ✗ CURL *curl = NULL;
1742 ✗ CURLcode res = CURLE_OK;
1743 ✗ struct curl_slist *headers = NULL;
1744 ✗ curl = curl_easy_init();
1745 ✗ if (!curl) {
1746 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Unable to init curl!");
1747 ✗ return;
1748 }
1749 ✗ res = curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
1750 ✗ if (proxy != "DIRECT") {
1751 ✗ res = curl_easy_setopt(curl, CURLOPT_PROXY, proxy.c_str());
1752 }
1753 ✗ res = curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 1l);
1754 ✗ res = curl_easy_setopt(curl, CURLOPT_TIMEOUT, 3l);
1755 ✗ res = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeFunction);
1756
1757 ✗ if (first) {
1758 ✗ headers = curl_slist_append(headers, "Cache-Control: no-cache");
1759 ✗ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
1760 } else {
1761 ✗ curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PURGE");
1762 }
1763
1764 ✗ res = curl_easy_perform(curl);
1765 ✗ if (res != CURLE_OK) {
1766 ✗ LogCvmfs(kLogCvmfs, kLogStdout,
1767 "Manifest invalidation failed: curl error = [%d] [%s] url = %s "
1768 "proxy = %s",
1769 res, curl_easy_strerror(res), url.c_str(), proxy.c_str());
1770 ✗ ok = false;
1771 }
1772 ✗ if (headers) {
1773 ✗ curl_slist_free_all(headers);
1774 }
1775 ✗ curl_easy_cleanup(curl);
1776 ✗ if (ok) {
1777 ✗ first = false;
1778 }
1779 }
1780 }
1781
1782
1783 ✗ static int wait_for_update(std::string path, long revision, int timeout_sec) {
1784 char val[101];
1785 ✗ memset(val, 0, 101);
1786 ✗ long current = -1;
1787 DIR *d;
1788 time_t waiting_since;
1789 ✗ if (timeout_sec >= 0) {
1790 ✗ waiting_since = time(NULL);
1791 }
1792
1793 ✗ while (-1 != getxattr(path.c_str(), "user.revision", val, 100)) {
1794 ✗ long const x = atol(val);
1795 ✗ if (x >= revision) {
1796 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Mount reached revision %ld", x);
1797 ✗ return 0;
1798 ✗ } else if (x != current) {
1799 ✗ current = x;
1800 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Mount at revision %ld, waiting..", x);
1801 }
1802
1803 ✗ if (timeout_sec >= 0) {
1804 ✗ int const waited_seconds = time(NULL) - waiting_since;
1805 ✗ if ((waited_seconds >= 0) && (waited_seconds > timeout_sec)) {
1806 ✗ LogCvmfs(kLogCvmfs, kLogStdout, "Out of time waiting for the mount to reach revision %ld (waited %u seconds)", revision, waited_seconds);
1807 ✗ return 1;
1808 }
1809 }
1810 ✗ sleep(1);
1811 ✗ d = opendir(path.c_str());
1812 ✗ if (d) {
1813 ✗ closedir(d);
1814 }
1815 }
1816 ✗ LogCvmfs(kLogCvmfs, kLogStdout,
1817 "Unable to query user.revision xattr of [%s]: errno: %d",
1818 path.c_str(), errno);
1819 ✗ return 2;
1820 }
1821