#include "swissknife_ingestsql.h"
#include <fcntl.h>
#include <stdio.h>
#include <unistd.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <sys/types.h>
#include <pwd.h>
#include <grp.h>
#include <csignal>
#include <cstdlib>
#include <fstream>
#include <sstream>
#include <unordered_map>
#include <unordered_set>
#include <stack>
#include "acl.h"
#include "catalog_mgr_rw.h"
#include "curl/curl.h"
#include "gateway_util.h"
#include "swissknife_lease_curl.h"
#include "swissknife_lease_json.h"
#include "swissknife_sync.h"
#include "upload.h"
#include "util/logging.h"
#include "catalog_downloader.h"
#include "shortstring.h"

Include dependency graph for swissknife_ingestsql.cc:

Macros
#define	CHECK_SQLITE_ERROR(ret, expected)

#define	CUSTOM_ASSERT(check, msg,...)

#define	SHOW_PROGRESS(item, freq, curr, total)

Functions
static string	get_lease_from_paths (vector< string > paths)

static vector< string >	get_all_dirs_from_sqlite (vector< string > &sqlite_db_vec, bool include_additions, bool include_deletions)

static string	get_parent (const string &path)

static string	get_basename (const string &path)

static XattrList	marshal_xattrs (const char *acl)

static string	sanitise_name (const char *name_cstr, bool allow_leading_slash)

static void	on_signal (int sig)

static string	acquire_lease (const string &key_id, const string &secret, const string &lease_path, const string &repo_service_url, bool force_cancel_lease, uint64_t *current_revision, string &current_root_hash, unsigned int refresh_interval)

static void	cancel_lease ()

static void	refresh_lease ()

static vector< string >	get_file_list (string &path)

static int	check_hash (const char *hash)

static void	recursively_delete_directory (PathString &path, catalog::WritableCatalogManager &catalog_manager)

static void	create_empty_database (string &filename)

static void	relax_db_locking (sqlite3 *db)

static bool	check_prefix (const std::string &path, const std::string &prefix)

static bool	isDatabaseMarkedComplete (const char *dbfile)

static void	setDatabaseMarkedComplete (const char *dbfile)

void *	lease_refresh_thread (void *payload)

static string	MakeCatalogPath (const std::string &relative_path)

static uint64_t	make_commit_on_gateway (const std::string &old_root_hash, const std::string &new_root_hash, int priority)

static int	get_db_schema_revision (sqlite3 *db, const std::string &db_name="")

static int	get_row_count (sqlite3 *db, const std::string &table_name)

static int	calculate_print_frequency (int total)

std::unordered_map< string, string >	load_config (const string &config_file)

string	retrieve_config (std::unordered_map< string, string > &config_map, const string &key)

size_t	writeFunction (void ptr, size_t size, size_t nmemb, std::string data)

void	replaceAllSubstrings (std::string &str, const std::string &from, const std::string &to)

void	add_dir_to_tree (std::string path, std::unordered_map< std::string, std::set< std::string >> &tree, const std::string &lease_path)

Variables
static const unsigned	kExternalChunkSize = 24 * 1024 * 1024

static const unsigned	kInternalChunkSize = 6 * 1024 * 1024

static const unsigned	kDefaultLeaseBusyRetryInterval = 10

static const unsigned	kLeaseRefreshInterval = 90

static bool	g_lease_acquired = false

static string	g_gateway_url

static string	g_gateway_key_id

static string	g_gateway_secret

static string	g_session_token

static string	g_session_token_file

static string	g_s3_file

static time_t	g_last_lease_refresh =0

static bool	g_stop_refresh = false

static int	g_priority =0

static bool	g_add_missing_catalogs = false

bool	g_log_with_time

const char *	schema []

Macro Definition Documentation

#define CHECK_SQLITE_ERROR	(	ret,
		expected
	)

Value:

do {                                                          \
                                            if (ret!=expected) {                                        \
                                              LogCvmfs(kLogCvmfs, kLogStderr, "SQLite error: %d", ret); \
                                              assert(0);                                                \
                                            }                                                           \
                                          } while (0)

Definition at line 32 of file swissknife_ingestsql.cc.

Referenced by create_empty_database(), swissknife::IngestSQL::do_deletions(), get_all_dirs_from_sqlite(), get_db_schema_revision(), get_row_count(), swissknife::IngestSQL::load_dirs(), swissknife::IngestSQL::load_files(), swissknife::IngestSQL::load_symlinks(), swissknife::IngestSQL::Main(), swissknife::IngestSQL::process_sqlite(), and relax_db_locking().

#define CUSTOM_ASSERT	(	check,
		msg,
		...
	)

Value:

do {                                                     \
                                         if (!(check)) {                                        \
                                           LogCvmfs(kLogCvmfs, kLogStderr, msg, ##__VA_ARGS__); \
                                           assert(0);                                           \
                                         }                                                      \
                                       } while (0)

Definition at line 39 of file swissknife_ingestsql.cc.

Referenced by acquire_lease(), swissknife::IngestSQL::add_files(), swissknife::IngestSQL::add_symlinks(), swissknife::IngestSQL::do_additions(), swissknife::IngestSQL::do_deletions(), get_file_list(), get_lease_from_paths(), swissknife::IngestSQL::load_dirs(), swissknife::IngestSQL::load_files(), swissknife::IngestSQL::load_symlinks(), swissknife::IngestSQL::Main(), marshal_xattrs(), recursively_delete_directory(), retrieve_config(), and sanitise_name().

#define SHOW_PROGRESS	(	item,
		freq,
		curr,
		total
	)

Value:

do {                                                                          \
                                                 if (curr % freq == 0 || curr == total) {                                    \
                                                   LogCvmfs(kLogCvmfs, kLogStdout, "Processed %d/%d %s", curr, total, item); \
                                                 }                                                                           \
                                               } while (0)

Definition at line 46 of file swissknife_ingestsql.cc.

Referenced by swissknife::IngestSQL::do_additions(), and swissknife::IngestSQL::do_deletions().

Function Documentation

static string acquire_lease	(	const string &	key_id,
		const string &	secret,
		const string &	lease_path,
		const string &	repo_service_url,
		bool	force_cancel_lease,
		uint64_t *	current_revision,
		string &	current_root_hash,
		unsigned int	refresh_interval
	)

static

Definition at line 162 of file swissknife_ingestsql.cc.

Referenced by swissknife::IngestSQL::Main().

Here is the call graph for this function: