| Line |
Branch |
Exec |
Source |
| 1 |
|
|
/** |
| 2 |
|
|
* This file is part of the CernVM File System. |
| 3 |
|
|
*/ |
| 4 |
|
|
|
| 5 |
|
|
|
| 6 |
|
|
#include "publish/repository.h" |
| 7 |
|
|
|
| 8 |
|
|
#include <cassert> |
| 9 |
|
|
#include <cstddef> |
| 10 |
|
|
#include <cstdlib> |
| 11 |
|
|
|
| 12 |
|
|
#include "catalog_mgr_ro.h" |
| 13 |
|
|
#include "catalog_mgr_rw.h" |
| 14 |
|
|
#include "crypto/hash.h" |
| 15 |
|
|
#include "crypto/signature.h" |
| 16 |
|
|
#include "gateway_util.h" |
| 17 |
|
|
#include "history_sqlite.h" |
| 18 |
|
|
#include "ingestion/ingestion_source.h" |
| 19 |
|
|
#include "manifest.h" |
| 20 |
|
|
#include "manifest_fetch.h" |
| 21 |
|
|
#include "network/download.h" |
| 22 |
|
|
#include "publish/except.h" |
| 23 |
|
|
#include "publish/repository_util.h" |
| 24 |
|
|
#include "publish/settings.h" |
| 25 |
|
|
#include "reflog.h" |
| 26 |
|
|
#include "statistics.h" |
| 27 |
|
|
#include "sync_mediator.h" |
| 28 |
|
|
#include "sync_union_aufs.h" |
| 29 |
|
|
#include "sync_union_overlayfs.h" |
| 30 |
|
|
#include "sync_union_tarball.h" |
| 31 |
|
|
#include "upload.h" |
| 32 |
|
|
#include "upload_spooler_definition.h" |
| 33 |
|
|
#include "util/logging.h" |
| 34 |
|
|
#include "util/pointer.h" |
| 35 |
|
|
#include "whitelist.h" |
| 36 |
|
|
|
| 37 |
|
|
// TODO(jblomer): Remove Me |
| 38 |
|
|
namespace swissknife { |
| 39 |
|
|
class CommandTag { |
| 40 |
|
|
static const std::string kHeadTag; |
| 41 |
|
|
static const std::string kPreviousHeadTag; |
| 42 |
|
|
}; |
| 43 |
|
|
const std::string CommandTag::kHeadTag = "trunk"; |
| 44 |
|
|
const std::string CommandTag::kPreviousHeadTag = "trunk-previous"; |
| 45 |
|
|
} // namespace swissknife |
| 46 |
|
|
|
| 47 |
|
|
namespace publish { |
| 48 |
|
|
|
| 49 |
|
✗ |
Repository::Repository(const SettingsRepository &settings, const bool exists) |
| 50 |
|
✗ |
: settings_(settings) |
| 51 |
|
✗ |
, statistics_(new perf::Statistics()) |
| 52 |
|
✗ |
, signature_mgr_(new signature::SignatureManager()) |
| 53 |
|
✗ |
, download_mgr_(NULL) |
| 54 |
|
✗ |
, simple_catalog_mgr_(NULL) |
| 55 |
|
✗ |
, whitelist_(NULL) |
| 56 |
|
✗ |
, reflog_(NULL) |
| 57 |
|
✗ |
, manifest_(NULL) |
| 58 |
|
✗ |
, history_(NULL) { |
| 59 |
|
✗ |
signature_mgr_->Init(); |
| 60 |
|
|
|
| 61 |
|
✗ |
if (exists) { |
| 62 |
|
|
int rvb; |
| 63 |
|
|
const std::string keys = JoinStrings( |
| 64 |
|
✗ |
FindFilesBySuffix(settings.keychain().keychain_dir(), ".pub"), ":"); |
| 65 |
|
✗ |
rvb = signature_mgr_->LoadPublicRsaKeys(keys); |
| 66 |
|
✗ |
if (!rvb) { |
| 67 |
|
✗ |
signature_mgr_->Fini(); |
| 68 |
|
✗ |
delete signature_mgr_; |
| 69 |
|
✗ |
delete statistics_; |
| 70 |
|
✗ |
throw EPublish("cannot load public rsa key"); |
| 71 |
|
|
} |
| 72 |
|
|
} |
| 73 |
|
|
|
| 74 |
|
✗ |
if (!settings.cert_bundle().empty()) { |
| 75 |
|
✗ |
const int rvi = setenv("X509_CERT_BUNDLE", settings.cert_bundle().c_str(), |
| 76 |
|
|
1 /* overwrite */); |
| 77 |
|
✗ |
if (rvi != 0) |
| 78 |
|
✗ |
throw EPublish("cannot set X509_CERT_BUNDLE environment variable"); |
| 79 |
|
|
} |
| 80 |
|
✗ |
download_mgr_ = new download::DownloadManager( |
| 81 |
|
✗ |
16, perf::StatisticsTemplate("download", statistics_)); |
| 82 |
|
✗ |
download_mgr_->UseSystemCertificatePath(); |
| 83 |
|
|
|
| 84 |
|
✗ |
if (settings.proxy() != "") { |
| 85 |
|
✗ |
download_mgr_->SetProxyChain(settings.proxy(), "", |
| 86 |
|
|
download::DownloadManager::kSetProxyBoth); |
| 87 |
|
|
} |
| 88 |
|
|
|
| 89 |
|
✗ |
if (exists) { |
| 90 |
|
|
try { |
| 91 |
|
✗ |
DownloadRootObjects(settings.url(), settings.fqrn(), settings.tmp_dir()); |
| 92 |
|
✗ |
} catch (const EPublish &e) { |
| 93 |
|
✗ |
signature_mgr_->Fini(); |
| 94 |
|
✗ |
delete signature_mgr_; |
| 95 |
|
✗ |
delete download_mgr_; |
| 96 |
|
✗ |
delete statistics_; |
| 97 |
|
✗ |
throw; |
| 98 |
|
|
} |
| 99 |
|
|
} |
| 100 |
|
|
} |
| 101 |
|
|
|
| 102 |
|
✗ |
Repository::~Repository() { |
| 103 |
|
✗ |
if (signature_mgr_ != NULL) |
| 104 |
|
✗ |
signature_mgr_->Fini(); |
| 105 |
|
|
|
| 106 |
|
✗ |
delete history_; |
| 107 |
|
✗ |
delete manifest_; |
| 108 |
|
✗ |
delete reflog_; |
| 109 |
|
✗ |
delete whitelist_; |
| 110 |
|
✗ |
delete signature_mgr_; |
| 111 |
|
✗ |
delete download_mgr_; |
| 112 |
|
✗ |
delete simple_catalog_mgr_; |
| 113 |
|
✗ |
delete statistics_; |
| 114 |
|
|
} |
| 115 |
|
|
|
| 116 |
|
✗ |
const history::History *Repository::history() const { return history_; } |
| 117 |
|
|
|
| 118 |
|
✗ |
catalog::SimpleCatalogManager *Repository::GetSimpleCatalogManager() { |
| 119 |
|
✗ |
if (simple_catalog_mgr_ != NULL) |
| 120 |
|
✗ |
return simple_catalog_mgr_; |
| 121 |
|
|
|
| 122 |
|
✗ |
simple_catalog_mgr_ = new catalog::SimpleCatalogManager( |
| 123 |
|
✗ |
manifest_->catalog_hash(), |
| 124 |
|
✗ |
settings_.url(), |
| 125 |
|
✗ |
settings_.tmp_dir(), |
| 126 |
|
|
download_mgr_, |
| 127 |
|
|
statistics_, |
| 128 |
|
✗ |
true /* manage_catalog_files */); |
| 129 |
|
✗ |
simple_catalog_mgr_->Init(); |
| 130 |
|
✗ |
return simple_catalog_mgr_; |
| 131 |
|
|
} |
| 132 |
|
|
|
| 133 |
|
|
|
| 134 |
|
✗ |
void Repository::DownloadRootObjects(const std::string &url, |
| 135 |
|
|
const std::string &fqrn, |
| 136 |
|
|
const std::string &tmp_dir) { |
| 137 |
|
✗ |
delete whitelist_; |
| 138 |
|
✗ |
whitelist_ = new whitelist::Whitelist(fqrn, download_mgr_, signature_mgr_); |
| 139 |
|
✗ |
const whitelist::Failures rv_whitelist = whitelist_->LoadUrl(url); |
| 140 |
|
✗ |
if (whitelist_->status() != whitelist::Whitelist::kStAvailable) { |
| 141 |
|
✗ |
throw EPublish(std::string("cannot load whitelist [") |
| 142 |
|
✗ |
+ whitelist::Code2Ascii(rv_whitelist) + "]"); |
| 143 |
|
|
} |
| 144 |
|
|
|
| 145 |
|
✗ |
manifest::ManifestEnsemble ensemble; |
| 146 |
|
✗ |
const uint64_t minimum_timestamp = 0; |
| 147 |
|
✗ |
const shash::Any *base_catalog = NULL; |
| 148 |
|
✗ |
const manifest::Failures rv_manifest = manifest::Fetch( |
| 149 |
|
|
url, fqrn, minimum_timestamp, base_catalog, signature_mgr_, download_mgr_, |
| 150 |
|
|
&ensemble); |
| 151 |
|
✗ |
if (rv_manifest != manifest::kFailOk) |
| 152 |
|
✗ |
throw EPublish("cannot load manifest"); |
| 153 |
|
✗ |
delete manifest_; |
| 154 |
|
✗ |
manifest_ = new manifest::Manifest(*ensemble.manifest); |
| 155 |
|
|
|
| 156 |
|
✗ |
std::string reflog_path; |
| 157 |
|
✗ |
FILE *reflog_fd = CreateTempFile(tmp_dir + "/reflog", kPrivateFileMode, "w", |
| 158 |
|
|
&reflog_path); |
| 159 |
|
✗ |
if (reflog_fd == NULL) |
| 160 |
|
✗ |
throw EPublish("cannot create reflog temp file (disk full?)"); |
| 161 |
|
✗ |
const std::string reflog_url = url + "/.cvmfsreflog"; |
| 162 |
|
|
// TODO(jblomer): verify reflog hash |
| 163 |
|
|
// shash::Any reflog_hash(manifest_->GetHashAlgorithm()); |
| 164 |
|
✗ |
cvmfs::FileSink filesink(reflog_fd); |
| 165 |
|
|
download::JobInfo download_reflog(&reflog_url, false /* compressed */, |
| 166 |
|
✗ |
false /* probe hosts */, NULL, &filesink); |
| 167 |
|
✗ |
download::Failures rv_dl = download_mgr_->Fetch(&download_reflog); |
| 168 |
|
✗ |
fclose(reflog_fd); |
| 169 |
|
✗ |
if (rv_dl == download::kFailOk) { |
| 170 |
|
✗ |
delete reflog_; |
| 171 |
|
✗ |
reflog_ = manifest::Reflog::Open(reflog_path); |
| 172 |
|
✗ |
if (reflog_ == NULL) |
| 173 |
|
✗ |
throw EPublish("cannot open reflog"); |
| 174 |
|
✗ |
reflog_->TakeDatabaseFileOwnership(); |
| 175 |
|
|
} else { |
| 176 |
|
✗ |
if (!download_reflog.IsFileNotFound()) { |
| 177 |
|
✗ |
throw EPublish(std::string("cannot load reflog [") |
| 178 |
|
✗ |
+ download::Code2Ascii(rv_dl) + "]"); |
| 179 |
|
|
} |
| 180 |
|
✗ |
assert(reflog_ == NULL); |
| 181 |
|
|
} |
| 182 |
|
|
|
| 183 |
|
✗ |
std::string tags_path; |
| 184 |
|
✗ |
FILE *tags_fd = CreateTempFile(tmp_dir + "/tags", kPrivateFileMode, "w", |
| 185 |
|
|
&tags_path); |
| 186 |
|
✗ |
if (tags_fd == NULL) |
| 187 |
|
✗ |
throw EPublish("cannot create tags temp file (disk full?)"); |
| 188 |
|
✗ |
if (!manifest_->history().IsNull()) { |
| 189 |
|
✗ |
const std::string tags_url = url + "/data/" |
| 190 |
|
✗ |
+ manifest_->history().MakePath(); |
| 191 |
|
✗ |
const shash::Any tags_hash(manifest_->history()); |
| 192 |
|
✗ |
cvmfs::FileSink filesink(tags_fd); |
| 193 |
|
|
download::JobInfo download_tags(&tags_url, true /* compressed */, |
| 194 |
|
|
true /* probe hosts */, &tags_hash, |
| 195 |
|
✗ |
&filesink); |
| 196 |
|
✗ |
rv_dl = download_mgr_->Fetch(&download_tags); |
| 197 |
|
✗ |
fclose(tags_fd); |
| 198 |
|
✗ |
if (rv_dl != download::kFailOk) |
| 199 |
|
✗ |
throw EPublish("cannot load tag database"); |
| 200 |
|
✗ |
delete history_; |
| 201 |
|
✗ |
history_ = history::SqliteHistory::OpenWritable(tags_path); |
| 202 |
|
✗ |
if (history_ == NULL) |
| 203 |
|
✗ |
throw EPublish("cannot open tag database"); |
| 204 |
|
✗ |
} else { |
| 205 |
|
✗ |
fclose(tags_fd); |
| 206 |
|
✗ |
delete history_; |
| 207 |
|
✗ |
history_ = history::SqliteHistory::Create(tags_path, fqrn); |
| 208 |
|
✗ |
if (history_ == NULL) |
| 209 |
|
✗ |
throw EPublish("cannot create tag database"); |
| 210 |
|
|
} |
| 211 |
|
✗ |
history_->TakeDatabaseFileOwnership(); |
| 212 |
|
|
|
| 213 |
|
✗ |
if (!manifest_->meta_info().IsNull()) { |
| 214 |
|
✗ |
const shash::Any info_hash(manifest_->meta_info()); |
| 215 |
|
✗ |
const std::string info_url = url + "/data/" + info_hash.MakePath(); |
| 216 |
|
✗ |
cvmfs::MemSink metainfo_memsink; |
| 217 |
|
|
download::JobInfo download_info(&info_url, true /* compressed */, |
| 218 |
|
|
true /* probe_hosts */, &info_hash, |
| 219 |
|
✗ |
&metainfo_memsink); |
| 220 |
|
✗ |
const download::Failures rv_info = download_mgr_->Fetch(&download_info); |
| 221 |
|
✗ |
if (rv_info != download::kFailOk) { |
| 222 |
|
✗ |
throw EPublish(std::string("cannot load meta info [") |
| 223 |
|
✗ |
+ download::Code2Ascii(rv_info) + "]"); |
| 224 |
|
|
} |
| 225 |
|
✗ |
meta_info_ = std::string(reinterpret_cast<char *>(metainfo_memsink.data()), |
| 226 |
|
✗ |
metainfo_memsink.pos()); |
| 227 |
|
✗ |
} else { |
| 228 |
|
✗ |
meta_info_ = "n/a"; |
| 229 |
|
|
} |
| 230 |
|
|
} |
| 231 |
|
|
|
| 232 |
|
|
|
| 233 |
|
✗ |
std::string Repository::GetFqrnFromUrl(const std::string &url) { |
| 234 |
|
✗ |
return GetFileName(MakeCanonicalPath(url)); |
| 235 |
|
|
} |
| 236 |
|
|
|
| 237 |
|
|
|
| 238 |
|
✗ |
bool Repository::IsMasterReplica() { |
| 239 |
|
✗ |
const std::string url = settings_.url() + "/.cvmfs_master_replica"; |
| 240 |
|
✗ |
download::JobInfo head(&url, false /* probe_hosts */); |
| 241 |
|
✗ |
const download::Failures retval = download_mgr_->Fetch(&head); |
| 242 |
|
✗ |
if (retval == download::kFailOk) { |
| 243 |
|
✗ |
return true; |
| 244 |
|
|
} |
| 245 |
|
✗ |
if (head.IsFileNotFound()) { |
| 246 |
|
✗ |
return false; |
| 247 |
|
|
} |
| 248 |
|
|
|
| 249 |
|
✗ |
throw EPublish(std::string("error looking for .cvmfs_master_replica [") |
| 250 |
|
✗ |
+ download::Code2Ascii(retval) + "]"); |
| 251 |
|
|
} |
| 252 |
|
|
|
| 253 |
|
|
|
| 254 |
|
|
//------------------------------------------------------------------------------ |
| 255 |
|
|
|
| 256 |
|
|
|
| 257 |
|
✗ |
void Publisher::ConstructSpoolers() { |
| 258 |
|
✗ |
if ((spooler_files_ != NULL) && (spooler_catalogs_ != NULL)) |
| 259 |
|
✗ |
return; |
| 260 |
|
✗ |
assert((spooler_files_ == NULL) && (spooler_catalogs_ == NULL)); |
| 261 |
|
|
|
| 262 |
|
✗ |
upload::SpoolerDefinition sd(settings_.storage().GetLocator(), |
| 263 |
|
✗ |
settings_.transaction().hash_algorithm(), |
| 264 |
|
✗ |
settings_.transaction().compression_algorithm()); |
| 265 |
|
✗ |
sd.session_token_file = settings_.transaction() |
| 266 |
|
✗ |
.spool_area() |
| 267 |
|
✗ |
.gw_session_token(); |
| 268 |
|
✗ |
sd.key_file = settings_.keychain().gw_key_path(); |
| 269 |
|
|
|
| 270 |
|
✗ |
spooler_files_ = upload::Spooler::Construct(sd, |
| 271 |
|
|
statistics_publish_.weak_ref()); |
| 272 |
|
✗ |
if (spooler_files_ == NULL) |
| 273 |
|
✗ |
throw EPublish("could not initialize file spooler"); |
| 274 |
|
|
|
| 275 |
|
✗ |
const upload::SpoolerDefinition sd_catalogs(sd.Dup2DefaultCompression()); |
| 276 |
|
✗ |
spooler_catalogs_ = upload::Spooler::Construct( |
| 277 |
|
|
sd_catalogs, statistics_publish_.weak_ref()); |
| 278 |
|
✗ |
if (spooler_catalogs_ == NULL) { |
| 279 |
|
✗ |
delete spooler_files_; |
| 280 |
|
✗ |
throw EPublish("could not initialize catalog spooler"); |
| 281 |
|
|
} |
| 282 |
|
|
} |
| 283 |
|
|
|
| 284 |
|
|
|
| 285 |
|
✗ |
void Publisher::CreateKeychain() { |
| 286 |
|
✗ |
if (settings_.keychain().HasDanglingMasterKeys()) { |
| 287 |
|
✗ |
throw EPublish("dangling master key pair"); |
| 288 |
|
|
} |
| 289 |
|
✗ |
if (settings_.keychain().HasDanglingRepositoryKeys()) { |
| 290 |
|
✗ |
throw EPublish("dangling repository keys"); |
| 291 |
|
|
} |
| 292 |
|
✗ |
if (!settings_.keychain().HasMasterKeys()) |
| 293 |
|
✗ |
signature_mgr_->GenerateMasterKeyPair(); |
| 294 |
|
✗ |
if (!settings_.keychain().HasRepositoryKeys()) |
| 295 |
|
✗ |
signature_mgr_->GenerateCertificate(settings_.fqrn()); |
| 296 |
|
|
|
| 297 |
|
✗ |
whitelist_ = new whitelist::Whitelist(settings_.fqrn(), NULL, signature_mgr_); |
| 298 |
|
|
const std::string whitelist_str = whitelist::Whitelist::CreateString( |
| 299 |
|
✗ |
settings_.fqrn(), settings_.whitelist_validity_days(), |
| 300 |
|
✗ |
settings_.transaction().hash_algorithm(), signature_mgr_); |
| 301 |
|
✗ |
const whitelist::Failures rv_wl = whitelist_->LoadMem(whitelist_str); |
| 302 |
|
✗ |
if (rv_wl != whitelist::kFailOk) |
| 303 |
|
✗ |
throw EPublish("whitelist generation failed"); |
| 304 |
|
|
} |
| 305 |
|
|
|
| 306 |
|
|
|
| 307 |
|
✗ |
void Publisher::CreateRootObjects() { |
| 308 |
|
|
// Reflog |
| 309 |
|
|
const std::string reflog_path = CreateTempPath( |
| 310 |
|
✗ |
settings_.transaction().spool_area().tmp_dir() + "/cvmfs_reflog", 0600); |
| 311 |
|
✗ |
reflog_ = manifest::Reflog::Create(reflog_path, settings_.fqrn()); |
| 312 |
|
✗ |
if (reflog_ == NULL) |
| 313 |
|
✗ |
throw EPublish("could not create reflog"); |
| 314 |
|
✗ |
reflog_->TakeDatabaseFileOwnership(); |
| 315 |
|
|
|
| 316 |
|
|
// Root file catalog and initial manifest |
| 317 |
|
✗ |
manifest_ = catalog::WritableCatalogManager::CreateRepository( |
| 318 |
|
✗ |
settings_.transaction().spool_area().tmp_dir(), |
| 319 |
|
✗ |
settings_.transaction().is_volatile(), |
| 320 |
|
✗ |
settings_.transaction().voms_authz(), |
| 321 |
|
|
spooler_catalogs_); |
| 322 |
|
✗ |
spooler_catalogs_->WaitForUpload(); |
| 323 |
|
✗ |
if (manifest_ == NULL) |
| 324 |
|
✗ |
throw EPublish("could not create initial file catalog"); |
| 325 |
|
✗ |
reflog_->AddCatalog(manifest_->catalog_hash()); |
| 326 |
|
|
|
| 327 |
|
✗ |
manifest_->set_repository_name(settings_.fqrn()); |
| 328 |
|
✗ |
manifest_->set_ttl(settings_.transaction().ttl_second()); |
| 329 |
|
|
const bool |
| 330 |
|
✗ |
needs_bootstrap_shortcuts = !settings_.transaction().voms_authz().empty(); |
| 331 |
|
✗ |
manifest_->set_has_alt_catalog_path(needs_bootstrap_shortcuts); |
| 332 |
|
✗ |
manifest_->set_garbage_collectability( |
| 333 |
|
✗ |
settings_.transaction().is_garbage_collectable()); |
| 334 |
|
|
|
| 335 |
|
|
// Tag database |
| 336 |
|
|
const std::string tags_path = CreateTempPath( |
| 337 |
|
✗ |
settings_.transaction().spool_area().tmp_dir() + "/cvmfs_tags", 0600); |
| 338 |
|
✗ |
history_ = history::SqliteHistory::Create(tags_path, settings_.fqrn()); |
| 339 |
|
✗ |
if (history_ == NULL) |
| 340 |
|
✗ |
throw EPublish("could not create tag database"); |
| 341 |
|
✗ |
history_->TakeDatabaseFileOwnership(); |
| 342 |
|
|
const history::History::Tag tag_trunk( |
| 343 |
|
✗ |
"trunk", manifest_->catalog_hash(), manifest_->catalog_size(), |
| 344 |
|
✗ |
manifest_->revision(), manifest_->publish_timestamp(), "empty repository", |
| 345 |
|
✗ |
"" /* branch */); |
| 346 |
|
✗ |
history_->Insert(tag_trunk); |
| 347 |
|
|
|
| 348 |
|
|
// Meta information, TODO(jblomer) |
| 349 |
|
✗ |
meta_info_ = "{}"; |
| 350 |
|
|
} |
| 351 |
|
|
|
| 352 |
|
|
|
| 353 |
|
✗ |
void Publisher::CreateStorage() { |
| 354 |
|
✗ |
ConstructSpoolers(); |
| 355 |
|
✗ |
if (!spooler_files_->Create()) |
| 356 |
|
✗ |
throw EPublish("could not initialize repository storage area"); |
| 357 |
|
|
} |
| 358 |
|
|
|
| 359 |
|
|
|
| 360 |
|
✗ |
void Publisher::PushCertificate() { |
| 361 |
|
✗ |
upload::Spooler::CallbackPtr callback = spooler_files_->RegisterListener( |
| 362 |
|
|
&Publisher::OnProcessCertificate, this); |
| 363 |
|
✗ |
spooler_files_->ProcessCertificate( |
| 364 |
|
✗ |
new StringIngestionSource(signature_mgr_->GetCertificate())); |
| 365 |
|
✗ |
spooler_files_->WaitForUpload(); |
| 366 |
|
✗ |
spooler_files_->UnregisterListener(callback); |
| 367 |
|
|
} |
| 368 |
|
|
|
| 369 |
|
|
|
| 370 |
|
✗ |
void Publisher::PushHistory() { |
| 371 |
|
✗ |
assert(history_ != NULL); |
| 372 |
|
✗ |
history_->SetPreviousRevision(manifest_->history()); |
| 373 |
|
✗ |
const string history_path = history_->filename(); |
| 374 |
|
✗ |
history_->DropDatabaseFileOwnership(); |
| 375 |
|
✗ |
delete history_; |
| 376 |
|
|
|
| 377 |
|
✗ |
upload::Spooler::CallbackPtr callback = spooler_files_->RegisterListener( |
| 378 |
|
|
&Publisher::OnProcessHistory, this); |
| 379 |
|
✗ |
spooler_files_->ProcessHistory(history_path); |
| 380 |
|
✗ |
spooler_files_->WaitForUpload(); |
| 381 |
|
✗ |
spooler_files_->UnregisterListener(callback); |
| 382 |
|
|
|
| 383 |
|
✗ |
history_ = history::SqliteHistory::OpenWritable(history_path); |
| 384 |
|
✗ |
assert(history_ != NULL); |
| 385 |
|
✗ |
history_->TakeDatabaseFileOwnership(); |
| 386 |
|
|
} |
| 387 |
|
|
|
| 388 |
|
|
|
| 389 |
|
✗ |
void Publisher::PushMetainfo() { |
| 390 |
|
✗ |
upload::Spooler::CallbackPtr callback = spooler_files_->RegisterListener( |
| 391 |
|
|
&Publisher::OnProcessMetainfo, this); |
| 392 |
|
✗ |
spooler_files_->ProcessMetainfo(new StringIngestionSource(meta_info_)); |
| 393 |
|
✗ |
spooler_files_->WaitForUpload(); |
| 394 |
|
✗ |
spooler_files_->UnregisterListener(callback); |
| 395 |
|
|
} |
| 396 |
|
|
|
| 397 |
|
|
|
| 398 |
|
✗ |
void Publisher::PushManifest() { |
| 399 |
|
✗ |
std::string signed_manifest = manifest_->ExportString(); |
| 400 |
|
✗ |
shash::Any manifest_hash(settings_.transaction().hash_algorithm()); |
| 401 |
|
✗ |
shash::HashMem( |
| 402 |
|
✗ |
reinterpret_cast<const unsigned char *>(signed_manifest.data()), |
| 403 |
|
✗ |
signed_manifest.length(), &manifest_hash); |
| 404 |
|
✗ |
signed_manifest += "--\n" + manifest_hash.ToString() + "\n"; |
| 405 |
|
|
unsigned char *signature; |
| 406 |
|
|
unsigned sig_size; |
| 407 |
|
✗ |
bool rvb = signature_mgr_->Sign( |
| 408 |
|
✗ |
reinterpret_cast<const unsigned char *>(manifest_hash.ToString().data()), |
| 409 |
|
|
manifest_hash.GetHexSize(), &signature, &sig_size); |
| 410 |
|
✗ |
if (!rvb) |
| 411 |
|
✗ |
throw EPublish("cannot sign manifest"); |
| 412 |
|
✗ |
signed_manifest += std::string(reinterpret_cast<char *>(signature), sig_size); |
| 413 |
|
✗ |
free(signature); |
| 414 |
|
|
|
| 415 |
|
|
// Create alternative bootstrapping symlinks for VOMS secured repos |
| 416 |
|
✗ |
if (manifest_->has_alt_catalog_path()) { |
| 417 |
|
✗ |
rvb = spooler_files_->PlaceBootstrappingShortcut(manifest_->certificate()) |
| 418 |
|
✗ |
&& spooler_files_->PlaceBootstrappingShortcut( |
| 419 |
|
✗ |
manifest_->catalog_hash()) |
| 420 |
|
✗ |
&& (manifest_->history().IsNull() |
| 421 |
|
✗ |
|| spooler_files_->PlaceBootstrappingShortcut( |
| 422 |
|
✗ |
manifest_->history())) |
| 423 |
|
✗ |
&& (manifest_->meta_info().IsNull() |
| 424 |
|
✗ |
|| spooler_files_->PlaceBootstrappingShortcut( |
| 425 |
|
✗ |
manifest_->meta_info())); |
| 426 |
|
✗ |
if (!rvb) |
| 427 |
|
✗ |
EPublish("cannot place VOMS bootstrapping symlinks"); |
| 428 |
|
|
} |
| 429 |
|
|
|
| 430 |
|
✗ |
upload::Spooler::CallbackPtr callback = spooler_files_->RegisterListener( |
| 431 |
|
|
&Publisher::OnUploadManifest, this); |
| 432 |
|
✗ |
spooler_files_->Upload(".cvmfspublished", |
| 433 |
|
✗ |
new StringIngestionSource(signed_manifest)); |
| 434 |
|
✗ |
spooler_files_->WaitForUpload(); |
| 435 |
|
✗ |
spooler_files_->UnregisterListener(callback); |
| 436 |
|
|
} |
| 437 |
|
|
|
| 438 |
|
|
|
| 439 |
|
✗ |
void Publisher::PushReflog() { |
| 440 |
|
✗ |
const string reflog_path = reflog_->database_file(); |
| 441 |
|
✗ |
reflog_->DropDatabaseFileOwnership(); |
| 442 |
|
✗ |
delete reflog_; |
| 443 |
|
|
|
| 444 |
|
✗ |
shash::Any hash_reflog(settings_.transaction().hash_algorithm()); |
| 445 |
|
✗ |
manifest::Reflog::HashDatabase(reflog_path, &hash_reflog); |
| 446 |
|
|
|
| 447 |
|
✗ |
upload::Spooler::CallbackPtr callback = spooler_files_->RegisterListener( |
| 448 |
|
|
&Publisher::OnUploadReflog, this); |
| 449 |
|
✗ |
spooler_files_->UploadReflog(reflog_path); |
| 450 |
|
✗ |
spooler_files_->WaitForUpload(); |
| 451 |
|
✗ |
spooler_files_->UnregisterListener(callback); |
| 452 |
|
|
|
| 453 |
|
✗ |
manifest_->set_reflog_hash(hash_reflog); |
| 454 |
|
|
|
| 455 |
|
✗ |
reflog_ = manifest::Reflog::Open(reflog_path); |
| 456 |
|
✗ |
assert(reflog_ != NULL); |
| 457 |
|
✗ |
reflog_->TakeDatabaseFileOwnership(); |
| 458 |
|
|
} |
| 459 |
|
|
|
| 460 |
|
|
|
| 461 |
|
✗ |
void Publisher::PushWhitelist() { |
| 462 |
|
|
// TODO(jblomer): PKCS7 handling |
| 463 |
|
✗ |
upload::Spooler::CallbackPtr callback = spooler_files_->RegisterListener( |
| 464 |
|
|
&Publisher::OnUploadWhitelist, this); |
| 465 |
|
✗ |
spooler_files_->Upload(".cvmfswhitelist", |
| 466 |
|
✗ |
new StringIngestionSource(whitelist_->ExportString())); |
| 467 |
|
✗ |
spooler_files_->WaitForUpload(); |
| 468 |
|
✗ |
spooler_files_->UnregisterListener(callback); |
| 469 |
|
|
} |
| 470 |
|
|
|
| 471 |
|
|
|
| 472 |
|
✗ |
Publisher *Publisher::Create(const SettingsPublisher &settings) { |
| 473 |
|
✗ |
UniquePtr<Publisher> publisher(new Publisher(settings, false)); |
| 474 |
|
|
|
| 475 |
|
✗ |
LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout | kLogNoLinebreak, |
| 476 |
|
|
"Creating Key Chain... "); |
| 477 |
|
✗ |
publisher->CreateKeychain(); |
| 478 |
|
✗ |
publisher->ExportKeychain(); |
| 479 |
|
✗ |
LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout, "done"); |
| 480 |
|
|
|
| 481 |
|
✗ |
LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout | kLogNoLinebreak, |
| 482 |
|
|
"Creating Backend Storage... "); |
| 483 |
|
✗ |
publisher->CreateStorage(); |
| 484 |
|
✗ |
publisher->PushWhitelist(); |
| 485 |
|
✗ |
LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout, "done"); |
| 486 |
|
|
|
| 487 |
|
✗ |
LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout | kLogNoLinebreak, |
| 488 |
|
|
"Creating Initial Repository... "); |
| 489 |
|
✗ |
publisher->InitSpoolArea(); |
| 490 |
|
✗ |
publisher->CreateRootObjects(); |
| 491 |
|
✗ |
publisher->PushHistory(); |
| 492 |
|
✗ |
publisher->PushCertificate(); |
| 493 |
|
✗ |
publisher->PushMetainfo(); |
| 494 |
|
✗ |
publisher->PushReflog(); |
| 495 |
|
✗ |
publisher->PushManifest(); |
| 496 |
|
|
// TODO(jblomer): meta-info |
| 497 |
|
|
|
| 498 |
|
|
// Re-create from empty repository in order to properly initialize |
| 499 |
|
|
// parent Repository object |
| 500 |
|
✗ |
publisher = new Publisher(settings); |
| 501 |
|
|
|
| 502 |
|
✗ |
LogCvmfs(kLogCvmfs, publisher->llvl_ | kLogStdout, "done"); |
| 503 |
|
|
|
| 504 |
|
✗ |
return publisher.Release(); |
| 505 |
|
|
} |
| 506 |
|
|
|
| 507 |
|
✗ |
void Publisher::ExportKeychain() { |
| 508 |
|
✗ |
CreateDirectoryAsOwner(settings_.keychain().keychain_dir(), kDefaultDirMode); |
| 509 |
|
|
|
| 510 |
|
|
bool rvb; |
| 511 |
|
✗ |
rvb = SafeWriteToFile(signature_mgr_->GetActivePubkeys(), |
| 512 |
|
✗ |
settings_.keychain().master_public_key_path(), 0644); |
| 513 |
|
✗ |
if (!rvb) |
| 514 |
|
✗ |
throw EPublish("cannot export public master key"); |
| 515 |
|
✗ |
rvb = SafeWriteToFile(signature_mgr_->GetCertificate(), |
| 516 |
|
✗ |
settings_.keychain().certificate_path(), 0644); |
| 517 |
|
✗ |
if (!rvb) |
| 518 |
|
✗ |
throw EPublish("cannot export certificate"); |
| 519 |
|
|
|
| 520 |
|
✗ |
rvb = SafeWriteToFile(signature_mgr_->GetPrivateKey(), |
| 521 |
|
✗ |
settings_.keychain().private_key_path(), 0600); |
| 522 |
|
✗ |
if (!rvb) |
| 523 |
|
✗ |
throw EPublish("cannot export private certificate key"); |
| 524 |
|
✗ |
rvb = SafeWriteToFile(signature_mgr_->GetPrivateMasterKey(), |
| 525 |
|
✗ |
settings_.keychain().master_private_key_path(), 0600); |
| 526 |
|
✗ |
if (!rvb) |
| 527 |
|
✗ |
throw EPublish("cannot export private master key"); |
| 528 |
|
|
|
| 529 |
|
|
int rvi; |
| 530 |
|
✗ |
rvi = chown(settings_.keychain().master_public_key_path().c_str(), |
| 531 |
|
|
settings_.owner_uid(), settings_.owner_gid()); |
| 532 |
|
✗ |
if (rvi != 0) |
| 533 |
|
✗ |
throw EPublish("cannot set key file ownership"); |
| 534 |
|
✗ |
rvi = chown(settings_.keychain().certificate_path().c_str(), |
| 535 |
|
|
settings_.owner_uid(), settings_.owner_gid()); |
| 536 |
|
✗ |
if (rvi != 0) |
| 537 |
|
✗ |
throw EPublish("cannot set key file ownership"); |
| 538 |
|
✗ |
rvi = chown(settings_.keychain().private_key_path().c_str(), |
| 539 |
|
|
settings_.owner_uid(), settings_.owner_gid()); |
| 540 |
|
✗ |
if (rvi != 0) |
| 541 |
|
✗ |
throw EPublish("cannot set key file ownership"); |
| 542 |
|
✗ |
rvi = chown(settings_.keychain().master_private_key_path().c_str(), |
| 543 |
|
|
settings_.owner_uid(), settings_.owner_gid()); |
| 544 |
|
✗ |
if (rvi != 0) |
| 545 |
|
✗ |
throw EPublish("cannot set key file ownership"); |
| 546 |
|
|
} |
| 547 |
|
|
|
| 548 |
|
✗ |
void Publisher::OnProcessCertificate(const upload::SpoolerResult &result) { |
| 549 |
|
✗ |
if (result.return_code != 0) { |
| 550 |
|
✗ |
throw EPublish("cannot write certificate to storage"); |
| 551 |
|
|
} |
| 552 |
|
✗ |
manifest_->set_certificate(result.content_hash); |
| 553 |
|
✗ |
reflog_->AddCertificate(result.content_hash); |
| 554 |
|
|
} |
| 555 |
|
|
|
| 556 |
|
✗ |
void Publisher::OnProcessHistory(const upload::SpoolerResult &result) { |
| 557 |
|
✗ |
if (result.return_code != 0) { |
| 558 |
|
✗ |
throw EPublish("cannot write tag database to storage"); |
| 559 |
|
|
} |
| 560 |
|
✗ |
manifest_->set_history(result.content_hash); |
| 561 |
|
✗ |
reflog_->AddHistory(result.content_hash); |
| 562 |
|
|
} |
| 563 |
|
|
|
| 564 |
|
✗ |
void Publisher::OnProcessMetainfo(const upload::SpoolerResult &result) { |
| 565 |
|
✗ |
if (result.return_code != 0) { |
| 566 |
|
✗ |
throw EPublish("cannot write repository meta info to storage"); |
| 567 |
|
|
} |
| 568 |
|
✗ |
manifest_->set_meta_info(result.content_hash); |
| 569 |
|
✗ |
reflog_->AddMetainfo(result.content_hash); |
| 570 |
|
|
} |
| 571 |
|
|
|
| 572 |
|
✗ |
void Publisher::OnUploadManifest(const upload::SpoolerResult &result) { |
| 573 |
|
✗ |
if (result.return_code != 0) { |
| 574 |
|
✗ |
throw EPublish("cannot write manifest to storage"); |
| 575 |
|
|
} |
| 576 |
|
|
} |
| 577 |
|
|
|
| 578 |
|
✗ |
void Publisher::OnUploadReflog(const upload::SpoolerResult &result) { |
| 579 |
|
✗ |
if (result.return_code != 0) { |
| 580 |
|
✗ |
throw EPublish("cannot write reflog to storage"); |
| 581 |
|
|
} |
| 582 |
|
|
} |
| 583 |
|
|
|
| 584 |
|
✗ |
void Publisher::OnUploadWhitelist(const upload::SpoolerResult &result) { |
| 585 |
|
✗ |
if (result.return_code != 0) { |
| 586 |
|
✗ |
throw EPublish("cannot write whitelist to storage"); |
| 587 |
|
|
} |
| 588 |
|
|
} |
| 589 |
|
|
|
| 590 |
|
✗ |
void Publisher::CreateDirectoryAsOwner(const std::string &path, int mode) { |
| 591 |
|
✗ |
const bool rvb = MkdirDeep(path, mode); |
| 592 |
|
✗ |
if (!rvb) |
| 593 |
|
✗ |
throw EPublish("cannot create directory " + path); |
| 594 |
|
✗ |
const int rvi = chown(path.c_str(), settings_.owner_uid(), |
| 595 |
|
|
settings_.owner_gid()); |
| 596 |
|
✗ |
if (rvi != 0) |
| 597 |
|
✗ |
throw EPublish("cannot set ownership on directory " + path); |
| 598 |
|
|
} |
| 599 |
|
|
|
| 600 |
|
✗ |
void Publisher::InitSpoolArea() { |
| 601 |
|
✗ |
CreateDirectoryAsOwner(settings_.transaction().spool_area().workspace(), |
| 602 |
|
|
kPrivateDirMode); |
| 603 |
|
✗ |
CreateDirectoryAsOwner(settings_.transaction().spool_area().tmp_dir(), |
| 604 |
|
|
kPrivateDirMode); |
| 605 |
|
✗ |
CreateDirectoryAsOwner(settings_.transaction().spool_area().cache_dir(), |
| 606 |
|
|
kPrivateDirMode); |
| 607 |
|
✗ |
CreateDirectoryAsOwner(settings_.transaction().spool_area().scratch_dir(), |
| 608 |
|
|
kDefaultDirMode); |
| 609 |
|
✗ |
CreateDirectoryAsOwner(settings_.transaction().spool_area().ovl_work_dir(), |
| 610 |
|
|
kPrivateDirMode); |
| 611 |
|
|
|
| 612 |
|
|
// On a managed node, the mount points are already mounted |
| 613 |
|
✗ |
if (!DirectoryExists(settings_.transaction().spool_area().readonly_mnt())) { |
| 614 |
|
✗ |
CreateDirectoryAsOwner(settings_.transaction().spool_area().readonly_mnt(), |
| 615 |
|
|
kDefaultDirMode); |
| 616 |
|
|
} |
| 617 |
|
✗ |
if (!DirectoryExists(settings_.transaction().spool_area().union_mnt())) { |
| 618 |
|
✗ |
CreateDirectoryAsOwner(settings_.transaction().spool_area().union_mnt(), |
| 619 |
|
|
kDefaultDirMode); |
| 620 |
|
|
} |
| 621 |
|
|
} |
| 622 |
|
|
|
| 623 |
|
✗ |
Publisher::Publisher(const SettingsPublisher &settings, const bool exists) |
| 624 |
|
✗ |
: Repository(SettingsRepository(settings), exists) |
| 625 |
|
✗ |
, settings_(settings) |
| 626 |
|
✗ |
, statistics_publish_(new perf::StatisticsTemplate("publish", statistics_)) |
| 627 |
|
✗ |
, llvl_(settings.is_silent() ? kLogNone : kLogNormal) |
| 628 |
|
✗ |
, in_transaction_(settings.transaction().spool_area().transaction_lock()) |
| 629 |
|
✗ |
, is_publishing_(settings.transaction().spool_area().publishing_lock()) |
| 630 |
|
✗ |
, spooler_files_(NULL) |
| 631 |
|
✗ |
, spooler_catalogs_(NULL) |
| 632 |
|
✗ |
, catalog_mgr_(NULL) |
| 633 |
|
✗ |
, sync_parameters_(NULL) |
| 634 |
|
✗ |
, sync_mediator_(NULL) |
| 635 |
|
✗ |
, sync_union_(NULL) { |
| 636 |
|
✗ |
if (settings.transaction().layout_revision() != kRequiredLayoutRevision) { |
| 637 |
|
✗ |
const unsigned layout_revision = settings.transaction().layout_revision(); |
| 638 |
|
✗ |
throw EPublish("This repository uses layout revision " |
| 639 |
|
✗ |
+ StringifyInt(layout_revision) |
| 640 |
|
✗ |
+ ".\n" |
| 641 |
|
|
"This version of CernVM-FS requires layout revision " |
| 642 |
|
✗ |
+ StringifyInt(kRequiredLayoutRevision) |
| 643 |
|
✗ |
+ ", which is\n" |
| 644 |
|
|
"incompatible to " |
| 645 |
|
✗ |
+ StringifyInt(layout_revision) |
| 646 |
|
✗ |
+ ".\n\n" |
| 647 |
|
|
"Please run `cvmfs_server migrate` to update your " |
| 648 |
|
|
"repository before " |
| 649 |
|
|
"proceeding.", |
| 650 |
|
✗ |
EPublish::kFailLayoutRevision); |
| 651 |
|
|
} |
| 652 |
|
|
|
| 653 |
|
|
// Session and managed node are needed even when skipping downloads (e.g. |
| 654 |
|
|
// for abort under disk-full conditions), so initialize them before the |
| 655 |
|
|
// early return below. |
| 656 |
|
✗ |
if (settings.is_managed()) |
| 657 |
|
✗ |
managed_node_ = new ManagedNode(this); |
| 658 |
|
✗ |
session_ = new Session(settings_, llvl_); |
| 659 |
|
|
|
| 660 |
|
✗ |
if (!exists) |
| 661 |
|
✗ |
return; |
| 662 |
|
|
|
| 663 |
|
✗ |
CreateDirectoryAsOwner(settings_.transaction().spool_area().tmp_dir(), |
| 664 |
|
|
kPrivateDirMode); |
| 665 |
|
|
|
| 666 |
|
✗ |
if (settings.storage().type() == upload::SpoolerDefinition::Gateway) { |
| 667 |
|
✗ |
if (!settings.keychain().HasGatewayKey()) { |
| 668 |
|
✗ |
throw EPublish("gateway key missing: " |
| 669 |
|
✗ |
+ settings.keychain().gw_key_path()); |
| 670 |
|
|
} |
| 671 |
|
✗ |
gw_key_ = gateway::ReadGatewayKey(settings.keychain().gw_key_path()); |
| 672 |
|
✗ |
if (!gw_key_.IsValid()) { |
| 673 |
|
✗ |
throw EPublish("cannot read gateway key: " |
| 674 |
|
✗ |
+ settings.keychain().gw_key_path()); |
| 675 |
|
|
} |
| 676 |
|
|
} |
| 677 |
|
|
|
| 678 |
|
✗ |
if ((settings.storage().type() != upload::SpoolerDefinition::Gateway) |
| 679 |
|
✗ |
&& !settings.transaction().in_enter_session()) { |
| 680 |
|
✗ |
int rvb = signature_mgr_->LoadCertificatePath( |
| 681 |
|
✗ |
settings.keychain().certificate_path()); |
| 682 |
|
✗ |
if (!rvb) |
| 683 |
|
✗ |
throw EPublish("cannot load certificate, thus cannot commit changes"); |
| 684 |
|
✗ |
rvb = signature_mgr_->LoadPrivateKeyPath( |
| 685 |
|
✗ |
settings.keychain().private_key_path(), ""); |
| 686 |
|
✗ |
if (!rvb) |
| 687 |
|
✗ |
throw EPublish("cannot load private key, thus cannot commit changes"); |
| 688 |
|
|
// The private master key might be on a key card instead |
| 689 |
|
✗ |
if (FileExists(settings.keychain().master_private_key_path())) { |
| 690 |
|
✗ |
rvb = signature_mgr_->LoadPrivateMasterKeyPath( |
| 691 |
|
✗ |
settings.keychain().master_private_key_path()); |
| 692 |
|
✗ |
if (!rvb) |
| 693 |
|
✗ |
throw EPublish("cannot load private master key"); |
| 694 |
|
|
} |
| 695 |
|
✗ |
if (!signature_mgr_->KeysMatch()) |
| 696 |
|
✗ |
throw EPublish("corrupted keychain"); |
| 697 |
|
|
} |
| 698 |
|
|
|
| 699 |
|
✗ |
if (in_transaction_.IsSet()) |
| 700 |
|
✗ |
ConstructSpoolers(); |
| 701 |
|
|
} |
| 702 |
|
|
|
| 703 |
|
✗ |
Publisher::~Publisher() { |
| 704 |
|
✗ |
delete sync_union_; |
| 705 |
|
✗ |
delete sync_mediator_; |
| 706 |
|
✗ |
delete sync_parameters_; |
| 707 |
|
✗ |
delete catalog_mgr_; |
| 708 |
|
✗ |
delete spooler_catalogs_; |
| 709 |
|
✗ |
delete spooler_files_; |
| 710 |
|
|
} |
| 711 |
|
|
|
| 712 |
|
|
|
| 713 |
|
✗ |
void Publisher::ConstructSyncManagers() { |
| 714 |
|
✗ |
ConstructSpoolers(); |
| 715 |
|
|
|
| 716 |
|
✗ |
if (catalog_mgr_ == NULL) { |
| 717 |
|
✗ |
catalog_mgr_ = new catalog::WritableCatalogManager( |
| 718 |
|
✗ |
settings_.transaction().base_hash(), |
| 719 |
|
✗ |
settings_.url(), |
| 720 |
|
✗ |
settings_.transaction().spool_area().tmp_dir(), |
| 721 |
|
|
spooler_catalogs_, |
| 722 |
|
|
download_mgr_, |
| 723 |
|
✗ |
settings_.transaction().enforce_limits(), |
| 724 |
|
✗ |
settings_.transaction().limit_nested_catalog_kentries(), |
| 725 |
|
✗ |
settings_.transaction().limit_root_catalog_kentries(), |
| 726 |
|
✗ |
settings_.transaction().limit_file_size_mb(), |
| 727 |
|
|
statistics_, |
| 728 |
|
✗ |
settings_.transaction().use_catalog_autobalance(), |
| 729 |
|
✗ |
settings_.transaction().autobalance_max_weight(), |
| 730 |
|
✗ |
settings_.transaction().autobalance_min_weight(), |
| 731 |
|
✗ |
""); |
| 732 |
|
✗ |
catalog_mgr_->Init(); |
| 733 |
|
|
} |
| 734 |
|
|
|
| 735 |
|
✗ |
if (sync_parameters_ == NULL) { |
| 736 |
|
✗ |
SyncParameters *p = new SyncParameters(); |
| 737 |
|
✗ |
p->spooler = spooler_files_; |
| 738 |
|
✗ |
p->repo_name = settings_.fqrn(); |
| 739 |
|
✗ |
p->dir_union = settings_.transaction().spool_area().union_mnt(); |
| 740 |
|
✗ |
p->dir_scratch = settings_.transaction().spool_area().scratch_dir(); |
| 741 |
|
✗ |
p->dir_rdonly = settings_.transaction().spool_area().readonly_mnt(); |
| 742 |
|
✗ |
p->dir_temp = settings_.transaction().spool_area().tmp_dir(); |
| 743 |
|
✗ |
p->base_hash = settings_.transaction().base_hash(); |
| 744 |
|
✗ |
p->stratum0 = settings_.url(); |
| 745 |
|
|
// p->manifest_path = SHOULD NOT BE NEEDED |
| 746 |
|
|
// p->spooler_definition = SHOULD NOT BE NEEDED; |
| 747 |
|
|
// p->union_fs_type = SHOULD NOT BE NEEDED |
| 748 |
|
✗ |
p->print_changeset = settings_.transaction().print_changeset(); |
| 749 |
|
✗ |
p->dry_run = settings_.transaction().dry_run(); |
| 750 |
|
✗ |
sync_parameters_ = p; |
| 751 |
|
|
} |
| 752 |
|
|
|
| 753 |
|
✗ |
if (sync_mediator_ == NULL) { |
| 754 |
|
✗ |
sync_mediator_ = new SyncMediator(catalog_mgr_, sync_parameters_, |
| 755 |
|
✗ |
*statistics_publish_); |
| 756 |
|
|
} |
| 757 |
|
|
|
| 758 |
|
✗ |
if (sync_union_ == NULL) { |
| 759 |
|
✗ |
switch (settings_.transaction().union_fs()) { |
| 760 |
|
✗ |
case kUnionFsAufs: |
| 761 |
|
✗ |
sync_union_ = new publish::SyncUnionAufs( |
| 762 |
|
|
sync_mediator_, |
| 763 |
|
✗ |
settings_.transaction().spool_area().readonly_mnt(), |
| 764 |
|
✗ |
settings_.transaction().spool_area().union_mnt(), |
| 765 |
|
✗ |
settings_.transaction().spool_area().scratch_dir()); |
| 766 |
|
✗ |
break; |
| 767 |
|
✗ |
case kUnionFsOverlay: |
| 768 |
|
✗ |
sync_union_ = new publish::SyncUnionOverlayfs( |
| 769 |
|
|
sync_mediator_, |
| 770 |
|
✗ |
settings_.transaction().spool_area().readonly_mnt(), |
| 771 |
|
✗ |
settings_.transaction().spool_area().union_mnt(), |
| 772 |
|
✗ |
settings_.transaction().spool_area().scratch_dir()); |
| 773 |
|
✗ |
break; |
| 774 |
|
✗ |
case kUnionFsTarball: |
| 775 |
|
✗ |
sync_union_ = new publish::SyncUnionTarball( |
| 776 |
|
✗ |
sync_mediator_, |
| 777 |
|
✗ |
settings_.transaction().spool_area().readonly_mnt(), |
| 778 |
|
|
// TODO(jblomer): get from settings |
| 779 |
|
|
"tar_file", |
| 780 |
|
|
"base_directory", |
| 781 |
|
|
-1u, |
| 782 |
|
|
-1u, |
| 783 |
|
|
"to_delete", |
| 784 |
|
✗ |
false /* create_catalog */); |
| 785 |
|
✗ |
break; |
| 786 |
|
✗ |
default: |
| 787 |
|
✗ |
throw EPublish("unknown union file system"); |
| 788 |
|
|
} |
| 789 |
|
✗ |
const bool rvb = sync_union_->Initialize(); |
| 790 |
|
✗ |
if (!rvb) { |
| 791 |
|
✗ |
delete sync_union_; |
| 792 |
|
✗ |
sync_union_ = NULL; |
| 793 |
|
✗ |
throw EPublish("cannot initialize union file system engine"); |
| 794 |
|
|
} |
| 795 |
|
|
} |
| 796 |
|
|
} |
| 797 |
|
|
|
| 798 |
|
✗ |
void Publisher::ExitShell() { |
| 799 |
|
✗ |
const std::string session_dir = Env::GetEnterSessionDir(); |
| 800 |
|
✗ |
const std::string session_pid_tmp = session_dir + "/session_pid"; |
| 801 |
|
✗ |
std::string session_pid; |
| 802 |
|
✗ |
const int fd_session_pid = open(session_pid_tmp.c_str(), O_RDONLY); |
| 803 |
|
✗ |
if (fd_session_pid < 0) |
| 804 |
|
✗ |
throw EPublish("Session pid cannot be retrieved"); |
| 805 |
|
✗ |
SafeReadToString(fd_session_pid, &session_pid); |
| 806 |
|
|
|
| 807 |
|
✗ |
const pid_t pid_child = String2Uint64(session_pid); |
| 808 |
|
✗ |
kill(pid_child, SIGUSR1); |
| 809 |
|
|
} |
| 810 |
|
|
|
| 811 |
|
✗ |
void Publisher::Sync() { |
| 812 |
|
✗ |
const ServerLockFileGuard g(is_publishing_); |
| 813 |
|
|
|
| 814 |
|
✗ |
ConstructSyncManagers(); |
| 815 |
|
|
|
| 816 |
|
✗ |
sync_union_->Traverse(); |
| 817 |
|
✗ |
bool rvb = sync_mediator_->Commit(manifest_); |
| 818 |
|
✗ |
if (!rvb) |
| 819 |
|
✗ |
throw EPublish("cannot write change set to storage"); |
| 820 |
|
|
|
| 821 |
|
✗ |
if (!settings_.transaction().dry_run()) { |
| 822 |
|
✗ |
spooler_files_->WaitForUpload(); |
| 823 |
|
✗ |
spooler_catalogs_->WaitForUpload(); |
| 824 |
|
✗ |
spooler_files_->FinalizeSession(false /* commit */); |
| 825 |
|
|
|
| 826 |
|
✗ |
const std::string old_root_hash = settings_.transaction() |
| 827 |
|
✗ |
.base_hash() |
| 828 |
|
✗ |
.ToString(true /* with_suffix */); |
| 829 |
|
✗ |
const std::string new_root_hash = manifest_->catalog_hash().ToString( |
| 830 |
|
✗ |
true /* with_suffix */); |
| 831 |
|
✗ |
rvb = spooler_catalogs_->FinalizeSession( |
| 832 |
|
|
true /* commit */, old_root_hash, new_root_hash, |
| 833 |
|
✗ |
/* TODO(jblomer) */ sync_parameters_->repo_tag); |
| 834 |
|
✗ |
if (!rvb) |
| 835 |
|
✗ |
throw EPublish("failed to commit transaction"); |
| 836 |
|
|
|
| 837 |
|
|
// Reset to the new catalog root hash |
| 838 |
|
✗ |
settings_.GetTransaction()->SetBaseHash(manifest_->catalog_hash()); |
| 839 |
|
|
// TODO(jblomer): think about how to deal with the scratch area at |
| 840 |
|
|
// this point |
| 841 |
|
|
// WipeScratchArea(); |
| 842 |
|
|
} |
| 843 |
|
|
|
| 844 |
|
✗ |
delete sync_union_; |
| 845 |
|
✗ |
delete sync_mediator_; |
| 846 |
|
✗ |
delete sync_parameters_; |
| 847 |
|
✗ |
delete catalog_mgr_; |
| 848 |
|
✗ |
sync_union_ = NULL; |
| 849 |
|
✗ |
sync_mediator_ = NULL; |
| 850 |
|
✗ |
sync_parameters_ = NULL; |
| 851 |
|
✗ |
catalog_mgr_ = NULL; |
| 852 |
|
|
|
| 853 |
|
✗ |
if (!settings_.transaction().dry_run()) { |
| 854 |
|
✗ |
LogCvmfs(kLogCvmfs, kLogStdout, "New revision: %" PRIu64, |
| 855 |
|
✗ |
manifest_->revision()); |
| 856 |
|
✗ |
reflog_->AddCatalog(manifest_->catalog_hash()); |
| 857 |
|
|
} |
| 858 |
|
|
} |
| 859 |
|
|
|
| 860 |
|
✗ |
void Publisher::Publish() { |
| 861 |
|
✗ |
if (!in_transaction_.IsSet()) |
| 862 |
|
✗ |
throw EPublish("cannot publish outside transaction"); |
| 863 |
|
|
|
| 864 |
|
✗ |
PushReflog(); |
| 865 |
|
✗ |
PushManifest(); |
| 866 |
|
✗ |
in_transaction_.Clear(); |
| 867 |
|
|
} |
| 868 |
|
|
|
| 869 |
|
|
|
| 870 |
|
✗ |
void Publisher::MarkReplicatible(bool value) { |
| 871 |
|
✗ |
ConstructSpoolers(); |
| 872 |
|
|
|
| 873 |
|
✗ |
if (value) { |
| 874 |
|
✗ |
spooler_files_->Upload("/dev/null", "/.cvmfs_master_replica"); |
| 875 |
|
|
} else { |
| 876 |
|
✗ |
spooler_files_->RemoveAsync("/.cvmfs_master_replica"); |
| 877 |
|
|
} |
| 878 |
|
✗ |
spooler_files_->WaitForUpload(); |
| 879 |
|
✗ |
if (spooler_files_->GetNumberOfErrors() > 0) |
| 880 |
|
✗ |
throw EPublish("cannot set replication mode"); |
| 881 |
|
|
} |
| 882 |
|
|
|
| 883 |
|
✗ |
void Publisher::Ingest() { } |
| 884 |
|
✗ |
void Publisher::Migrate() { } |
| 885 |
|
✗ |
void Publisher::Resign() { } |
| 886 |
|
✗ |
void Publisher::Rollback() { } |
| 887 |
|
✗ |
void Publisher::UpdateMetaInfo() { } |
| 888 |
|
|
|
| 889 |
|
✗ |
void Publisher::Transaction() { |
| 890 |
|
✗ |
TransactionRetry(); |
| 891 |
|
✗ |
session()->SetKeepAlive(true); |
| 892 |
|
|
} |
| 893 |
|
|
|
| 894 |
|
|
//------------------------------------------------------------------------------ |
| 895 |
|
|
|
| 896 |
|
|
|
| 897 |
|
✗ |
Replica::Replica(const SettingsReplica &settings) |
| 898 |
|
✗ |
: Repository(SettingsRepository(settings)) { } |
| 899 |
|
|
|
| 900 |
|
|
|
| 901 |
|
✗ |
Replica::~Replica() { } |
| 902 |
|
|
|
| 903 |
|
|
} // namespace publish |
| 904 |
|
|
|