GCC Code Coverage Report


Directory: cvmfs/
File: cvmfs/publish/settings.h
Date: 2024-04-28 02:33:07
Exec Total Coverage
Lines: 0 168 0.0%
Branches: 0 120 0.0%

Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 */
4
5 #ifndef CVMFS_PUBLISH_SETTINGS_H_
6 #define CVMFS_PUBLISH_SETTINGS_H_
7
8 #include <stdint.h>
9 #include <unistd.h>
10
11 #include <map>
12 #include <string>
13
14 #include "compression.h"
15 #include "crypto/hash.h"
16 #include "sync_union.h"
17 #include "upload_spooler_definition.h"
18
19 class OptionsManager;
20
21 namespace publish {
22
23 /**
24 * Allows for settings that remember whether they have been explicitly
25 * overwritten. Otherwise, default values can be changed to upstream repository
26 * settings.
27 */
28 template <class T>
29 class Setting {
30 public:
31 Setting() : value_(), is_default_(true) { }
32 explicit Setting(const T &v) : value_(v), is_default_(true) { }
33
34 Setting& operator=(const T &v) {
35 value_ = v;
36 is_default_ = false;
37 return *this;
38 }
39
40 const T& operator()() const {
41 return value_;
42 }
43
44 bool SetIfDefault(const T &v) {
45 if (!is_default_) return false;
46 value_ = v;
47 is_default_ = false;
48 return true;
49 }
50
51 bool is_default() const { return is_default_; }
52
53 T* GetPtr() { return &value_; }
54
55 private:
56 T value_;
57 bool is_default_;
58 }; // Setting
59
60
61 /**
62 * Steers the aggressiveness of Publisher::ManagedNode::Check()
63 */
64 enum EUnionMountRepairMode {
65 kUnionMountRepairNever = 0,
66 kUnionMountRepairSafe,
67 kUnionMountRepairAlways
68 };
69
70
71 // Settings from the point of construction always represent a valid
72 // configuration. The constructor sets default values, which can be overwritten
73 // by setters. The setters throw errors when invalid options are detected.
74
75 class SettingsSpoolArea {
76 public:
77 explicit SettingsSpoolArea(const std::string &fqrn)
78 : workspace_(std::string("/var/spool/cvmfs/") + fqrn)
79 , tmp_dir_(workspace_() + "/tmp")
80 , union_mnt_(std::string("/cvmfs/") + fqrn)
81 , repair_mode_(kUnionMountRepairSafe)
82 { }
83
84 void UseSystemTempDir();
85 void SetSpoolArea(const std::string &path);
86 void SetUnionMount(const std::string &path);
87 void SetRepairMode(const EUnionMountRepairMode val);
88
89 // Creates, if necessary, all the directories in the spool area and the temp
90 // directory. Does not take care of the union mount point.
91 void EnsureDirectories();
92
93 std::string workspace() const { return workspace_(); }
94 std::string tmp_dir() const { return tmp_dir_(); }
95 std::string readonly_mnt() const { return workspace_() + "/rdonly"; }
96 std::string readonly_talk_socket() const {
97 return workspace_() + "/cvmfs_io";
98 }
99 std::string union_mnt() const { return union_mnt_(); }
100 std::string scratch_base() const { return workspace_() + "/scratch"; }
101 std::string scratch_dir() const { return scratch_base() + "/current"; }
102 std::string scratch_wastebin() const { return scratch_base() + "/wastebin"; }
103 std::string log_dir() const { return workspace() + "/logs"; }
104 // TODO(jblomer): shouldn't this be in /etc/cvmfs/repository.../client.conf
105 std::string client_config() const { return workspace_() + "/client.config"; }
106 std::string client_lconfig() const { return workspace_() + "/client.local"; }
107 std::string client_log() const { return log_dir() + "/cvmfs.log"; }
108 std::string cache_dir() const { return workspace_() + "/cache"; }
109 std::string ovl_work_dir() const { return workspace_() + "/ovl_work"; }
110 std::string checkout_marker() const { return workspace_() + "/checkout"; }
111 std::string gw_session_token() const {
112 return workspace_() + "/session_token";
113 }
114 std::string transaction_lock() const {
115 return workspace_() + "/in_transaction.lock";
116 }
117 std::string publishing_lock() const {
118 return workspace_() + "/is_publishing.lock";
119 }
120 EUnionMountRepairMode repair_mode() const { return repair_mode_(); }
121
122 private:
123 Setting<std::string> workspace_;
124 Setting<std::string> tmp_dir_;
125 Setting<std::string> union_mnt_;
126 /**
127 * How aggressively should the mount point stack be repaired
128 */
129 Setting<EUnionMountRepairMode> repair_mode_;
130 }; // SettingsSpoolArea
131
132
133 class SettingsTransaction {
134 public:
135 explicit SettingsTransaction(const std::string &fqrn)
136 : layout_revision_(0)
137 , in_enter_session_(false)
138 , hash_algorithm_(shash::kShake128)
139 , compression_algorithm_(zlib::kZlibDefault)
140 , ttl_second_(240)
141 , is_garbage_collectable_(true)
142 , is_volatile_(false)
143 , enforce_limits_(false)
144 // SyncParameters::kDefaultNestedKcatalogLimit
145 , limit_nested_catalog_kentries_(500)
146 // SyncParameters::kDefaultRootKcatalogLimit
147 , limit_root_catalog_kentries_(500)
148 // SyncParameters::kDefaultFileMbyteLimit
149 , limit_file_size_mb_(1024)
150 , use_catalog_autobalance_(false)
151 // SyncParameters::kDefaultMaxWeight
152 , autobalance_max_weight_(100000)
153 // SyncParameters::kDefaultMinWeight
154 , autobalance_min_weight_(1000)
155 , print_changeset_(false)
156 , dry_run_(false)
157 , union_fs_(kUnionFsUnknown)
158 , timeout_s_(0)
159 , spool_area_(fqrn)
160 {}
161
162 void SetLayoutRevision(const unsigned revision);
163 void SetInEnterSession(const bool value);
164 void SetBaseHash(const shash::Any &hash);
165 void SetUnionFsType(const std::string &union_fs);
166 void SetHashAlgorithm(const std::string &algorithm);
167 void SetCompressionAlgorithm(const std::string &algorithm);
168 void SetEnforceLimits(bool value);
169 void SetLimitNestedCatalogKentries(unsigned value);
170 void SetLimitRootCatalogKentries(unsigned value);
171 void SetLimitFileSizeMb(unsigned value);
172 void SetUseCatalogAutobalance(bool value);
173 void SetAutobalanceMaxWeight(unsigned value);
174 void SetAutobalanceMinWeight(unsigned value);
175 void SetPrintChangeset(bool value);
176 void SetDryRun(bool value);
177 void SetTimeout(unsigned seconds);
178 void SetLeasePath(const std::string &path);
179 void SetTemplate(const std::string &from, const std::string &to);
180 void DetectUnionFsType();
181
182 /**
183 * 0 - wait infinitely
184 * <0: unset, fail immediately
185 */
186 int GetTimeoutS() const;
187
188 unsigned layout_revision() const { return layout_revision_(); }
189 bool in_enter_session() const { return in_enter_session_(); }
190 shash::Any base_hash() const { return base_hash_(); }
191 shash::Algorithms hash_algorithm() const { return hash_algorithm_(); }
192 zlib::Algorithms compression_algorithm() const {
193 return compression_algorithm_();
194 }
195 uint32_t ttl_second() const { return ttl_second_(); }
196 bool is_garbage_collectable() const { return is_garbage_collectable_(); }
197 bool is_volatile() const { return is_volatile_(); }
198 bool enforce_limits() const { return enforce_limits_(); }
199 unsigned limit_nested_catalog_kentries() const {
200 return limit_nested_catalog_kentries_();
201 }
202 unsigned limit_root_catalog_kentries() const {
203 return limit_root_catalog_kentries_();
204 }
205 unsigned limit_file_size_mb() const { return limit_file_size_mb_(); }
206 bool use_catalog_autobalance() const { return use_catalog_autobalance_(); }
207 unsigned autobalance_max_weight() const { return autobalance_max_weight_(); }
208 unsigned autobalance_min_weight() const { return autobalance_min_weight_(); }
209 bool print_changeset() const { return print_changeset_(); }
210 bool dry_run() const { return dry_run_(); }
211 std::string voms_authz() const { return voms_authz_(); }
212 UnionFsType union_fs() const { return union_fs_(); }
213 std::string lease_path() const { return lease_path_(); }
214 std::string template_from() const { return template_from_(); }
215 std::string template_to() const { return template_to_(); }
216
217 const SettingsSpoolArea &spool_area() const { return spool_area_; }
218 SettingsSpoolArea *GetSpoolArea() { return &spool_area_; }
219
220 bool HasTemplate() const { return !template_to().empty(); }
221
222 private:
223 bool ValidateUnionFs();
224
225 /**
226 * See CVMFS_CREATOR_VERSION
227 */
228 Setting<unsigned> layout_revision_;
229 /**
230 * Set to true if the settings have been created from the environment of
231 * the ephemeral writable shell (cvmfs_server enter command).
232 */
233 Setting<bool> in_enter_session_;
234 /**
235 * The root catalog hash based on which the transaction takes place.
236 * Usually the current root catalog from the manifest, which should be equal
237 * to the root hash of the mounted read-only volume. In some cases, this
238 * can be different though, e.g. for checked out branches or after silent
239 * transactions such as template transactions.
240 */
241 Setting<shash::Any> base_hash_;
242 Setting<shash::Algorithms> hash_algorithm_;
243 Setting<zlib::Algorithms> compression_algorithm_;
244 Setting<uint32_t> ttl_second_;
245 Setting<bool> is_garbage_collectable_;
246 Setting<bool> is_volatile_;
247 Setting<bool> enforce_limits_;
248 Setting<unsigned> limit_nested_catalog_kentries_;
249 Setting<unsigned> limit_root_catalog_kentries_;
250 Setting<unsigned> limit_file_size_mb_;
251 Setting<bool> use_catalog_autobalance_;
252 Setting<unsigned> autobalance_max_weight_;
253 Setting<unsigned> autobalance_min_weight_;
254 Setting<bool> print_changeset_;
255 Setting<bool> dry_run_;
256 Setting<std::string> voms_authz_;
257 Setting<UnionFsType> union_fs_;
258 /**
259 * How long to retry taking a lease before giving up
260 */
261 Setting<unsigned> timeout_s_;
262 Setting<std::string> lease_path_;
263 /**
264 * Used for template transactions where a directory tree gets cloned
265 * (from --> to) as part of opening the transaction
266 */
267 Setting<std::string> template_from_;
268 Setting<std::string> template_to_;
269
270 SettingsSpoolArea spool_area_;
271 }; // class SettingsTransaction
272
273
274 class SettingsGc {
275 }; // class SettingsGc
276
277
278 class SettingsStorage {
279 public:
280 explicit SettingsStorage(const std::string &fqrn)
281 : fqrn_(fqrn)
282 , type_(upload::SpoolerDefinition::Local)
283 , tmp_dir_(std::string("/srv/cvmfs/") + fqrn + "/data/txn")
284 , endpoint_(std::string("/srv/cvmfs/") + fqrn)
285 { }
286
287 std::string GetLocator() const;
288 void SetLocator(const std::string &locator);
289 void MakeLocal(const std::string &path);
290 void MakeS3(const std::string &s3_config, const std::string &tmp_dir);
291 void MakeGateway(const std::string &host, unsigned port,
292 const std::string &tmp_dir);
293
294 upload::SpoolerDefinition::DriverType type() const { return type_(); }
295 std::string endpoint() const { return endpoint_(); }
296
297 private:
298 Setting<std::string> fqrn_;
299 Setting<upload::SpoolerDefinition::DriverType> type_;
300 Setting<std::string> tmp_dir_;
301 Setting<std::string> endpoint_;
302 }; // class SettingsStorage
303
304
305 class SettingsKeychain {
306 public:
307 explicit SettingsKeychain(const std::string &fqrn)
308 : fqrn_(fqrn)
309 , keychain_dir_("/etc/cvmfs/keys")
310 , master_private_key_path_(keychain_dir_() + "/" + fqrn + ".masterkey")
311 , master_public_key_path_(keychain_dir_() + "/" + fqrn + ".pub")
312 , private_key_path_(keychain_dir_() + "/" + fqrn + ".key")
313 , certificate_path_(keychain_dir_() + "/" + fqrn + ".crt")
314 , gw_key_path_(keychain_dir_() + "/" + fqrn + ".gw")
315 {}
316
317 void SetKeychainDir(const std::string &keychain_dir);
318
319 bool HasDanglingMasterKeys() const;
320 bool HasMasterKeys() const;
321 bool HasDanglingRepositoryKeys() const;
322 bool HasRepositoryKeys() const;
323 bool HasGatewayKey() const;
324
325 std::string keychain_dir() const { return keychain_dir_(); }
326 std::string master_private_key_path() const {
327 return master_private_key_path_();
328 }
329 std::string master_public_key_path() const {
330 return master_public_key_path_();
331 }
332 std::string private_key_path() const { return private_key_path_(); }
333 std::string certificate_path() const { return certificate_path_(); }
334 std::string gw_key_path() const { return gw_key_path_(); }
335
336 private:
337 Setting<std::string> fqrn_;
338 Setting<std::string> keychain_dir_;
339 Setting<std::string> master_private_key_path_;
340 Setting<std::string> master_public_key_path_;
341 Setting<std::string> private_key_path_;
342 Setting<std::string> certificate_path_;
343 Setting<std::string> gw_key_path_;
344 }; // class SettingsKeychain
345
346
347 class SettingsPublisher;
348 class SettingsReplica;
349
350 /**
351 * Description of a read-only repository
352 */
353 class SettingsRepository {
354 public:
355 explicit SettingsRepository(const std::string &fqrn)
356 : fqrn_(fqrn)
357 , url_(std::string("http://localhost/cvmfs/") + fqrn_())
358 , proxy_("")
359 , tmp_dir_("/tmp")
360 , keychain_(fqrn)
361 {}
362 explicit SettingsRepository(const SettingsPublisher &settings_publisher);
363 explicit SettingsRepository(const SettingsReplica &settings_replica);
364
365 void SetUrl(const std::string &url);
366 void SetProxy(const std::string &proxy);
367 void SetTmpDir(const std::string &tmp_dir);
368 void SetCertBundle(const std::string &cert_bundle);
369
370 std::string fqrn() const { return fqrn_(); }
371 std::string url() const { return url_(); }
372 std::string proxy() const { return proxy_(); }
373 std::string tmp_dir() const { return tmp_dir_(); }
374 std::string cert_bundle() const { return cert_bundle_(); }
375
376 const SettingsKeychain &keychain() const { return keychain_; }
377 SettingsKeychain *GetKeychain() { return &keychain_; }
378
379 private:
380 Setting<std::string> fqrn_;
381 Setting<std::string> url_;
382 Setting<std::string> proxy_;
383 Setting<std::string> tmp_dir_;
384 // Currently only used for testing, steered by X509_CERT_BUNDLE
385 // in /etc/cvmfs/server.local
386 Setting<std::string> cert_bundle_;
387
388 SettingsKeychain keychain_;
389 }; // class SettingsRepository
390
391
392 /**
393 * Description of an editable repository.
394 */
395 class SettingsPublisher {
396 public:
397 static const unsigned kDefaultWhitelistValidity; // 30 days
398
399 explicit SettingsPublisher(const std::string &fqrn)
400 : fqrn_(fqrn)
401 , url_(std::string("http://localhost/cvmfs/") + fqrn)
402 , proxy_("")
403 , owner_uid_(0)
404 , owner_gid_(0)
405 , whitelist_validity_days_(kDefaultWhitelistValidity)
406 , is_silent_(false)
407 , is_managed_(false)
408 , ignore_invalid_lease_(false)
409 , storage_(fqrn_())
410 , transaction_(fqrn_())
411 , keychain_(fqrn_())
412 { }
413 explicit SettingsPublisher(const SettingsRepository &settings_repository);
414
415 void SetUrl(const std::string &url);
416 void SetProxy(const std::string &proxy);
417 void SetOwner(const std::string &user_name);
418 void SetOwner(uid_t uid, gid_t gid);
419 void SetIsSilent(bool value);
420 void SetIsManaged(bool value);
421 void SetIgnoreInvalidLease(bool value);
422
423 std::string GetReadOnlyXAttr(const std::string &attr);
424
425 std::string fqrn() const { return fqrn_(); }
426 std::string url() const { return url_(); }
427 std::string proxy() const { return proxy_(); }
428 unsigned whitelist_validity_days() const {
429 return whitelist_validity_days_();
430 }
431 uid_t owner_uid() const { return owner_uid_(); }
432 uid_t owner_gid() const { return owner_gid_(); }
433 bool is_silent() const { return is_silent_(); }
434 bool is_managed() const { return is_managed_(); }
435 bool ignore_invalid_lease() const { return ignore_invalid_lease_(); }
436
437 const SettingsStorage &storage() const { return storage_; }
438 const SettingsTransaction &transaction() const { return transaction_; }
439 const SettingsKeychain &keychain() const { return keychain_; }
440 SettingsStorage *GetStorage() { return &storage_; }
441 SettingsTransaction *GetTransaction() { return &transaction_; }
442 SettingsKeychain *GetKeychain() { return &keychain_; }
443
444 private:
445 Setting<std::string> fqrn_;
446 Setting<std::string> url_;
447 Setting<std::string> proxy_;
448 Setting<uid_t> owner_uid_;
449 Setting<gid_t> owner_gid_;
450 Setting<unsigned> whitelist_validity_days_;
451 Setting<bool> is_silent_;
452 Setting<bool> is_managed_;
453 // When trying to drop the session, ignore an invalid lease failure. Useful
454 // to recover a publisher with abort -f.
455 Setting<bool> ignore_invalid_lease_;
456
457 SettingsStorage storage_;
458 SettingsTransaction transaction_;
459 SettingsKeychain keychain_;
460 }; // SettingsPublisher
461
462
463 /**
464 * Description of a stratum 1
465 */
466 class SettingsReplica {
467 public:
468 explicit SettingsReplica(const std::string &fqrn)
469 : fqrn_(fqrn)
470 , alias_(fqrn)
471 , url_(std::string("http://localhost/cvmfs/") + alias_())
472 {}
473
474 std::string fqrn() const { return fqrn_(); }
475 std::string url() const { return url_(); }
476
477 private:
478 Setting<std::string> fqrn_;
479 Setting<std::string> alias_;
480 Setting<std::string> url_;
481 }; // class SettingsReplica
482
483
484 /**
485 * Create Settings objects from the system configuration in
486 * /etc/cvmfs/repositories.d
487 */
488 class SettingsBuilder : SingleCopy {
489 public:
490 SettingsBuilder()
491 : config_path_("/etc/cvmfs/repositories.d")
492 , options_mgr_(NULL)
493 {}
494 ~SettingsBuilder();
495 /**
496 * Used in unit tests.
497 */
498 explicit SettingsBuilder(const std::string &c) : config_path_(c) {}
499
500 /**
501 * If ident is a url, creates a generic settings object inferring the fqrn
502 * from the url.
503 * Otherwise, looks in the config files in /etc/cvmfs/repositories.d/<alias>/
504 * If alias is an empty string, the command still succeeds iff there is a
505 * single repository under /etc/cvmfs/repositories.d
506 */
507 SettingsRepository CreateSettingsRepository(const std::string &ident);
508
509 /**
510 * If ident is a url, creates a generic settings object inferring the fqrn
511 * from the url.
512 * Otherwise, looks in the config files in /etc/cvmfs/repositories.d/<alias>/
513 * If alias is an empty string, the command still succeeds iff there is a
514 * single repository under /etc/cvmfs/repositories.d
515 * If needs_managed is true, remote repositories are rejected
516 * In an "enter environment" (see cmd_enter), the spool area of the enter
517 * environment is applied.
518 */
519
520 SettingsPublisher* CreateSettingsPublisher(
521 const std::string &ident, bool needs_managed = false);
522
523 OptionsManager *options_mgr() const { return options_mgr_; }
524 bool IsManagedRepository() const { return options_mgr_ != NULL; }
525
526 /**
527 * Get the values from the server configuration and set them to the publisher settings.
528 */
529 void ApplyOptionsFromServerPath(const OptionsManager &options_mgr_,
530 SettingsPublisher* settings_publisher);
531
532 void SetConfigPath(const std::string &config_path) {
533 config_path_ = config_path;
534 }
535
536 private:
537 /**
538 * For non locally managed repositories, a configuration file should be
539 * provided in order to publish from the ephemeral shell through the gateway
540 */
541 std::string config_path_;
542
543 /**
544 * For locally managed repositories, the options manager is non NULL and
545 * contains the configuration after a call to CreateSettingsRepository()
546 */
547 OptionsManager *options_mgr_;
548 // TODO(avalenzu): options_mgr_ should become a unique pointer.
549
550 /**
551 * Returns the name of the one and only repository under kConfigPath
552 * Throws an exception if there are none or multiple repositories.
553 * The alias is usually the fqrn except for a replica with an explicit
554 * alias set different from the fqrn (e.g. if Stratum 0 and 1 are hosted)
555 * on the same node.
556 */
557 std::string GetSingleAlias();
558
559 /**
560 * If in a ephemeral writable shell, parse $session_dir/env.conf
561 * Otherwise return an empty map. A non-empty map has at least CVMFS_FQRN set.
562 */
563 std::map<std::string, std::string> GetSessionEnvironment();
564
565 /**
566 * Create settings from an ephermal writable shell
567 */
568 SettingsPublisher* CreateSettingsPublisherFromSession();
569 }; // class SettingsBuilder
570
571 } // namespace publish
572
573 #endif // CVMFS_PUBLISH_SETTINGS_H_
574