GCC Code Coverage Report
Directory: cvmfs/ Exec Total Coverage
File: cvmfs/upload_s3.h Lines: 4 6 66.7 %
Date: 2019-02-03 02:48:13 Branches: 1 3 33.3 %

Line Branch Exec Source
1
/**
2
 * This file is part of the CernVM File System.
3
 */
4
5
#ifndef CVMFS_UPLOAD_S3_H_
6
#define CVMFS_UPLOAD_S3_H_
7
8
#include <pthread.h>
9
10
#include <string>
11
#include <utility>
12
#include <vector>
13
14
#include "atomic.h"
15
#include "s3fanout.h"
16
#include "upload_facility.h"
17
18
namespace upload {
19
20
1814
struct S3StreamHandle : public UploadStreamHandle {
21
907
  S3StreamHandle(
22
    const CallbackTN *commit_callback,
23
    const int tmp_fd,
24
    const std::string &tmp_path)
25
    : UploadStreamHandle(commit_callback)
26
    , file_descriptor(tmp_fd)
27
907
    , temporary_path(tmp_path)
28
  { }
29
30
  const int file_descriptor;
31
  const std::string temporary_path;
32
};
33
34
/**
35
 * The S3Spooler implements the AbstractSpooler interface to push files
36
 * into a S3 CVMFS repository backend.
37
 * For a detailed description of the classes interface please have a look into
38
 * the AbstractSpooler base class.
39
 */
40
class S3Uploader : public AbstractUploader {
41
 public:
42
  explicit S3Uploader(const SpoolerDefinition &spooler_definition);
43
  virtual ~S3Uploader();
44
  static bool WillHandle(const SpoolerDefinition &spooler_definition);
45
46
  virtual std::string name() const { return "S3"; }
47
48
  /**
49
   * Upload() is not done concurrently in the current implementation of the
50
   * S3Spooler, since it is a simple move or copy of a file without CPU
51
   * intensive operation
52
   * This method calls NotifyListeners and invokes a callback for all
53
   * registered listeners (see the Observable template for details).
54
   */
55
  virtual void FileUpload(const std::string &local_path,
56
                          const std::string &remote_path,
57
                          const CallbackTN *callback = NULL);
58
59
  virtual UploadStreamHandle *InitStreamedUpload(
60
    const CallbackTN *callback = NULL);
61
  virtual void StreamedUpload(UploadStreamHandle *handle, UploadBuffer buffer,
62
                              const CallbackTN *callback = NULL);
63
  virtual void FinalizeStreamedUpload(UploadStreamHandle *handle,
64
                                      const shash::Any &content_hash);
65
66
  virtual void DoRemoveAsync(const std::string &file_to_delete);
67
  virtual bool Peek(const std::string &path) const;
68
  virtual bool PlaceBootstrappingShortcut(const shash::Any &object) const;
69
70
  virtual unsigned int GetNumberOfErrors() const;
71
  int64_t DoGetObjectSize(const std::string &file_name);
72
73
  // Only for testing
74
36
  s3fanout::S3FanoutManager *GetS3FanoutManager() { return &s3fanout_mgr_; }
75
76
 private:
77
  static const unsigned kDefaultPort = 80;
78
  static const unsigned kDefaultNumParallelUploads = 16;
79
  static const unsigned kDefaultNumRetries = 3;
80
  static const unsigned kDefaultTimeoutSec = 60;
81
  static const unsigned kDefaultBackoffInitMs = 100;
82
  static const unsigned kDefaultBackoffMaxMs = 2000;
83
84
  static void *MainCollectResults(void *data);
85
86
  bool ParseSpoolerDefinition(const SpoolerDefinition &spooler_definition);
87
  void UploadJobInfo(s3fanout::JobInfo *info);
88
89
  s3fanout::JobInfo *CreateJobInfo(const std::string &path) const;
90
91
  s3fanout::S3FanoutManager s3fanout_mgr_;
92
  std::string repository_alias_;
93
  std::string host_name_port_;
94
  std::string host_name_;
95
  std::string region_;
96
  std::string bucket_;
97
  bool dns_buckets_;
98
  int num_parallel_uploads_;
99
  unsigned num_retries_;
100
  unsigned timeout_sec_;
101
  std::string access_key_;
102
  std::string secret_key_;
103
  s3fanout::AuthzMethods authz_method_;
104
  bool peek_before_put_;
105
106
  const std::string temporary_path_;
107
  mutable atomic_int32 io_errors_;
108
  /**
109
   * Signals the CollectResults thread to quit
110
   */
111
  atomic_int32 terminate_;
112
  pthread_t thread_collect_results_;
113
};  // S3Uploader
114
115
}  // namespace upload
116
117
#endif  // CVMFS_UPLOAD_S3_H_