CernVM-FS  2.13.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
upload_s3.h
Go to the documentation of this file.
1 
5 #ifndef CVMFS_UPLOAD_S3_H_
6 #define CVMFS_UPLOAD_S3_H_
7 
8 #include <pthread.h>
9 
10 #include <string>
11 #include <utility>
12 #include <vector>
13 
14 #include "network/s3fanout.h"
15 #include "upload_facility.h"
16 #include "util/atomic.h"
18 #include "util/pointer.h"
19 #include "util/single_copy.h"
20 
21 namespace upload {
22 
25  uint64_t in_memory_threshold,
26  const std::string &tmp_dir = "/tmp/")
27  : UploadStreamHandle(commit_callback) {
28  buffer = FileBackedBuffer::Create(in_memory_threshold, tmp_dir);
29  }
30 
31  // Ownership is later transferred to the S3 fanout
33 };
34 
41 class S3Uploader : public AbstractUploader {
42  public:
44  virtual ~S3Uploader();
45  static bool WillHandle(const SpoolerDefinition &spooler_definition);
46 
47  virtual std::string name() const { return "S3"; }
48 
49  virtual bool Create();
50 
58  virtual void DoUpload(const std::string &remote_path,
60  const CallbackTN *callback);
61 
63  const CallbackTN *callback = NULL);
64  virtual void StreamedUpload(UploadStreamHandle *handle, UploadBuffer buffer,
65  const CallbackTN *callback);
66  virtual void FinalizeStreamedUpload(UploadStreamHandle *handle,
67  const shash::Any &content_hash);
68 
69  virtual void DoRemoveAsync(const std::string &file_to_delete);
70  virtual bool Peek(const std::string &path);
71  virtual bool Mkdir(const std::string &path);
72  virtual bool PlaceBootstrappingShortcut(const shash::Any &object);
73 
74  virtual unsigned int GetNumberOfErrors() const;
75  int64_t DoGetObjectSize(const std::string &file_name);
76 
77  // Only for testing
79  return s3fanout_mgr_.weak_ref();
80  }
81 
82  private:
83  static const unsigned kDefaultPort = 80;
84  static const unsigned kHttpsPort = 443;
85  static const unsigned kDefaultNumParallelUploads = 16;
86  static const unsigned kDefaultNumRetries = 3;
87  static const unsigned kDefaultTimeoutSec = 60;
88  static const unsigned kDefaultBackoffInitMs = 100;
89  static const unsigned kDefaultBackoffMaxMs = 2000;
90  static const unsigned kInMemoryObjectThreshold = 500 * 1024; // 500KiB
91 
92  // Used to make the async HTTP requests synchronous in Peek() Create(),
93  // and Upload() of single bits
96  pipe_wait[0] = pipe_wait[1] = -1;
97  }
98 
99  void WaitFor();
100 
103  std::string original_path;
104  int pipe_wait[2];
105  };
106 
107  void OnReqComplete(const upload::UploaderResults &results, RequestCtrl *ctrl);
108 
109  static void *MainCollectResults(void *data);
110 
111  bool ParseSpoolerDefinition(const SpoolerDefinition &spooler_definition);
112  void UploadJobInfo(s3fanout::JobInfo *info);
113 
114  s3fanout::JobInfo *CreateJobInfo(const std::string &path) const;
115 
117  std::string repository_alias_;
118  std::string host_name_port_;
119  std::string host_name_;
120  std::string region_;
121  std::string flavor_;
122  std::string bucket_;
125  unsigned num_retries_;
126  unsigned timeout_sec_;
127  std::string access_key_;
128  std::string secret_key_;
132  std::string proxy_;
133 
134  const std::string temporary_path_;
137 
138  std::string x_amz_acl_;
139 }; // S3Uploader
140 
141 } // namespace upload
142 
143 #endif // CVMFS_UPLOAD_S3_H_
std::string host_name_
Definition: upload_s3.h:119
virtual bool Create()
Definition: upload_s3.cc:231
s3fanout::S3FanoutManager * GetS3FanoutManager()
Definition: upload_s3.h:78
std::string proxy_
Definition: upload_s3.h:132
static const unsigned kDefaultPort
Definition: upload_s3.h:83
static FileBackedBuffer * Create(uint64_t in_memory_threshold, const std::string &tmp_dir="/tmp/")
T * weak_ref() const
Definition: pointer.h:46
virtual void DoUpload(const std::string &remote_path, IngestionSource *source, const CallbackTN *callback)
Definition: upload_s3.cc:322
CVMFS_EXPORT const LogSource source
Definition: exception.h:33
virtual void FinalizeStreamedUpload(UploadStreamHandle *handle, const shash::Any &content_hash)
Definition: upload_s3.cc:409
std::string secret_key_
Definition: upload_s3.h:128
int64_t DoGetObjectSize(const std::string &file_name)
Definition: upload_s3.cc:509
S3Uploader(const SpoolerDefinition &spooler_definition)
Definition: upload_s3.cc:45
const SpoolerDefinition & spooler_definition() const
S3StreamHandle(const CallbackTN *commit_callback, uint64_t in_memory_threshold, const std::string &tmp_dir="/tmp/")
Definition: upload_s3.h:24
static const unsigned kDefaultNumRetries
Definition: upload_s3.h:86
s3fanout::JobInfo * CreateJobInfo(const std::string &path) const
Definition: upload_s3.cc:449
static bool WillHandle(const SpoolerDefinition &spooler_definition)
Definition: upload_s3.cc:226
s3fanout::AuthzMethods authz_method_
Definition: upload_s3.h:129
static const unsigned kDefaultTimeoutSec
Definition: upload_s3.h:87
static void * MainCollectResults(void *data)
Definition: upload_s3.cc:270
virtual ~S3Uploader()
Definition: upload_s3.cc:98
std::string repository_alias_
Definition: upload_s3.h:117
AuthzMethods
Definition: s3fanout.h:32
void OnReqComplete(const upload::UploaderResults &results, RequestCtrl *ctrl)
Definition: upload_s3.cc:467
int32_t atomic_int32
Definition: atomic.h:17
const CallbackTN * commit_callback
atomic_int32 io_errors_
Definition: upload_s3.h:135
UniquePtr< s3fanout::S3FanoutManager > s3fanout_mgr_
Definition: upload_s3.h:116
virtual void DoRemoveAsync(const std::string &file_to_delete)
Definition: upload_s3.cc:455
virtual bool Peek(const std::string &path)
Definition: upload_s3.cc:482
virtual UploadStreamHandle * InitStreamedUpload(const CallbackTN *callback=NULL)
Definition: upload_s3.cc:393
std::string x_amz_acl_
Definition: upload_s3.h:138
std::string access_key_
Definition: upload_s3.h:127
virtual bool Mkdir(const std::string &path)
Definition: upload_s3.cc:501
std::string region_
Definition: upload_s3.h:120
UniquePtr< FileBackedBuffer > buffer
Definition: upload_s3.h:32
pthread_t thread_collect_results_
Definition: upload_s3.h:136
static const unsigned kDefaultBackoffMaxMs
Definition: upload_s3.h:89
static const unsigned kDefaultNumParallelUploads
Definition: upload_s3.h:85
void UploadJobInfo(s3fanout::JobInfo *info)
Definition: upload_s3.cc:381
AbstractUploader::CallbackTN CallbackTN
virtual bool PlaceBootstrappingShortcut(const shash::Any &object)
Definition: upload_s3.cc:504
const CallbackTN * callback_forward
Definition: upload_s3.h:102
static const unsigned kDefaultBackoffInitMs
Definition: upload_s3.h:88
virtual std::string name() const
Definition: upload_s3.h:47
bool ParseSpoolerDefinition(const SpoolerDefinition &spooler_definition)
Definition: upload_s3.cc:105
static const unsigned kHttpsPort
Definition: upload_s3.h:84
virtual unsigned int GetNumberOfErrors() const
Definition: upload_s3.cc:262
static const unsigned kInMemoryObjectThreshold
Definition: upload_s3.h:90
virtual void StreamedUpload(UploadStreamHandle *handle, UploadBuffer buffer, const CallbackTN *callback)
Definition: upload_s3.cc:399
const std::string temporary_path_
Definition: upload_s3.h:134
unsigned num_retries_
Definition: upload_s3.h:125
CallbackBase< UploaderResults > CallbackTN
Definition: async.h:185
std::string host_name_port_
Definition: upload_s3.h:118
std::string flavor_
Definition: upload_s3.h:121
unsigned timeout_sec_
Definition: upload_s3.h:126
std::string bucket_
Definition: upload_s3.h:122