CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
upload_s3.h
Go to the documentation of this file.
1 
5 #ifndef CVMFS_UPLOAD_S3_H_
6 #define CVMFS_UPLOAD_S3_H_
7 
8 #include <pthread.h>
9 
10 #include <string>
11 #include <utility>
12 #include <vector>
13 
14 #include "atomic.h"
15 #include "s3fanout.h"
16 #include "upload_facility.h"
18 #include "util/pointer.h"
19 #include "util/single_copy.h"
20 
21 namespace upload {
22 
26  uint64_t in_memory_threshold,
27  const std::string &tmp_dir = "/tmp/")
28  : UploadStreamHandle(commit_callback)
29  {
30  buffer = FileBackedBuffer::Create(in_memory_threshold, tmp_dir);
31  }
32 
33  // Ownership is later transferred to the S3 fanout
35 };
36 
43 class S3Uploader : public AbstractUploader {
44  public:
46  virtual ~S3Uploader();
47  static bool WillHandle(const SpoolerDefinition &spooler_definition);
48 
49  virtual std::string name() const { return "S3"; }
50 
51  virtual bool Create();
52 
60  virtual void DoUpload(const std::string &remote_path,
61  IngestionSource *source,
62  const CallbackTN *callback = NULL);
63 
65  const CallbackTN *callback = NULL);
66  virtual void StreamedUpload(UploadStreamHandle *handle, UploadBuffer buffer,
67  const CallbackTN *callback = NULL);
68  virtual void FinalizeStreamedUpload(UploadStreamHandle *handle,
69  const shash::Any &content_hash);
70 
71  virtual void DoRemoveAsync(const std::string &file_to_delete);
72  virtual bool Peek(const std::string &path);
73  virtual bool Mkdir(const std::string &path);
74  virtual bool PlaceBootstrappingShortcut(const shash::Any &object);
75 
76  virtual unsigned int GetNumberOfErrors() const;
77  int64_t DoGetObjectSize(const std::string &file_name);
78 
79  // Only for testing
81 
82  private:
83  static const unsigned kDefaultPort = 80;
84  static const unsigned kDefaultNumParallelUploads = 16;
85  static const unsigned kDefaultNumRetries = 3;
86  static const unsigned kDefaultTimeoutSec = 60;
87  static const unsigned kDefaultBackoffInitMs = 100;
88  static const unsigned kDefaultBackoffMaxMs = 2000;
89  static const unsigned kInMemoryObjectThreshold = 500*1024; // 500KiB
90 
91  // Used to make the async HTTP requests synchronous in Peek() Create(),
92  // and Upload() of single bits
95  pipe_wait[0] = pipe_wait[1] = -1;
96  }
97 
98  void WaitFor();
99 
102  std::string original_path;
103  int pipe_wait[2];
104  };
105 
106  void OnReqComplete(const upload::UploaderResults &results, RequestCtrl *ctrl);
107 
108  static void *MainCollectResults(void *data);
109 
110  bool ParseSpoolerDefinition(const SpoolerDefinition &spooler_definition);
111  void UploadJobInfo(s3fanout::JobInfo *info);
112 
113  s3fanout::JobInfo *CreateJobInfo(const std::string &path) const;
114 
116  std::string repository_alias_;
117  std::string host_name_port_;
118  std::string host_name_;
119  std::string region_;
120  std::string flavor_;
121  std::string bucket_;
124  unsigned num_retries_;
125  unsigned timeout_sec_;
126  std::string access_key_;
127  std::string secret_key_;
130 
131  const std::string temporary_path_;
134 }; // S3Uploader
135 
136 } // namespace upload
137 
138 #endif // CVMFS_UPLOAD_S3_H_
std::string host_name_
Definition: upload_s3.h:118
virtual bool Create()
Definition: upload_s3.cc:178
s3fanout::S3FanoutManager * GetS3FanoutManager()
Definition: upload_s3.h:80
static const unsigned kDefaultPort
Definition: upload_s3.h:83
static FileBackedBuffer * Create(uint64_t in_memory_threshold, const std::string &tmp_dir="/tmp/")
virtual void FinalizeStreamedUpload(UploadStreamHandle *handle, const shash::Any &content_hash)
Definition: upload_s3.cc:363
std::string secret_key_
Definition: upload_s3.h:127
int64_t DoGetObjectSize(const std::string &file_name)
Definition: upload_s3.cc:471
S3Uploader(const SpoolerDefinition &spooler_definition)
Definition: upload_s3.cc:33
const SpoolerDefinition & spooler_definition() const
S3StreamHandle(const CallbackTN *commit_callback, uint64_t in_memory_threshold, const std::string &tmp_dir="/tmp/")
Definition: upload_s3.h:24
static const unsigned kDefaultNumRetries
Definition: upload_s3.h:85
s3fanout::JobInfo * CreateJobInfo(const std::string &path) const
Definition: upload_s3.cc:407
static bool WillHandle(const SpoolerDefinition &spooler_definition)
Definition: upload_s3.cc:173
s3fanout::AuthzMethods authz_method_
Definition: upload_s3.h:128
static const unsigned kDefaultTimeoutSec
Definition: upload_s3.h:86
static void * MainCollectResults(void *data)
Definition: upload_s3.cc:216
virtual ~S3Uploader()
Definition: upload_s3.cc:76
std::string repository_alias_
Definition: upload_s3.h:116
AuthzMethods
Definition: s3fanout.h:28
void OnReqComplete(const upload::UploaderResults &results, RequestCtrl *ctrl)
Definition: upload_s3.cc:425
int32_t atomic_int32
Definition: atomic.h:17
const CallbackTN * commit_callback
virtual void DoUpload(const std::string &remote_path, IngestionSource *source, const CallbackTN *callback=NULL)
Definition: upload_s3.cc:269
atomic_int32 io_errors_
Definition: upload_s3.h:132
UniquePtr< s3fanout::S3FanoutManager > s3fanout_mgr_
Definition: upload_s3.h:115
virtual void DoRemoveAsync(const std::string &file_to_delete)
Definition: upload_s3.cc:413
virtual bool Peek(const std::string &path)
Definition: upload_s3.cc:442
virtual UploadStreamHandle * InitStreamedUpload(const CallbackTN *callback=NULL)
Definition: upload_s3.cc:345
std::string access_key_
Definition: upload_s3.h:126
virtual bool Mkdir(const std::string &path)
Definition: upload_s3.cc:461
std::string region_
Definition: upload_s3.h:119
UniquePtr< FileBackedBuffer > buffer
Definition: upload_s3.h:34
pthread_t thread_collect_results_
Definition: upload_s3.h:133
static const unsigned kDefaultBackoffMaxMs
Definition: upload_s3.h:88
static const unsigned kDefaultNumParallelUploads
Definition: upload_s3.h:84
virtual void StreamedUpload(UploadStreamHandle *handle, UploadBuffer buffer, const CallbackTN *callback=NULL)
Definition: upload_s3.cc:351
void UploadJobInfo(s3fanout::JobInfo *info)
Definition: upload_s3.cc:331
AbstractUploader::CallbackTN CallbackTN
virtual bool PlaceBootstrappingShortcut(const shash::Any &object)
Definition: upload_s3.cc:466
const CallbackTN * callback_forward
Definition: upload_s3.h:101
static const unsigned kDefaultBackoffInitMs
Definition: upload_s3.h:87
virtual std::string name() const
Definition: upload_s3.h:49
bool ParseSpoolerDefinition(const SpoolerDefinition &spooler_definition)
Definition: upload_s3.cc:83
virtual unsigned int GetNumberOfErrors() const
Definition: upload_s3.cc:208
static const unsigned kInMemoryObjectThreshold
Definition: upload_s3.h:89
const std::string temporary_path_
Definition: upload_s3.h:131
unsigned num_retries_
Definition: upload_s3.h:124
CallbackBase< UploaderResults > CallbackTN
Definition: async.h:192
std::string host_name_port_
Definition: upload_s3.h:117
std::string flavor_
Definition: upload_s3.h:120
unsigned timeout_sec_
Definition: upload_s3.h:125
std::string bucket_
Definition: upload_s3.h:121