CernVM-FS  2.10.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
pack.h
Go to the documentation of this file.
1 
5 #ifndef CVMFS_PACK_H_
6 #define CVMFS_PACK_H_
7 
8 #include <inttypes.h>
9 #include <pthread.h>
10 
11 #include <cstdio>
12 #include <set>
13 #include <string>
14 #include <vector>
15 
16 #include "gtest/gtest_prod.h"
17 #include "hash.h"
18 #include "util/single_copy.h"
19 #include "util_concurrency.h"
20 
36  FRIEND_TEST(T_Pack, Bucket);
37  FRIEND_TEST(T_Pack, ObjectPack);
38  FRIEND_TEST(T_Pack, ObjectPackTransfer);
39 
40  private:
41  struct Bucket;
42 
43  public:
44  typedef Bucket *BucketHandle;
45 
54 
55  static const uint64_t kDefaultLimit = 200 * 1024 * 1024; // 200MB
56 
62  static const uint64_t kMaxObjects = 100000;
63 
64  explicit ObjectPack(const uint64_t limit = kDefaultLimit);
65  ~ObjectPack();
66 
67  static void AddToBucket(const void *buf, const uint64_t size,
68  const BucketHandle handle);
69 
71 
72  bool CommitBucket(const BucketContentType type, const shash::Any &id,
73  const BucketHandle handle, const std::string &name = "");
74 
75  void DiscardBucket(const BucketHandle handle);
76  void TransferBucket(const BucketHandle handle, ObjectPack *other);
77 
78  unsigned char *BucketContent(size_t idx) const;
79  uint64_t BucketSize(size_t idx) const;
80  const shash::Any &BucketId(size_t idx) const;
81 
82  uint64_t size() const { return size_; }
83 
84  // This returns the number of objects in the pack (equal to the number of
85  // committed buckets)
86  size_t GetNoObjects() const { return buckets_.size(); }
87 
88  private:
93  struct Bucket : SingleCopy {
94  static const unsigned kInitialSize = 128;
95 
96  Bucket();
97  ~Bucket();
98  void Add(const void *buf, const uint64_t buf_size);
99 
100  unsigned char *content;
101  uint64_t size;
102  uint64_t capacity;
105  std::string name;
106  };
107 
108  void InitLock();
109 
113  pthread_mutex_t *lock_;
114 
118  uint64_t limit_;
122  uint64_t size_;
126  std::set<BucketHandle> open_buckets_;
130  std::vector<BucketHandle> buckets_;
131 };
132 
138 namespace ObjectPackBuild {
139 struct Event {
140  Event(const shash::Any &id, uint64_t size, unsigned buf_size, const void *buf,
141  ObjectPack::BucketContentType type, const std::string &name)
142  : id(id),
143  size(size),
144  buf_size(buf_size),
145  buf(buf),
146  object_type(type),
147  object_name(name) {}
148 
150  uint64_t size;
151  unsigned buf_size;
152  const void *buf;
154  std::string object_name;
155 };
156 
157 enum State {
164 };
165 } // namespace ObjectPackBuild
166 
183  public:
184  explicit ObjectPackProducer(ObjectPack *pack);
185  ObjectPackProducer(const shash::Any &id, FILE *big_file,
186  const std::string &file_name = "");
187  unsigned ProduceNext(const unsigned buf_size, unsigned char *buf);
188  void GetDigest(shash::Any *hash);
189  unsigned GetHeaderSize() { return header_.size(); }
190 
191  private:
196 
200  FILE *big_file_;
201 
205  uint64_t pos_;
206 
210  size_t idx_;
211 
216 
220  std::string header_;
221 };
222 
230 class ObjectPackConsumer : public Observable<ObjectPackBuild::Event> {
231  public:
232  explicit ObjectPackConsumer(const shash::Any &expected_digest,
233  const unsigned expected_header_size);
234  ObjectPackBuild::State ConsumeNext(const unsigned buf_size,
235  const unsigned char *buf);
236 
237  private:
241  static const unsigned kAccuSize = 128 * 1024;
242 
243  struct IndexEntry {
245  IndexEntry(const shash::Any &id, const uint64_t size,
246  ObjectPack::BucketContentType type, const std::string &name)
247  : id(id), size(size), entry_type(type), entry_name(name) {}
249  uint64_t size;
251  std::string entry_name;
252  };
253 
254  bool ParseHeader();
255  bool ParseItem(const std::string &line, IndexEntry *entry,
256  uint64_t *sum_size);
257 
258  ObjectPackBuild::State ConsumePayload(const unsigned buf_size,
259  const unsigned char *buf);
260 
263 
267  uint64_t pos_;
268 
272  unsigned idx_;
273 
277  unsigned pos_in_object_;
278 
283  unsigned char accumulator_[kAccuSize];
284 
288  unsigned pos_in_accu_;
289 
295 
300  std::string raw_header_;
301 
305  uint64_t size_;
306 
310  std::vector<IndexEntry> index_;
311 };
312 
313 #endif // CVMFS_PACK_H_
Event(const shash::Any &id, uint64_t size, unsigned buf_size, const void *buf, ObjectPack::BucketContentType type, const std::string &name)
Definition: pack.h:140
unsigned char accumulator_[kAccuSize]
Definition: pack.h:283
BucketContentType
Definition: pack.h:53
ObjectPackConsumer(const shash::Any &expected_digest, const unsigned expected_header_size)
Definition: pack.cc:258
std::string header_
Definition: pack.h:220
unsigned idx_
Definition: pack.h:272
void TransferBucket(const BucketHandle handle, ObjectPack *other)
Definition: pack.cc:149
void Add(const void *buf, const uint64_t buf_size)
Definition: pack.cc:66
uint64_t size_
Definition: pack.h:122
static const uint64_t kMaxObjects
Definition: pack.h:62
std::string entry_name
Definition: pack.h:251
FILE * big_file_
Definition: pack.h:200
const shash::Any & BucketId(size_t idx) const
Definition: pack.cc:166
pthread_mutex_t * lock_
Definition: pack.h:113
ObjectPackProducer(ObjectPack *pack)
Definition: pack.cc:181
std::string object_name
Definition: pack.h:154
unsigned char * BucketContent(size_t idx) const
Definition: pack.cc:156
size_t idx_
Definition: pack.h:210
static const unsigned kInitialSize
Definition: pack.h:94
uint64_t size() const
Definition: pack.h:82
Bucket * BucketHandle
Definition: pack.h:41
unsigned pos_in_object_
Definition: pack.h:277
uint64_t size
Definition: pack.h:249
IndexEntry()
Definition: pack.h:244
BucketHandle NewBucket()
Definition: pack.cc:102
unsigned expected_header_size_
Definition: pack.h:262
unsigned char * content
Definition: pack.h:100
static const unsigned kAccuSize
Definition: pack.h:241
size_t pos_in_bucket_
Definition: pack.h:215
unsigned pos_in_accu_
Definition: pack.h:288
std::string name
Definition: pack.h:105
static const uint64_t kDefaultLimit
Definition: pack.h:55
ObjectPackBuild::State ConsumePayload(const unsigned buf_size, const unsigned char *buf)
Definition: pack.cc:338
uint64_t BucketSize(size_t idx) const
Definition: pack.cc:161
ObjectPackBuild::State ConsumeNext(const unsigned buf_size, const unsigned char *buf)
Definition: pack.cc:281
Definition: pack.h:243
uint64_t capacity
Definition: pack.h:102
void InitLock()
Definition: pack.cc:139
bool ParseHeader()
Definition: pack.cc:388
BucketContentType content_type
Definition: pack.h:104
std::vector< IndexEntry > index_
Definition: pack.h:310
shash::Any id
Definition: pack.h:149
std::vector< BucketHandle > buckets_
Definition: pack.h:130
void GetDigest(shash::Any *hash)
Definition: pack.cc:176
const void * buf
Definition: pack.h:152
void DiscardBucket(const BucketHandle handle)
Definition: pack.cc:133
bool ParseItem(const std::string &line, IndexEntry *entry, uint64_t *sum_size)
Definition: pack.cc:425
uint64_t size
Definition: pack.h:101
ObjectPack::BucketContentType object_type
Definition: pack.h:153
std::set< BucketHandle > open_buckets_
Definition: pack.h:126
ObjectPack::BucketContentType entry_type
Definition: pack.h:250
shash::Any id
Definition: pack.h:248
~ObjectPack()
Definition: pack.cc:85
ObjectPack(const uint64_t limit=kDefaultLimit)
Definition: pack.cc:81
uint64_t limit_
Definition: pack.h:118
shash::Any id
Definition: pack.h:103
uint64_t size_
Definition: pack.h:305
IndexEntry(const shash::Any &id, const uint64_t size, ObjectPack::BucketContentType type, const std::string &name)
Definition: pack.h:245
unsigned GetHeaderSize()
Definition: pack.h:189
bool CommitBucket(const BucketContentType type, const shash::Any &id, const BucketHandle handle, const std::string &name="")
Definition: pack.cc:113
ObjectPack * pack_
Definition: pack.h:195
static void AddToBucket(const void *buf, const uint64_t size, const BucketHandle handle)
Definition: pack.cc:97
uint64_t pos_
Definition: pack.h:205
size_t GetNoObjects() const
Definition: pack.h:86
std::string raw_header_
Definition: pack.h:300
ObjectPackBuild::State state_
Definition: pack.h:294
unsigned buf_size
Definition: pack.h:151
unsigned ProduceNext(const unsigned buf_size, unsigned char *buf)
Definition: pack.cc:216
shash::Any expected_digest_
Definition: pack.h:261
uint64_t pos_
Definition: pack.h:267
FRIEND_TEST(T_Pack, Bucket)