CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
item.cc
Go to the documentation of this file.
1 
5 #include "item.h"
6 
7 #include <algorithm>
8 #include <cassert>
9 #include <cstdlib>
10 #include <cstring>
11 
13 #include "item_mem.h"
14 #include "smalloc.h"
15 #include "util_concurrency.h"
16 
18  IngestionSource* source,
19  uint64_t min_chunk_size,
20  uint64_t avg_chunk_size,
21  uint64_t max_chunk_size,
22  zlib::Algorithms compression_algorithm,
23  shash::Algorithms hash_algorithm,
24  shash::Suffix hash_suffix,
25  bool may_have_chunks,
26  bool has_legacy_bulk_chunk)
27  : source_(source)
28  , compression_algorithm_(compression_algorithm)
29  , hash_algorithm_(hash_algorithm)
30  , hash_suffix_(hash_suffix)
31  , has_legacy_bulk_chunk_(has_legacy_bulk_chunk)
32  , size_(kSizeUnknown)
33  , may_have_chunks_(may_have_chunks)
34  , chunk_detector_(min_chunk_size, avg_chunk_size, max_chunk_size)
35  , bulk_hash_(hash_algorithm)
36  , chunks_(1)
37 {
38  int retval = pthread_mutex_init(&lock_, NULL);
39  assert(retval == 0);
40  atomic_init64(&nchunks_in_fly_);
41  atomic_init32(&is_fully_chunked_);
42 }
43 
45  pthread_mutex_destroy(&lock_);
46 }
47 
48 void FileItem::RegisterChunk(const FileChunk &file_chunk) {
49  MutexLockGuard lock_guard(lock_);
50 
51  switch (file_chunk.content_hash().suffix) {
53  chunks_.PushBack(file_chunk);
54  break;
55 
56  default:
57  assert(file_chunk.offset() == 0);
58  assert(file_chunk.size() == size_);
59  bulk_hash_ = file_chunk.content_hash();
60  break;
61  }
62  atomic_dec64(&nchunks_in_fly_);
63 }
64 
65 
66 //------------------------------------------------------------------------------
67 
68 
69 ChunkItem::ChunkItem(FileItem *file_item, uint64_t offset)
70  : file_item_(file_item)
71  , offset_(offset)
72  , size_(0)
73  , is_bulk_chunk_(false)
74  , upload_handle_(NULL)
75  , compressor_(NULL)
76 {
77  hash_ctx_.algorithm = file_item->hash_algorithm();
84 }
85 
86 
88  is_bulk_chunk_ = true;
90 }
91 
92 
94  if (!compressor_.IsValid()) {
95  compressor_ =
97  }
98  return compressor_.weak_ref();
99 }
100 
101 
104 }
105 
106 
107 //------------------------------------------------------------------------------
108 
110 
111 
113  : allocator_(allocator)
114  , type_(kBlockHollow)
115  , tag_(-1)
116  , file_item_(NULL)
117  , chunk_item_(NULL)
118  , data_(NULL)
119  , capacity_(0)
120  , size_(0)
121 { }
122 
123 
124 BlockItem::BlockItem(int64_t tag, ItemAllocator *allocator)
125  : allocator_(allocator)
126  , type_(kBlockHollow)
127  , tag_(tag)
128  , file_item_(NULL)
129  , chunk_item_(NULL)
130  , data_(NULL)
131  , capacity_(0)
132  , size_(0)
133 {
134  assert(tag_ >= 0);
135 }
136 
137 
139  if (data_)
141  atomic_xadd64(&managed_bytes_, -static_cast<int64_t>(capacity_));
142 }
143 
144 
146  data_ = NULL;
147  size_ = capacity_ = 0;
148 }
149 
150 
153  type_ = kBlockStop;
154 }
155 
156 
157 void BlockItem::MakeData(uint32_t capacity) {
159  assert(allocator_ != NULL);
160  assert(capacity > 0);
161 
162  type_ = kBlockData;
164  data_ = reinterpret_cast<unsigned char *>(allocator_->Malloc(capacity_));
165  atomic_xadd64(&managed_bytes_, static_cast<int64_t>(capacity_));
166 }
167 
168 
174  assert(other->type_ == kBlockData);
175  assert(other->size_ > 0);
176 
177  type_ = kBlockData;
178  capacity_ = size_ = other->size_;
179  data_ = other->data_;
180  allocator_ = other->allocator_;
181 
182  other->Discharge();
183 }
184 
185 
190  const unsigned char *data,
191  uint32_t size)
192 {
194  assert(allocator_ != NULL);
195  assert(size > 0);
196 
197  type_ = kBlockData;
198  capacity_ = size_ = size;
199  data_ = reinterpret_cast<unsigned char *>(allocator_->Malloc(capacity_));
200  memcpy(data_, data, size);
201  atomic_xadd64(&managed_bytes_, static_cast<int64_t>(capacity_));
202 }
203 
204 
206  assert(type_ == kBlockData);
207 
208  atomic_xadd64(&managed_bytes_, -static_cast<int64_t>(capacity_));
210  data_ = NULL;
211  size_ = capacity_ = 0;
213 }
214 
215 
217  assert(value != NULL);
218  assert(chunk_item_ == NULL);
219  chunk_item_ = value;
220 }
221 
222 
224  assert(value != NULL);
225  assert(file_item_ == NULL);
226  file_item_ = value;
227 }
228 
229 
230 uint32_t BlockItem::Write(void *buf, uint32_t count) {
231  assert(type_ == kBlockData);
232 
233  uint32_t remaining = capacity_ - size_;
234  uint32_t nbytes = std::min(remaining, count);
235  memcpy(data_ + size_, buf, nbytes);
236  size_ += nbytes;
237  return nbytes;
238 }
unsigned char * data_
Definition: item.h:253
void RegisterChunk(const FileChunk &file_chunk)
Definition: item.cc:48
void MakeDataMove(BlockItem *other)
Definition: item.cc:172
int64_t atomic_int64
Definition: atomic.h:18
uint32_t size_
Definition: item.h:255
void Discharge()
Definition: item.cc:145
shash::ContextPtr hash_ctx_
Definition: item.h:168
T * weak_ref() const
Definition: pointer.h:42
void SetChunkItem(ChunkItem *item)
Definition: item.cc:216
FileItem(IngestionSource *source, uint64_t min_chunk_size=4 *1024 *1024, uint64_t avg_chunk_size=8 *1024 *1024, uint64_t max_chunk_size=16 *1024 *1024, zlib::Algorithms compression_algorithm=zlib::kZlibDefault, shash::Algorithms hash_algorithm=shash::kSha1, shash::Suffix hash_suffix=shash::kSuffixNone, bool may_have_chunks=true, bool has_legacy_bulk_chunk=false)
Definition: item.cc:17
void Init(ContextPtr context)
Definition: hash.cc:164
shash::Algorithms hash_algorithm()
Definition: item.h:62
void * Malloc(unsigned size)
Definition: item_mem.cc:55
unsigned char hash_ctx_buffer_[shash::kMaxContextSize]
Definition: item.h:170
zlib::Compressor * GetCompressor()
Definition: item.cc:93
void Reset()
Definition: item.cc:205
ChunkItem * chunk_item_
Definition: item.h:248
pthread_mutex_t lock_
Definition: item.h:115
shash::Any hash_value_
Definition: item.h:169
const shash::Any & content_hash() const
Definition: file_chunk.h:41
BlockItem(ItemAllocator *allocator)
Definition: item.cc:112
assert((mem||(size==0))&&"Out Of Memory")
void SetFileItem(FileItem *item)
Definition: item.cc:223
uint32_t capacity()
Definition: item.h:214
bool is_bulk_chunk_
Definition: item.h:162
Algorithms algorithm
Definition: hash.h:123
void MakeBulkChunk()
Definition: item.cc:87
void ReleaseCompressor()
Definition: item.cc:102
ChunkItem(FileItem *file_item, uint64_t offset)
Definition: item.cc:69
Algorithms
Definition: hash.h:39
static Compressor * Construct(const Algorithms &param)
Definition: plugin.h:188
FileChunkList chunks_
Definition: item.h:105
Algorithms
Definition: compression.h:44
const char kSuffixPartial
Definition: hash.h:55
Algorithms algorithm
Definition: hash.h:486
int64_t tag_
Definition: item.h:242
off_t offset() const
Definition: file_chunk.h:42
uint32_t capacity_
Definition: item.h:254
uint32_t size()
Definition: item.h:215
void MakeData(uint32_t capacity)
Definition: item.cc:157
unsigned GetContextSize(const Algorithms algorithm)
Definition: hash.cc:148
~BlockItem()
Definition: item.cc:138
Definition: item.h:34
bool IsValid() const
Definition: pointer.h:43
atomic_int64 nchunks_in_fly_
Definition: item.h:109
void * buffer
Definition: hash.h:487
BlockType type_
Definition: item.h:234
void IncNchunksInFly()
Definition: item.h:84
shash::Suffix hash_suffix()
Definition: item.h:63
char Suffix
Definition: hash.h:112
void PushBack(const Item &item)
Definition: bigvector.h:60
static atomic_int64 managed_bytes_
Definition: item.h:228
size_t size() const
Definition: file_chunk.h:43
FileItem * file_item_
Definition: item.h:247
uint64_t size_
Definition: item.h:100
zlib::Algorithms compression_algorithm()
Definition: item.h:61
ItemAllocator * allocator_
Definition: item.h:233
atomic_int32 is_fully_chunked_
Definition: item.h:114
~FileItem()
Definition: item.cc:44
void MakeDataCopy(const unsigned char *data, uint32_t size)
Definition: item.cc:189
Suffix suffix
Definition: hash.h:124
unsigned size
Definition: hash.h:488
uint32_t Write(void *buf, uint32_t count)
Definition: item.cc:230
shash::Any bulk_hash_
Definition: item.h:104
void MakeStop()
Definition: item.cc:151
static void size_t size
Definition: smalloc.h:47
UniquePtr< zlib::Compressor > compressor_
Definition: item.h:167
void Destroy()
Definition: pointer.h:45
FileItem * file_item_
Definition: item.h:155
void Free(void *ptr)
Definition: item_mem.cc:16