Directory: | cvmfs/ |
---|---|
File: | cvmfs/ingestion/item.cc |
Date: | 2025-10-19 02:35:28 |
Exec | Total | Coverage | |
---|---|---|---|
Lines: | 142 | 142 | 100.0% |
Branches: | 35 | 64 | 54.7% |
Line | Branch | Exec | Source |
---|---|---|---|
1 | /** | ||
2 | * This file is part of the CernVM File System. | ||
3 | */ | ||
4 | |||
5 | #include "item.h" | ||
6 | |||
7 | #include <algorithm> | ||
8 | #include <cassert> | ||
9 | #include <cstdlib> | ||
10 | #include <cstring> | ||
11 | |||
12 | #include "ingestion/ingestion_source.h" | ||
13 | #include "item_mem.h" | ||
14 | #include "util/concurrency.h" | ||
15 | #include "util/smalloc.h" | ||
16 | |||
17 | 1075583 | FileItem::FileItem(IngestionSource *source, | |
18 | uint64_t min_chunk_size, | ||
19 | uint64_t avg_chunk_size, | ||
20 | uint64_t max_chunk_size, | ||
21 | zlib::Algorithms compression_algorithm, | ||
22 | shash::Algorithms hash_algorithm, | ||
23 | shash::Suffix hash_suffix, | ||
24 | bool may_have_chunks, | ||
25 | 1075583 | bool has_legacy_bulk_chunk) | |
26 | 1075583 | : source_(source) | |
27 | 1075583 | , compression_algorithm_(compression_algorithm) | |
28 | 1075583 | , hash_algorithm_(hash_algorithm) | |
29 | 1075583 | , hash_suffix_(hash_suffix) | |
30 | 1075583 | , has_legacy_bulk_chunk_(has_legacy_bulk_chunk) | |
31 | 1075583 | , size_(kSizeUnknown) | |
32 | 1075583 | , may_have_chunks_(may_have_chunks) | |
33 |
1/2✓ Branch 1 taken 1075583 times.
✗ Branch 2 not taken.
|
1075583 | , chunk_detector_(min_chunk_size, avg_chunk_size, max_chunk_size) |
34 |
1/2✓ Branch 1 taken 1075583 times.
✗ Branch 2 not taken.
|
1075583 | , bulk_hash_(hash_algorithm) |
35 |
1/2✓ Branch 2 taken 1075583 times.
✗ Branch 3 not taken.
|
2151166 | , chunks_(1) { |
36 | 1075583 | const int retval = pthread_mutex_init(&lock_, NULL); | |
37 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1075583 times.
|
1075583 | assert(retval == 0); |
38 | 1075583 | atomic_init64(&nchunks_in_fly_); | |
39 | 1075583 | atomic_init32(&is_fully_chunked_); | |
40 | 1075583 | } | |
41 | |||
42 | 1074407 | FileItem::~FileItem() { pthread_mutex_destroy(&lock_); } | |
43 | |||
44 | 1004416 | void FileItem::RegisterChunk(const FileChunk &file_chunk) { | |
45 | 1004416 | const MutexLockGuard lock_guard(lock_); | |
46 | |||
47 |
2/2✓ Branch 1 taken 2628 times.
✓ Branch 2 taken 1001788 times.
|
1004416 | switch (file_chunk.content_hash().suffix) { |
48 | 2628 | case shash::kSuffixPartial: | |
49 |
1/2✓ Branch 1 taken 2628 times.
✗ Branch 2 not taken.
|
2628 | chunks_.PushBack(file_chunk); |
50 | 2628 | break; | |
51 | |||
52 | 1001788 | default: | |
53 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 1001788 times.
|
1001788 | assert(file_chunk.offset() == 0); |
54 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 1001788 times.
|
1001788 | assert(file_chunk.size() == size_); |
55 | 1001788 | bulk_hash_ = file_chunk.content_hash(); | |
56 | 1001788 | break; | |
57 | } | ||
58 | 1004416 | atomic_dec64(&nchunks_in_fly_); | |
59 | 1004416 | } | |
60 | |||
61 | |||
62 | //------------------------------------------------------------------------------ | ||
63 | |||
64 | |||
65 | 1158357 | ChunkItem::ChunkItem(FileItem *file_item, uint64_t offset) | |
66 | 1157685 | : file_item_(file_item) | |
67 | 1157685 | , offset_(offset) | |
68 | 1157685 | , size_(0) | |
69 | 1157685 | , is_bulk_chunk_(false) | |
70 | 1157685 | , upload_handle_(NULL) | |
71 |
1/2✓ Branch 4 taken 1157505 times.
✗ Branch 5 not taken.
|
1158357 | , compressor_(NULL) { |
72 | 1157505 | hash_ctx_.algorithm = file_item->hash_algorithm(); | |
73 |
1/2✓ Branch 1 taken 1157373 times.
✗ Branch 2 not taken.
|
1157421 | hash_ctx_.size = shash::GetContextSize(hash_ctx_.algorithm); |
74 | 1157373 | hash_ctx_.buffer = hash_ctx_buffer_; | |
75 |
1/2✓ Branch 1 taken 1157405 times.
✗ Branch 2 not taken.
|
1157373 | shash::Init(hash_ctx_); |
76 | 1157405 | hash_value_.algorithm = hash_ctx_.algorithm; | |
77 | 1157405 | hash_value_.suffix = shash::kSuffixPartial; | |
78 | 1157405 | file_item_->IncNchunksInFly(); | |
79 | 1159141 | } | |
80 | |||
81 | |||
82 | 999885 | void ChunkItem::MakeBulkChunk() { | |
83 | 999885 | is_bulk_chunk_ = true; | |
84 | 999885 | hash_value_.suffix = file_item_->hash_suffix(); | |
85 | 999665 | } | |
86 | |||
87 | |||
88 | 2326785 | zlib::Compressor *ChunkItem::GetCompressor() { | |
89 |
2/2✓ Branch 1 taken 999067 times.
✓ Branch 2 taken 1328550 times.
|
2326785 | if (!compressor_.IsValid()) { |
90 | compressor_ = zlib::Compressor::Construct( | ||
91 |
2/4✓ Branch 2 taken 1002299 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 1001471 times.
✗ Branch 6 not taken.
|
999067 | file_item_->compression_algorithm()); |
92 | } | ||
93 | 2330021 | return compressor_.weak_ref(); | |
94 | } | ||
95 | |||
96 | |||
97 | 1000947 | void ChunkItem::ReleaseCompressor() { compressor_.Destroy(); } | |
98 | |||
99 | |||
100 | //------------------------------------------------------------------------------ | ||
101 | |||
102 | atomic_int64 BlockItem::managed_bytes_ = 0; | ||
103 | |||
104 | |||
105 | 102504 | BlockItem::BlockItem(ItemAllocator *allocator) | |
106 | 102504 | : allocator_(allocator) | |
107 | 102504 | , type_(kBlockHollow) | |
108 | 102504 | , tag_(-1) | |
109 | 102504 | , file_item_(NULL) | |
110 | 102504 | , chunk_item_(NULL) | |
111 | 102504 | , data_(NULL) | |
112 | 102504 | , capacity_(0) | |
113 | 102504 | , size_(0) { } | |
114 | |||
115 | |||
116 | 9006399 | BlockItem::BlockItem(int64_t tag, ItemAllocator *allocator) | |
117 | 8968496 | : allocator_(allocator) | |
118 | 8968496 | , type_(kBlockHollow) | |
119 | 8968496 | , tag_(tag) | |
120 | 8968496 | , file_item_(NULL) | |
121 | 8968496 | , chunk_item_(NULL) | |
122 | 8968496 | , data_(NULL) | |
123 | 8968496 | , capacity_(0) | |
124 | 9006399 | , size_(0) { | |
125 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 8968496 times.
|
8968496 | assert(tag_ >= 0); |
126 | 8968496 | } | |
127 | |||
128 | |||
129 | 9067931 | BlockItem::~BlockItem() { | |
130 |
2/2✓ Branch 0 taken 4403441 times.
✓ Branch 1 taken 4664490 times.
|
9067931 | if (data_) |
131 | 4403441 | allocator_->Free(data_); | |
132 | 9072391 | atomic_xadd64(&managed_bytes_, -static_cast<int64_t>(capacity_)); | |
133 | 9128743 | } | |
134 | |||
135 | |||
136 | 1103766 | void BlockItem::Discharge() { | |
137 | 1103766 | data_ = NULL; | |
138 | 1103766 | size_ = capacity_ = 0; | |
139 | 1103766 | } | |
140 | |||
141 | |||
142 | 3143118 | void BlockItem::MakeStop() { | |
143 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 3143118 times.
|
3143118 | assert(type_ == kBlockHollow); |
144 | 3143118 | type_ = kBlockStop; | |
145 | 3143118 | } | |
146 | |||
147 | |||
148 | 1580112 | void BlockItem::MakeData(uint32_t capacity) { | |
149 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1580112 times.
|
1580112 | assert(type_ == kBlockHollow); |
150 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1580112 times.
|
1580112 | assert(allocator_ != NULL); |
151 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1580112 times.
|
1580112 | assert(capacity > 0); |
152 | |||
153 | 1580112 | type_ = kBlockData; | |
154 | 1580112 | capacity_ = capacity; | |
155 | 1580112 | data_ = reinterpret_cast<unsigned char *>(allocator_->Malloc(capacity_)); | |
156 | 1584232 | atomic_xadd64(&managed_bytes_, static_cast<int64_t>(capacity_)); | |
157 | 1584776 | } | |
158 | |||
159 | |||
160 | /** | ||
161 | * Move data from one block to another. | ||
162 | */ | ||
163 | 1103470 | void BlockItem::MakeDataMove(BlockItem *other) { | |
164 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1103470 times.
|
1103470 | assert(type_ == kBlockHollow); |
165 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1103470 times.
|
1103470 | assert(other->type_ == kBlockData); |
166 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1103470 times.
|
1103470 | assert(other->size_ > 0); |
167 | |||
168 | 1103470 | type_ = kBlockData; | |
169 | 1103470 | capacity_ = size_ = other->size_; | |
170 | 1103470 | data_ = other->data_; | |
171 | 1103470 | allocator_ = other->allocator_; | |
172 | |||
173 | 1103470 | other->Discharge(); | |
174 | 1103878 | } | |
175 | |||
176 | |||
177 | /** | ||
178 | * Copy a piece of one block's data into a new block. | ||
179 | */ | ||
180 | 3241691 | void BlockItem::MakeDataCopy(const unsigned char *data, uint32_t size) { | |
181 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 3241691 times.
|
3241691 | assert(type_ == kBlockHollow); |
182 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 3241691 times.
|
3241691 | assert(allocator_ != NULL); |
183 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 3241691 times.
|
3241691 | assert(size > 0); |
184 | |||
185 | 3241691 | type_ = kBlockData; | |
186 | 3241691 | capacity_ = size_ = size; | |
187 | 3241691 | data_ = reinterpret_cast<unsigned char *>(allocator_->Malloc(capacity_)); | |
188 | 3244050 | memcpy(data_, data, size); | |
189 | 3244050 | atomic_xadd64(&managed_bytes_, static_cast<int64_t>(capacity_)); | |
190 | 3240916 | } | |
191 | |||
192 | |||
193 | 419976 | void BlockItem::Reset() { | |
194 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 419976 times.
|
419976 | assert(type_ == kBlockData); |
195 | |||
196 | 419976 | atomic_xadd64(&managed_bytes_, -static_cast<int64_t>(capacity_)); | |
197 | 420044 | allocator_->Free(data_); | |
198 | 420044 | data_ = NULL; | |
199 | 420044 | size_ = capacity_ = 0; | |
200 | 420044 | type_ = kBlockHollow; | |
201 | 420044 | } | |
202 | |||
203 | |||
204 | 6492702 | void BlockItem::SetChunkItem(ChunkItem *value) { | |
205 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 6492702 times.
|
6492702 | assert(value != NULL); |
206 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 6492702 times.
|
6492702 | assert(chunk_item_ == NULL); |
207 | 6492702 | chunk_item_ = value; | |
208 | 6492702 | } | |
209 | |||
210 | |||
211 | 8966445 | void BlockItem::SetFileItem(FileItem *value) { | |
212 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 8966445 times.
|
8966445 | assert(value != NULL); |
213 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 8966445 times.
|
8966445 | assert(file_item_ == NULL); |
214 | 8966445 | file_item_ = value; | |
215 | 8966445 | } | |
216 | |||
217 | |||
218 | 15872 | uint32_t BlockItem::Write(void *buf, uint32_t count) { | |
219 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 15872 times.
|
15872 | assert(type_ == kBlockData); |
220 | |||
221 | 15872 | const uint32_t remaining = capacity_ - size_; | |
222 | 15872 | const uint32_t nbytes = std::min(remaining, count); | |
223 | 15872 | memcpy(data_ + size_, buf, nbytes); | |
224 | 15872 | size_ += nbytes; | |
225 | 15872 | return nbytes; | |
226 | } | ||
227 |