Directory: | cvmfs/ |
---|---|
File: | cvmfs/ingestion/item.cc |
Date: | 2025-09-28 02:35:26 |
Exec | Total | Coverage | |
---|---|---|---|
Lines: | 142 | 142 | 100.0% |
Branches: | 35 | 64 | 54.7% |
Line | Branch | Exec | Source |
---|---|---|---|
1 | /** | ||
2 | * This file is part of the CernVM File System. | ||
3 | */ | ||
4 | |||
5 | #include "item.h" | ||
6 | |||
7 | #include <algorithm> | ||
8 | #include <cassert> | ||
9 | #include <cstdlib> | ||
10 | #include <cstring> | ||
11 | |||
12 | #include "ingestion/ingestion_source.h" | ||
13 | #include "item_mem.h" | ||
14 | #include "util/concurrency.h" | ||
15 | #include "util/smalloc.h" | ||
16 | |||
17 | 806075 | FileItem::FileItem(IngestionSource *source, | |
18 | uint64_t min_chunk_size, | ||
19 | uint64_t avg_chunk_size, | ||
20 | uint64_t max_chunk_size, | ||
21 | zlib::Algorithms compression_algorithm, | ||
22 | shash::Algorithms hash_algorithm, | ||
23 | shash::Suffix hash_suffix, | ||
24 | bool may_have_chunks, | ||
25 | 806075 | bool has_legacy_bulk_chunk) | |
26 | 806075 | : source_(source) | |
27 | 806075 | , compression_algorithm_(compression_algorithm) | |
28 | 806075 | , hash_algorithm_(hash_algorithm) | |
29 | 806075 | , hash_suffix_(hash_suffix) | |
30 | 806075 | , has_legacy_bulk_chunk_(has_legacy_bulk_chunk) | |
31 | 806075 | , size_(kSizeUnknown) | |
32 | 806075 | , may_have_chunks_(may_have_chunks) | |
33 |
1/2✓ Branch 1 taken 806075 times.
✗ Branch 2 not taken.
|
806075 | , chunk_detector_(min_chunk_size, avg_chunk_size, max_chunk_size) |
34 |
1/2✓ Branch 1 taken 806075 times.
✗ Branch 2 not taken.
|
806075 | , bulk_hash_(hash_algorithm) |
35 |
1/2✓ Branch 2 taken 806075 times.
✗ Branch 3 not taken.
|
1612150 | , chunks_(1) { |
36 | 806075 | const int retval = pthread_mutex_init(&lock_, NULL); | |
37 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 806075 times.
|
806075 | assert(retval == 0); |
38 | 806075 | atomic_init64(&nchunks_in_fly_); | |
39 | 806075 | atomic_init32(&is_fully_chunked_); | |
40 | 806075 | } | |
41 | |||
42 | 805418 | FileItem::~FileItem() { pthread_mutex_destroy(&lock_); } | |
43 | |||
44 | 754340 | void FileItem::RegisterChunk(const FileChunk &file_chunk) { | |
45 | 754340 | const MutexLockGuard lock_guard(lock_); | |
46 | |||
47 |
2/2✓ Branch 1 taken 2475 times.
✓ Branch 2 taken 751865 times.
|
754340 | switch (file_chunk.content_hash().suffix) { |
48 | 2475 | case shash::kSuffixPartial: | |
49 |
1/2✓ Branch 1 taken 2475 times.
✗ Branch 2 not taken.
|
2475 | chunks_.PushBack(file_chunk); |
50 | 2475 | break; | |
51 | |||
52 | 751865 | default: | |
53 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 751865 times.
|
751865 | assert(file_chunk.offset() == 0); |
54 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 751865 times.
|
751865 | assert(file_chunk.size() == size_); |
55 | 751865 | bulk_hash_ = file_chunk.content_hash(); | |
56 | 751865 | break; | |
57 | } | ||
58 | 754340 | atomic_dec64(&nchunks_in_fly_); | |
59 | 754340 | } | |
60 | |||
61 | |||
62 | //------------------------------------------------------------------------------ | ||
63 | |||
64 | |||
65 | 948743 | ChunkItem::ChunkItem(FileItem *file_item, uint64_t offset) | |
66 | 948227 | : file_item_(file_item) | |
67 | 948227 | , offset_(offset) | |
68 | 948227 | , size_(0) | |
69 | 948227 | , is_bulk_chunk_(false) | |
70 | 948227 | , upload_handle_(NULL) | |
71 |
1/2✓ Branch 4 taken 948254 times.
✗ Branch 5 not taken.
|
948743 | , compressor_(NULL) { |
72 | 948254 | hash_ctx_.algorithm = file_item->hash_algorithm(); | |
73 |
1/2✓ Branch 1 taken 948173 times.
✗ Branch 2 not taken.
|
948203 | hash_ctx_.size = shash::GetContextSize(hash_ctx_.algorithm); |
74 | 948173 | hash_ctx_.buffer = hash_ctx_buffer_; | |
75 |
1/2✓ Branch 1 taken 948374 times.
✗ Branch 2 not taken.
|
948173 | shash::Init(hash_ctx_); |
76 | 948374 | hash_value_.algorithm = hash_ctx_.algorithm; | |
77 | 948374 | hash_value_.suffix = shash::kSuffixPartial; | |
78 | 948374 | file_item_->IncNchunksInFly(); | |
79 | 949370 | } | |
80 | |||
81 | |||
82 | 750686 | void ChunkItem::MakeBulkChunk() { | |
83 | 750686 | is_bulk_chunk_ = true; | |
84 | 750686 | hash_value_.suffix = file_item_->hash_suffix(); | |
85 | 750530 | } | |
86 | |||
87 | |||
88 | 1754374 | zlib::Compressor *ChunkItem::GetCompressor() { | |
89 |
2/2✓ Branch 1 taken 750152 times.
✓ Branch 2 taken 1004864 times.
|
1754374 | if (!compressor_.IsValid()) { |
90 | compressor_ = zlib::Compressor::Construct( | ||
91 |
2/4✓ Branch 2 taken 752408 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 751790 times.
✗ Branch 6 not taken.
|
750152 | file_item_->compression_algorithm()); |
92 | } | ||
93 | 1756654 | return compressor_.weak_ref(); | |
94 | } | ||
95 | |||
96 | |||
97 | 751298 | void ChunkItem::ReleaseCompressor() { compressor_.Destroy(); } | |
98 | |||
99 | |||
100 | //------------------------------------------------------------------------------ | ||
101 | |||
102 | atomic_int64 BlockItem::managed_bytes_ = 0; | ||
103 | |||
104 | |||
105 | 101320 | BlockItem::BlockItem(ItemAllocator *allocator) | |
106 | 101320 | : allocator_(allocator) | |
107 | 101320 | , type_(kBlockHollow) | |
108 | 101320 | , tag_(-1) | |
109 | 101320 | , file_item_(NULL) | |
110 | 101320 | , chunk_item_(NULL) | |
111 | 101320 | , data_(NULL) | |
112 | 101320 | , capacity_(0) | |
113 | 101320 | , size_(0) { } | |
114 | |||
115 | |||
116 | 7751844 | BlockItem::BlockItem(int64_t tag, ItemAllocator *allocator) | |
117 | 7717941 | : allocator_(allocator) | |
118 | 7717941 | , type_(kBlockHollow) | |
119 | 7717941 | , tag_(tag) | |
120 | 7717941 | , file_item_(NULL) | |
121 | 7717941 | , chunk_item_(NULL) | |
122 | 7717941 | , data_(NULL) | |
123 | 7717941 | , capacity_(0) | |
124 | 7751844 | , size_(0) { | |
125 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 7717941 times.
|
7717941 | assert(tag_ >= 0); |
126 | 7717941 | } | |
127 | |||
128 | |||
129 | 7826794 | BlockItem::~BlockItem() { | |
130 |
2/2✓ Branch 0 taken 4091071 times.
✓ Branch 1 taken 3735723 times.
|
7826794 | if (data_) |
131 | 4091071 | allocator_->Free(data_); | |
132 | 7829815 | atomic_xadd64(&managed_bytes_, -static_cast<int64_t>(capacity_)); | |
133 | 7869538 | } | |
134 | |||
135 | |||
136 | 828173 | void BlockItem::Discharge() { | |
137 | 828173 | data_ = NULL; | |
138 | 828173 | size_ = capacity_ = 0; | |
139 | 828173 | } | |
140 | |||
141 | |||
142 | 2440260 | void BlockItem::MakeStop() { | |
143 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 2440260 times.
|
2440260 | assert(type_ == kBlockHollow); |
144 | 2440260 | type_ = kBlockStop; | |
145 | 2440260 | } | |
146 | |||
147 | |||
148 | 1212632 | void BlockItem::MakeData(uint32_t capacity) { | |
149 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1212632 times.
|
1212632 | assert(type_ == kBlockHollow); |
150 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1212632 times.
|
1212632 | assert(allocator_ != NULL); |
151 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1212632 times.
|
1212632 | assert(capacity > 0); |
152 | |||
153 | 1212632 | type_ = kBlockData; | |
154 | 1212632 | capacity_ = capacity; | |
155 | 1212632 | data_ = reinterpret_cast<unsigned char *>(allocator_->Malloc(capacity_)); | |
156 | 1215245 | atomic_xadd64(&managed_bytes_, static_cast<int64_t>(capacity_)); | |
157 | 1215599 | } | |
158 | |||
159 | |||
160 | /** | ||
161 | * Move data from one block to another. | ||
162 | */ | ||
163 | 827990 | void BlockItem::MakeDataMove(BlockItem *other) { | |
164 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 827990 times.
|
827990 | assert(type_ == kBlockHollow); |
165 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 827990 times.
|
827990 | assert(other->type_ == kBlockData); |
166 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 827990 times.
|
827990 | assert(other->size_ > 0); |
167 | |||
168 | 827990 | type_ = kBlockData; | |
169 | 827990 | capacity_ = size_ = other->size_; | |
170 | 827990 | data_ = other->data_; | |
171 | 827990 | allocator_ = other->allocator_; | |
172 | |||
173 | 827990 | other->Discharge(); | |
174 | 828203 | } | |
175 | |||
176 | |||
177 | /** | ||
178 | * Copy a piece of one block's data into a new block. | ||
179 | */ | ||
180 | 3347840 | void BlockItem::MakeDataCopy(const unsigned char *data, uint32_t size) { | |
181 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 3347840 times.
|
3347840 | assert(type_ == kBlockHollow); |
182 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 3347840 times.
|
3347840 | assert(allocator_ != NULL); |
183 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 3347840 times.
|
3347840 | assert(size > 0); |
184 | |||
185 | 3347840 | type_ = kBlockData; | |
186 | 3347840 | capacity_ = size_ = size; | |
187 | 3347840 | data_ = reinterpret_cast<unsigned char *>(allocator_->Malloc(capacity_)); | |
188 | 3352220 | memcpy(data_, data, size); | |
189 | 3352220 | atomic_xadd64(&managed_bytes_, static_cast<int64_t>(capacity_)); | |
190 | 3349001 | } | |
191 | |||
192 | |||
193 | 472779 | void BlockItem::Reset() { | |
194 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 472779 times.
|
472779 | assert(type_ == kBlockData); |
195 | |||
196 | 472779 | atomic_xadd64(&managed_bytes_, -static_cast<int64_t>(capacity_)); | |
197 | 472794 | allocator_->Free(data_); | |
198 | 472794 | data_ = NULL; | |
199 | 472794 | size_ = capacity_ = 0; | |
200 | 472794 | type_ = kBlockHollow; | |
201 | 472794 | } | |
202 | |||
203 | |||
204 | 5712398 | void BlockItem::SetChunkItem(ChunkItem *value) { | |
205 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 5712398 times.
|
5712398 | assert(value != NULL); |
206 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 5712398 times.
|
5712398 | assert(chunk_item_ == NULL); |
207 | 5712398 | chunk_item_ = value; | |
208 | 5712398 | } | |
209 | |||
210 | |||
211 | 7718325 | void BlockItem::SetFileItem(FileItem *value) { | |
212 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 7718325 times.
|
7718325 | assert(value != NULL); |
213 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 7718325 times.
|
7718325 | assert(file_item_ == NULL); |
214 | 7718325 | file_item_ = value; | |
215 | 7718325 | } | |
216 | |||
217 | |||
218 | 19968 | uint32_t BlockItem::Write(void *buf, uint32_t count) { | |
219 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 19968 times.
|
19968 | assert(type_ == kBlockData); |
220 | |||
221 | 19968 | const uint32_t remaining = capacity_ - size_; | |
222 | 19968 | const uint32_t nbytes = std::min(remaining, count); | |
223 | 19968 | memcpy(data_ + size_, buf, nbytes); | |
224 | 19968 | size_ += nbytes; | |
225 | 19968 | return nbytes; | |
226 | } | ||
227 |