Directory: | cvmfs/ |
---|---|
File: | cvmfs/ingestion/item.cc |
Date: | 2025-07-27 02:42:09 |
Exec | Total | Coverage | |
---|---|---|---|
Lines: | 142 | 142 | 100.0% |
Branches: | 35 | 64 | 54.7% |
Line | Branch | Exec | Source |
---|---|---|---|
1 | /** | ||
2 | * This file is part of the CernVM File System. | ||
3 | */ | ||
4 | |||
5 | #include "item.h" | ||
6 | |||
7 | #include <algorithm> | ||
8 | #include <cassert> | ||
9 | #include <cstdlib> | ||
10 | #include <cstring> | ||
11 | |||
12 | #include "ingestion/ingestion_source.h" | ||
13 | #include "item_mem.h" | ||
14 | #include "util/concurrency.h" | ||
15 | #include "util/smalloc.h" | ||
16 | |||
17 | 4052199 | FileItem::FileItem(IngestionSource *source, | |
18 | uint64_t min_chunk_size, | ||
19 | uint64_t avg_chunk_size, | ||
20 | uint64_t max_chunk_size, | ||
21 | zlib::Algorithms compression_algorithm, | ||
22 | shash::Algorithms hash_algorithm, | ||
23 | shash::Suffix hash_suffix, | ||
24 | bool may_have_chunks, | ||
25 | 4052199 | bool has_legacy_bulk_chunk) | |
26 | 4052199 | : source_(source) | |
27 | 4052199 | , compression_algorithm_(compression_algorithm) | |
28 | 4052199 | , hash_algorithm_(hash_algorithm) | |
29 | 4052199 | , hash_suffix_(hash_suffix) | |
30 | 4052199 | , has_legacy_bulk_chunk_(has_legacy_bulk_chunk) | |
31 | 4052199 | , size_(kSizeUnknown) | |
32 | 4052199 | , may_have_chunks_(may_have_chunks) | |
33 |
1/2✓ Branch 1 taken 4052199 times.
✗ Branch 2 not taken.
|
4052199 | , chunk_detector_(min_chunk_size, avg_chunk_size, max_chunk_size) |
34 |
1/2✓ Branch 1 taken 4052199 times.
✗ Branch 2 not taken.
|
4052199 | , bulk_hash_(hash_algorithm) |
35 |
1/2✓ Branch 2 taken 4052199 times.
✗ Branch 3 not taken.
|
8104398 | , chunks_(1) { |
36 | 4052199 | const int retval = pthread_mutex_init(&lock_, NULL); | |
37 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 4052199 times.
|
4052199 | assert(retval == 0); |
38 | 4052199 | atomic_init64(&nchunks_in_fly_); | |
39 | 4052199 | atomic_init32(&is_fully_chunked_); | |
40 | 4052199 | } | |
41 | |||
42 | 4050665 | FileItem::~FileItem() { pthread_mutex_destroy(&lock_); } | |
43 | |||
44 | 4008062 | void FileItem::RegisterChunk(const FileChunk &file_chunk) { | |
45 | 4008062 | const MutexLockGuard lock_guard(lock_); | |
46 | |||
47 |
2/2✓ Branch 1 taken 6576 times.
✓ Branch 2 taken 4001486 times.
|
4008062 | switch (file_chunk.content_hash().suffix) { |
48 | 6576 | case shash::kSuffixPartial: | |
49 |
1/2✓ Branch 1 taken 6576 times.
✗ Branch 2 not taken.
|
6576 | chunks_.PushBack(file_chunk); |
50 | 6576 | break; | |
51 | |||
52 | 4001486 | default: | |
53 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 4001486 times.
|
4001486 | assert(file_chunk.offset() == 0); |
54 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 4001486 times.
|
4001486 | assert(file_chunk.size() == size_); |
55 | 4001486 | bulk_hash_ = file_chunk.content_hash(); | |
56 | 4001486 | break; | |
57 | } | ||
58 | 4008062 | atomic_dec64(&nchunks_in_fly_); | |
59 | 4008062 | } | |
60 | |||
61 | |||
62 | //------------------------------------------------------------------------------ | ||
63 | |||
64 | |||
65 | 4006897 | ChunkItem::ChunkItem(FileItem *file_item, uint64_t offset) | |
66 | 4003825 | : file_item_(file_item) | |
67 | 4003825 | , offset_(offset) | |
68 | 4003825 | , size_(0) | |
69 | 4003825 | , is_bulk_chunk_(false) | |
70 | 4003825 | , upload_handle_(NULL) | |
71 |
1/2✓ Branch 4 taken 4006305 times.
✗ Branch 5 not taken.
|
4006897 | , compressor_(NULL) { |
72 | 4006305 | hash_ctx_.algorithm = file_item->hash_algorithm(); | |
73 |
1/2✓ Branch 1 taken 4004193 times.
✗ Branch 2 not taken.
|
4004513 | hash_ctx_.size = shash::GetContextSize(hash_ctx_.algorithm); |
74 | 4004193 | hash_ctx_.buffer = hash_ctx_buffer_; | |
75 |
1/2✓ Branch 1 taken 4002481 times.
✗ Branch 2 not taken.
|
4004193 | shash::Init(hash_ctx_); |
76 | 4002481 | hash_value_.algorithm = hash_ctx_.algorithm; | |
77 | 4002481 | hash_value_.suffix = shash::kSuffixPartial; | |
78 | 4002481 | file_item_->IncNchunksInFly(); | |
79 | 4010561 | } | |
80 | |||
81 | |||
82 | 3993121 | void ChunkItem::MakeBulkChunk() { | |
83 | 3993121 | is_bulk_chunk_ = true; | |
84 | 3993121 | hash_value_.suffix = file_item_->hash_suffix(); | |
85 | 3992033 | } | |
86 | |||
87 | |||
88 | 9239066 | zlib::Compressor *ChunkItem::GetCompressor() { | |
89 |
2/2✓ Branch 1 taken 3994351 times.
✓ Branch 2 taken 5247131 times.
|
9239066 | if (!compressor_.IsValid()) { |
90 | compressor_ = zlib::Compressor::Construct( | ||
91 |
2/4✓ Branch 2 taken 4001471 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 3999135 times.
✗ Branch 6 not taken.
|
3994351 | file_item_->compression_algorithm()); |
92 | } | ||
93 | 9246266 | return compressor_.weak_ref(); | |
94 | } | ||
95 | |||
96 | |||
97 | 3997631 | void ChunkItem::ReleaseCompressor() { compressor_.Destroy(); } | |
98 | |||
99 | |||
100 | //------------------------------------------------------------------------------ | ||
101 | |||
102 | atomic_int64 BlockItem::managed_bytes_ = 0; | ||
103 | |||
104 | |||
105 | 125784 | BlockItem::BlockItem(ItemAllocator *allocator) | |
106 | 125784 | : allocator_(allocator) | |
107 | 125784 | , type_(kBlockHollow) | |
108 | 125784 | , tag_(-1) | |
109 | 125784 | , file_item_(NULL) | |
110 | 125784 | , chunk_item_(NULL) | |
111 | 125784 | , data_(NULL) | |
112 | 125784 | , capacity_(0) | |
113 | 125784 | , size_(0) { } | |
114 | |||
115 | |||
116 | 28128250 | BlockItem::BlockItem(int64_t tag, ItemAllocator *allocator) | |
117 | 27932845 | : allocator_(allocator) | |
118 | 27932845 | , type_(kBlockHollow) | |
119 | 27932845 | , tag_(tag) | |
120 | 27932845 | , file_item_(NULL) | |
121 | 27932845 | , chunk_item_(NULL) | |
122 | 27932845 | , data_(NULL) | |
123 | 27932845 | , capacity_(0) | |
124 | 28128250 | , size_(0) { | |
125 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 27932845 times.
|
27932845 | assert(tag_ >= 0); |
126 | 27932845 | } | |
127 | |||
128 | |||
129 | 28107352 | BlockItem::~BlockItem() { | |
130 |
2/2✓ Branch 0 taken 11586514 times.
✓ Branch 1 taken 16520838 times.
|
28107352 | if (data_) |
131 | 11586514 | allocator_->Free(data_); | |
132 | 28121928 | atomic_xadd64(&managed_bytes_, -static_cast<int64_t>(capacity_)); | |
133 | 28377498 | } | |
134 | |||
135 | |||
136 | 4407131 | void BlockItem::Discharge() { | |
137 | 4407131 | data_ = NULL; | |
138 | 4407131 | size_ = capacity_ = 0; | |
139 | 4407131 | } | |
140 | |||
141 | |||
142 | 11935496 | void BlockItem::MakeStop() { | |
143 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 11935496 times.
|
11935496 | assert(type_ == kBlockHollow); |
144 | 11935496 | type_ = kBlockStop; | |
145 | 11935496 | } | |
146 | |||
147 | |||
148 | 6235890 | void BlockItem::MakeData(uint32_t capacity) { | |
149 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 6235890 times.
|
6235890 | assert(type_ == kBlockHollow); |
150 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 6235890 times.
|
6235890 | assert(allocator_ != NULL); |
151 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 6235890 times.
|
6235890 | assert(capacity > 0); |
152 | |||
153 | 6235890 | type_ = kBlockData; | |
154 | 6235890 | capacity_ = capacity; | |
155 | 6235890 | data_ = reinterpret_cast<unsigned char *>(allocator_->Malloc(capacity_)); | |
156 | 6254946 | atomic_xadd64(&managed_bytes_, static_cast<int64_t>(capacity_)); | |
157 | 6257890 | } | |
158 | |||
159 | |||
160 | /** | ||
161 | * Move data from one block to another. | ||
162 | */ | ||
163 | 4408203 | void BlockItem::MakeDataMove(BlockItem *other) { | |
164 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 4408203 times.
|
4408203 | assert(type_ == kBlockHollow); |
165 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 4408203 times.
|
4408203 | assert(other->type_ == kBlockData); |
166 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 4408203 times.
|
4408203 | assert(other->size_ > 0); |
167 | |||
168 | 4408203 | type_ = kBlockData; | |
169 | 4408203 | capacity_ = size_ = other->size_; | |
170 | 4408203 | data_ = other->data_; | |
171 | 4408203 | allocator_ = other->allocator_; | |
172 | |||
173 | 4408203 | other->Discharge(); | |
174 | 4406251 | } | |
175 | |||
176 | |||
177 | /** | ||
178 | * Copy a piece of one block's data into a new block. | ||
179 | */ | ||
180 | 5789749 | void BlockItem::MakeDataCopy(const unsigned char *data, uint32_t size) { | |
181 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 5789749 times.
|
5789749 | assert(type_ == kBlockHollow); |
182 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 5789749 times.
|
5789749 | assert(allocator_ != NULL); |
183 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 5789749 times.
|
5789749 | assert(size > 0); |
184 | |||
185 | 5789749 | type_ = kBlockData; | |
186 | 5789749 | capacity_ = size_ = size; | |
187 | 5789749 | data_ = reinterpret_cast<unsigned char *>(allocator_->Malloc(capacity_)); | |
188 | 5798445 | memcpy(data_, data, size); | |
189 | 5798445 | atomic_xadd64(&managed_bytes_, static_cast<int64_t>(capacity_)); | |
190 | 5799234 | } | |
191 | |||
192 | |||
193 | 448160 | void BlockItem::Reset() { | |
194 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 448160 times.
|
448160 | assert(type_ == kBlockData); |
195 | |||
196 | 448160 | atomic_xadd64(&managed_bytes_, -static_cast<int64_t>(capacity_)); | |
197 | 448272 | allocator_->Free(data_); | |
198 | 448224 | data_ = NULL; | |
199 | 448224 | size_ = capacity_ = 0; | |
200 | 448224 | type_ = kBlockHollow; | |
201 | 448224 | } | |
202 | |||
203 | |||
204 | 19281653 | void BlockItem::SetChunkItem(ChunkItem *value) { | |
205 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 19281653 times.
|
19281653 | assert(value != NULL); |
206 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 19281653 times.
|
19281653 | assert(chunk_item_ == NULL); |
207 | 19281653 | chunk_item_ = value; | |
208 | 19281653 | } | |
209 | |||
210 | |||
211 | 27853941 | void BlockItem::SetFileItem(FileItem *value) { | |
212 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 27853941 times.
|
27853941 | assert(value != NULL); |
213 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 27853941 times.
|
27853941 | assert(file_item_ == NULL); |
214 | 27853941 | file_item_ = value; | |
215 | 27853941 | } | |
216 | |||
217 | |||
218 | 512 | uint32_t BlockItem::Write(void *buf, uint32_t count) { | |
219 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 512 times.
|
512 | assert(type_ == kBlockData); |
220 | |||
221 | 512 | const uint32_t remaining = capacity_ - size_; | |
222 | 512 | const uint32_t nbytes = std::min(remaining, count); | |
223 | 512 | memcpy(data_ + size_, buf, nbytes); | |
224 | 512 | size_ += nbytes; | |
225 | 512 | return nbytes; | |
226 | } | ||
227 |