Directory: | cvmfs/ |
---|---|
File: | cvmfs/ingestion/task_chunk.cc |
Date: | 2025-10-19 02:35:28 |
Exec | Total | Coverage | |
---|---|---|---|
Lines: | 89 | 91 | 97.8% |
Branches: | 92 | 149 | 61.7% |
Line | Branch | Exec | Source |
---|---|---|---|
1 | /** | ||
2 | * This file is part of the CernVM File System. | ||
3 | */ | ||
4 | |||
5 | #include "ingestion/task_chunk.h" | ||
6 | |||
7 | #include <unistd.h> | ||
8 | |||
9 | #include <cassert> | ||
10 | |||
11 | #include "util/exception.h" | ||
12 | |||
13 | /** | ||
14 | * The tags from the read stage in the pipeline and the tags given in the | ||
15 | * chunking stage can safely overlap. Nevertheless, debugging might be easier | ||
16 | * if they don't. So let's start with a high number. | ||
17 | */ | ||
18 | atomic_int64 TaskChunk::tag_seq_ = 2 << 28; | ||
19 | |||
20 | /** | ||
21 | * Consumes the stream of input blocks and produces new output blocks according | ||
22 | * to cut marks. The output blocks correspond to chunks. | ||
23 | */ | ||
24 | 2513517 | void TaskChunk::Process(BlockItem *input_block) { | |
25 | 2513517 | FileItem *file_item = input_block->file_item(); | |
26 | 2515253 | const int64_t input_tag = input_block->tag(); | |
27 |
3/4✓ Branch 0 taken 2517857 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2516213 times.
✓ Branch 3 taken 1644 times.
|
2516293 | assert((file_item != NULL) && (input_tag >= 0)); |
28 | |||
29 | 2516213 | ChunkInfo chunk_info; | |
30 | // Do we see blocks of the file for the first time? | ||
31 |
3/4✓ Branch 1 taken 2519993 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1000595 times.
✓ Branch 4 taken 1519398 times.
|
2515989 | if (!tag_map_.Lookup(input_tag, &chunk_info)) { |
32 | // We may have only regular chunks, only a bulk chunk, or both. We may | ||
33 | // end up in a situation where we produced only a single non-bulk chunk. | ||
34 | // This needs to be fixed up later in the pipeline by the write task. | ||
35 |
2/2✓ Branch 1 taken 168 times.
✓ Branch 2 taken 1000015 times.
|
1000595 | if (file_item->may_have_chunks()) { |
36 |
2/4✓ Branch 1 taken 168 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 168 times.
✗ Branch 5 not taken.
|
168 | chunk_info.next_chunk = new ChunkItem(file_item, 0); |
37 | 168 | chunk_info.output_tag_chunk = atomic_xadd64(&tag_seq_, 1); | |
38 |
2/2✓ Branch 1 taken 71 times.
✓ Branch 2 taken 97 times.
|
168 | if (file_item->has_legacy_bulk_chunk()) { |
39 |
2/4✓ Branch 1 taken 71 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 71 times.
✗ Branch 5 not taken.
|
71 | chunk_info.bulk_chunk = new ChunkItem(file_item, 0); |
40 | } | ||
41 | } else { | ||
42 |
2/4✓ Branch 1 taken 1000711 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1000059 times.
✗ Branch 5 not taken.
|
1000015 | chunk_info.bulk_chunk = new ChunkItem(file_item, 0); |
43 | } | ||
44 | |||
45 |
2/2✓ Branch 0 taken 1000154 times.
✓ Branch 1 taken 73 times.
|
1000227 | if (chunk_info.bulk_chunk != NULL) { |
46 |
1/2✓ Branch 1 taken 999390 times.
✗ Branch 2 not taken.
|
1000154 | chunk_info.bulk_chunk->MakeBulkChunk(); |
47 | 999390 | chunk_info.bulk_chunk->set_size(file_item->size()); | |
48 | 999006 | chunk_info.output_tag_bulk = atomic_xadd64(&tag_seq_, 1); | |
49 | } | ||
50 |
1/2✓ Branch 1 taken 999111 times.
✗ Branch 2 not taken.
|
1001499 | tag_map_.Insert(input_tag, chunk_info); |
51 | } | ||
52 |
3/4✓ Branch 0 taken 311617 times.
✓ Branch 1 taken 2206892 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 311617 times.
|
2518509 | assert((chunk_info.bulk_chunk != NULL) || (chunk_info.next_chunk != NULL)); |
53 | |||
54 | 2518509 | BlockItem *output_block_bulk = NULL; | |
55 |
2/2✓ Branch 0 taken 2209428 times.
✓ Branch 1 taken 309081 times.
|
2518509 | if (chunk_info.bulk_chunk != NULL) { |
56 |
2/4✓ Branch 1 taken 2209940 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 2206728 times.
✗ Branch 5 not taken.
|
2209428 | output_block_bulk = new BlockItem(chunk_info.output_tag_bulk, allocator_); |
57 |
1/2✓ Branch 1 taken 2206108 times.
✗ Branch 2 not taken.
|
2206728 | output_block_bulk->SetFileItem(file_item); |
58 |
1/2✓ Branch 1 taken 2206844 times.
✗ Branch 2 not taken.
|
2206108 | output_block_bulk->SetChunkItem(chunk_info.bulk_chunk); |
59 | } | ||
60 | |||
61 | 2515925 | ChunkDetector *chunk_detector = file_item->chunk_detector(); | |
62 |
2/3✓ Branch 1 taken 997859 times.
✓ Branch 2 taken 1523810 times.
✗ Branch 3 not taken.
|
2518105 | switch (input_block->type()) { |
63 | 997859 | case BlockItem::kBlockStop: | |
64 | // End of the file, no more new chunks | ||
65 | 997859 | file_item->set_is_fully_chunked(); | |
66 |
2/2✓ Branch 0 taken 1000298 times.
✓ Branch 1 taken 65 times.
|
1000363 | if (output_block_bulk) |
67 |
1/2✓ Branch 1 taken 997954 times.
✗ Branch 2 not taken.
|
1000298 | output_block_bulk->MakeStop(); |
68 |
2/2✓ Branch 0 taken 168 times.
✓ Branch 1 taken 997851 times.
|
998019 | if (chunk_info.next_chunk != NULL) { |
69 |
1/2✗ Branch 2 not taken.
✓ Branch 3 taken 168 times.
|
168 | assert(file_item->size() >= chunk_info.next_chunk->offset()); |
70 | 336 | chunk_info.next_chunk->set_size(file_item->size() | |
71 | 168 | - chunk_info.next_chunk->offset()); | |
72 | BlockItem *block_stop = new BlockItem(chunk_info.output_tag_chunk, | ||
73 |
2/4✓ Branch 1 taken 168 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 168 times.
✗ Branch 5 not taken.
|
168 | allocator_); |
74 |
1/2✓ Branch 1 taken 168 times.
✗ Branch 2 not taken.
|
168 | block_stop->SetFileItem(file_item); |
75 |
1/2✓ Branch 1 taken 168 times.
✗ Branch 2 not taken.
|
168 | block_stop->SetChunkItem(chunk_info.next_chunk); |
76 |
1/2✓ Branch 1 taken 168 times.
✗ Branch 2 not taken.
|
168 | block_stop->MakeStop(); |
77 |
1/2✓ Branch 1 taken 168 times.
✗ Branch 2 not taken.
|
168 | tubes_out_->Dispatch(block_stop); |
78 | } | ||
79 |
1/2✓ Branch 1 taken 998315 times.
✗ Branch 2 not taken.
|
998019 | tag_map_.Erase(input_tag); |
80 | 998315 | break; | |
81 | |||
82 | 1523810 | case BlockItem::kBlockData: | |
83 |
2/2✓ Branch 0 taken 1212234 times.
✓ Branch 1 taken 311576 times.
|
1523810 | if (output_block_bulk) { |
84 |
2/2✓ Branch 0 taken 108440 times.
✓ Branch 1 taken 1103794 times.
|
1212234 | if (chunk_info.next_chunk != NULL) { |
85 | // Reserve zero-copy for the regular chunk | ||
86 |
1/2✓ Branch 3 taken 108572 times.
✗ Branch 4 not taken.
|
108440 | output_block_bulk->MakeDataCopy(input_block->data(), |
87 | input_block->size()); | ||
88 | } else { | ||
89 | // There is only the bulk chunk, zero copy | ||
90 |
1/2✓ Branch 1 taken 1103846 times.
✗ Branch 2 not taken.
|
1103794 | output_block_bulk->MakeDataMove(input_block); |
91 | } | ||
92 | } | ||
93 | |||
94 |
2/2✓ Branch 0 taken 420044 times.
✓ Branch 1 taken 1103950 times.
|
1523994 | if (chunk_info.next_chunk != NULL) { |
95 | 420044 | unsigned offset_in_block = 0; | |
96 | 420044 | uint64_t cut_mark = 0; | |
97 |
3/4✓ Branch 1 taken 576631 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 156623 times.
✓ Branch 4 taken 420008 times.
|
576667 | while ((cut_mark = chunk_detector->FindNextCutMark(input_block)) != 0) { |
98 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 156623 times.
|
156623 | assert(cut_mark >= chunk_info.offset + offset_in_block); |
99 | 156623 | const uint64_t cut_mark_in_block = cut_mark - chunk_info.offset; | |
100 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 156623 times.
|
156623 | assert(cut_mark_in_block >= offset_in_block); |
101 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 156623 times.
|
156623 | assert(cut_mark_in_block <= input_block->size()); |
102 | 156623 | const unsigned tail_size = cut_mark_in_block - offset_in_block; | |
103 | |||
104 |
1/2✓ Branch 0 taken 156623 times.
✗ Branch 1 not taken.
|
156623 | if (tail_size > 0) { |
105 | BlockItem *block_tail = new BlockItem(chunk_info.output_tag_chunk, | ||
106 |
2/4✓ Branch 1 taken 156623 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 156623 times.
✗ Branch 5 not taken.
|
156623 | allocator_); |
107 |
1/2✓ Branch 1 taken 156623 times.
✗ Branch 2 not taken.
|
156623 | block_tail->SetFileItem(file_item); |
108 |
1/2✓ Branch 1 taken 156623 times.
✗ Branch 2 not taken.
|
156623 | block_tail->SetChunkItem(chunk_info.next_chunk); |
109 |
1/2✓ Branch 2 taken 156623 times.
✗ Branch 3 not taken.
|
156623 | block_tail->MakeDataCopy(input_block->data() + offset_in_block, |
110 | tail_size); | ||
111 |
1/2✓ Branch 1 taken 156623 times.
✗ Branch 2 not taken.
|
156623 | tubes_out_->Dispatch(block_tail); |
112 | } | ||
113 | |||
114 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 156623 times.
|
156623 | assert(cut_mark >= chunk_info.next_chunk->offset()); |
115 | // If the cut mark happens to at the end of file, let the final | ||
116 | // incoming stop block schedule dispatch of the chunk stop block | ||
117 |
2/2✓ Branch 1 taken 156592 times.
✓ Branch 2 taken 31 times.
|
156623 | if (cut_mark < file_item->size()) { |
118 | 156592 | chunk_info.next_chunk->set_size(cut_mark | |
119 | 156592 | - chunk_info.next_chunk->offset()); | |
120 | BlockItem *block_stop = new BlockItem(chunk_info.output_tag_chunk, | ||
121 |
2/4✓ Branch 1 taken 156592 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 156592 times.
✗ Branch 5 not taken.
|
156592 | allocator_); |
122 |
1/2✓ Branch 1 taken 156592 times.
✗ Branch 2 not taken.
|
156592 | block_stop->SetFileItem(file_item); |
123 |
1/2✓ Branch 1 taken 156592 times.
✗ Branch 2 not taken.
|
156592 | block_stop->SetChunkItem(chunk_info.next_chunk); |
124 |
1/2✓ Branch 1 taken 156592 times.
✗ Branch 2 not taken.
|
156592 | block_stop->MakeStop(); |
125 |
1/2✓ Branch 1 taken 156592 times.
✗ Branch 2 not taken.
|
156592 | tubes_out_->Dispatch(block_stop); |
126 | |||
127 |
2/4✓ Branch 1 taken 156592 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 156592 times.
✗ Branch 5 not taken.
|
156592 | chunk_info.next_chunk = new ChunkItem(file_item, cut_mark); |
128 | 156592 | chunk_info.output_tag_chunk = atomic_xadd64(&tag_seq_, 1); | |
129 | } | ||
130 | 156623 | offset_in_block = cut_mark_in_block; | |
131 | } | ||
132 | 420008 | chunk_info.offset += offset_in_block; | |
133 | |||
134 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 420012 times.
|
420008 | assert(input_block->size() >= offset_in_block); |
135 | 420012 | const unsigned tail_size = input_block->size() - offset_in_block; | |
136 |
2/2✓ Branch 0 taken 419742 times.
✓ Branch 1 taken 274 times.
|
420016 | if (tail_size > 0) { |
137 | BlockItem *block_tail = new BlockItem(chunk_info.output_tag_chunk, | ||
138 |
2/4✓ Branch 1 taken 419722 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 419710 times.
✗ Branch 5 not taken.
|
419742 | allocator_); |
139 |
1/2✓ Branch 1 taken 419714 times.
✗ Branch 2 not taken.
|
419710 | block_tail->SetFileItem(file_item); |
140 |
1/2✓ Branch 1 taken 419706 times.
✗ Branch 2 not taken.
|
419714 | block_tail->SetChunkItem(chunk_info.next_chunk); |
141 |
1/2✓ Branch 2 taken 419758 times.
✗ Branch 3 not taken.
|
419706 | block_tail->MakeDataCopy(input_block->data() + offset_in_block, |
142 | tail_size); | ||
143 |
1/2✓ Branch 1 taken 419726 times.
✗ Branch 2 not taken.
|
419758 | tubes_out_->Dispatch(block_tail); |
144 | 419726 | chunk_info.offset += tail_size; | |
145 | } | ||
146 | |||
147 | // Delete data from incoming block | ||
148 |
1/2✓ Branch 1 taken 420044 times.
✗ Branch 2 not taken.
|
420000 | input_block->Reset(); |
149 | } | ||
150 | |||
151 |
1/2✓ Branch 1 taken 1525306 times.
✗ Branch 2 not taken.
|
1523994 | tag_map_.Insert(input_tag, chunk_info); |
152 | 1525306 | break; | |
153 | |||
154 | ✗ | default: | |
155 | ✗ | PANIC(NULL); | |
156 | } | ||
157 | |||
158 |
2/2✓ Branch 0 taken 2520973 times.
✓ Branch 1 taken 2648 times.
|
2523621 | delete input_block; |
159 |
2/2✓ Branch 0 taken 2213864 times.
✓ Branch 1 taken 311617 times.
|
2525481 | if (output_block_bulk) |
160 |
1/2✓ Branch 1 taken 2205048 times.
✗ Branch 2 not taken.
|
2213864 | tubes_out_->Dispatch(output_block_bulk); |
161 | 2516665 | } | |
162 |