GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/ingestion/task_chunk.cc
Date: 2026-02-01 02:35:56
Exec Total Coverage
Lines: 89 91 97.8%
Branches: 90 149 60.4%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 */
4
5 #include "ingestion/task_chunk.h"
6
7 #include <unistd.h>
8
9 #include <cassert>
10
11 #include "util/exception.h"
12
13 /**
14 * The tags from the read stage in the pipeline and the tags given in the
15 * chunking stage can safely overlap. Nevertheless, debugging might be easier
16 * if they don't. So let's start with a high number.
17 */
18 atomic_int64 TaskChunk::tag_seq_ = 2 << 28;
19
20 /**
21 * Consumes the stream of input blocks and produces new output blocks according
22 * to cut marks. The output blocks correspond to chunks.
23 */
24 21923603 void TaskChunk::Process(BlockItem *input_block) {
25 21923603 FileItem *file_item = input_block->file_item();
26 21922043 const int64_t input_tag = input_block->tag();
27
3/4
✓ Branch 0 taken 21942674 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 21926996 times.
✓ Branch 3 taken 15678 times.
 21940919 assert((file_item != NULL) && (input_tag >= 0));
28
29 21926996 ChunkInfo chunk_info;
30 // Do we see blocks of the file for the first time?
31
3/4
✓ Branch 1 taken 21990761 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 9737315 times.
✓ Branch 4 taken 12253446 times.
 21932222 if (!tag_map_.Lookup(input_tag, &chunk_info)) {
32 // We may have only regular chunks, only a bulk chunk, or both. We may
33 // end up in a situation where we produced only a single non-bulk chunk.
34 // This needs to be fixed up later in the pipeline by the write task.
35
2/2
✓ Branch 1 taken 625 times.
✓ Branch 2 taken 9733999 times.
 9737315 if (file_item->may_have_chunks()) {
36
2/4
✓ Branch 1 taken 625 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 625 times.
✗ Branch 5 not taken.
 625 chunk_info.next_chunk = new ChunkItem(file_item, 0);
37 625 chunk_info.output_tag_chunk = atomic_xadd64(&tag_seq_, 1);
38
2/2
✓ Branch 1 taken 439 times.
✓ Branch 2 taken 186 times.
 625 if (file_item->has_legacy_bulk_chunk()) {
39
2/4
✓ Branch 1 taken 439 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 439 times.
✗ Branch 5 not taken.
 439 chunk_info.bulk_chunk = new ChunkItem(file_item, 0);
40 }
41 } else {
42
2/4
✓ Branch 1 taken 9738952 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 9731971 times.
✗ Branch 5 not taken.
 9733999 chunk_info.bulk_chunk = new ChunkItem(file_item, 0);
43 }
44
45
1/2
✓ Branch 0 taken 9732605 times.
✗ Branch 1 not taken.
 9732596 if (chunk_info.bulk_chunk != NULL) {
46
1/2
✓ Branch 1 taken 9725234 times.
✗ Branch 2 not taken.
 9732605 chunk_info.bulk_chunk->MakeBulkChunk();
47 9725234 chunk_info.bulk_chunk->set_size(file_item->size());
48 9723947 chunk_info.output_tag_bulk = atomic_xadd64(&tag_seq_, 1);
49 }
50
1/2
✓ Branch 1 taken 9727097 times.
✗ Branch 2 not taken.
 9748079 tag_map_.Insert(input_tag, chunk_info);
51 }
52
3/4
✓ Branch 0 taken 500954 times.
✓ Branch 1 taken 21479589 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 500954 times.
 21980543 assert((chunk_info.bulk_chunk != NULL) || (chunk_info.next_chunk != NULL));
53
54 21980543 BlockItem *output_block_bulk = NULL;
55
2/2
✓ Branch 0 taken 21509736 times.
✓ Branch 1 taken 470807 times.
 21980543 if (chunk_info.bulk_chunk != NULL) {
56
2/4
✓ Branch 1 taken 21515118 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 21476976 times.
✗ Branch 5 not taken.
 21509736 output_block_bulk = new BlockItem(chunk_info.output_tag_bulk, allocator_);
57
1/2
✓ Branch 1 taken 21470541 times.
✗ Branch 2 not taken.
 21476976 output_block_bulk->SetFileItem(file_item);
58
1/2
✓ Branch 1 taken 21476079 times.
✗ Branch 2 not taken.
 21470541 output_block_bulk->SetChunkItem(chunk_info.bulk_chunk);
59 }
60
61 21946886 ChunkDetector *chunk_detector = file_item->chunk_detector();
62
2/3
✓ Branch 1 taken 9714539 times.
✓ Branch 2 taken 12302586 times.
✗ Branch 3 not taken.
 21973523 switch (input_block->type()) {
63 9714539 case BlockItem::kBlockStop:
64 // End of the file, no more new chunks
65 9714539 file_item->set_is_fully_chunked();
66
1/2
✓ Branch 0 taken 9738260 times.
✗ Branch 1 not taken.
 9738173 if (output_block_bulk)
67
1/2
✓ Branch 1 taken 9712676 times.
✗ Branch 2 not taken.
 9738260 output_block_bulk->MakeStop();
68
2/2
✓ Branch 0 taken 625 times.
✓ Branch 1 taken 9711964 times.
 9712589 if (chunk_info.next_chunk != NULL) {
69
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 625 times.
 625 assert(file_item->size() >= chunk_info.next_chunk->offset());
70 1250 chunk_info.next_chunk->set_size(file_item->size()
71 625 - chunk_info.next_chunk->offset());
72 BlockItem *block_stop = new BlockItem(chunk_info.output_tag_chunk,
73
2/4
✓ Branch 1 taken 625 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 625 times.
✗ Branch 5 not taken.
 625 allocator_);
74
1/2
✓ Branch 1 taken 625 times.
✗ Branch 2 not taken.
 625 block_stop->SetFileItem(file_item);
75
1/2
✓ Branch 1 taken 625 times.
✗ Branch 2 not taken.
 625 block_stop->SetChunkItem(chunk_info.next_chunk);
76
1/2
✓ Branch 1 taken 625 times.
✗ Branch 2 not taken.
 625 block_stop->MakeStop();
77
1/2
✓ Branch 1 taken 625 times.
✗ Branch 2 not taken.
 625 tubes_out_->Dispatch(block_stop);
78 }
79
1/2
✓ Branch 1 taken 9701942 times.
✗ Branch 2 not taken.
 9712589 tag_map_.Erase(input_tag);
80 9701942 break;
81
82 12302586 case BlockItem::kBlockData:
83
2/2
✓ Branch 0 taken 11800882 times.
✓ Branch 1 taken 501704 times.
 12302586 if (output_block_bulk) {
84
2/2
✓ Branch 0 taken 1057368 times.
✓ Branch 1 taken 10743514 times.
 11800882 if (chunk_info.next_chunk != NULL) {
85 // Reserve zero-copy for the regular chunk
86
1/2
✓ Branch 3 taken 1057914 times.
✗ Branch 4 not taken.
 1057368 output_block_bulk->MakeDataCopy(input_block->data(),
87 input_block->size());
88 } else {
89 // There is only the bulk chunk, zero copy
90
1/2
✓ Branch 1 taken 10745230 times.
✗ Branch 2 not taken.
 10743514 output_block_bulk->MakeDataMove(input_block);
91 }
92 }
93
94
2/2
✓ Branch 0 taken 1558916 times.
✓ Branch 1 taken 10745932 times.
 12304848 if (chunk_info.next_chunk != NULL) {
95 1558916 unsigned offset_in_block = 0;
96 1558916 uint64_t cut_mark = 0;
97
3/4
✓ Branch 1 taken 1819253 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 260571 times.
✓ Branch 4 taken 1558682 times.
 1819487 while ((cut_mark = chunk_detector->FindNextCutMark(input_block)) != 0) {
98
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 260571 times.
 260571 assert(cut_mark >= chunk_info.offset + offset_in_block);
99 260571 const uint64_t cut_mark_in_block = cut_mark - chunk_info.offset;
100
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 260571 times.
 260571 assert(cut_mark_in_block >= offset_in_block);
101
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 260571 times.
 260571 assert(cut_mark_in_block <= input_block->size());
102 260571 const unsigned tail_size = cut_mark_in_block - offset_in_block;
103
104
1/2
✓ Branch 0 taken 260571 times.
✗ Branch 1 not taken.
 260571 if (tail_size > 0) {
105 BlockItem *block_tail = new BlockItem(chunk_info.output_tag_chunk,
106
2/4
✓ Branch 1 taken 260571 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 260571 times.
✗ Branch 5 not taken.
 260571 allocator_);
107
1/2
✓ Branch 1 taken 260571 times.
✗ Branch 2 not taken.
 260571 block_tail->SetFileItem(file_item);
108
1/2
✓ Branch 1 taken 260571 times.
✗ Branch 2 not taken.
 260571 block_tail->SetChunkItem(chunk_info.next_chunk);
109
1/2
✓ Branch 2 taken 260571 times.
✗ Branch 3 not taken.
 260571 block_tail->MakeDataCopy(input_block->data() + offset_in_block,
110 tail_size);
111
1/2
✓ Branch 1 taken 260571 times.
✗ Branch 2 not taken.
 260571 tubes_out_->Dispatch(block_tail);
112 }
113
114
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 260571 times.
 260571 assert(cut_mark >= chunk_info.next_chunk->offset());
115 // If the cut mark happens to at the end of file, let the final
116 // incoming stop block schedule dispatch of the chunk stop block
117
2/2
✓ Branch 1 taken 260522 times.
✓ Branch 2 taken 49 times.
 260571 if (cut_mark < file_item->size()) {
118 260522 chunk_info.next_chunk->set_size(cut_mark
119 260522 - chunk_info.next_chunk->offset());
120 BlockItem *block_stop = new BlockItem(chunk_info.output_tag_chunk,
121
2/4
✓ Branch 1 taken 260522 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 260522 times.
✗ Branch 5 not taken.
 260522 allocator_);
122
1/2
✓ Branch 1 taken 260522 times.
✗ Branch 2 not taken.
 260522 block_stop->SetFileItem(file_item);
123
1/2
✓ Branch 1 taken 260522 times.
✗ Branch 2 not taken.
 260522 block_stop->SetChunkItem(chunk_info.next_chunk);
124
1/2
✓ Branch 1 taken 260522 times.
✗ Branch 2 not taken.
 260522 block_stop->MakeStop();
125
1/2
✓ Branch 1 taken 260522 times.
✗ Branch 2 not taken.
 260522 tubes_out_->Dispatch(block_stop);
126
127
2/4
✓ Branch 1 taken 260522 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 260522 times.
✗ Branch 5 not taken.
 260522 chunk_info.next_chunk = new ChunkItem(file_item, cut_mark);
128 260522 chunk_info.output_tag_chunk = atomic_xadd64(&tag_seq_, 1);
129 }
130 260571 offset_in_block = cut_mark_in_block;
131 }
132 1558682 chunk_info.offset += offset_in_block;
133
134
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1558721 times.
 1558682 assert(input_block->size() >= offset_in_block);
135 1558721 const unsigned tail_size = input_block->size() - offset_in_block;
136
2/2
✓ Branch 0 taken 1556517 times.
✓ Branch 1 taken 2165 times.
 1558682 if (tail_size > 0) {
137 BlockItem *block_tail = new BlockItem(chunk_info.output_tag_chunk,
138
2/4
✓ Branch 1 taken 1556361 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 1556400 times.
✗ Branch 5 not taken.
 1556517 allocator_);
139
1/2
✓ Branch 1 taken 1556283 times.
✗ Branch 2 not taken.
 1556400 block_tail->SetFileItem(file_item);
140
1/2
✓ Branch 1 taken 1556517 times.
✗ Branch 2 not taken.
 1556283 block_tail->SetChunkItem(chunk_info.next_chunk);
141
1/2
✓ Branch 2 taken 1556751 times.
✗ Branch 3 not taken.
 1556517 block_tail->MakeDataCopy(input_block->data() + offset_in_block,
142 tail_size);
143
1/2
✓ Branch 1 taken 1556634 times.
✗ Branch 2 not taken.
 1556751 tubes_out_->Dispatch(block_tail);
144 1556634 chunk_info.offset += tail_size;
145 }
146
147 // Delete data from incoming block
148
1/2
✓ Branch 1 taken 1558643 times.
✗ Branch 2 not taken.
 1558799 input_block->Reset();
149 }
150
151
1/2
✓ Branch 1 taken 12318342 times.
✗ Branch 2 not taken.
 12304575 tag_map_.Insert(input_tag, chunk_info);
152 12318342 break;
153
154 ✗ default:
155 ✗ PANIC(NULL);
156 }
157
158
2/2
✓ Branch 0 taken 21984716 times.
✓ Branch 1 taken 35568 times.
 22020284 delete input_block;
159
2/2
✓ Branch 0 taken 21551700 times.
✓ Branch 1 taken 500954 times.
 22052654 if (output_block_bulk)
160
1/2
✓ Branch 1 taken 21458178 times.
✗ Branch 2 not taken.
 21551700 tubes_out_->Dispatch(output_block_bulk);
161 21959132 }
162