GCC Code Coverage Report

Directory: cvmfs/
File: cvmfs/ingestion/task_chunk.cc
Date: 2024-04-21 02:33:16
Exec Total Coverage
Lines: 87 89 97.8%
Branches: 92 149 61.7%
Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 */
4
5 #include <unistd.h>
6
7 #include <cassert>
8
9 #include "ingestion/task_chunk.h"
10 #include "util/exception.h"
11
12 /**
13 * The tags from the read stage in the pipeline and the tags given in the
14 * chunking stage can safely overlap. Nevertheless, debugging might be easier
15 * if they don't. So let's start with a high number.
16 */
17 atomic_int64 TaskChunk::tag_seq_ = 2 << 28;
18
19 /**
20 * Consumes the stream of input blocks and produces new output blocks according
21 * to cut marks. The output blocks correspond to chunks.
22 */
23 560261 void TaskChunk::Process(BlockItem *input_block) {
24 560261 FileItem *file_item = input_block->file_item();
25 562496 int64_t input_tag = input_block->tag();
26
3/4
✓ Branch 0 taken 563248 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 561672 times.
✓ Branch 3 taken 1576 times.
 562367 assert((file_item != NULL) && (input_tag >= 0));
27
28 561672 ChunkInfo chunk_info;
29 // Do we see blocks of the file for the first time?
30
3/4
✓ Branch 1 taken 561864 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 249836 times.
✓ Branch 4 taken 312028 times.
 561158 if (!tag_map_.Lookup(input_tag, &chunk_info)) {
31 // We may have only regular chunks, only a bulk chunk, or both. We may
32 // end up in a situation where we produced only a single non-bulk chunk.
33 // This needs to be fixed up later in the pipeline by the write task.
34
2/2
✓ Branch 1 taken 15 times.
✓ Branch 2 taken 249730 times.
 249836 if (file_item->may_have_chunks()) {
35
2/4
✓ Branch 1 taken 15 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 15 times.
✗ Branch 5 not taken.
 15 chunk_info.next_chunk = new ChunkItem(file_item, 0);
36 15 chunk_info.output_tag_chunk = atomic_xadd64(&tag_seq_, 1);
37
2/2
✓ Branch 1 taken 11 times.
✓ Branch 2 taken 4 times.
 15 if (file_item->has_legacy_bulk_chunk()) {
38
2/4
✓ Branch 1 taken 11 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 11 times.
✗ Branch 5 not taken.
 11 chunk_info.bulk_chunk = new ChunkItem(file_item, 0);
39 }
40 } else {
41
2/4
✓ Branch 1 taken 249852 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 249746 times.
✗ Branch 5 not taken.
 249730 chunk_info.bulk_chunk = new ChunkItem(file_item, 0);
42 }
43
44
2/2
✓ Branch 0 taken 249758 times.
✓ Branch 1 taken 3 times.
 249761 if (chunk_info.bulk_chunk != NULL) {
45
1/2
✓ Branch 1 taken 249600 times.
✗ Branch 2 not taken.
 249758 chunk_info.bulk_chunk->MakeBulkChunk();
46 249600 chunk_info.bulk_chunk->set_size(file_item->size());
47 249594 chunk_info.output_tag_bulk = atomic_xadd64(&tag_seq_, 1);
48 }
49
1/2
✓ Branch 1 taken 249590 times.
✗ Branch 2 not taken.
 250014 tag_map_.Insert(input_tag, chunk_info);
50 }
51
3/4
✓ Branch 0 taken 10276 times.
✓ Branch 1 taken 551342 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 10276 times.
 561618 assert((chunk_info.bulk_chunk != NULL) || (chunk_info.next_chunk != NULL));
52
53 561618 BlockItem *output_block_bulk = NULL;
54
2/2
✓ Branch 0 taken 552067 times.
✓ Branch 1 taken 9551 times.
 561618 if (chunk_info.bulk_chunk != NULL) {
55
2/4
✓ Branch 1 taken 552116 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 551425 times.
✗ Branch 5 not taken.
 552067 output_block_bulk = new BlockItem(chunk_info.output_tag_bulk, allocator_);
56
1/2
✓ Branch 1 taken 551323 times.
✗ Branch 2 not taken.
 551425 output_block_bulk->SetFileItem(file_item);
57
1/2
✓ Branch 1 taken 551355 times.
✗ Branch 2 not taken.
 551323 output_block_bulk->SetChunkItem(chunk_info.bulk_chunk);
58 }
59
60 560906 ChunkDetector *chunk_detector = file_item->chunk_detector();
61
2/3
✓ Branch 1 taken 249371 times.
✓ Branch 2 taken 313002 times.
✗ Branch 3 not taken.
 561601 switch (input_block->type()) {
62 249371 case BlockItem::kBlockStop:
63 // End of the file, no more new chunks
64 249371 file_item->set_is_fully_chunked();
65
3/4
✓ Branch 0 taken 249778 times.
✓ Branch 1 taken 1 times.
✓ Branch 3 taken 249403 times.
✗ Branch 4 not taken.
 249779 if (output_block_bulk) output_block_bulk->MakeStop();
66
2/2
✓ Branch 0 taken 15 times.
✓ Branch 1 taken 249389 times.
 249404 if (chunk_info.next_chunk != NULL) {
67
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 15 times.
 15 assert(file_item->size() >= chunk_info.next_chunk->offset());
68 15 chunk_info.next_chunk->set_size(
69 15 file_item->size() - chunk_info.next_chunk->offset());
70 BlockItem *block_stop =
71
2/4
✓ Branch 1 taken 15 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 15 times.
✗ Branch 5 not taken.
 15 new BlockItem(chunk_info.output_tag_chunk, allocator_);
72
1/2
✓ Branch 1 taken 15 times.
✗ Branch 2 not taken.
 15 block_stop->SetFileItem(file_item);
73
1/2
✓ Branch 1 taken 15 times.
✗ Branch 2 not taken.
 15 block_stop->SetChunkItem(chunk_info.next_chunk);
74
1/2
✓ Branch 1 taken 15 times.
✗ Branch 2 not taken.
 15 block_stop->MakeStop();
75
1/2
✓ Branch 1 taken 15 times.
✗ Branch 2 not taken.
 15 tubes_out_->Dispatch(block_stop);
76 }
77
1/2
✓ Branch 1 taken 249444 times.
✗ Branch 2 not taken.
 249404 tag_map_.Erase(input_tag);
78 249444 break;
79
80 313002 case BlockItem::kBlockData:
81
2/2
✓ Branch 0 taken 302719 times.
✓ Branch 1 taken 10283 times.
 313002 if (output_block_bulk) {
82
2/2
✓ Branch 0 taken 27107 times.
✓ Branch 1 taken 275612 times.
 302719 if (chunk_info.next_chunk != NULL) {
83 // Reserve zero-copy for the regular chunk
84
1/2
✓ Branch 3 taken 27132 times.
✗ Branch 4 not taken.
 27107 output_block_bulk->MakeDataCopy(input_block->data(),
85 input_block->size());
86 } else {
87 // There is only the bulk chunk, zero copy
88
1/2
✓ Branch 1 taken 275660 times.
✗ Branch 2 not taken.
 275612 output_block_bulk->MakeDataMove(input_block);
89 }
90 }
91
92
2/2
✓ Branch 0 taken 37404 times.
✓ Branch 1 taken 275671 times.
 313075 if (chunk_info.next_chunk != NULL) {
93 37404 unsigned offset_in_block = 0;
94 37404 uint64_t cut_mark = 0;
95
3/4
✓ Branch 1 taken 42794 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 5399 times.
✓ Branch 4 taken 37395 times.
 42803 while ((cut_mark = chunk_detector->FindNextCutMark(input_block)) != 0) {
96
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5399 times.
 5399 assert(cut_mark >= chunk_info.offset + offset_in_block);
97 5399 uint64_t cut_mark_in_block = cut_mark - chunk_info.offset;
98
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5399 times.
 5399 assert(cut_mark_in_block >= offset_in_block);
99
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5399 times.
 5399 assert(cut_mark_in_block <= input_block->size());
100 5399 unsigned tail_size = cut_mark_in_block - offset_in_block;
101
102
1/2
✓ Branch 0 taken 5399 times.
✗ Branch 1 not taken.
 5399 if (tail_size > 0) {
103 BlockItem *block_tail =
104
2/4
✓ Branch 1 taken 5399 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 5399 times.
✗ Branch 5 not taken.
 5399 new BlockItem(chunk_info.output_tag_chunk, allocator_);
105
1/2
✓ Branch 1 taken 5399 times.
✗ Branch 2 not taken.
 5399 block_tail->SetFileItem(file_item);
106
1/2
✓ Branch 1 taken 5399 times.
✗ Branch 2 not taken.
 5399 block_tail->SetChunkItem(chunk_info.next_chunk);
107
1/2
✓ Branch 2 taken 5399 times.
✗ Branch 3 not taken.
 5399 block_tail->MakeDataCopy(input_block->data() + offset_in_block,
108 tail_size);
109
1/2
✓ Branch 1 taken 5399 times.
✗ Branch 2 not taken.
 5399 tubes_out_->Dispatch(block_tail);
110 }
111
112
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5399 times.
 5399 assert(cut_mark >= chunk_info.next_chunk->offset());
113 // If the cut mark happens to at the end of file, let the final
114 // incoming stop block schedule dispatch of the chunk stop block
115
2/2
✓ Branch 1 taken 5398 times.
✓ Branch 2 taken 1 times.
 5399 if (cut_mark < file_item->size()) {
116 5398 chunk_info.next_chunk->set_size(
117 5398 cut_mark - chunk_info.next_chunk->offset());
118 BlockItem *block_stop =
119
2/4
✓ Branch 1 taken 5398 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 5398 times.
✗ Branch 5 not taken.
 5398 new BlockItem(chunk_info.output_tag_chunk, allocator_);
120
1/2
✓ Branch 1 taken 5398 times.
✗ Branch 2 not taken.
 5398 block_stop->SetFileItem(file_item);
121
1/2
✓ Branch 1 taken 5398 times.
✗ Branch 2 not taken.
 5398 block_stop->SetChunkItem(chunk_info.next_chunk);
122
1/2
✓ Branch 1 taken 5398 times.
✗ Branch 2 not taken.
 5398 block_stop->MakeStop();
123
1/2
✓ Branch 1 taken 5398 times.
✗ Branch 2 not taken.
 5398 tubes_out_->Dispatch(block_stop);
124
125
2/4
✓ Branch 1 taken 5398 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 5398 times.
✗ Branch 5 not taken.
 5398 chunk_info.next_chunk = new ChunkItem(file_item, cut_mark);
126 5398 chunk_info.output_tag_chunk = atomic_xadd64(&tag_seq_, 1);
127 }
128 5399 offset_in_block = cut_mark_in_block;
129 }
130 37395 chunk_info.offset += offset_in_block;
131
132
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 37395 times.
 37395 assert(input_block->size() >= offset_in_block);
133 37395 unsigned tail_size = input_block->size() - offset_in_block;
134
2/2
✓ Branch 0 taken 37341 times.
✓ Branch 1 taken 55 times.
 37396 if (tail_size > 0) {
135 BlockItem *block_tail =
136
2/4
✓ Branch 1 taken 37341 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 37338 times.
✗ Branch 5 not taken.
 37341 new BlockItem(chunk_info.output_tag_chunk, allocator_);
137
1/2
✓ Branch 1 taken 37336 times.
✗ Branch 2 not taken.
 37338 block_tail->SetFileItem(file_item);
138
1/2
✓ Branch 1 taken 37336 times.
✗ Branch 2 not taken.
 37336 block_tail->SetChunkItem(chunk_info.next_chunk);
139
1/2
✓ Branch 2 taken 37350 times.
✗ Branch 3 not taken.
 37336 block_tail->MakeDataCopy(input_block->data() + offset_in_block,
140 tail_size);
141
1/2
✓ Branch 1 taken 37343 times.
✗ Branch 2 not taken.
 37350 tubes_out_->Dispatch(block_tail);
142 37343 chunk_info.offset += tail_size;
143 }
144
145 // Delete data from incoming block
146
1/2
✓ Branch 1 taken 37403 times.
✗ Branch 2 not taken.
 37398 input_block->Reset();
147 }
148
149
1/2
✓ Branch 1 taken 313410 times.
✗ Branch 2 not taken.
 313074 tag_map_.Insert(input_tag, chunk_info);
150 313410 break;
151
152 ✗ default:
153 ✗ PANIC(NULL);
154 }
155
156
2/2
✓ Branch 0 taken 562384 times.
✓ Branch 1 taken 470 times.
 562854 delete input_block;
157
3/4
✓ Branch 0 taken 552670 times.
✓ Branch 1 taken 10276 times.
✓ Branch 3 taken 550685 times.
✗ Branch 4 not taken.
 562946 if (output_block_bulk) tubes_out_->Dispatch(output_block_bulk);
158 560961 }
159