| Directory: | cvmfs/ |
|---|---|
| File: | cvmfs/ingestion/chunk_detector.cc |
| Date: | 2025-09-14 02:35:40 |
| Exec | Total | Coverage | |
|---|---|---|---|
| Lines: | 56 | 56 | 100.0% |
| Branches: | 33 | 46 | 71.7% |
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /** | ||
| 2 | * This file is part of the CernVM File System. | ||
| 3 | */ | ||
| 4 | |||
| 5 | |||
| 6 | #include "chunk_detector.h" | ||
| 7 | |||
| 8 | #include <algorithm> | ||
| 9 | #include <cassert> | ||
| 10 | #include <limits> | ||
| 11 | |||
| 12 | #include "ingestion/item.h" | ||
| 13 | |||
| 14 | |||
| 15 | 6533704 | uint64_t ChunkDetector::FindNextCutMark(BlockItem *block) { | |
| 16 | 6533704 | const uint64_t result = DoFindNextCutMark(block); | |
| 17 |
2/2✓ Branch 0 taken 1498982 times.
✓ Branch 1 taken 5034262 times.
|
6533244 | if (result == 0) |
| 18 | 1498982 | offset_ += block->size(); | |
| 19 | 6533244 | return result; | |
| 20 | } | ||
| 21 | |||
| 22 | |||
| 23 | //------------------------------------------------------------------------------ | ||
| 24 | |||
| 25 | |||
| 26 | 4920192 | uint64_t StaticOffsetDetector::DoFindNextCutMark(BlockItem *buffer) { | |
| 27 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 4920192 times.
|
4920192 | assert(buffer->type() == BlockItem::kBlockData); |
| 28 | |||
| 29 | 4920192 | const uint64_t beginning = offset(); | |
| 30 | 4920192 | const uint64_t end = offset() + buffer->size(); | |
| 31 | |||
| 32 | 4920192 | const uint64_t next_cut = last_cut() + chunk_size_; | |
| 33 |
3/4✓ Branch 0 taken 4920192 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 4915248 times.
✓ Branch 3 taken 4944 times.
|
4920192 | if (next_cut >= beginning && next_cut < end) { |
| 34 | 4915248 | return DoCut(next_cut); | |
| 35 | } | ||
| 36 | |||
| 37 | 4944 | return NoCut(next_cut); | |
| 38 | } | ||
| 39 | |||
| 40 | |||
| 41 | //------------------------------------------------------------------------------ | ||
| 42 | |||
| 43 | |||
| 44 | // This defines the center of the interval where the xor32 rolling checksum is | ||
| 45 | // queried. You should never change this number, since it affects the definition | ||
| 46 | // of cut marks. | ||
| 47 | const int32_t Xor32Detector::kMagicNumber = std::numeric_limits<uint32_t>::max() | ||
| 48 | / 2; | ||
| 49 | |||
| 50 | |||
| 51 | 11629965 | Xor32Detector::Xor32Detector(const uint64_t minimal_chunk_size, | |
| 52 | const uint64_t average_chunk_size, | ||
| 53 | 11629965 | const uint64_t maximal_chunk_size) | |
| 54 | 11629965 | : minimal_chunk_size_(minimal_chunk_size) | |
| 55 | 11629965 | , average_chunk_size_(average_chunk_size) | |
| 56 | 11629965 | , maximal_chunk_size_(maximal_chunk_size) | |
| 57 | 16 | , threshold_( | |
| 58 | (average_chunk_size > 0) | ||
| 59 | 11629949 | ? (std::numeric_limits<uint32_t>::max() / average_chunk_size) | |
| 60 | : 0) | ||
| 61 | 11629965 | , xor32_ptr_(0) | |
| 62 |
2/2✓ Branch 1 taken 11629949 times.
✓ Branch 2 taken 16 times.
|
11629965 | , xor32_(0) { |
| 63 |
3/4✓ Branch 0 taken 11629949 times.
✓ Branch 1 taken 16 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 11629949 times.
|
11629965 | assert((average_chunk_size_ == 0) || (minimal_chunk_size_ > 0)); |
| 64 |
2/2✓ Branch 0 taken 11629949 times.
✓ Branch 1 taken 16 times.
|
11629965 | if (minimal_chunk_size_ > 0) { |
| 65 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 11629949 times.
|
11629949 | assert(minimal_chunk_size_ >= kXor32Window); |
| 66 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 11629949 times.
|
11629949 | assert(minimal_chunk_size_ < average_chunk_size_); |
| 67 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 11629949 times.
|
11629949 | assert(average_chunk_size_ < maximal_chunk_size_); |
| 68 | } | ||
| 69 | 11629965 | } | |
| 70 | |||
| 71 | |||
| 72 | 1613512 | uint64_t Xor32Detector::DoFindNextCutMark(BlockItem *buffer) { | |
| 73 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1613512 times.
|
1613512 | assert(minimal_chunk_size_ > 0); |
| 74 | 1613512 | const unsigned char *data = buffer->data(); | |
| 75 | |||
| 76 | // Get the offset where the next xor32 computation needs to be continued | ||
| 77 | // Note: this could be after collecting at least kMinChunkSize bytes in the | ||
| 78 | // current chunk, or directly at the beginning of the buffer, when a | ||
| 79 | // cut mark is currently searched | ||
| 80 | 1613512 | const uint64_t global_offset = std::max( | |
| 81 | 1613512 | last_cut() + static_cast<uint64_t>(minimal_chunk_size_ - kXor32Window), | |
| 82 | 1613512 | xor32_ptr_); | |
| 83 | |||
| 84 | // Check if the next xor32 computation is taking place in the current buffer | ||
| 85 |
2/2✓ Branch 2 taken 704172 times.
✓ Branch 3 taken 909340 times.
|
1613512 | if (global_offset >= offset() + static_cast<uint64_t>(buffer->size())) { |
| 86 |
1/2✓ Branch 1 taken 704126 times.
✗ Branch 2 not taken.
|
704172 | return NoCut(global_offset); |
| 87 | } | ||
| 88 | |||
| 89 | // get the byte offset in the current buffer | ||
| 90 | 909340 | uint64_t internal_offset = global_offset - offset(); | |
| 91 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 909340 times.
|
909340 | assert(internal_offset < static_cast<uint64_t>(buffer->size())); |
| 92 | |||
| 93 | // Precompute the xor32 rolling checksum for finding the next cut mark | ||
| 94 | // Note: this might be skipped, if the precomputation was already performed | ||
| 95 | // for the current rolling checksum | ||
| 96 | // (internal_precompute_end will be negative --> loop is not entered) | ||
| 97 | 909340 | const uint64_t precompute_end = last_cut() + minimal_chunk_size_; | |
| 98 | 909340 | const int64_t internal_precompute_end = std::min( | |
| 99 | 909340 | static_cast<int64_t>(precompute_end - offset()), | |
| 100 | 909340 | static_cast<int64_t>(buffer->size())); | |
| 101 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 909340 times.
|
909340 | assert(internal_precompute_end - static_cast<int64_t>(internal_offset) |
| 102 | <= static_cast<int64_t>(kXor32Window)); | ||
| 103 |
2/2✓ Branch 0 taken 3829344 times.
✓ Branch 1 taken 909340 times.
|
4738684 | for (; static_cast<int64_t>(internal_offset) < internal_precompute_end; |
| 104 | ++internal_offset) { | ||
| 105 | 3829344 | xor32(data[internal_offset]); | |
| 106 | } | ||
| 107 | |||
| 108 | // Do the actual computation and try to find a xor32 based cut mark | ||
| 109 | // Note: this loop is bound either by kMaxChunkSize or by the size of the | ||
| 110 | // current buffer, thus the computation would continue later | ||
| 111 | 909340 | const uint64_t internal_max_chunk_size_end = last_cut() + maximal_chunk_size_ | |
| 112 | 909340 | - offset(); | |
| 113 | 909294 | const uint64_t internal_compute_end = std::min( | |
| 114 | 909294 | internal_max_chunk_size_end, static_cast<uint64_t>(buffer->size())); | |
| 115 |
2/2✓ Branch 0 taken 23431856994 times.
✓ Branch 1 taken 794784 times.
|
23432651778 | for (; internal_offset < internal_compute_end; ++internal_offset) { |
| 116 | 23431856994 | xor32(data[internal_offset]); | |
| 117 | |||
| 118 | // check if we found a cut mark | ||
| 119 |
2/2✓ Branch 1 taken 114556 times.
✓ Branch 2 taken 23388844770 times.
|
23383264940 | if (CheckThreshold()) { |
| 120 |
1/2✓ Branch 2 taken 114556 times.
✗ Branch 3 not taken.
|
114556 | return DoCut(internal_offset + offset()); |
| 121 | } | ||
| 122 | } | ||
| 123 | |||
| 124 | // Check if the loop was exited because we reached kMaxChunkSize and do a | ||
| 125 | // hard cut in this case. If not, it exited because we ran out of data in this | ||
| 126 | // buffer --> continue computation with the next buffer | ||
| 127 |
2/2✓ Branch 0 taken 4504 times.
✓ Branch 1 taken 790280 times.
|
794784 | if (internal_offset == internal_max_chunk_size_end) { |
| 128 |
1/2✓ Branch 2 taken 4504 times.
✗ Branch 3 not taken.
|
4504 | return DoCut(internal_offset + offset()); |
| 129 | } else { | ||
| 130 |
1/2✓ Branch 2 taken 790188 times.
✗ Branch 3 not taken.
|
790280 | return NoCut(internal_offset + offset()); |
| 131 | } | ||
| 132 | } | ||
| 133 |