1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339
// Copyright 2023 The rust-ggstd authors. All rights reserved.
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//! This encoding algorithm, which prioritizes speed over output size, is
//! based on Snappy's LZ77-style encoder: github.com/golang/snappy
use super::deflate::{
BASE_MATCH_LENGTH, BASE_MATCH_OFFSET, MAX_MATCH_LENGTH, MAX_MATCH_OFFSET, MAX_STORE_BLOCK_SIZE,
};
use super::token::{literal_token, match_token, Token};
const TABLE_BITS: usize = 14; // Bits used in the table.
pub(super) const TABLE_SIZE: u32 = 1 << TABLE_BITS; // Size of the table.
const TABLE_MASK: u32 = TABLE_SIZE - 1; // Mask for table indices. Redundant, but can eliminate bounds checks.
const TABLE_SHIFT: usize = 32 - TABLE_BITS; // Right-shift to get the tableBits most significant bits of a u32.
// Reset the buffer offset when reaching this.
// Offsets are stored between blocks as i32 values.
// Since the offset we are checking against is at the beginning
// of the buffer, we need to subtract the current and input
// buffer to not risk overflowing the i32.
pub(super) const BUFFER_RESET: i32 = i32::MAX - (MAX_STORE_BLOCK_SIZE as i32) * 2;
fn load32(b: &[u8], i: usize) -> u32 {
// b = b[i : i+4 : b.len()] // Help the compiler eliminate bounds checks on the next line.
let b = &b[i..i + 4];
(b[0] as u32) | (b[1] as u32) << 8 | (b[2] as u32) << 16 | (b[3] as u32) << 24
}
fn load64(b: &[u8], i: usize) -> u64 {
// b = b[i : i+8 : b.len()] // Help the compiler eliminate bounds checks on the next line.
let b = &b[i..i + 8];
(b[0] as u64)
| (b[1] as u64) << 8
| (b[2] as u64) << 16
| (b[3] as u64) << 24
| (b[4] as u64) << 32
| (b[5] as u64) << 40
| (b[6] as u64) << 48
| (b[7] as u64) << 56
}
fn hash(u: u32) -> u32 {
u.overflowing_mul(0x1e35a7bd).0 >> TABLE_SHIFT
// return (u * 0x1e35a7bd) >> tableShift;
}
// These constants are defined by the Snappy implementation so that its
// assembly implementation can fast-path some 16-bytes-at-a-time copies. They
// aren't necessary in the pure Go implementation, as we don't use those same
// optimizations, but using the same thresholds doesn't really hurt.
const INPUT_MARGIN: usize = 16 - 1;
const MIN_NON_LITERAL_BLOCK_SIZE: usize = 1 + 1 + INPUT_MARGIN;
#[derive(Default, Clone, Copy)]
pub(super) struct TableEntry {
val: u32, // Value at destination
offset: i32,
}
/// DeflateFast maintains the table for matches,
/// and the previous byte block for cross block matching.
pub(super) struct DeflateFast {
pub(super) table: Vec<TableEntry>,
pub(super) prev: Vec<u8>, // Previous block, zero length if unknown.
pub(super) cur: i32, // Current match offset.
}
impl DeflateFast {
pub(super) fn new() -> Self {
Self {
table: vec![TableEntry::default(); TABLE_SIZE as usize],
prev: vec![0; MAX_STORE_BLOCK_SIZE],
cur: MAX_STORE_BLOCK_SIZE as i32,
}
}
}
impl DeflateFast {
/// encode encodes a block given in src and appends tokens
/// to dst.
pub(super) fn encode(&mut self, dst: &mut Vec<Token>, src: &[u8]) {
// Ensure that self.cur doesn't wrap.
if self.cur >= BUFFER_RESET {
self.shift_offsets();
}
// This check isn't in the Snappy implementation, but there, the caller
// instead of the callee handles this case.
if src.len() < MIN_NON_LITERAL_BLOCK_SIZE {
self.cur += MAX_STORE_BLOCK_SIZE as i32;
self.prev.truncate(0);
emit_literal(dst, src);
return;
}
// sLimit is when to stop looking for offset/length copies. The inputMargin
// lets us use a fast path for emitLiteral in the main loop, while we are
// looking for copies.
let s_limit = src.len() - INPUT_MARGIN;
// nextEmit is where in src the next emitLiteral should start from.
let mut next_emit = 0_usize;
let mut s = 0_usize;
let mut cv = load32(src, s);
let mut next_hash = hash(cv);
'outer: loop {
// Copied from the C++ snappy implementation:
//
// Heuristic match skipping: If 32 bytes are scanned with no matches
// found, start looking only at every other byte. If 32 more bytes are
// scanned (or skipped), look at every third byte, etc.. When a match
// is found, immediately go back to looking at every byte. This is a
// small loss (~5% performance, ~0.1% density) for compressible data
// due to more bookkeeping, but for non-compressible data (such as
// JPEG) it's a huge win since the compressor quickly "realizes" the
// data is incompressible and doesn't bother looking for matches
// everywhere.
//
// The "skip" variable keeps track of how many bytes there are since
// the last match; dividing it by 32 (ie. right-shifting by five) gives
// the number of bytes to move ahead for each iteration.
let mut skip = 32_usize;
let mut next_s = s;
let mut candidate: TableEntry;
loop {
s = next_s;
let bytes_between_hash_lookups = skip >> 5;
next_s = s + bytes_between_hash_lookups;
skip += bytes_between_hash_lookups;
if next_s > s_limit {
break 'outer;
}
candidate = self.table[(next_hash & TABLE_MASK) as usize];
let now = load32(src, next_s);
self.table[(next_hash & TABLE_MASK) as usize] = TableEntry {
offset: s as i32 + self.cur,
val: cv,
};
next_hash = hash(now);
let offset = s as i32 - (candidate.offset - self.cur);
if offset > MAX_MATCH_OFFSET as i32 || cv != candidate.val {
// Out of range or not matched.
cv = now;
continue;
}
break;
}
// A 4-byte match has been found. We'll later see if more than 4 bytes
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
// them as literal bytes.
emit_literal(dst, &src[next_emit..s]);
// Call emitCopy, and then see if another emitCopy could be our next
// move. Repeat until we find no match for the input immediately after
// what was consumed by the last emitCopy call.
//
// If we exit this loop normally then we need to call emitLiteral next,
// though we don't yet know how big the literal will be. We handle that
// by proceeding to the next iteration of the main loop. We also can
// exit this loop via goto if we get close to exhausting the input.
loop {
// Invariant: we have a 4-byte match at s, and no need to emit any
// literal bytes prior to s.
// Extend the 4-byte match as long as possible.
s += 4;
let t = candidate.offset - self.cur + 4;
let l = self.match_len(s, t, src);
// matchToken is flate's equivalent of Snappy's emitCopy. (length,offset)
dst.push(match_token(
(l + 4 - BASE_MATCH_LENGTH) as u32,
(s as i32 - t - BASE_MATCH_OFFSET as i32) as u32,
));
s += l;
next_emit = s;
if s >= s_limit {
break 'outer;
}
// We could immediately start working at s now, but to improve
// compression we first update the hash table at s-1 and at s. If
// another emitCopy is not our next move, also calculate nextHash
// at s+1. At least on GOARCH=amd64, these three hash calculations
// are faster as one load64 call (with some shifts) instead of
// three load32 calls.
let mut x = load64(src, s - 1);
let prev_hash = hash(x as u32);
self.table[(prev_hash & TABLE_MASK) as usize] = TableEntry {
offset: self.cur + s as i32 - 1,
val: x as u32,
};
x >>= 8;
let curr_hash = hash(x as u32);
candidate = self.table[(curr_hash & TABLE_MASK) as usize];
self.table[(curr_hash & TABLE_MASK) as usize] = TableEntry {
offset: self.cur + s as i32,
val: x as u32,
};
let offset = (s as i32) - (candidate.offset - self.cur);
if offset > MAX_MATCH_OFFSET as i32 || (x as u32) != candidate.val {
cv = (x >> 8) as u32;
next_hash = hash(cv);
s += 1;
break;
}
}
}
// emitRemainder:
if next_emit < src.len() {
emit_literal(dst, &src[next_emit..])
}
self.cur += src.len() as i32;
self.prev.resize(src.len(), 0);
self.prev.copy_from_slice(src);
}
}
fn emit_literal(dst: &mut Vec<Token>, lit: &[u8]) {
for v in lit {
dst.push(literal_token(*v as u32));
}
}
impl DeflateFast {
/// matchLen returns the match length between src[s..] and src[t..].
/// t can be negative to indicate the match is starting in self.prev.
/// We assume that src[s-4:s] and src[t-4:t] already match.
pub(super) fn match_len(&mut self, s: usize, t: i32, src: &[u8]) -> usize {
let mut s1 = s + MAX_MATCH_LENGTH - 4;
if s1 > src.len() {
s1 = src.len();
}
// If we are inside the current block
if t >= 0 {
let t = t as usize;
let b = &src[t..];
let a = &src[s..s1];
let b = &b[..a.len()];
// Extend the match to be as long as possible.
for i in 0..a.len() {
if a[i] != b[i] {
return i;
}
}
return a.len();
}
// We found a match in the previous block.
let tp = self.prev.len() as i32 + t;
if tp < 0 {
return 0;
}
// Extend the match to be as long as possible.
let tp = tp as usize;
let mut a = &src[s..s1];
let mut b = &self.prev[tp..];
if b.len() > a.len() {
b = &b[..a.len()];
}
a = &a[..b.len()];
for i in 0..b.len() {
if a[i] != b[i] {
return i;
}
}
// If we reached our limit, we matched everything we are
// allowed to in the previous block and we return.
let n = b.len();
if (s + n) == s1 {
return n;
}
// Continue looking for more matches in the current block.
a = &src[s + n..s1];
b = &src[..a.len()];
for i in 0..a.len() {
if a[i] != b[i] {
return i + n;
}
}
a.len() + n
}
/// reset resets the encoding history.
/// This ensures that no matches are made to the previous block.
pub(super) fn reset(&mut self) {
self.prev.truncate(0);
// Bump the offset, so all matches will fail distance check.
// Nothing should be >= self.cur in the table.
self.cur += MAX_MATCH_OFFSET as i32;
// Protect against self.cur wraparound.
if self.cur >= BUFFER_RESET {
self.shift_offsets();
}
}
/// shiftOffsets will shift down all match offset.
/// This is only called in rare situations to prevent integer overflow.
///
/// See https://golang.org/issue/18636 and https://github.com/golang/go/issues/34121.
pub(super) fn shift_offsets(&mut self) {
if self.prev.is_empty() {
// We have no history; just clear the table.
for i in 0..self.table.len() {
self.table[i] = TableEntry::default();
}
self.cur = MAX_MATCH_OFFSET as i32 + 1;
return;
}
// Shift down everything in the table that isn't already too far away.
for i in 0..self.table.len() {
let mut v = self.table[i].offset - self.cur + MAX_MATCH_OFFSET as i32 + 1;
if v < 0 {
// We want to reset self.cur to maxMatchOffset + 1, so we need to shift
// all table entries down by (self.cur - (maxMatchOffset + 1)).
// Because we ignore matches > maxMatchOffset, we can cap
// any negative offsets at 0.
v = 0;
}
self.table[i].offset = v;
}
self.cur = MAX_MATCH_OFFSET as i32 + 1;
}
}