libzstd_rs_sys/lib/compress/
zstd_fast.rs

1use core::arch::asm;
2pub type ZSTD_longLengthType_e = core::ffi::c_uint;
3pub const ZSTD_llt_matchLength: ZSTD_longLengthType_e = 2;
4pub const ZSTD_llt_literalLength: ZSTD_longLengthType_e = 1;
5pub const ZSTD_llt_none: ZSTD_longLengthType_e = 0;
6#[repr(C)]
7pub struct optState_t {
8    pub litFreq: *mut core::ffi::c_uint,
9    pub litLengthFreq: *mut core::ffi::c_uint,
10    pub matchLengthFreq: *mut core::ffi::c_uint,
11    pub offCodeFreq: *mut core::ffi::c_uint,
12    pub matchTable: *mut ZSTD_match_t,
13    pub priceTable: *mut ZSTD_optimal_t,
14    pub litSum: u32,
15    pub litLengthSum: u32,
16    pub matchLengthSum: u32,
17    pub offCodeSum: u32,
18    pub litSumBasePrice: u32,
19    pub litLengthSumBasePrice: u32,
20    pub matchLengthSumBasePrice: u32,
21    pub offCodeSumBasePrice: u32,
22    pub priceType: ZSTD_OptPrice_e,
23    pub symbolCosts: *const ZSTD_entropyCTables_t,
24    pub literalCompressionMode: ZSTD_ParamSwitch_e,
25}
26pub type ZSTD_ParamSwitch_e = core::ffi::c_uint;
27pub const ZSTD_ps_disable: ZSTD_ParamSwitch_e = 2;
28pub const ZSTD_ps_enable: ZSTD_ParamSwitch_e = 1;
29pub const ZSTD_ps_auto: ZSTD_ParamSwitch_e = 0;
30#[repr(C)]
31pub struct ZSTD_entropyCTables_t {
32    pub huf: ZSTD_hufCTables_t,
33    pub fse: ZSTD_fseCTables_t,
34}
35#[repr(C)]
36pub struct ZSTD_fseCTables_t {
37    pub offcodeCTable: [FSE_CTable; 193],
38    pub matchlengthCTable: [FSE_CTable; 363],
39    pub litlengthCTable: [FSE_CTable; 329],
40    pub offcode_repeatMode: FSE_repeat,
41    pub matchlength_repeatMode: FSE_repeat,
42    pub litlength_repeatMode: FSE_repeat,
43}
44#[repr(C)]
45pub struct ZSTD_hufCTables_t {
46    pub CTable: [HUF_CElt; 257],
47    pub repeatMode: HUF_repeat,
48}
49pub type ZSTD_OptPrice_e = core::ffi::c_uint;
50pub const zop_predef: ZSTD_OptPrice_e = 1;
51pub const zop_dynamic: ZSTD_OptPrice_e = 0;
52#[repr(C)]
53pub struct ZSTD_window_t {
54    pub nextSrc: *const u8,
55    pub base: *const u8,
56    pub dictBase: *const u8,
57    pub dictLimit: u32,
58    pub lowLimit: u32,
59    pub nbOverflowCorrections: u32,
60}
61pub type ZSTD_dictTableLoadMethod_e = core::ffi::c_uint;
62pub const ZSTD_dtlm_full: ZSTD_dictTableLoadMethod_e = 1;
63pub const ZSTD_dtlm_fast: ZSTD_dictTableLoadMethod_e = 0;
64pub type ZSTD_tableFillPurpose_e = core::ffi::c_uint;
65pub const ZSTD_tfp_forCDict: ZSTD_tableFillPurpose_e = 1;
66pub const ZSTD_tfp_forCCtx: ZSTD_tableFillPurpose_e = 0;
67pub type ZSTD_match4Found = Option<unsafe fn(*const u8, *const u8, u32, u32) -> core::ffi::c_int>;
68pub const CACHELINE_SIZE: core::ffi::c_int = 64;
69
70use libc::size_t;
71
72use crate::lib::common::fse::{FSE_CTable, FSE_repeat};
73use crate::lib::common::huf::{HUF_CElt, HUF_repeat};
74use crate::lib::common::mem::{
75    MEM_64bits, MEM_isLittleEndian, MEM_read16, MEM_read32, MEM_readLE32, MEM_readLE64, MEM_readST,
76};
77use crate::lib::common::zstd_internal::{
78    Overlap, ZSTD_copy16, ZSTD_wildcopy, MINMATCH, WILDCOPY_OVERLENGTH, ZSTD_REP_NUM,
79};
80use crate::lib::compress::zstd_compress::{
81    SeqStore_t, ZSTD_MatchState_t, ZSTD_match_t, ZSTD_optimal_t,
82};
83use crate::lib::zstd::*;
84pub const kSearchStrength: core::ffi::c_int = 8;
85pub const HASH_READ_SIZE: core::ffi::c_int = 8;
86
87unsafe fn ZSTD_safecopyLiterals(
88    mut op: *mut u8,
89    mut ip: *const u8,
90    iend: *const u8,
91    ilimit_w: *const u8,
92) {
93    if ip <= ilimit_w {
94        ZSTD_wildcopy(
95            op as *mut core::ffi::c_void,
96            ip as *const core::ffi::c_void,
97            ilimit_w.offset_from(ip) as size_t,
98            Overlap::NoOverlap,
99        );
100        op = op.offset(ilimit_w.offset_from(ip) as core::ffi::c_long as isize);
101        ip = ilimit_w;
102    }
103    while ip < iend {
104        let fresh0 = ip;
105        ip = ip.offset(1);
106        let fresh1 = op;
107        op = op.offset(1);
108        *fresh1 = *fresh0;
109    }
110}
111pub const REPCODE1_TO_OFFBASE: core::ffi::c_int = 1;
112#[inline(always)]
113unsafe fn ZSTD_storeSeqOnly(
114    seqStorePtr: *mut SeqStore_t,
115    litLength: size_t,
116    offBase: u32,
117    matchLength: size_t,
118) {
119    if (litLength > 0xffff as core::ffi::c_int as size_t) as core::ffi::c_int as core::ffi::c_long
120        != 0
121    {
122        (*seqStorePtr).longLengthType = ZSTD_llt_literalLength;
123        (*seqStorePtr).longLengthPos = ((*seqStorePtr).sequences)
124            .offset_from((*seqStorePtr).sequencesStart)
125            as core::ffi::c_long as u32;
126    }
127    (*((*seqStorePtr).sequences).offset(0)).litLength = litLength as u16;
128    (*((*seqStorePtr).sequences).offset(0)).offBase = offBase;
129    let mlBase = matchLength.wrapping_sub(MINMATCH as size_t);
130    if (mlBase > 0xffff as core::ffi::c_int as size_t) as core::ffi::c_int as core::ffi::c_long != 0
131    {
132        (*seqStorePtr).longLengthType = ZSTD_llt_matchLength;
133        (*seqStorePtr).longLengthPos = ((*seqStorePtr).sequences)
134            .offset_from((*seqStorePtr).sequencesStart)
135            as core::ffi::c_long as u32;
136    }
137    (*((*seqStorePtr).sequences).offset(0)).mlBase = mlBase as u16;
138    (*seqStorePtr).sequences = ((*seqStorePtr).sequences).offset(1);
139    (*seqStorePtr).sequences;
140}
141#[inline(always)]
142unsafe fn ZSTD_storeSeq(
143    seqStorePtr: *mut SeqStore_t,
144    litLength: size_t,
145    literals: *const u8,
146    litLimit: *const u8,
147    offBase: u32,
148    matchLength: size_t,
149) {
150    let litLimit_w = litLimit.sub(WILDCOPY_OVERLENGTH);
151    let litEnd = literals.add(litLength);
152    if litEnd <= litLimit_w {
153        ZSTD_copy16(
154            (*seqStorePtr).lit as *mut core::ffi::c_void,
155            literals as *const core::ffi::c_void,
156        );
157        if litLength > 16 {
158            ZSTD_wildcopy(
159                ((*seqStorePtr).lit).offset(16) as *mut core::ffi::c_void,
160                literals.offset(16) as *const core::ffi::c_void,
161                litLength.wrapping_sub(16),
162                Overlap::NoOverlap,
163            );
164        }
165    } else {
166        ZSTD_safecopyLiterals((*seqStorePtr).lit, literals, litEnd, litLimit_w);
167    }
168    (*seqStorePtr).lit = ((*seqStorePtr).lit).add(litLength);
169    ZSTD_storeSeqOnly(seqStorePtr, litLength, offBase, matchLength);
170}
171#[inline]
172unsafe fn ZSTD_count(mut pIn: *const u8, mut pMatch: *const u8, pInLimit: *const u8) -> size_t {
173    let pStart = pIn;
174    let pInLoopLimit = pInLimit.offset(
175        -((::core::mem::size_of::<size_t>() as core::ffi::c_ulong).wrapping_sub(1) as isize),
176    );
177    if pIn < pInLoopLimit {
178        let diff = MEM_readST(pMatch as *const core::ffi::c_void)
179            ^ MEM_readST(pIn as *const core::ffi::c_void);
180        if diff != 0 {
181            return ZSTD_NbCommonBytes(diff) as size_t;
182        }
183        pIn = pIn.offset(::core::mem::size_of::<size_t>() as core::ffi::c_ulong as isize);
184        pMatch = pMatch.offset(::core::mem::size_of::<size_t>() as core::ffi::c_ulong as isize);
185        while pIn < pInLoopLimit {
186            let diff_0 = MEM_readST(pMatch as *const core::ffi::c_void)
187                ^ MEM_readST(pIn as *const core::ffi::c_void);
188            if diff_0 == 0 {
189                pIn = pIn.offset(::core::mem::size_of::<size_t>() as core::ffi::c_ulong as isize);
190                pMatch =
191                    pMatch.offset(::core::mem::size_of::<size_t>() as core::ffi::c_ulong as isize);
192            } else {
193                pIn = pIn.offset(ZSTD_NbCommonBytes(diff_0) as isize);
194                return pIn.offset_from(pStart) as size_t;
195            }
196        }
197    }
198    if MEM_64bits() != 0
199        && pIn < pInLimit.offset(-(3))
200        && MEM_read32(pMatch as *const core::ffi::c_void)
201            == MEM_read32(pIn as *const core::ffi::c_void)
202    {
203        pIn = pIn.offset(4);
204        pMatch = pMatch.offset(4);
205    }
206    if pIn < pInLimit.offset(-(1))
207        && MEM_read16(pMatch as *const core::ffi::c_void) as core::ffi::c_int
208            == MEM_read16(pIn as *const core::ffi::c_void) as core::ffi::c_int
209    {
210        pIn = pIn.offset(2);
211        pMatch = pMatch.offset(2);
212    }
213    if pIn < pInLimit && *pMatch as core::ffi::c_int == *pIn as core::ffi::c_int {
214        pIn = pIn.offset(1);
215    }
216    pIn.offset_from(pStart) as size_t
217}
218#[inline]
219unsafe fn ZSTD_count_2segments(
220    ip: *const u8,
221    match_0: *const u8,
222    iEnd: *const u8,
223    mEnd: *const u8,
224    iStart: *const u8,
225) -> size_t {
226    let vEnd = if ip.offset(mEnd.offset_from(match_0) as core::ffi::c_long as isize) < iEnd {
227        ip.offset(mEnd.offset_from(match_0) as core::ffi::c_long as isize)
228    } else {
229        iEnd
230    };
231    let matchLength = ZSTD_count(ip, match_0, vEnd);
232    if match_0.add(matchLength) != mEnd {
233        return matchLength;
234    }
235    matchLength.wrapping_add(ZSTD_count(ip.add(matchLength), iStart, iEnd))
236}
237static prime4bytes: u32 = 2654435761;
238unsafe fn ZSTD_hash4(u: u32, h: u32, s: u32) -> u32 {
239    ((u * prime4bytes) ^ s) >> 32u32.wrapping_sub(h)
240}
241unsafe fn ZSTD_hash4Ptr(ptr: *const core::ffi::c_void, h: u32) -> size_t {
242    ZSTD_hash4(MEM_readLE32(ptr), h, 0) as size_t
243}
244static prime5bytes: u64 = 889523592379;
245unsafe fn ZSTD_hash5(u: u64, h: u32, s: u64) -> size_t {
246    ((((u << (64 - 40)) * prime5bytes) ^ s) >> 64u32.wrapping_sub(h)) as size_t
247}
248unsafe fn ZSTD_hash5Ptr(p: *const core::ffi::c_void, h: u32) -> size_t {
249    ZSTD_hash5(MEM_readLE64(p), h, 0)
250}
251static prime6bytes: u64 = 227718039650203;
252unsafe fn ZSTD_hash6(u: u64, h: u32, s: u64) -> size_t {
253    ((((u << (64 - 48)) * prime6bytes) ^ s) >> (64u32).wrapping_sub(h)) as size_t
254}
255unsafe fn ZSTD_hash6Ptr(p: *const core::ffi::c_void, h: u32) -> size_t {
256    ZSTD_hash6(MEM_readLE64(p), h, 0)
257}
258static prime7bytes: u64 = 58295818150454627;
259unsafe fn ZSTD_hash7(u: u64, h: u32, s: u64) -> size_t {
260    ((((u << (64 - 56)) * prime7bytes) ^ s) >> 64u32.wrapping_sub(h)) as size_t
261}
262unsafe fn ZSTD_hash7Ptr(p: *const core::ffi::c_void, h: u32) -> size_t {
263    ZSTD_hash7(MEM_readLE64(p), h, 0)
264}
265static prime8bytes: u64 = 0xcf1bbcdcb7a56463 as core::ffi::c_ulonglong;
266unsafe fn ZSTD_hash8(u: u64, h: u32, s: u64) -> size_t {
267    (((u * prime8bytes) ^ s) >> 64u32.wrapping_sub(h)) as size_t
268}
269unsafe fn ZSTD_hash8Ptr(p: *const core::ffi::c_void, h: u32) -> size_t {
270    ZSTD_hash8(MEM_readLE64(p), h, 0)
271}
272#[inline(always)]
273unsafe fn ZSTD_hashPtr(p: *const core::ffi::c_void, hBits: u32, mls: u32) -> size_t {
274    match mls {
275        5 => ZSTD_hash5Ptr(p, hBits),
276        6 => ZSTD_hash6Ptr(p, hBits),
277        7 => ZSTD_hash7Ptr(p, hBits),
278        8 => ZSTD_hash8Ptr(p, hBits),
279        4 | _ => ZSTD_hash4Ptr(p, hBits),
280    }
281}
282#[inline]
283unsafe fn ZSTD_getLowestMatchIndex(
284    ms: *const ZSTD_MatchState_t,
285    curr: u32,
286    windowLog: core::ffi::c_uint,
287) -> u32 {
288    let maxDistance = (1) << windowLog;
289    let lowestValid = (*ms).window.lowLimit;
290    let withinWindow = if curr.wrapping_sub(lowestValid) > maxDistance {
291        curr.wrapping_sub(maxDistance)
292    } else {
293        lowestValid
294    };
295    let isDictionary = ((*ms).loadedDictEnd != 0) as core::ffi::c_int as u32;
296
297    if isDictionary != 0 {
298        lowestValid
299    } else {
300        withinWindow
301    }
302}
303#[inline]
304unsafe fn ZSTD_getLowestPrefixIndex(
305    ms: *const ZSTD_MatchState_t,
306    curr: u32,
307    windowLog: core::ffi::c_uint,
308) -> u32 {
309    let maxDistance = (1) << windowLog;
310    let lowestValid = (*ms).window.dictLimit;
311    let withinWindow = if curr.wrapping_sub(lowestValid) > maxDistance {
312        curr.wrapping_sub(maxDistance)
313    } else {
314        lowestValid
315    };
316    let isDictionary = ((*ms).loadedDictEnd != 0) as core::ffi::c_int as u32;
317
318    if isDictionary != 0 {
319        lowestValid
320    } else {
321        withinWindow
322    }
323}
324#[inline]
325unsafe fn ZSTD_index_overlap_check(prefixLowestIndex: u32, repIndex: u32) -> core::ffi::c_int {
326    (prefixLowestIndex.wrapping_sub(1).wrapping_sub(repIndex) >= 3) as core::ffi::c_int
327}
328pub const ZSTD_SHORT_CACHE_TAG_BITS: core::ffi::c_int = 8;
329pub const ZSTD_SHORT_CACHE_TAG_MASK: core::ffi::c_uint =
330    ((1 as core::ffi::c_uint) << ZSTD_SHORT_CACHE_TAG_BITS).wrapping_sub(1);
331#[inline]
332unsafe fn ZSTD_writeTaggedIndex(hashTable: *mut u32, hashAndTag: size_t, index: u32) {
333    let hash = hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS;
334    let tag = (hashAndTag & ZSTD_SHORT_CACHE_TAG_MASK as size_t) as u32;
335    *hashTable.add(hash) = index << ZSTD_SHORT_CACHE_TAG_BITS | tag;
336}
337#[inline]
338unsafe fn ZSTD_comparePackedTags(packedTag1: size_t, packedTag2: size_t) -> core::ffi::c_int {
339    let tag1 = (packedTag1 & ZSTD_SHORT_CACHE_TAG_MASK as size_t) as u32;
340    let tag2 = (packedTag2 & ZSTD_SHORT_CACHE_TAG_MASK as size_t) as u32;
341    (tag1 == tag2) as core::ffi::c_int
342}
343#[inline]
344unsafe fn ZSTD_countTrailingZeros32(val: u32) -> core::ffi::c_uint {
345    val.trailing_zeros() as i32 as core::ffi::c_uint
346}
347#[inline]
348unsafe fn ZSTD_countLeadingZeros32(val: u32) -> core::ffi::c_uint {
349    val.leading_zeros() as i32 as core::ffi::c_uint
350}
351#[inline]
352unsafe fn ZSTD_countTrailingZeros64(val: u64) -> core::ffi::c_uint {
353    (val as core::ffi::c_ulonglong).trailing_zeros() as i32 as core::ffi::c_uint
354}
355#[inline]
356unsafe fn ZSTD_countLeadingZeros64(val: u64) -> core::ffi::c_uint {
357    (val as core::ffi::c_ulonglong).leading_zeros() as i32 as core::ffi::c_uint
358}
359#[inline]
360unsafe fn ZSTD_NbCommonBytes(val: size_t) -> core::ffi::c_uint {
361    if MEM_isLittleEndian() != 0 {
362        if MEM_64bits() != 0 {
363            ZSTD_countTrailingZeros64(val as u64) >> 3
364        } else {
365            ZSTD_countTrailingZeros32(val as u32) >> 3
366        }
367    } else if MEM_64bits() != 0 {
368        ZSTD_countLeadingZeros64(val as u64) >> 3
369    } else {
370        ZSTD_countLeadingZeros32(val as u32) >> 3
371    }
372}
373unsafe fn ZSTD_fillHashTableForCDict(
374    ms: *mut ZSTD_MatchState_t,
375    end: *const core::ffi::c_void,
376    dtlm: ZSTD_dictTableLoadMethod_e,
377) {
378    let cParams: *const ZSTD_compressionParameters = &mut (*ms).cParams;
379    let hashTable = (*ms).hashTable;
380    let hBits = ((*cParams).hashLog).wrapping_add(ZSTD_SHORT_CACHE_TAG_BITS as core::ffi::c_uint);
381    let mls = (*cParams).minMatch;
382    let base = (*ms).window.base;
383    let mut ip = base.offset((*ms).nextToUpdate as isize);
384    let iend = (end as *const u8).offset(-(HASH_READ_SIZE as isize));
385    let fastHashFillStep = 3;
386    while ip.offset(fastHashFillStep as isize) < iend.offset(2) {
387        let curr = ip.offset_from(base) as core::ffi::c_long as u32;
388        let hashAndTag = ZSTD_hashPtr(ip as *const core::ffi::c_void, hBits, mls);
389        ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr);
390        if dtlm as core::ffi::c_uint != ZSTD_dtlm_fast as core::ffi::c_int as core::ffi::c_uint {
391            let mut p: u32 = 0;
392            p = 1;
393            while p < fastHashFillStep {
394                let hashAndTag_0 = ZSTD_hashPtr(
395                    ip.offset(p as isize) as *const core::ffi::c_void,
396                    hBits,
397                    mls,
398                );
399                if *hashTable.add(hashAndTag_0 >> ZSTD_SHORT_CACHE_TAG_BITS) == 0 {
400                    ZSTD_writeTaggedIndex(hashTable, hashAndTag_0, curr.wrapping_add(p));
401                }
402                p = p.wrapping_add(1);
403            }
404        }
405        ip = ip.offset(fastHashFillStep as isize);
406    }
407}
408unsafe fn ZSTD_fillHashTableForCCtx(
409    ms: *mut ZSTD_MatchState_t,
410    end: *const core::ffi::c_void,
411    dtlm: ZSTD_dictTableLoadMethod_e,
412) {
413    let cParams: *const ZSTD_compressionParameters = &mut (*ms).cParams;
414    let hashTable = (*ms).hashTable;
415    let hBits = (*cParams).hashLog;
416    let mls = (*cParams).minMatch;
417    let base = (*ms).window.base;
418    let mut ip = base.offset((*ms).nextToUpdate as isize);
419    let iend = (end as *const u8).offset(-(HASH_READ_SIZE as isize));
420    let fastHashFillStep = 3;
421    while ip.offset(fastHashFillStep as isize) < iend.offset(2) {
422        let curr = ip.offset_from(base) as core::ffi::c_long as u32;
423        let hash0 = ZSTD_hashPtr(ip as *const core::ffi::c_void, hBits, mls);
424        *hashTable.add(hash0) = curr;
425        if dtlm as core::ffi::c_uint != ZSTD_dtlm_fast as core::ffi::c_int as core::ffi::c_uint {
426            let mut p: u32 = 0;
427            p = 1;
428            while p < fastHashFillStep {
429                let hash = ZSTD_hashPtr(
430                    ip.offset(p as isize) as *const core::ffi::c_void,
431                    hBits,
432                    mls,
433                );
434                if *hashTable.add(hash) == 0 {
435                    *hashTable.add(hash) = curr.wrapping_add(p);
436                }
437                p = p.wrapping_add(1);
438            }
439        }
440        ip = ip.offset(fastHashFillStep as isize);
441    }
442}
443pub unsafe fn ZSTD_fillHashTable(
444    ms: *mut ZSTD_MatchState_t,
445    end: *const core::ffi::c_void,
446    dtlm: ZSTD_dictTableLoadMethod_e,
447    tfp: ZSTD_tableFillPurpose_e,
448) {
449    if tfp as core::ffi::c_uint == ZSTD_tfp_forCDict as core::ffi::c_int as core::ffi::c_uint {
450        ZSTD_fillHashTableForCDict(ms, end, dtlm);
451    } else {
452        ZSTD_fillHashTableForCCtx(ms, end, dtlm);
453    };
454}
455
456unsafe fn ZSTD_match4Found_cmov(
457    currentPtr: *const u8,
458    matchAddress: *const u8,
459    matchIdx: u32,
460    idxLowLimit: u32,
461) -> core::ffi::c_int {
462    // Array of ~random data, should have low probability of matching data.
463    // Load from here if the index is invalid.
464    // Used to avoid unpredictable branches.
465    static dummy: [u8; 4] = [0x12, 0x34, 0x56, 0x78];
466
467    // currentIdx >= lowLimit is a (somewhat) unpredictable branch.
468    // However expression below compiles into conditional move.
469    let mvalAddr =
470        core::hint::select_unpredictable(matchIdx >= idxLowLimit, matchAddress, dummy.as_ptr());
471
472    // Note: this used to be written as : return test1 && test2;
473    // Unfortunately, once inlined, these tests become branches,
474    // in which case it becomes critical that they are executed in the right order (test1 then test2).
475    // So we have to write these tests in a specific manner to ensure their ordering.
476    if MEM_read32(currentPtr as *const core::ffi::c_void)
477        != MEM_read32(mvalAddr as *const core::ffi::c_void)
478    {
479        return 0;
480    }
481
482    // force ordering of these tests, which matters once the function is inlined, as they become branches.
483    #[cfg(not(target_family = "wasm"))]
484    asm!("", options(preserves_flags));
485
486    (matchIdx >= idxLowLimit) as core::ffi::c_int
487}
488
489unsafe fn ZSTD_match4Found_branch(
490    currentPtr: *const u8,
491    matchAddress: *const u8,
492    matchIdx: u32,
493    idxLowLimit: u32,
494) -> core::ffi::c_int {
495    let mut mval: u32 = 0;
496    if matchIdx >= idxLowLimit {
497        mval = MEM_read32(matchAddress as *const core::ffi::c_void);
498    } else {
499        mval = MEM_read32(currentPtr as *const core::ffi::c_void) ^ 1;
500    }
501    (MEM_read32(currentPtr as *const core::ffi::c_void) == mval) as core::ffi::c_int
502}
503#[inline(always)]
504unsafe fn ZSTD_compressBlock_fast_noDict_generic(
505    ms: *mut ZSTD_MatchState_t,
506    seqStore: *mut SeqStore_t,
507    rep: *mut u32,
508    src: *const core::ffi::c_void,
509    srcSize: size_t,
510    mls: u32,
511    useCmov: core::ffi::c_int,
512) -> size_t {
513    let mut current_block: u64;
514    let cParams: *const ZSTD_compressionParameters = &mut (*ms).cParams;
515    let hashTable = (*ms).hashTable;
516    let hlog = (*cParams).hashLog;
517    let stepSize = ((*cParams).targetLength)
518        .wrapping_add(((*cParams).targetLength == 0) as core::ffi::c_int as core::ffi::c_uint)
519        .wrapping_add(1) as size_t;
520    let base = (*ms).window.base;
521    let istart = src as *const u8;
522    let endIndex = (istart.offset_from(base) as size_t).wrapping_add(srcSize) as u32;
523    let prefixStartIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, (*cParams).windowLog);
524    let prefixStart = base.offset(prefixStartIndex as isize);
525    let iend = istart.add(srcSize);
526    let ilimit = iend.offset(-(HASH_READ_SIZE as isize));
527    let mut anchor = istart;
528    let mut ip0 = istart;
529    let mut ip1 = core::ptr::null::<u8>();
530    let mut ip2 = core::ptr::null::<u8>();
531    let mut ip3 = core::ptr::null::<u8>();
532    let mut current0: u32 = 0;
533    let mut rep_offset1 = *rep.offset(0);
534    let mut rep_offset2 = *rep.offset(1);
535    let mut offsetSaved1 = 0;
536    let mut offsetSaved2 = 0;
537    let mut hash0: size_t = 0;
538    let mut hash1: size_t = 0;
539    let mut matchIdx: u32 = 0;
540    let mut offcode: u32 = 0;
541    let mut match0 = core::ptr::null::<u8>();
542    let mut mLength: size_t = 0;
543    let mut step: size_t = 0;
544    let mut nextStep = core::ptr::null::<u8>();
545    let kStepIncr = ((1) << (kSearchStrength - 1)) as size_t;
546    let matchFound: ZSTD_match4Found = if useCmov != 0 {
547        Some(ZSTD_match4Found_cmov as unsafe fn(*const u8, *const u8, u32, u32) -> core::ffi::c_int)
548    } else {
549        Some(
550            ZSTD_match4Found_branch
551                as unsafe fn(*const u8, *const u8, u32, u32) -> core::ffi::c_int,
552        )
553    };
554    ip0 = ip0.offset((ip0 == prefixStart) as core::ffi::c_int as isize);
555    let curr = ip0.offset_from(base) as core::ffi::c_long as u32;
556    let windowLow = ZSTD_getLowestPrefixIndex(ms, curr, (*cParams).windowLog);
557    let maxRep = curr.wrapping_sub(windowLow);
558    if rep_offset2 > maxRep {
559        offsetSaved2 = rep_offset2;
560        rep_offset2 = 0;
561    }
562    if rep_offset1 > maxRep {
563        offsetSaved1 = rep_offset1;
564        rep_offset1 = 0;
565    }
566    '__start: loop {
567        step = stepSize;
568        nextStep = ip0.add(kStepIncr);
569        ip1 = ip0.offset(1);
570        ip2 = ip0.add(step);
571        ip3 = ip2.offset(1);
572        if ip3 >= ilimit {
573            break;
574        }
575        hash0 = ZSTD_hashPtr(ip0 as *const core::ffi::c_void, hlog, mls);
576        hash1 = ZSTD_hashPtr(ip1 as *const core::ffi::c_void, hlog, mls);
577        matchIdx = *hashTable.add(hash0);
578        loop {
579            let rval = MEM_read32(ip2.offset(-(rep_offset1 as isize)) as *const core::ffi::c_void);
580            current0 = ip0.offset_from(base) as core::ffi::c_long as u32;
581            *hashTable.add(hash0) = current0;
582            if (MEM_read32(ip2 as *const core::ffi::c_void) == rval) as core::ffi::c_int
583                & (rep_offset1 > 0) as core::ffi::c_int
584                != 0
585            {
586                ip0 = ip2;
587                match0 = ip0.offset(-(rep_offset1 as isize));
588                mLength = (*ip0.offset(-1_isize) as core::ffi::c_int
589                    == *match0.offset(-1_isize) as core::ffi::c_int)
590                    as core::ffi::c_int as size_t;
591                ip0 = ip0.offset(-(mLength as isize));
592                match0 = match0.offset(-(mLength as isize));
593                offcode = REPCODE1_TO_OFFBASE as u32;
594                mLength = mLength.wrapping_add(4);
595                *hashTable.add(hash1) = ip1.offset_from(base) as core::ffi::c_long as u32;
596                current_block = 4391991184774404966;
597                break;
598            } else if matchFound.unwrap_unchecked()(
599                ip0,
600                base.offset(matchIdx as isize),
601                matchIdx,
602                prefixStartIndex,
603            ) != 0
604            {
605                *hashTable.add(hash1) = ip1.offset_from(base) as core::ffi::c_long as u32;
606                current_block = 11113405673187116881;
607                break;
608            } else {
609                matchIdx = *hashTable.add(hash1);
610                hash0 = hash1;
611                hash1 = ZSTD_hashPtr(ip2 as *const core::ffi::c_void, hlog, mls);
612                ip0 = ip1;
613                ip1 = ip2;
614                ip2 = ip3;
615                current0 = ip0.offset_from(base) as core::ffi::c_long as u32;
616                *hashTable.add(hash0) = current0;
617                if matchFound.unwrap_unchecked()(
618                    ip0,
619                    base.offset(matchIdx as isize),
620                    matchIdx,
621                    prefixStartIndex,
622                ) != 0
623                {
624                    if step <= 4 {
625                        *hashTable.add(hash1) = ip1.offset_from(base) as core::ffi::c_long as u32;
626                    }
627                    current_block = 11113405673187116881;
628                    break;
629                } else {
630                    matchIdx = *hashTable.add(hash1);
631                    hash0 = hash1;
632                    hash1 = ZSTD_hashPtr(ip2 as *const core::ffi::c_void, hlog, mls);
633                    ip0 = ip1;
634                    ip1 = ip2;
635                    ip2 = ip0.add(step);
636                    ip3 = ip1.add(step);
637                    if ip2 >= nextStep {
638                        step = step.wrapping_add(1);
639                        nextStep = nextStep.add(kStepIncr);
640                    }
641                    if ip3 >= ilimit {
642                        break '__start;
643                    }
644                }
645            }
646        }
647        if current_block == 11113405673187116881 {
648            match0 = base.offset(matchIdx as isize);
649            rep_offset2 = rep_offset1;
650            rep_offset1 = ip0.offset_from(match0) as core::ffi::c_long as u32;
651            offcode = rep_offset1.wrapping_add(ZSTD_REP_NUM as u32);
652            mLength = 4;
653            while (ip0 > anchor) as core::ffi::c_int & (match0 > prefixStart) as core::ffi::c_int
654                != 0
655                && *ip0.offset(-1_isize) as core::ffi::c_int
656                    == *match0.offset(-1_isize) as core::ffi::c_int
657            {
658                ip0 = ip0.offset(-1);
659                match0 = match0.offset(-1);
660                mLength = mLength.wrapping_add(1);
661            }
662        }
663        mLength = mLength.wrapping_add(ZSTD_count(ip0.add(mLength), match0.add(mLength), iend));
664        ZSTD_storeSeq(
665            seqStore,
666            ip0.offset_from(anchor) as size_t,
667            anchor,
668            iend,
669            offcode,
670            mLength,
671        );
672        ip0 = ip0.add(mLength);
673        anchor = ip0;
674        if ip0 <= ilimit {
675            *hashTable.add(ZSTD_hashPtr(
676                base.offset(current0 as isize).offset(2) as *const core::ffi::c_void,
677                hlog,
678                mls,
679            )) = current0.wrapping_add(2);
680            *hashTable.add(ZSTD_hashPtr(
681                ip0.offset(-(2)) as *const core::ffi::c_void,
682                hlog,
683                mls,
684            )) = ip0.offset(-(2)).offset_from(base) as core::ffi::c_long as u32;
685            if rep_offset2 > 0 {
686                while ip0 <= ilimit
687                    && MEM_read32(ip0 as *const core::ffi::c_void)
688                        == MEM_read32(
689                            ip0.offset(-(rep_offset2 as isize)) as *const core::ffi::c_void
690                        )
691                {
692                    let rLength = (ZSTD_count(
693                        ip0.offset(4),
694                        ip0.offset(4).offset(-(rep_offset2 as isize)),
695                        iend,
696                    ))
697                    .wrapping_add(4);
698                    core::mem::swap(&mut rep_offset2, &mut rep_offset1);
699                    *hashTable.add(ZSTD_hashPtr(ip0 as *const core::ffi::c_void, hlog, mls)) =
700                        ip0.offset_from(base) as core::ffi::c_long as u32;
701                    ip0 = ip0.add(rLength);
702                    ZSTD_storeSeq(
703                        seqStore,
704                        0,
705                        anchor,
706                        iend,
707                        REPCODE1_TO_OFFBASE as u32,
708                        rLength,
709                    );
710                    anchor = ip0;
711                }
712            }
713        }
714    }
715    offsetSaved2 = if offsetSaved1 != 0 && rep_offset1 != 0 {
716        offsetSaved1
717    } else {
718        offsetSaved2
719    };
720    *rep.offset(0) = if rep_offset1 != 0 {
721        rep_offset1
722    } else {
723        offsetSaved1
724    };
725    *rep.offset(1) = if rep_offset2 != 0 {
726        rep_offset2
727    } else {
728        offsetSaved2
729    };
730    iend.offset_from(anchor) as size_t
731}
732unsafe fn ZSTD_compressBlock_fast_noDict_4_1(
733    ms: *mut ZSTD_MatchState_t,
734    seqStore: *mut SeqStore_t,
735    rep: *mut u32,
736    src: *const core::ffi::c_void,
737    srcSize: size_t,
738) -> size_t {
739    ZSTD_compressBlock_fast_noDict_generic(ms, seqStore, rep, src, srcSize, 4, 1)
740}
741unsafe fn ZSTD_compressBlock_fast_noDict_5_1(
742    ms: *mut ZSTD_MatchState_t,
743    seqStore: *mut SeqStore_t,
744    rep: *mut u32,
745    src: *const core::ffi::c_void,
746    srcSize: size_t,
747) -> size_t {
748    ZSTD_compressBlock_fast_noDict_generic(ms, seqStore, rep, src, srcSize, 5, 1)
749}
750unsafe fn ZSTD_compressBlock_fast_noDict_6_1(
751    ms: *mut ZSTD_MatchState_t,
752    seqStore: *mut SeqStore_t,
753    rep: *mut u32,
754    src: *const core::ffi::c_void,
755    srcSize: size_t,
756) -> size_t {
757    ZSTD_compressBlock_fast_noDict_generic(ms, seqStore, rep, src, srcSize, 6, 1)
758}
759unsafe fn ZSTD_compressBlock_fast_noDict_7_1(
760    ms: *mut ZSTD_MatchState_t,
761    seqStore: *mut SeqStore_t,
762    rep: *mut u32,
763    src: *const core::ffi::c_void,
764    srcSize: size_t,
765) -> size_t {
766    ZSTD_compressBlock_fast_noDict_generic(ms, seqStore, rep, src, srcSize, 7, 1)
767}
768unsafe fn ZSTD_compressBlock_fast_noDict_4_0(
769    ms: *mut ZSTD_MatchState_t,
770    seqStore: *mut SeqStore_t,
771    rep: *mut u32,
772    src: *const core::ffi::c_void,
773    srcSize: size_t,
774) -> size_t {
775    ZSTD_compressBlock_fast_noDict_generic(ms, seqStore, rep, src, srcSize, 4, 0)
776}
777unsafe fn ZSTD_compressBlock_fast_noDict_5_0(
778    ms: *mut ZSTD_MatchState_t,
779    seqStore: *mut SeqStore_t,
780    rep: *mut u32,
781    src: *const core::ffi::c_void,
782    srcSize: size_t,
783) -> size_t {
784    ZSTD_compressBlock_fast_noDict_generic(ms, seqStore, rep, src, srcSize, 5, 0)
785}
786unsafe fn ZSTD_compressBlock_fast_noDict_6_0(
787    ms: *mut ZSTD_MatchState_t,
788    seqStore: *mut SeqStore_t,
789    rep: *mut u32,
790    src: *const core::ffi::c_void,
791    srcSize: size_t,
792) -> size_t {
793    ZSTD_compressBlock_fast_noDict_generic(ms, seqStore, rep, src, srcSize, 6, 0)
794}
795unsafe fn ZSTD_compressBlock_fast_noDict_7_0(
796    ms: *mut ZSTD_MatchState_t,
797    seqStore: *mut SeqStore_t,
798    rep: *mut u32,
799    src: *const core::ffi::c_void,
800    srcSize: size_t,
801) -> size_t {
802    ZSTD_compressBlock_fast_noDict_generic(ms, seqStore, rep, src, srcSize, 7, 0)
803}
804pub unsafe fn ZSTD_compressBlock_fast(
805    ms: *mut ZSTD_MatchState_t,
806    seqStore: *mut SeqStore_t,
807    rep: *mut u32,
808    src: *const core::ffi::c_void,
809    srcSize: size_t,
810) -> size_t {
811    let mml = (*ms).cParams.minMatch;
812    let useCmov = ((*ms).cParams.windowLog < 19) as core::ffi::c_int;
813    if useCmov != 0 {
814        match mml {
815            5 => ZSTD_compressBlock_fast_noDict_5_1(ms, seqStore, rep, src, srcSize),
816            6 => ZSTD_compressBlock_fast_noDict_6_1(ms, seqStore, rep, src, srcSize),
817            7 => ZSTD_compressBlock_fast_noDict_7_1(ms, seqStore, rep, src, srcSize),
818            4 | _ => ZSTD_compressBlock_fast_noDict_4_1(ms, seqStore, rep, src, srcSize),
819        }
820    } else {
821        match mml {
822            5 => ZSTD_compressBlock_fast_noDict_5_0(ms, seqStore, rep, src, srcSize),
823            6 => ZSTD_compressBlock_fast_noDict_6_0(ms, seqStore, rep, src, srcSize),
824            7 => ZSTD_compressBlock_fast_noDict_7_0(ms, seqStore, rep, src, srcSize),
825            4 | _ => ZSTD_compressBlock_fast_noDict_4_0(ms, seqStore, rep, src, srcSize),
826        }
827    }
828}
829#[inline(always)]
830unsafe fn ZSTD_compressBlock_fast_dictMatchState_generic(
831    ms: *mut ZSTD_MatchState_t,
832    seqStore: *mut SeqStore_t,
833    rep: *mut u32,
834    src: *const core::ffi::c_void,
835    srcSize: size_t,
836    mls: u32,
837    hasStep: u32,
838) -> size_t {
839    let cParams: *const ZSTD_compressionParameters = &mut (*ms).cParams;
840    let hashTable = (*ms).hashTable;
841    let hlog = (*cParams).hashLog;
842    let stepSize = ((*cParams).targetLength)
843        .wrapping_add(((*cParams).targetLength == 0) as core::ffi::c_int as core::ffi::c_uint);
844    let base = (*ms).window.base;
845    let istart = src as *const u8;
846    let mut ip0 = istart;
847    let mut ip1 = ip0.offset(stepSize as isize);
848    let mut anchor = istart;
849    let prefixStartIndex = (*ms).window.dictLimit;
850    let prefixStart = base.offset(prefixStartIndex as isize);
851    let iend = istart.add(srcSize);
852    let ilimit = iend.offset(-(HASH_READ_SIZE as isize));
853    let mut offset_1 = *rep.offset(0);
854    let mut offset_2 = *rep.offset(1);
855    let dms = (*ms).dictMatchState;
856    let dictCParams: *const ZSTD_compressionParameters = &(*dms).cParams;
857    let dictHashTable: *const u32 = (*dms).hashTable;
858    let dictStartIndex = (*dms).window.dictLimit;
859    let dictBase = (*dms).window.base;
860    let dictStart = dictBase.offset(dictStartIndex as isize);
861    let dictEnd = (*dms).window.nextSrc;
862    let dictIndexDelta =
863        prefixStartIndex.wrapping_sub(dictEnd.offset_from(dictBase) as core::ffi::c_long as u32);
864    let dictAndPrefixLength = dictEnd
865        .offset(istart.offset_from(prefixStart) as core::ffi::c_long as isize)
866        .offset_from(dictStart) as core::ffi::c_long as u32;
867    let dictHBits =
868        ((*dictCParams).hashLog).wrapping_add(ZSTD_SHORT_CACHE_TAG_BITS as core::ffi::c_uint);
869    let maxDistance = (1) << (*cParams).windowLog;
870    let endIndex = (istart.offset_from(base) as size_t).wrapping_add(srcSize) as u32;
871    if (*ms).prefetchCDictTables != 0 {
872        let hashTableBytes = ((1 as core::ffi::c_int as size_t) << (*dictCParams).hashLog)
873            .wrapping_mul(::core::mem::size_of::<u32>());
874        let _ptr = dictHashTable as *const core::ffi::c_char;
875        let _size = hashTableBytes;
876        let mut _pos: size_t = 0;
877        _pos = 0;
878        while _pos < _size {
879            _pos = _pos.wrapping_add(CACHELINE_SIZE as size_t);
880        }
881    }
882    ip0 = ip0.offset((dictAndPrefixLength == 0) as core::ffi::c_int as isize);
883    's_135: while ip1 <= ilimit {
884        let mut mLength: size_t = 0;
885        let mut hash0 = ZSTD_hashPtr(ip0 as *const core::ffi::c_void, hlog, mls);
886        let dictHashAndTag0 = ZSTD_hashPtr(ip0 as *const core::ffi::c_void, dictHBits, mls);
887        let mut dictMatchIndexAndTag =
888            *dictHashTable.add(dictHashAndTag0 >> ZSTD_SHORT_CACHE_TAG_BITS);
889        let mut dictTagsMatch =
890            ZSTD_comparePackedTags(dictMatchIndexAndTag as size_t, dictHashAndTag0);
891        let mut matchIndex = *hashTable.add(hash0);
892        let mut curr = ip0.offset_from(base) as core::ffi::c_long as u32;
893        let mut step = stepSize as size_t;
894        let kStepIncr = ((1) << kSearchStrength) as size_t;
895        let mut nextStep = ip0.add(kStepIncr);
896        loop {
897            let mut match_0 = base.offset(matchIndex as isize);
898            let repIndex = curr.wrapping_add(1).wrapping_sub(offset_1);
899            let repMatch = if repIndex < prefixStartIndex {
900                dictBase.offset(repIndex.wrapping_sub(dictIndexDelta) as isize)
901            } else {
902                base.offset(repIndex as isize)
903            };
904            let hash1 = ZSTD_hashPtr(ip1 as *const core::ffi::c_void, hlog, mls);
905            let dictHashAndTag1 = ZSTD_hashPtr(ip1 as *const core::ffi::c_void, dictHBits, mls);
906            *hashTable.add(hash0) = curr;
907            if ZSTD_index_overlap_check(prefixStartIndex, repIndex) != 0
908                && MEM_read32(repMatch as *const core::ffi::c_void)
909                    == MEM_read32(ip0.offset(1) as *const core::ffi::c_void)
910            {
911                let repMatchEnd = if repIndex < prefixStartIndex {
912                    dictEnd
913                } else {
914                    iend
915                };
916                mLength = (ZSTD_count_2segments(
917                    ip0.offset(1).offset(4),
918                    repMatch.offset(4),
919                    iend,
920                    repMatchEnd,
921                    prefixStart,
922                ))
923                .wrapping_add(4);
924                ip0 = ip0.offset(1);
925                ZSTD_storeSeq(
926                    seqStore,
927                    ip0.offset_from(anchor) as size_t,
928                    anchor,
929                    iend,
930                    REPCODE1_TO_OFFBASE as u32,
931                    mLength,
932                );
933                break;
934            } else {
935                if dictTagsMatch != 0 {
936                    let dictMatchIndex = dictMatchIndexAndTag >> ZSTD_SHORT_CACHE_TAG_BITS;
937                    let mut dictMatch = dictBase.offset(dictMatchIndex as isize);
938                    if dictMatchIndex > dictStartIndex
939                        && MEM_read32(dictMatch as *const core::ffi::c_void)
940                            == MEM_read32(ip0 as *const core::ffi::c_void)
941                        && matchIndex <= prefixStartIndex
942                    {
943                        let offset = curr
944                            .wrapping_sub(dictMatchIndex)
945                            .wrapping_sub(dictIndexDelta);
946                        mLength = (ZSTD_count_2segments(
947                            ip0.offset(4),
948                            dictMatch.offset(4),
949                            iend,
950                            dictEnd,
951                            prefixStart,
952                        ))
953                        .wrapping_add(4);
954                        while (ip0 > anchor) as core::ffi::c_int
955                            & (dictMatch > dictStart) as core::ffi::c_int
956                            != 0
957                            && *ip0.offset(-1_isize) as core::ffi::c_int
958                                == *dictMatch.offset(-1_isize) as core::ffi::c_int
959                        {
960                            ip0 = ip0.offset(-1);
961                            dictMatch = dictMatch.offset(-1);
962                            mLength = mLength.wrapping_add(1);
963                        }
964                        offset_2 = offset_1;
965                        offset_1 = offset;
966                        ZSTD_storeSeq(
967                            seqStore,
968                            ip0.offset_from(anchor) as size_t,
969                            anchor,
970                            iend,
971                            offset.wrapping_add(ZSTD_REP_NUM as u32),
972                            mLength,
973                        );
974                        break;
975                    }
976                }
977                if ZSTD_match4Found_cmov(ip0, match_0, matchIndex, prefixStartIndex) != 0 {
978                    let offset_0 = ip0.offset_from(match_0) as core::ffi::c_long as u32;
979                    mLength = (ZSTD_count(ip0.offset(4), match_0.offset(4), iend)).wrapping_add(4);
980                    while (ip0 > anchor) as core::ffi::c_int
981                        & (match_0 > prefixStart) as core::ffi::c_int
982                        != 0
983                        && *ip0.offset(-1_isize) as core::ffi::c_int
984                            == *match_0.offset(-1_isize) as core::ffi::c_int
985                    {
986                        ip0 = ip0.offset(-1);
987                        match_0 = match_0.offset(-1);
988                        mLength = mLength.wrapping_add(1);
989                    }
990                    offset_2 = offset_1;
991                    offset_1 = offset_0;
992                    ZSTD_storeSeq(
993                        seqStore,
994                        ip0.offset_from(anchor) as size_t,
995                        anchor,
996                        iend,
997                        offset_0.wrapping_add(ZSTD_REP_NUM as u32),
998                        mLength,
999                    );
1000                    break;
1001                } else {
1002                    dictMatchIndexAndTag =
1003                        *dictHashTable.add(dictHashAndTag1 >> ZSTD_SHORT_CACHE_TAG_BITS);
1004                    dictTagsMatch =
1005                        ZSTD_comparePackedTags(dictMatchIndexAndTag as size_t, dictHashAndTag1);
1006                    matchIndex = *hashTable.add(hash1);
1007                    if ip1 >= nextStep {
1008                        step = step.wrapping_add(1);
1009                        nextStep = nextStep.add(kStepIncr);
1010                    }
1011                    ip0 = ip1;
1012                    ip1 = ip1.add(step);
1013                    if ip1 > ilimit {
1014                        break 's_135;
1015                    }
1016                    curr = ip0.offset_from(base) as core::ffi::c_long as u32;
1017                    hash0 = hash1;
1018                }
1019            }
1020        }
1021        ip0 = ip0.add(mLength);
1022        anchor = ip0;
1023        if ip0 <= ilimit {
1024            *hashTable.add(ZSTD_hashPtr(
1025                base.offset(curr as isize).offset(2) as *const core::ffi::c_void,
1026                hlog,
1027                mls,
1028            )) = curr.wrapping_add(2);
1029            *hashTable.add(ZSTD_hashPtr(
1030                ip0.offset(-(2)) as *const core::ffi::c_void,
1031                hlog,
1032                mls,
1033            )) = ip0.offset(-(2)).offset_from(base) as core::ffi::c_long as u32;
1034            while ip0 <= ilimit {
1035                let current2 = ip0.offset_from(base) as core::ffi::c_long as u32;
1036                let repIndex2 = current2.wrapping_sub(offset_2);
1037                let repMatch2 = if repIndex2 < prefixStartIndex {
1038                    dictBase
1039                        .offset(-(dictIndexDelta as isize))
1040                        .offset(repIndex2 as isize)
1041                } else {
1042                    base.offset(repIndex2 as isize)
1043                };
1044                if !(ZSTD_index_overlap_check(prefixStartIndex, repIndex2) != 0
1045                    && MEM_read32(repMatch2 as *const core::ffi::c_void)
1046                        == MEM_read32(ip0 as *const core::ffi::c_void))
1047                {
1048                    break;
1049                }
1050                let repEnd2 = if repIndex2 < prefixStartIndex {
1051                    dictEnd
1052                } else {
1053                    iend
1054                };
1055                let repLength2 = (ZSTD_count_2segments(
1056                    ip0.offset(4),
1057                    repMatch2.offset(4),
1058                    iend,
1059                    repEnd2,
1060                    prefixStart,
1061                ))
1062                .wrapping_add(4);
1063                core::mem::swap(&mut offset_2, &mut offset_1);
1064                ZSTD_storeSeq(
1065                    seqStore,
1066                    0,
1067                    anchor,
1068                    iend,
1069                    REPCODE1_TO_OFFBASE as u32,
1070                    repLength2,
1071                );
1072                *hashTable.add(ZSTD_hashPtr(ip0 as *const core::ffi::c_void, hlog, mls)) = current2;
1073                ip0 = ip0.add(repLength2);
1074                anchor = ip0;
1075            }
1076        }
1077        ip1 = ip0.offset(stepSize as isize);
1078    }
1079    *rep.offset(0) = offset_1;
1080    *rep.offset(1) = offset_2;
1081    iend.offset_from(anchor) as size_t
1082}
1083unsafe fn ZSTD_compressBlock_fast_dictMatchState_4_0(
1084    ms: *mut ZSTD_MatchState_t,
1085    seqStore: *mut SeqStore_t,
1086    rep: *mut u32,
1087    src: *const core::ffi::c_void,
1088    srcSize: size_t,
1089) -> size_t {
1090    ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 4, 0)
1091}
1092unsafe fn ZSTD_compressBlock_fast_dictMatchState_5_0(
1093    ms: *mut ZSTD_MatchState_t,
1094    seqStore: *mut SeqStore_t,
1095    rep: *mut u32,
1096    src: *const core::ffi::c_void,
1097    srcSize: size_t,
1098) -> size_t {
1099    ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 5, 0)
1100}
1101unsafe fn ZSTD_compressBlock_fast_dictMatchState_6_0(
1102    ms: *mut ZSTD_MatchState_t,
1103    seqStore: *mut SeqStore_t,
1104    rep: *mut u32,
1105    src: *const core::ffi::c_void,
1106    srcSize: size_t,
1107) -> size_t {
1108    ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 6, 0)
1109}
1110unsafe fn ZSTD_compressBlock_fast_dictMatchState_7_0(
1111    ms: *mut ZSTD_MatchState_t,
1112    seqStore: *mut SeqStore_t,
1113    rep: *mut u32,
1114    src: *const core::ffi::c_void,
1115    srcSize: size_t,
1116) -> size_t {
1117    ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 7, 0)
1118}
1119pub unsafe fn ZSTD_compressBlock_fast_dictMatchState(
1120    ms: *mut ZSTD_MatchState_t,
1121    seqStore: *mut SeqStore_t,
1122    rep: *mut u32,
1123    src: *const core::ffi::c_void,
1124    srcSize: size_t,
1125) -> size_t {
1126    let mls = (*ms).cParams.minMatch;
1127    match mls {
1128        5 => ZSTD_compressBlock_fast_dictMatchState_5_0(ms, seqStore, rep, src, srcSize),
1129        6 => ZSTD_compressBlock_fast_dictMatchState_6_0(ms, seqStore, rep, src, srcSize),
1130        7 => ZSTD_compressBlock_fast_dictMatchState_7_0(ms, seqStore, rep, src, srcSize),
1131        4 | _ => ZSTD_compressBlock_fast_dictMatchState_4_0(ms, seqStore, rep, src, srcSize),
1132    }
1133}
1134unsafe fn ZSTD_compressBlock_fast_extDict_generic(
1135    ms: *mut ZSTD_MatchState_t,
1136    seqStore: *mut SeqStore_t,
1137    rep: *mut u32,
1138    src: *const core::ffi::c_void,
1139    srcSize: size_t,
1140    mls: u32,
1141    hasStep: u32,
1142) -> size_t {
1143    let mut current_block: u64;
1144    let cParams: *const ZSTD_compressionParameters = &mut (*ms).cParams;
1145    let hashTable = (*ms).hashTable;
1146    let hlog = (*cParams).hashLog;
1147    let stepSize = ((*cParams).targetLength)
1148        .wrapping_add(((*cParams).targetLength == 0) as core::ffi::c_int as core::ffi::c_uint)
1149        .wrapping_add(1) as size_t;
1150    let base = (*ms).window.base;
1151    let dictBase = (*ms).window.dictBase;
1152    let istart = src as *const u8;
1153    let mut anchor = istart;
1154    let endIndex = (istart.offset_from(base) as size_t).wrapping_add(srcSize) as u32;
1155    let lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, (*cParams).windowLog);
1156    let dictStartIndex = lowLimit;
1157    let dictStart = dictBase.offset(dictStartIndex as isize);
1158    let dictLimit = (*ms).window.dictLimit;
1159    let prefixStartIndex = if dictLimit < lowLimit {
1160        lowLimit
1161    } else {
1162        dictLimit
1163    };
1164    let prefixStart = base.offset(prefixStartIndex as isize);
1165    let dictEnd = dictBase.offset(prefixStartIndex as isize);
1166    let iend = istart.add(srcSize);
1167    let ilimit = iend.offset(-(8));
1168    let mut offset_1 = *rep.offset(0);
1169    let mut offset_2 = *rep.offset(1);
1170    let mut offsetSaved1 = 0;
1171    let mut offsetSaved2 = 0;
1172    let mut ip0 = istart;
1173    let mut ip1 = core::ptr::null::<u8>();
1174    let mut ip2 = core::ptr::null::<u8>();
1175    let mut ip3 = core::ptr::null::<u8>();
1176    let mut current0: u32 = 0;
1177    let mut hash0: size_t = 0;
1178    let mut hash1: size_t = 0;
1179    let mut idx: u32 = 0;
1180    let mut idxBase = core::ptr::null::<u8>();
1181    let mut offcode: u32 = 0;
1182    let mut match0 = core::ptr::null::<u8>();
1183    let mut mLength: size_t = 0;
1184    let mut matchEnd = core::ptr::null::<u8>();
1185    let mut step: size_t = 0;
1186    let mut nextStep = core::ptr::null::<u8>();
1187    let kStepIncr = ((1) << (kSearchStrength - 1)) as size_t;
1188    if prefixStartIndex == dictStartIndex {
1189        return ZSTD_compressBlock_fast(ms, seqStore, rep, src, srcSize);
1190    }
1191    let curr = ip0.offset_from(base) as core::ffi::c_long as u32;
1192    let maxRep = curr.wrapping_sub(dictStartIndex);
1193    if offset_2 >= maxRep {
1194        offsetSaved2 = offset_2;
1195        offset_2 = 0;
1196    }
1197    if offset_1 >= maxRep {
1198        offsetSaved1 = offset_1;
1199        offset_1 = 0;
1200    }
1201    '__start: loop {
1202        step = stepSize;
1203        nextStep = ip0.add(kStepIncr);
1204        ip1 = ip0.offset(1);
1205        ip2 = ip0.add(step);
1206        ip3 = ip2.offset(1);
1207        if ip3 >= ilimit {
1208            break;
1209        }
1210        hash0 = ZSTD_hashPtr(ip0 as *const core::ffi::c_void, hlog, mls);
1211        hash1 = ZSTD_hashPtr(ip1 as *const core::ffi::c_void, hlog, mls);
1212        idx = *hashTable.add(hash0);
1213        idxBase = if idx < prefixStartIndex {
1214            dictBase
1215        } else {
1216            base
1217        };
1218        loop {
1219            let current2 = ip2.offset_from(base) as core::ffi::c_long as u32;
1220            let repIndex = current2.wrapping_sub(offset_1);
1221            let repBase = if repIndex < prefixStartIndex {
1222                dictBase
1223            } else {
1224                base
1225            };
1226            let mut rval: u32 = 0;
1227            if (prefixStartIndex.wrapping_sub(repIndex) >= 4) as core::ffi::c_int
1228                & (offset_1 > 0) as core::ffi::c_int
1229                != 0
1230            {
1231                rval = MEM_read32(repBase.offset(repIndex as isize) as *const core::ffi::c_void);
1232            } else {
1233                rval = MEM_read32(ip2 as *const core::ffi::c_void) ^ 1;
1234            }
1235            current0 = ip0.offset_from(base) as core::ffi::c_long as u32;
1236            *hashTable.add(hash0) = current0;
1237            if MEM_read32(ip2 as *const core::ffi::c_void) == rval {
1238                ip0 = ip2;
1239                match0 = repBase.offset(repIndex as isize);
1240                matchEnd = if repIndex < prefixStartIndex {
1241                    dictEnd
1242                } else {
1243                    iend
1244                };
1245                mLength = (*ip0.offset(-1_isize) as core::ffi::c_int
1246                    == *match0.offset(-1_isize) as core::ffi::c_int)
1247                    as core::ffi::c_int as size_t;
1248                ip0 = ip0.offset(-(mLength as isize));
1249                match0 = match0.offset(-(mLength as isize));
1250                offcode = REPCODE1_TO_OFFBASE as u32;
1251                mLength = mLength.wrapping_add(4);
1252                current_block = 1352918242886884122;
1253                break;
1254            } else {
1255                let mval = if idx >= dictStartIndex {
1256                    MEM_read32(idxBase.offset(idx as isize) as *const core::ffi::c_void)
1257                } else {
1258                    MEM_read32(ip0 as *const core::ffi::c_void) ^ 1
1259                };
1260                if MEM_read32(ip0 as *const core::ffi::c_void) == mval {
1261                    current_block = 934346911184053177;
1262                    break;
1263                } else {
1264                    idx = *hashTable.add(hash1);
1265                    idxBase = if idx < prefixStartIndex {
1266                        dictBase
1267                    } else {
1268                        base
1269                    };
1270                    hash0 = hash1;
1271                    hash1 = ZSTD_hashPtr(ip2 as *const core::ffi::c_void, hlog, mls);
1272                    ip0 = ip1;
1273                    ip1 = ip2;
1274                    ip2 = ip3;
1275                    current0 = ip0.offset_from(base) as core::ffi::c_long as u32;
1276                    *hashTable.add(hash0) = current0;
1277                    let mval_0 = if idx >= dictStartIndex {
1278                        MEM_read32(idxBase.offset(idx as isize) as *const core::ffi::c_void)
1279                    } else {
1280                        MEM_read32(ip0 as *const core::ffi::c_void) ^ 1
1281                    };
1282                    if MEM_read32(ip0 as *const core::ffi::c_void) == mval_0 {
1283                        current_block = 934346911184053177;
1284                        break;
1285                    }
1286                    idx = *hashTable.add(hash1);
1287                    idxBase = if idx < prefixStartIndex {
1288                        dictBase
1289                    } else {
1290                        base
1291                    };
1292                    hash0 = hash1;
1293                    hash1 = ZSTD_hashPtr(ip2 as *const core::ffi::c_void, hlog, mls);
1294                    ip0 = ip1;
1295                    ip1 = ip2;
1296                    ip2 = ip0.add(step);
1297                    ip3 = ip1.add(step);
1298                    if ip2 >= nextStep {
1299                        step = step.wrapping_add(1);
1300                        nextStep = nextStep.add(kStepIncr);
1301                    }
1302                    if ip3 >= ilimit {
1303                        break '__start;
1304                    }
1305                }
1306            }
1307        }
1308        if current_block == 934346911184053177 {
1309            let offset = current0.wrapping_sub(idx);
1310            let lowMatchPtr = if idx < prefixStartIndex {
1311                dictStart
1312            } else {
1313                prefixStart
1314            };
1315            matchEnd = if idx < prefixStartIndex {
1316                dictEnd
1317            } else {
1318                iend
1319            };
1320            match0 = idxBase.offset(idx as isize);
1321            offset_2 = offset_1;
1322            offset_1 = offset;
1323            offcode = offset.wrapping_add(ZSTD_REP_NUM as u32);
1324            mLength = 4;
1325            while (ip0 > anchor) as core::ffi::c_int & (match0 > lowMatchPtr) as core::ffi::c_int
1326                != 0
1327                && *ip0.offset(-1_isize) as core::ffi::c_int
1328                    == *match0.offset(-1_isize) as core::ffi::c_int
1329            {
1330                ip0 = ip0.offset(-1);
1331                match0 = match0.offset(-1);
1332                mLength = mLength.wrapping_add(1);
1333            }
1334        }
1335        mLength = mLength.wrapping_add(ZSTD_count_2segments(
1336            ip0.add(mLength),
1337            match0.add(mLength),
1338            iend,
1339            matchEnd,
1340            prefixStart,
1341        ));
1342        ZSTD_storeSeq(
1343            seqStore,
1344            ip0.offset_from(anchor) as size_t,
1345            anchor,
1346            iend,
1347            offcode,
1348            mLength,
1349        );
1350        ip0 = ip0.add(mLength);
1351        anchor = ip0;
1352        if ip1 < ip0 {
1353            *hashTable.add(hash1) = ip1.offset_from(base) as core::ffi::c_long as u32;
1354        }
1355        if ip0 <= ilimit {
1356            *hashTable.add(ZSTD_hashPtr(
1357                base.offset(current0 as isize).offset(2) as *const core::ffi::c_void,
1358                hlog,
1359                mls,
1360            )) = current0.wrapping_add(2);
1361            *hashTable.add(ZSTD_hashPtr(
1362                ip0.offset(-(2)) as *const core::ffi::c_void,
1363                hlog,
1364                mls,
1365            )) = ip0.offset(-(2)).offset_from(base) as core::ffi::c_long as u32;
1366            while ip0 <= ilimit {
1367                let repIndex2 =
1368                    (ip0.offset_from(base) as core::ffi::c_long as u32).wrapping_sub(offset_2);
1369                let repMatch2 = if repIndex2 < prefixStartIndex {
1370                    dictBase.offset(repIndex2 as isize)
1371                } else {
1372                    base.offset(repIndex2 as isize)
1373                };
1374                if !(ZSTD_index_overlap_check(prefixStartIndex, repIndex2)
1375                    & (offset_2 > 0) as core::ffi::c_int
1376                    != 0
1377                    && MEM_read32(repMatch2 as *const core::ffi::c_void)
1378                        == MEM_read32(ip0 as *const core::ffi::c_void))
1379                {
1380                    break;
1381                }
1382                let repEnd2 = if repIndex2 < prefixStartIndex {
1383                    dictEnd
1384                } else {
1385                    iend
1386                };
1387                let repLength2 = (ZSTD_count_2segments(
1388                    ip0.offset(4),
1389                    repMatch2.offset(4),
1390                    iend,
1391                    repEnd2,
1392                    prefixStart,
1393                ))
1394                .wrapping_add(4);
1395                core::mem::swap(&mut offset_2, &mut offset_1);
1396                ZSTD_storeSeq(
1397                    seqStore,
1398                    0,
1399                    anchor,
1400                    iend,
1401                    REPCODE1_TO_OFFBASE as u32,
1402                    repLength2,
1403                );
1404                *hashTable.add(ZSTD_hashPtr(ip0 as *const core::ffi::c_void, hlog, mls)) =
1405                    ip0.offset_from(base) as core::ffi::c_long as u32;
1406                ip0 = ip0.add(repLength2);
1407                anchor = ip0;
1408            }
1409        }
1410    }
1411    offsetSaved2 = if offsetSaved1 != 0 && offset_1 != 0 {
1412        offsetSaved1
1413    } else {
1414        offsetSaved2
1415    };
1416    *rep.offset(0) = if offset_1 != 0 {
1417        offset_1
1418    } else {
1419        offsetSaved1
1420    };
1421    *rep.offset(1) = if offset_2 != 0 {
1422        offset_2
1423    } else {
1424        offsetSaved2
1425    };
1426    iend.offset_from(anchor) as size_t
1427}
1428unsafe fn ZSTD_compressBlock_fast_extDict_4_0(
1429    ms: *mut ZSTD_MatchState_t,
1430    seqStore: *mut SeqStore_t,
1431    rep: *mut u32,
1432    src: *const core::ffi::c_void,
1433    srcSize: size_t,
1434) -> size_t {
1435    ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 4, 0)
1436}
1437unsafe fn ZSTD_compressBlock_fast_extDict_5_0(
1438    ms: *mut ZSTD_MatchState_t,
1439    seqStore: *mut SeqStore_t,
1440    rep: *mut u32,
1441    src: *const core::ffi::c_void,
1442    srcSize: size_t,
1443) -> size_t {
1444    ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 5, 0)
1445}
1446unsafe fn ZSTD_compressBlock_fast_extDict_6_0(
1447    ms: *mut ZSTD_MatchState_t,
1448    seqStore: *mut SeqStore_t,
1449    rep: *mut u32,
1450    src: *const core::ffi::c_void,
1451    srcSize: size_t,
1452) -> size_t {
1453    ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 6, 0)
1454}
1455unsafe fn ZSTD_compressBlock_fast_extDict_7_0(
1456    ms: *mut ZSTD_MatchState_t,
1457    seqStore: *mut SeqStore_t,
1458    rep: *mut u32,
1459    src: *const core::ffi::c_void,
1460    srcSize: size_t,
1461) -> size_t {
1462    ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 7, 0)
1463}
1464pub unsafe fn ZSTD_compressBlock_fast_extDict(
1465    ms: *mut ZSTD_MatchState_t,
1466    seqStore: *mut SeqStore_t,
1467    rep: *mut u32,
1468    src: *const core::ffi::c_void,
1469    srcSize: size_t,
1470) -> size_t {
1471    let mls = (*ms).cParams.minMatch;
1472    match mls {
1473        5 => ZSTD_compressBlock_fast_extDict_5_0(ms, seqStore, rep, src, srcSize),
1474        6 => ZSTD_compressBlock_fast_extDict_6_0(ms, seqStore, rep, src, srcSize),
1475        7 => ZSTD_compressBlock_fast_extDict_7_0(ms, seqStore, rep, src, srcSize),
1476        4 | _ => ZSTD_compressBlock_fast_extDict_4_0(ms, seqStore, rep, src, srcSize),
1477    }
1478}
libzstd_rs_sys/lib/compress/zstd_fast.rs

libzstd_rs_sys/lib/compress/
zstd_fast.rs
