Skip to main content

hekate_math/towers/
block64.rs

1// SPDX-License-Identifier: Apache-2.0
2// This file is part of the hekate-math project.
3// Copyright (C) 2026 Andrei Kochergin <andrei@oumuamua.dev>
4// Copyright (C) 2026 Oumuamua Labs <info@oumuamua.dev>. All rights reserved.
5//
6// Licensed under the Apache License, Version 2.0 (the "License");
7// you may not use this file except in compliance with the License.
8// You may obtain a copy of the License at
9//
10//     http://www.apache.org/licenses/LICENSE-2.0
11//
12// Unless required by applicable law or agreed to in writing, software
13// distributed under the License is distributed on an "AS IS" BASIS,
14// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15// See the License for the specific language governing permissions and
16// limitations under the License.
17
18//! GF(2^64) as `Block32[X] / (X^2 + X + τ)`, its flat basis
19//! modulo x^64 + x^4 + x^3 + x + 1, and the aarch64 NEON kernels.
20
21use crate::algebra::impl_binary_field_extras;
22use crate::constants::FLAT_TO_TOWER_BIT_MASKS_64;
23use crate::towers::bit::Bit;
24use crate::towers::block8::Block8;
25use crate::towers::block16::Block16;
26use crate::towers::block32::Block32;
27use crate::{
28    BinaryFieldExtras, CanonicalDeserialize, CanonicalSerialize, Flat, FlatPromote, HardwareField,
29    PackableField, PackedFlat, TowerField, constants,
30};
31use core::ops::{Add, AddAssign, BitXor, BitXorAssign, Mul, MulAssign, Sub, SubAssign};
32use serde::{Deserialize, Serialize};
33use zeroize::Zeroize;
34
35#[cfg(not(feature = "table-math"))]
36#[repr(align(64))]
37struct CtConvertBasisU64<const N: usize>([u64; N]);
38
39#[cfg(not(feature = "table-math"))]
40static TOWER_TO_FLAT_BASIS_64: CtConvertBasisU64<64> =
41    CtConvertBasisU64(constants::RAW_TOWER_TO_FLAT_64);
42
43#[cfg(not(feature = "table-math"))]
44static FLAT_TO_TOWER_BASIS_64: CtConvertBasisU64<64> =
45    CtConvertBasisU64(constants::RAW_FLAT_TO_TOWER_64);
46
47/// GF(2^64) in the tower basis: `lo + hi·X`
48/// over [`Block32`], `lo` in the low 32 bits.
49#[derive(Copy, Clone, Default, Debug, Eq, PartialEq, Serialize, Deserialize, Zeroize)]
50#[repr(transparent)]
51pub struct Block64(pub u64);
52
53impl Block64 {
54    /// τ of the next level, `Block64[X] / (X^2 + X + τ)`;
55    /// equals [`TowerField::EXTENSION_TAU`].
56    pub const TAU: Self = Block64(0x2000_0000_0000_0000);
57
58    /// Builds `lo + hi·X` from its halves.
59    pub fn new(lo: Block32, hi: Block32) -> Self {
60        Self((hi.0 as u64) << 32 | (lo.0 as u64))
61    }
62
63    /// Returns the halves `(lo, hi)` of `lo + hi·X`.
64    #[inline(always)]
65    pub fn split(self) -> (Block32, Block32) {
66        (Block32(self.0 as u32), Block32((self.0 >> 32) as u32))
67    }
68
69    #[inline(always)]
70    pub(crate) fn mul_tau(self) -> Self {
71        let (a0, a1) = self.split();
72        let t = a1.mul_tau();
73
74        Self::new(t.mul_tau(), (a0 + a1).mul_tau())
75    }
76}
77
78impl TowerField for Block64 {
79    const BITS: usize = 64;
80    const ZERO: Self = Block64(0);
81    const ONE: Self = Block64(1);
82
83    const EXTENSION_TAU: Self = Self::TAU;
84
85    fn invert(&self) -> Self {
86        let (l, h) = self.split();
87
88        let norm = h.square().mul_tau() + h * l + l.square();
89        let norm_inv = norm.invert();
90
91        let res_hi = h * norm_inv;
92        let res_lo = (h + l) * norm_inv;
93
94        Self::new(res_lo, res_hi)
95    }
96
97    fn from_uniform_bytes(bytes: &[u8; 32]) -> Self {
98        let mut buf = [0u8; 8];
99        buf.copy_from_slice(&bytes[0..8]);
100
101        Self(u64::from_le_bytes(buf))
102    }
103}
104
105impl Add for Block64 {
106    type Output = Self;
107
108    fn add(self, rhs: Self) -> Self {
109        Self(self.0.bitxor(rhs.0))
110    }
111}
112
113impl Sub for Block64 {
114    type Output = Self;
115
116    fn sub(self, rhs: Self) -> Self {
117        self.add(rhs)
118    }
119}
120
121impl Mul for Block64 {
122    type Output = Self;
123
124    #[inline]
125    fn mul(self, rhs: Self) -> Self {
126        #[cfg(pmull)]
127        {
128            mul_iso_64(self, rhs)
129        }
130
131        #[cfg(not(pmull))]
132        {
133            let (a0, a1) = self.split();
134            let (b0, b1) = rhs.split();
135
136            let v0 = a0 * b0;
137            let v1 = a1 * b1;
138            let vs = (a0 + a1) * (b0 + b1);
139
140            Self::new(v0 + v1.mul_tau(), v0 + vs)
141        }
142    }
143}
144
145impl AddAssign for Block64 {
146    fn add_assign(&mut self, rhs: Self) {
147        self.0.bitxor_assign(rhs.0);
148    }
149}
150
151impl SubAssign for Block64 {
152    fn sub_assign(&mut self, rhs: Self) {
153        self.0.bitxor_assign(rhs.0);
154    }
155}
156
157impl MulAssign for Block64 {
158    fn mul_assign(&mut self, rhs: Self) {
159        *self = *self * rhs;
160    }
161}
162
163impl CanonicalSerialize for Block64 {
164    fn serialized_size(&self) -> usize {
165        8
166    }
167
168    fn serialize(&self, writer: &mut [u8]) -> Result<(), ()> {
169        if writer.len() < 8 {
170            return Err(());
171        }
172
173        writer[..8].copy_from_slice(&self.0.to_le_bytes());
174
175        Ok(())
176    }
177}
178
179impl CanonicalDeserialize for Block64 {
180    fn deserialize(bytes: &[u8]) -> Result<Self, ()> {
181        if bytes.len() < 8 {
182            return Err(());
183        }
184
185        let mut buf = [0u8; 8];
186        buf.copy_from_slice(&bytes[0..8]);
187
188        Ok(Self(u64::from_le_bytes(buf)))
189    }
190}
191
192impl From<u8> for Block64 {
193    #[inline(always)]
194    fn from(val: u8) -> Self {
195        Self(val as u64)
196    }
197}
198
199impl From<u32> for Block64 {
200    #[inline(always)]
201    fn from(val: u32) -> Self {
202        Self::from(val as u64)
203    }
204}
205
206impl From<u64> for Block64 {
207    #[inline(always)]
208    fn from(val: u64) -> Self {
209        Self(val)
210    }
211}
212
213impl From<u128> for Block64 {
214    #[inline(always)]
215    fn from(val: u128) -> Self {
216        Self(val as u64)
217    }
218}
219
220// ========================================
221// FIELD LIFTING
222// ========================================
223
224impl From<Bit> for Block64 {
225    #[inline(always)]
226    fn from(val: Bit) -> Self {
227        Self(val.get() as u64)
228    }
229}
230
231impl From<Block8> for Block64 {
232    #[inline(always)]
233    fn from(val: Block8) -> Self {
234        Self(val.0 as u64)
235    }
236}
237
238impl From<Block16> for Block64 {
239    #[inline(always)]
240    fn from(val: Block16) -> Self {
241        Self(val.0 as u64)
242    }
243}
244
245impl From<Block32> for Block64 {
246    #[inline(always)]
247    fn from(val: Block32) -> Self {
248        Self(val.0 as u64)
249    }
250}
251
252// ===================================
253// PACKED BLOCK 64 (Width = 2)
254// ===================================
255
256/// Lanes in [`PackedBlock64`].
257pub const PACKED_WIDTH_64: usize = 2;
258
259/// Two [`Block64`] lanes; operators act lane by lane.
260#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)]
261#[repr(C, align(16))] // 128-bit alignment
262pub struct PackedBlock64(pub [Block64; PACKED_WIDTH_64]);
263
264impl PackedBlock64 {
265    /// Returns the packed value with every lane zero.
266    #[inline(always)]
267    pub fn zero() -> Self {
268        Self([Block64::ZERO; PACKED_WIDTH_64])
269    }
270}
271
272impl PackableField for Block64 {
273    type Packed = PackedBlock64;
274
275    const WIDTH: usize = PACKED_WIDTH_64;
276
277    #[inline(always)]
278    fn pack(chunk: &[Self]) -> Self::Packed {
279        assert!(
280            chunk.len() >= PACKED_WIDTH_64,
281            "PackableField::pack: input slice too short",
282        );
283
284        let mut arr = [Self::ZERO; PACKED_WIDTH_64];
285        arr.copy_from_slice(&chunk[..PACKED_WIDTH_64]);
286
287        PackedBlock64(arr)
288    }
289
290    #[inline(always)]
291    fn unpack(packed: Self::Packed, output: &mut [Self]) {
292        assert!(
293            output.len() >= PACKED_WIDTH_64,
294            "PackableField::unpack: output slice too short",
295        );
296
297        output[..PACKED_WIDTH_64].copy_from_slice(&packed.0);
298    }
299}
300
301impl Add for PackedBlock64 {
302    type Output = Self;
303
304    #[inline(always)]
305    fn add(self, rhs: Self) -> Self {
306        let mut res = [Block64::ZERO; PACKED_WIDTH_64];
307        for ((out, l), r) in res.iter_mut().zip(self.0.iter()).zip(rhs.0.iter()) {
308            *out = *l + *r;
309        }
310
311        Self(res)
312    }
313}
314
315impl AddAssign for PackedBlock64 {
316    #[inline(always)]
317    fn add_assign(&mut self, rhs: Self) {
318        for (l, r) in self.0.iter_mut().zip(rhs.0.iter()) {
319            *l += *r;
320        }
321    }
322}
323
324impl Sub for PackedBlock64 {
325    type Output = Self;
326
327    #[inline(always)]
328    fn sub(self, rhs: Self) -> Self {
329        self.add(rhs)
330    }
331}
332
333impl SubAssign for PackedBlock64 {
334    #[inline(always)]
335    fn sub_assign(&mut self, rhs: Self) {
336        self.add_assign(rhs);
337    }
338}
339
340impl Mul for PackedBlock64 {
341    type Output = Self;
342
343    #[inline(always)]
344    fn mul(self, rhs: Self) -> Self {
345        let mut res = [Block64::ZERO; PACKED_WIDTH_64];
346        for ((out, l), r) in res.iter_mut().zip(self.0.iter()).zip(rhs.0.iter()) {
347            *out = *l * *r;
348        }
349
350        Self(res)
351    }
352}
353
354impl MulAssign for PackedBlock64 {
355    #[inline(always)]
356    fn mul_assign(&mut self, rhs: Self) {
357        for (l, r) in self.0.iter_mut().zip(rhs.0.iter()) {
358            *l *= *r;
359        }
360    }
361}
362
363impl Mul<Block64> for PackedBlock64 {
364    type Output = Self;
365
366    #[inline(always)]
367    fn mul(self, rhs: Block64) -> Self {
368        let mut res = [Block64::ZERO; PACKED_WIDTH_64];
369        for (out, v) in res.iter_mut().zip(self.0.iter()) {
370            *out = *v * rhs;
371        }
372
373        Self(res)
374    }
375}
376
377// ===================================
378// Hardware Field
379// ===================================
380
381impl HardwareField for Block64 {
382    #[inline(always)]
383    fn to_hardware(self) -> Flat<Self> {
384        #[cfg(feature = "table-math")]
385        {
386            Flat::from_raw(apply_matrix_64(self, &constants::TOWER_TO_FLAT_64))
387        }
388
389        #[cfg(not(feature = "table-math"))]
390        {
391            Flat::from_raw(Block64(map_ct_64(self.0, &TOWER_TO_FLAT_BASIS_64.0)))
392        }
393    }
394
395    #[inline(always)]
396    fn from_hardware(value: Flat<Self>) -> Self {
397        let value = value.into_raw();
398
399        #[cfg(feature = "table-math")]
400        {
401            apply_matrix_64(value, &constants::FLAT_TO_TOWER_64)
402        }
403
404        #[cfg(not(feature = "table-math"))]
405        {
406            Block64(map_ct_64(value.0, &FLAT_TO_TOWER_BASIS_64.0))
407        }
408    }
409
410    #[inline(always)]
411    fn add_hardware(lhs: Flat<Self>, rhs: Flat<Self>) -> Flat<Self> {
412        Flat::from_raw(lhs.into_raw() + rhs.into_raw())
413    }
414
415    #[inline(always)]
416    fn add_hardware_packed(lhs: PackedFlat<Self>, rhs: PackedFlat<Self>) -> PackedFlat<Self> {
417        let lhs = lhs.into_raw();
418        let rhs = rhs.into_raw();
419
420        #[cfg(target_arch = "aarch64")]
421        {
422            PackedFlat::from_raw(neon::add_packed_64(lhs, rhs))
423        }
424
425        #[cfg(not(target_arch = "aarch64"))]
426        {
427            PackedFlat::from_raw(lhs + rhs)
428        }
429    }
430
431    #[inline(always)]
432    fn mul_hardware(lhs: Flat<Self>, rhs: Flat<Self>) -> Flat<Self> {
433        let lhs = lhs.into_raw();
434        let rhs = rhs.into_raw();
435
436        #[cfg(pmull)]
437        {
438            Flat::from_raw(neon::mul_flat_64(lhs, rhs))
439        }
440
441        #[cfg(not(pmull))]
442        {
443            let a_tower = Self::from_hardware(Flat::from_raw(lhs));
444            let b_tower = Self::from_hardware(Flat::from_raw(rhs));
445
446            (a_tower * b_tower).to_hardware()
447        }
448    }
449
450    #[inline(always)]
451    fn mul_hardware_packed(lhs: PackedFlat<Self>, rhs: PackedFlat<Self>) -> PackedFlat<Self> {
452        let lhs = lhs.into_raw();
453        let rhs = rhs.into_raw();
454
455        #[cfg(pmull)]
456        {
457            PackedFlat::from_raw(neon::mul_flat_packed_64(lhs, rhs))
458        }
459
460        #[cfg(not(pmull))]
461        {
462            let mut l = [Self::ZERO; <Self as PackableField>::WIDTH];
463            let mut r = [Self::ZERO; <Self as PackableField>::WIDTH];
464            let mut res = [Self::ZERO; <Self as PackableField>::WIDTH];
465
466            Self::unpack(lhs, &mut l);
467            Self::unpack(rhs, &mut r);
468
469            for i in 0..<Self as PackableField>::WIDTH {
470                res[i] = Self::mul_hardware(Flat::from_raw(l[i]), Flat::from_raw(r[i])).into_raw();
471            }
472
473            PackedFlat::from_raw(Self::pack(&res))
474        }
475    }
476
477    #[inline(always)]
478    fn mul_hardware_scalar_packed(lhs: PackedFlat<Self>, rhs: Flat<Self>) -> PackedFlat<Self> {
479        let broadcasted = PackedBlock64([rhs.into_raw(); PACKED_WIDTH_64]);
480        Self::mul_hardware_packed(lhs, PackedFlat::from_raw(broadcasted))
481    }
482
483    #[inline(always)]
484    fn tower_bit_from_hardware(value: Flat<Self>, bit_idx: usize) -> u8 {
485        let mask = FLAT_TO_TOWER_BIT_MASKS_64[bit_idx];
486
487        // Parity of (x & mask) without popcount
488        // Folds 64 bits down to 4,
489        // then uses a lookup table.
490        let mut v = value.into_raw().0 & mask;
491        v ^= v >> 32;
492        v ^= v >> 16;
493        v ^= v >> 8;
494        v ^= v >> 4;
495
496        let idx = (v & 0xF) as u8;
497
498        // Nibble parity lookup encoded
499        // in a 16-bit constant (0x6996).
500        ((0x6996u16 >> idx) & 1) as u8
501    }
502}
503
504impl FlatPromote<Block8> for Block64 {
505    #[inline(always)]
506    fn promote_flat(val: Flat<Block8>) -> Flat<Self> {
507        let val = val.into_raw();
508
509        #[cfg(not(feature = "table-math"))]
510        {
511            Flat::from_raw(Block64(lift_ct_64::<8>(
512                val.0 as u64,
513                &constants::LIFT_BASIS_8_TO_64,
514            )))
515        }
516
517        #[cfg(feature = "table-math")]
518        {
519            Flat::from_raw(Block64(constants::LIFT_TABLE_8_TO_64[val.0 as usize]))
520        }
521    }
522}
523
524impl FlatPromote<Block16> for Block64 {
525    #[inline(always)]
526    fn promote_flat(val: Flat<Block16>) -> Flat<Self> {
527        let val = val.into_raw();
528
529        #[cfg(not(feature = "table-math"))]
530        {
531            Flat::from_raw(Block64(lift_ct_64::<16>(
532                val.0 as u64,
533                &constants::LIFT_BASIS_16_TO_64,
534            )))
535        }
536
537        #[cfg(feature = "table-math")]
538        {
539            let v = val.0;
540            let res = constants::PROMOTE_16_BYTE_0_TO_64[(v & 0xFF) as usize]
541                ^ constants::PROMOTE_16_BYTE_1_TO_64[(v >> 8) as usize];
542
543            Flat::from_raw(Block64(res))
544        }
545    }
546}
547
548impl FlatPromote<Block32> for Block64 {
549    #[inline(always)]
550    fn promote_flat(val: Flat<Block32>) -> Flat<Self> {
551        let val = val.into_raw();
552
553        #[cfg(not(feature = "table-math"))]
554        {
555            Flat::from_raw(Block64(lift_ct_64::<32>(
556                val.0 as u64,
557                &constants::LIFT_BASIS_32_TO_64,
558            )))
559        }
560
561        #[cfg(feature = "table-math")]
562        {
563            let v = val.0;
564            let res = constants::PROMOTE_32_BYTE_0_TO_64[(v & 0xFF) as usize]
565                ^ constants::PROMOTE_32_BYTE_1_TO_64[((v >> 8) & 0xFF) as usize]
566                ^ constants::PROMOTE_32_BYTE_2_TO_64[((v >> 16) & 0xFF) as usize]
567                ^ constants::PROMOTE_32_BYTE_3_TO_64[(v >> 24) as usize];
568
569            Flat::from_raw(Block64(res))
570        }
571    }
572}
573
574// ===========================================
575// Binary Field Extras
576// ===========================================
577
578impl_binary_field_extras!(Block64, map_ct_64, TRACE_MASK_64, SOLVE_QUADRATIC_BASIS_64);
579
580// ===========================================
581// UTILS
582// ===========================================
583
584/// Returns `a * b` through the flat basis, φ⁻¹(φ(a) · φ(b)), with
585/// the PMULL kernel; aarch64 with the `aes` target feature only.
586#[cfg(pmull)]
587#[inline(always)]
588pub fn mul_iso_64(a: Block64, b: Block64) -> Block64 {
589    let a_flat = a.to_hardware();
590    let b_flat = b.to_hardware();
591
592    let c_flat = Flat::from_raw(neon::mul_flat_64(a_flat.into_raw(), b_flat.into_raw()));
593
594    c_flat.to_tower()
595}
596
597#[cfg(feature = "table-math")]
598#[inline(always)]
599pub(crate) fn apply_matrix_64(val: Block64, table: &[u64; 2048]) -> Block64 {
600    let mut res = 0u64;
601    let v = val.0;
602
603    // 8 lookups (8-bit window)
604    for i in 0..8 {
605        let byte = (v >> (i * 8)) & 0xFF;
606        let idx = (i * 256) + (byte as usize);
607
608        // SAFETY:
609        // idx = i * 256 + byte < 8 * 256 = table.len().
610        res ^= unsafe { *table.get_unchecked(idx) };
611    }
612
613    Block64(res)
614}
615
616#[cfg(not(feature = "table-math"))]
617#[inline(always)]
618fn lift_ct_64<const N: usize>(x: u64, basis: &[u64; N]) -> u64 {
619    let mut acc = 0u64;
620    let mut i = 0usize;
621
622    while i < N {
623        let bit = (x >> i) & 1;
624        let mask = 0u64.wrapping_sub(bit);
625
626        acc ^= basis[i] & mask;
627        i += 1;
628    }
629
630    acc
631}
632
633#[inline(always)]
634fn map_ct_64(x: u64, basis: &[u64; 64]) -> u64 {
635    let mut acc = 0u64;
636    let mut i = 0usize;
637
638    while i < 64 {
639        let bit = (x >> i) & 1;
640        let mask = 0u64.wrapping_sub(bit);
641
642        acc ^= basis[i] & mask;
643        i += 1;
644    }
645
646    acc
647}
648
649// ===========================================
650// SIMD INSTRUCTIONS
651// ===========================================
652
653#[cfg(target_arch = "aarch64")]
654mod neon {
655    use super::*;
656    use core::arch::aarch64::*;
657    use core::mem::transmute;
658
659    const _: () = assert!(constants::POLY_64 == 0x1b, "verus twins hardcode R = 0x1b");
660
661    #[inline(always)]
662    pub fn add_packed_64(lhs: PackedBlock64, rhs: PackedBlock64) -> PackedBlock64 {
663        unsafe {
664            let l: uint8x16_t = transmute::<[Block64; PACKED_WIDTH_64], uint8x16_t>(lhs.0);
665            let r: uint8x16_t = transmute::<[Block64; PACKED_WIDTH_64], uint8x16_t>(rhs.0);
666            let res = veorq_u8(l, r);
667            let out: [Block64; PACKED_WIDTH_64] =
668                transmute::<uint8x16_t, [Block64; PACKED_WIDTH_64]>(res);
669
670            PackedBlock64(out)
671        }
672    }
673
674    #[cfg(pmull)]
675    #[inline(always)]
676    pub fn mul_flat_packed_64(lhs: PackedBlock64, rhs: PackedBlock64) -> PackedBlock64 {
677        unsafe {
678            let ap: poly64x2_t = transmute(lhs.0);
679            let bp: poly64x2_t = transmute(rhs.0);
680
681            // vdupq_n_p64, not [c; 2]: the repeat
682            // form lowers to a memcpy libcall.
683            let rv: poly64x2_t = vdupq_n_p64(constants::POLY_64);
684
685            // Both lanes per stage: PMULL/PMULL2 pairs,
686            // uzp1/uzp2 regroup [lo, hi] products by lane.
687            let p0: uint64x2_t =
688                transmute(vmull_p64(vgetq_lane_p64::<0>(ap), vgetq_lane_p64::<0>(bp)));
689            let p1: uint64x2_t = transmute(vmull_high_p64(ap, bp));
690
691            let los = vuzp1q_u64(p0, p1);
692            let his = vuzp2q_u64(p0, p1);
693            let hisp: poly64x2_t = transmute(his);
694
695            let hr0: uint64x2_t = transmute(vmull_p64(
696                vgetq_lane_p64::<0>(hisp),
697                vgetq_lane_p64::<0>(rv),
698            ));
699            let hr1: uint64x2_t = transmute(vmull_high_p64(hisp, rv));
700
701            let folded = vuzp1q_u64(hr0, hr1);
702            let carries = vuzp2q_u64(hr0, hr1);
703            let cp: poly64x2_t = transmute(carries);
704
705            let cr0: uint64x2_t =
706                transmute(vmull_p64(vgetq_lane_p64::<0>(cp), vgetq_lane_p64::<0>(rv)));
707            let cr1: uint64x2_t = transmute(vmull_high_p64(cp, rv));
708            let carry_red = vuzp1q_u64(cr0, cr1);
709
710            let res = veorq_u64(veorq_u64(los, folded), carry_red);
711
712            PackedBlock64(transmute::<uint64x2_t, [Block64; 2]>(res))
713        }
714    }
715
716    #[cfg(pmull)]
717    #[inline(always)]
718    pub fn mul_flat_64(a: Block64, b: Block64) -> Block64 {
719        unsafe {
720            // vdupq_n_p64, not [c; 2]: the repeat
721            // form lowers to a memcpy libcall.
722            let rv: poly64x2_t = vdupq_n_p64(constants::POLY_64);
723
724            // Reduction P(x) = x^64 + R(x), R(x) = 0x1b
725            let prod: uint8x16_t = transmute(vmull_p64(a.0, b.0));
726            let h_red: uint8x16_t = transmute(vmull_high_p64(
727                transmute::<uint8x16_t, poly64x2_t>(prod),
728                rv,
729            ));
730
731            // carry = hi lane of h_red < 2^3;
732            // deg(carry·R) < 7: no third fold.
733            let c_red: uint8x16_t = transmute(vmull_high_p64(
734                transmute::<uint8x16_t, poly64x2_t>(h_red),
735                rv,
736            ));
737
738            let res = veorq_u8(veorq_u8(prod, h_red), c_red);
739
740            Block64(vgetq_lane_u64(vreinterpretq_u64_u8(res), 0))
741        }
742    }
743}
744
745// ==================================
746// BLOCK 64 TESTS
747// ==================================
748
749#[cfg(test)]
750mod tests {
751    use super::*;
752    use proptest::prelude::*;
753    use rand::{RngExt, rng};
754
755    // ==================================
756    // BASIC
757    // ==================================
758
759    #[test]
760    fn tower_constants() {
761        // Check that tau is propagated correctly
762        // For Block64, tau must be (0, 1) from Block32.
763        let tau64 = Block64::EXTENSION_TAU;
764        let (lo64, hi64) = tau64.split();
765        assert_eq!(lo64, Block32::ZERO);
766        assert_eq!(hi64, Block32::TAU);
767    }
768
769    #[test]
770    fn add_truth() {
771        let zero = Block64::ZERO;
772        let one = Block64::ONE;
773
774        assert_eq!(zero + zero, zero);
775        assert_eq!(zero + one, one);
776        assert_eq!(one + zero, one);
777        assert_eq!(one + one, zero);
778    }
779
780    #[test]
781    fn mul_truth() {
782        let zero = Block64::ZERO;
783        let one = Block64::ONE;
784
785        assert_eq!(zero * zero, zero);
786        assert_eq!(zero * one, zero);
787        assert_eq!(one * one, one);
788    }
789
790    #[test]
791    fn add() {
792        // 5 ^ 3 = 6
793        // 101 ^ 011 = 110
794        assert_eq!(Block64(5) + Block64(3), Block64(6));
795    }
796
797    #[test]
798    fn mul_simple() {
799        // Check for prime numbers (without overflow)
800        // x^1 * x^1 = x^2 (2 * 2 = 4)
801        assert_eq!(Block64(2) * Block64(2), Block64(4));
802    }
803
804    #[test]
805    fn mul_overflow() {
806        // Reduction verification (AES test vectors)
807        // Example from the AES specification:
808        // 0x57 * 0x83 = 0xC1
809        assert_eq!(Block64(0x57) * Block64(0x83), Block64(0xC1));
810    }
811
812    #[test]
813    fn karatsuba_correctness() {
814        // Let's check using Block64 as an example
815        // Let A = X (hi=1, lo=0)
816        // Let B = X (hi=1, lo=0)
817        // A * B = X^2
818        // According to the rule:
819        // X^2 = X + tau
820        // Where tau for Block32 = 0x2000_0000.
821        // So the result should be:
822        // hi=1 (X), lo=0x20 (tau)
823
824        // Construct X manually
825        let x = Block64::new(Block32::ZERO, Block32::ONE);
826        let squared = x * x;
827
828        // Verify result via splitting
829        let (res_lo, res_hi) = squared.split();
830
831        assert_eq!(res_hi, Block32::ONE, "X^2 should contain X component");
832        assert_eq!(
833            res_lo,
834            Block32(0x2000_0000),
835            "X^2 should contain tau component (0x2000_0000)"
836        );
837    }
838
839    #[test]
840    fn mul_tau_matches_mul() {
841        let mut rng = rng();
842        let edges = [Block64::ZERO, Block64::ONE, Block64::TAU, Block64(u64::MAX)];
843        let random = (0..1000).map(|_| Block64(rng.random()));
844
845        for x in edges.into_iter().chain(random) {
846            assert_eq!(
847                x.mul_tau(),
848                x * Block64::EXTENSION_TAU,
849                "Block64 mul_tau mismatch at {:#018x}",
850                x.0
851            );
852        }
853    }
854
855    #[test]
856    fn security_zeroize() {
857        let mut secret_val = Block64::from(0xDEAD_BEEF_CAFE_BABE_u64);
858        assert_ne!(secret_val, Block64::ZERO);
859
860        secret_val.zeroize();
861
862        assert_eq!(secret_val, Block64::ZERO);
863        assert_eq!(secret_val.0, 0, "Block64 memory leak detected");
864    }
865
866    #[test]
867    fn invert_zero() {
868        // Zero check
869        assert_eq!(
870            Block64::ZERO.invert(),
871            Block64::ZERO,
872            "invert(0) must return 0"
873        );
874    }
875
876    #[test]
877    fn inversion_random() {
878        let mut rng = rng();
879        for _ in 0..1000 {
880            let val = Block64(rng.random());
881            if val != Block64::ZERO {
882                let inv = val.invert();
883                assert_eq!(
884                    val * inv,
885                    Block64::ONE,
886                    "Inversion identity failed: a * a^-1 != 1"
887                );
888            }
889        }
890    }
891
892    #[test]
893    fn tower_embedding() {
894        let mut rng = rng();
895        for _ in 0..100 {
896            let a = Block32(rng.random());
897            let b = Block32(rng.random());
898
899            // 1. Structure check
900            let a_lifted: Block64 = a.into();
901            let (lo, hi) = a_lifted.split();
902
903            assert_eq!(lo, a, "Embedding structure failed: low part mismatch");
904            assert_eq!(
905                hi,
906                Block32::ZERO,
907                "Embedding structure failed: high part must be zero"
908            );
909
910            // 2. Addition Homomorphism
911            let sum_sub = a + b;
912            let sum_lifted: Block64 = sum_sub.into();
913            let sum_in_super = Block64::from(a) + Block64::from(b);
914
915            assert_eq!(sum_lifted, sum_in_super, "Homomorphism failed: add");
916
917            // 3. Multiplication Homomorphism
918            let prod_sub = a * b;
919            let prod_lifted: Block64 = prod_sub.into();
920            let prod_in_super = Block64::from(a) * Block64::from(b);
921
922            assert_eq!(prod_lifted, prod_in_super, "Homomorphism failed: mul");
923        }
924    }
925
926    // ==================================
927    // HARDWARE
928    // ==================================
929
930    #[test]
931    fn isomorphism_roundtrip() {
932        let mut rng = rng();
933        for _ in 0..1000 {
934            let val = Block64(rng.random::<u64>());
935            assert_eq!(val.to_hardware().to_tower(), val);
936        }
937    }
938
939    #[test]
940    fn flat_mul_homomorphism() {
941        let mut rng = rng();
942        for _ in 0..1000 {
943            let a = Block64(rng.random());
944            let b = Block64(rng.random());
945
946            let expected_flat = (a * b).to_hardware();
947            let actual_flat = a.to_hardware() * b.to_hardware();
948
949            assert_eq!(
950                actual_flat, expected_flat,
951                "Block64 flat multiplication mismatch: (a*b)^H != a^H * b^H"
952            );
953        }
954    }
955
956    #[test]
957    fn promote_block8_matches_embedding() {
958        for v in 0u16..256 {
959            let x = Block8(v as u8);
960            assert_eq!(
961                <Block64 as FlatPromote<Block8>>::promote_flat(x.to_hardware()),
962                Block64::from(x).to_hardware(),
963                "Block8 -> Block64 promote mismatch at {v:#04x}"
964            );
965        }
966    }
967
968    #[test]
969    fn promote_block16_matches_embedding() {
970        for v in 0u32..0x1_0000 {
971            let x = Block16(v as u16);
972            assert_eq!(
973                <Block64 as FlatPromote<Block16>>::promote_flat(x.to_hardware()),
974                Block64::from(x).to_hardware(),
975                "Block16 -> Block64 promote mismatch at {v:#06x}"
976            );
977        }
978    }
979
980    #[test]
981    fn promote_block32_matches_embedding() {
982        let mut rng = rng();
983        let edges = [Block32::ZERO, Block32::ONE, Block32::TAU, Block32(u32::MAX)];
984        let random = (0..100_000).map(|_| Block32(rng.random()));
985
986        for x in edges.into_iter().chain(random) {
987            assert_eq!(
988                <Block64 as FlatPromote<Block32>>::promote_flat(x.to_hardware()),
989                Block64::from(x).to_hardware(),
990                "Block32 -> Block64 promote mismatch at {:#010x}",
991                x.0
992            );
993        }
994    }
995
996    #[test]
997    fn packed_consistency() {
998        let mut rng = rng();
999        for _ in 0..100 {
1000            let a_vals = [Block64(rng.random()), Block64(rng.random())];
1001            let b_vals = [Block64(rng.random()), Block64(rng.random())];
1002
1003            let a_flat_vals = a_vals.map(|x| x.to_hardware());
1004            let b_flat_vals = b_vals.map(|x| x.to_hardware());
1005            let a_packed = Flat::<Block64>::pack(&a_flat_vals);
1006            let b_packed = Flat::<Block64>::pack(&b_flat_vals);
1007
1008            // 1. Test SIMD Add (XOR)
1009            let add_res = Block64::add_hardware_packed(a_packed, b_packed);
1010
1011            let mut add_out = [Block64::ZERO.to_hardware(); 2];
1012            Flat::<Block64>::unpack(add_res, &mut add_out);
1013
1014            assert_eq!(add_out[0], (a_vals[0] + b_vals[0]).to_hardware());
1015            assert_eq!(add_out[1], (a_vals[1] + b_vals[1]).to_hardware());
1016
1017            // 2. Test SIMD Mul (Isomorphic/Flat basis)
1018            let mul_res = Block64::mul_hardware_packed(a_packed, b_packed);
1019
1020            let mut mul_out = [Block64::ZERO.to_hardware(); 2];
1021            Flat::<Block64>::unpack(mul_res, &mut mul_out);
1022
1023            assert_eq!(
1024                mul_out[0],
1025                (a_vals[0] * b_vals[0]).to_hardware(),
1026                "Block64 SIMD mul mismatch at index 0"
1027            );
1028            assert_eq!(
1029                mul_out[1],
1030                (a_vals[1] * b_vals[1]).to_hardware(),
1031                "Block64 SIMD mul mismatch at index 1"
1032            );
1033        }
1034    }
1035
1036    // ==================================
1037    // PACKED
1038    // ==================================
1039
1040    #[test]
1041    fn pack_unpack_roundtrip() {
1042        let mut rng = rng();
1043        let data = [Block64(rng.random()), Block64(rng.random())];
1044
1045        let packed = Block64::pack(&data);
1046        let mut unpacked = [Block64::ZERO; 2];
1047
1048        Block64::unpack(packed, &mut unpacked);
1049        assert_eq!(data, unpacked);
1050    }
1051
1052    #[test]
1053    fn packed_add_consistency() {
1054        let mut rng = rng();
1055        let a_vals = [Block64(rng.random()), Block64(rng.random())];
1056        let b_vals = [Block64(rng.random()), Block64(rng.random())];
1057
1058        let res_packed = Block64::pack(&a_vals) + Block64::pack(&b_vals);
1059        let mut res_unpacked = [Block64::ZERO; 2];
1060        Block64::unpack(res_packed, &mut res_unpacked);
1061
1062        assert_eq!(res_unpacked[0], a_vals[0] + b_vals[0]);
1063        assert_eq!(res_unpacked[1], a_vals[1] + b_vals[1]);
1064    }
1065
1066    #[test]
1067    fn packed_mul_consistency() {
1068        let mut rng = rng();
1069
1070        for _ in 0..1000 {
1071            let mut a_arr = [Block64::ZERO; PACKED_WIDTH_64];
1072            let mut b_arr = [Block64::ZERO; PACKED_WIDTH_64];
1073
1074            for i in 0..PACKED_WIDTH_64 {
1075                let val_a: u64 = rng.random();
1076                let val_b: u64 = rng.random();
1077                a_arr[i] = Block64(val_a);
1078                b_arr[i] = Block64(val_b);
1079            }
1080
1081            let a_packed = PackedBlock64(a_arr);
1082            let b_packed = PackedBlock64(b_arr);
1083
1084            // Perform SIMD multiplication
1085            let c_packed = a_packed * b_packed;
1086
1087            // Verify against Scalar
1088            let mut c_expected = [Block64::ZERO; PACKED_WIDTH_64];
1089            for i in 0..PACKED_WIDTH_64 {
1090                c_expected[i] = a_arr[i] * b_arr[i];
1091            }
1092
1093            assert_eq!(c_packed.0, c_expected, "SIMD Block64 mismatch!");
1094        }
1095    }
1096
1097    proptest! {
1098        #[test]
1099        fn parity_masks_match_from_hardware(x_flat in any::<u64>()) {
1100            let tower = Block64::from_hardware(Flat::from_raw(Block64(x_flat))).0;
1101
1102            for (k, &mask) in FLAT_TO_TOWER_BIT_MASKS_64.iter().enumerate() {
1103                // Ensure the static masks
1104                // themselves are correct.
1105                let parity = ((x_flat & mask).count_ones() & 1) as u8;
1106                let bit = ((tower >> k) & 1) as u8;
1107                prop_assert_eq!(parity, bit, "Block64 static mask mismatch at k={}", k);
1108
1109                // Ensure XOR-tree implementation matches.
1110                let via_api = Flat::from_raw(Block64(x_flat)).tower_bit(k);
1111                prop_assert_eq!(
1112                    via_api, bit,
1113                    "Block64 tower_bit_from_hardware mismatch at x_flat={:#018x}, bit_idx={}",
1114                    x_flat, k
1115                );
1116            }
1117        }
1118    }
1119}