Skip to main content

EngramHash

cortiq_engine::dsv41

Struct EngramHash

pub struct EngramHash {
    pub layer_ids: Vec<usize>,
    pub max_ngram: usize,
    pub n_heads: usize,
    pub compressed_vocab: usize,
    pub pad_id: i64,
    pub primes: Vec<Vec<Vec<u64>>>,
    pub offsets: Vec<Vec<u64>>,
    pub token_map: Vec<u32>,
    pub multipliers: Vec<[u64; 4]>,
    /* private fields */
}

Expand description

The exact source layout of the V4.1 n-gram tables. The two configured Engram layers have disjoint prime ranges, even though both tables use the same nominal 16M vocabulary.

Fields§

§layer_ids: Vec<usize>§max_ngram: usize§n_heads: usize§compressed_vocab: usize§pad_id: i64§primes: Vec<Vec<Vec<u64>>>§offsets: Vec<Vec<u64>>§token_map: Vec<u32>§multipliers: Vec<[u64; 4]>

Implementations§

impl EngramHash

pub fn new( layer_ids: Vec<usize>, max_ngram: usize, n_heads: usize, table_vocab: usize, compressed_vocab: usize, pad_token: usize, token_map: Vec<u32>, ) -> Result<Self, String>

Examples found in repository ?

examples/dsv41_engram_proof.rs (lines 196-204)

164fn verify_hash_reference(
165    model_path: &Path,
166    reference_path: &Path,
167    map_path: &Path,
168) -> Result<(Value, Vec<Vec<Vec<Vec<usize>>>>), Box<dyn Error>> {
169    let reference: Value = serde_json::from_slice(&fs::read(reference_path)?)?;
170    let model = CmfModel::open(model_path)?;
171    let vocab = model.header.arch.vocab_size;
172    let token_map = token_map_from_model(&model, vocab);
173    let mut bytes = Vec::with_capacity(token_map.len() * 4);
174    for value in &token_map {
175        bytes.extend_from_slice(&value.to_le_bytes());
176    }
177    fs::write(map_path, &bytes)?;
178    let got_sha = sha256sum(map_path)?;
179    let expected_sha = reference["token_map_le_u32_sha256"]
180        .as_str()
181        .ok_or("reference token map SHA is missing")?;
182    println!(
183        "hash token_map vocab={} compressed_vocab={} sha256={got_sha}",
184        token_map.len(),
185        token_map.iter().copied().max().unwrap_or(0) + 1
186    );
187    if got_sha != expected_sha {
188        return Err(
189            format!("token map SHA mismatch: runtime={got_sha} oracle={expected_sha}").into(),
190        );
191    }
192
193    let compressed_vocab = reference["compressed_vocab_size"]
194        .as_u64()
195        .ok_or("reference compressed vocab is missing")? as usize;
196    let hash = EngramHash::new(
197        vec![1, 14],
198        4,
199        8,
200        16_000_000,
201        compressed_vocab,
202        2,
203        token_map,
204    )?;
205    let expected_pad = reference["compressed_pad_id"]
206        .as_i64()
207        .ok_or("reference compressed pad is missing")?;
208    if hash.pad_id != expected_pad {
209        return Err(format!(
210            "pad id mismatch: runtime={} oracle={expected_pad}",
211            hash.pad_id
212        )
213        .into());
214    }
215    let expected_multipliers: Vec<[u64; 4]> =
216        serde_json::from_value(reference["multipliers"].clone())?;
217    let expected_primes: Vec<Vec<Vec<u64>>> = serde_json::from_value(reference["primes"].clone())?;
218    let expected_offsets: Vec<Vec<u64>> = serde_json::from_value(reference["offsets"].clone())?;
219    if hash.multipliers != expected_multipliers {
220        return Err(format!(
221            "multiplier mismatch: runtime={:?} oracle={expected_multipliers:?}",
222            hash.multipliers
223        )
224        .into());
225    }
226    if hash.primes != expected_primes {
227        return Err("prime layout mismatch".into());
228    }
229    let actual_offsets: Vec<Vec<u64>> = hash
230        .offsets
231        .iter()
232        .zip(&hash.primes)
233        .map(|(starts, per_ngram)| {
234            per_ngram
235                .iter()
236                .zip(starts)
237                .flat_map(|(primes, &start)| {
238                    let mut offset = start;
239                    primes.iter().map(move |&prime| {
240                        let current = offset;
241                        offset += prime;
242                        current
243                    })
244                })
245                .collect()
246        })
247        .collect();
248    if actual_offsets != expected_offsets {
249        return Err("offset layout mismatch".into());
250    }
251    println!(
252        "hash layout layers={} primes={} offsets={} multipliers=exact",
253        hash.layer_ids.len(),
254        hash.primes
255            .iter()
256            .map(|x| x.iter().map(Vec::len).sum::<usize>())
257            .sum::<usize>(),
258        hash.offsets.iter().map(Vec::len).sum::<usize>()
259    );
260
261    let cases = reference["cases"]
262        .as_array()
263        .ok_or("reference cases missing")?;
264    let mut all_hashes = Vec::with_capacity(cases.len());
265    for (case_no, case) in cases.iter().enumerate() {
266        let ids: Vec<u32> = serde_json::from_value(case["input_ids"].clone())?;
267        let mask: Option<Vec<bool>> = if case["token_mask"].is_null() {
268            None
269        } else {
270            Some(serde_json::from_value(case["token_mask"].clone())?)
271        };
272        let expected_hashes: Vec<Vec<Vec<usize>>> = serde_json::from_value(case["hashes"].clone())?;
273        let mut state = hash.clone();
274        state.reset();
275        let mut got_hashes = Vec::with_capacity(ids.len());
276        for (pos, &id) in ids.iter().enumerate() {
277            got_hashes.push(state.push(id, mask.as_ref().map(|m| m[pos]).unwrap_or(true)));
278        }
279        if got_hashes != expected_hashes {
280            let first = got_hashes
281                .iter()
282                .zip(&expected_hashes)
283                .enumerate()
284                .find(|(_, (a, b))| a != b)
285                .map(|(i, (a, b))| (i, a, b));
286            return Err(format!("hash case {case_no} mismatch: {first:?}").into());
287        }
288        let cuts: Vec<usize> = serde_json::from_value(case["chunk_ends"].clone())?;
289        let mut chunk_state = hash.clone();
290        chunk_state.reset();
291        let mut chunk_hashes = Vec::with_capacity(ids.len());
292        let mut start = 0;
293        for end in cuts {
294            for pos in start..end {
295                chunk_hashes.push(
296                    chunk_state.push(ids[pos], mask.as_ref().map(|m| m[pos]).unwrap_or(true)),
297                );
298            }
299            start = end;
300        }
301        if start != ids.len() || chunk_hashes != expected_hashes {
302            return Err(format!("hash case {case_no} chunked sequence mismatch").into());
303        }
304        println!(
305            "hash case={} name={} seq={} exact_indices=true chunked=true",
306            case_no,
307            case["name"].as_str().unwrap_or("?"),
308            ids.len()
309        );
310        all_hashes.push(expected_hashes);
311    }
312    Ok((reference, all_hashes))
313}

pub fn reset(&mut self)

Examples found in repository ?

examples/dsv41_engram_proof.rs (line 274)

164fn verify_hash_reference(
165    model_path: &Path,
166    reference_path: &Path,
167    map_path: &Path,
168) -> Result<(Value, Vec<Vec<Vec<Vec<usize>>>>), Box<dyn Error>> {
169    let reference: Value = serde_json::from_slice(&fs::read(reference_path)?)?;
170    let model = CmfModel::open(model_path)?;
171    let vocab = model.header.arch.vocab_size;
172    let token_map = token_map_from_model(&model, vocab);
173    let mut bytes = Vec::with_capacity(token_map.len() * 4);
174    for value in &token_map {
175        bytes.extend_from_slice(&value.to_le_bytes());
176    }
177    fs::write(map_path, &bytes)?;
178    let got_sha = sha256sum(map_path)?;
179    let expected_sha = reference["token_map_le_u32_sha256"]
180        .as_str()
181        .ok_or("reference token map SHA is missing")?;
182    println!(
183        "hash token_map vocab={} compressed_vocab={} sha256={got_sha}",
184        token_map.len(),
185        token_map.iter().copied().max().unwrap_or(0) + 1
186    );
187    if got_sha != expected_sha {
188        return Err(
189            format!("token map SHA mismatch: runtime={got_sha} oracle={expected_sha}").into(),
190        );
191    }
192
193    let compressed_vocab = reference["compressed_vocab_size"]
194        .as_u64()
195        .ok_or("reference compressed vocab is missing")? as usize;
196    let hash = EngramHash::new(
197        vec![1, 14],
198        4,
199        8,
200        16_000_000,
201        compressed_vocab,
202        2,
203        token_map,
204    )?;
205    let expected_pad = reference["compressed_pad_id"]
206        .as_i64()
207        .ok_or("reference compressed pad is missing")?;
208    if hash.pad_id != expected_pad {
209        return Err(format!(
210            "pad id mismatch: runtime={} oracle={expected_pad}",
211            hash.pad_id
212        )
213        .into());
214    }
215    let expected_multipliers: Vec<[u64; 4]> =
216        serde_json::from_value(reference["multipliers"].clone())?;
217    let expected_primes: Vec<Vec<Vec<u64>>> = serde_json::from_value(reference["primes"].clone())?;
218    let expected_offsets: Vec<Vec<u64>> = serde_json::from_value(reference["offsets"].clone())?;
219    if hash.multipliers != expected_multipliers {
220        return Err(format!(
221            "multiplier mismatch: runtime={:?} oracle={expected_multipliers:?}",
222            hash.multipliers
223        )
224        .into());
225    }
226    if hash.primes != expected_primes {
227        return Err("prime layout mismatch".into());
228    }
229    let actual_offsets: Vec<Vec<u64>> = hash
230        .offsets
231        .iter()
232        .zip(&hash.primes)
233        .map(|(starts, per_ngram)| {
234            per_ngram
235                .iter()
236                .zip(starts)
237                .flat_map(|(primes, &start)| {
238                    let mut offset = start;
239                    primes.iter().map(move |&prime| {
240                        let current = offset;
241                        offset += prime;
242                        current
243                    })
244                })
245                .collect()
246        })
247        .collect();
248    if actual_offsets != expected_offsets {
249        return Err("offset layout mismatch".into());
250    }
251    println!(
252        "hash layout layers={} primes={} offsets={} multipliers=exact",
253        hash.layer_ids.len(),
254        hash.primes
255            .iter()
256            .map(|x| x.iter().map(Vec::len).sum::<usize>())
257            .sum::<usize>(),
258        hash.offsets.iter().map(Vec::len).sum::<usize>()
259    );
260
261    let cases = reference["cases"]
262        .as_array()
263        .ok_or("reference cases missing")?;
264    let mut all_hashes = Vec::with_capacity(cases.len());
265    for (case_no, case) in cases.iter().enumerate() {
266        let ids: Vec<u32> = serde_json::from_value(case["input_ids"].clone())?;
267        let mask: Option<Vec<bool>> = if case["token_mask"].is_null() {
268            None
269        } else {
270            Some(serde_json::from_value(case["token_mask"].clone())?)
271        };
272        let expected_hashes: Vec<Vec<Vec<usize>>> = serde_json::from_value(case["hashes"].clone())?;
273        let mut state = hash.clone();
274        state.reset();
275        let mut got_hashes = Vec::with_capacity(ids.len());
276        for (pos, &id) in ids.iter().enumerate() {
277            got_hashes.push(state.push(id, mask.as_ref().map(|m| m[pos]).unwrap_or(true)));
278        }
279        if got_hashes != expected_hashes {
280            let first = got_hashes
281                .iter()
282                .zip(&expected_hashes)
283                .enumerate()
284                .find(|(_, (a, b))| a != b)
285                .map(|(i, (a, b))| (i, a, b));
286            return Err(format!("hash case {case_no} mismatch: {first:?}").into());
287        }
288        let cuts: Vec<usize> = serde_json::from_value(case["chunk_ends"].clone())?;
289        let mut chunk_state = hash.clone();
290        chunk_state.reset();
291        let mut chunk_hashes = Vec::with_capacity(ids.len());
292        let mut start = 0;
293        for end in cuts {
294            for pos in start..end {
295                chunk_hashes.push(
296                    chunk_state.push(ids[pos], mask.as_ref().map(|m| m[pos]).unwrap_or(true)),
297                );
298            }
299            start = end;
300        }
301        if start != ids.len() || chunk_hashes != expected_hashes {
302            return Err(format!("hash case {case_no} chunked sequence mismatch").into());
303        }
304        println!(
305            "hash case={} name={} seq={} exact_indices=true chunked=true",
306            case_no,
307            case["name"].as_str().unwrap_or("?"),
308            ids.len()
309        );
310        all_hashes.push(expected_hashes);
311    }
312    Ok((reference, all_hashes))
313}

pub fn push(&mut self, token: u32, participates: bool) -> Vec<Vec<usize>>

Append one token and return one flattened hash column vector per configured layer. participates=false is used for image span tokens; dead tokens break every n-gram lookback just like the reference.

Examples found in repository ?

examples/dsv41_engram_proof.rs (line 277)

164fn verify_hash_reference(
165    model_path: &Path,
166    reference_path: &Path,
167    map_path: &Path,
168) -> Result<(Value, Vec<Vec<Vec<Vec<usize>>>>), Box<dyn Error>> {
169    let reference: Value = serde_json::from_slice(&fs::read(reference_path)?)?;
170    let model = CmfModel::open(model_path)?;
171    let vocab = model.header.arch.vocab_size;
172    let token_map = token_map_from_model(&model, vocab);
173    let mut bytes = Vec::with_capacity(token_map.len() * 4);
174    for value in &token_map {
175        bytes.extend_from_slice(&value.to_le_bytes());
176    }
177    fs::write(map_path, &bytes)?;
178    let got_sha = sha256sum(map_path)?;
179    let expected_sha = reference["token_map_le_u32_sha256"]
180        .as_str()
181        .ok_or("reference token map SHA is missing")?;
182    println!(
183        "hash token_map vocab={} compressed_vocab={} sha256={got_sha}",
184        token_map.len(),
185        token_map.iter().copied().max().unwrap_or(0) + 1
186    );
187    if got_sha != expected_sha {
188        return Err(
189            format!("token map SHA mismatch: runtime={got_sha} oracle={expected_sha}").into(),
190        );
191    }
192
193    let compressed_vocab = reference["compressed_vocab_size"]
194        .as_u64()
195        .ok_or("reference compressed vocab is missing")? as usize;
196    let hash = EngramHash::new(
197        vec![1, 14],
198        4,
199        8,
200        16_000_000,
201        compressed_vocab,
202        2,
203        token_map,
204    )?;
205    let expected_pad = reference["compressed_pad_id"]
206        .as_i64()
207        .ok_or("reference compressed pad is missing")?;
208    if hash.pad_id != expected_pad {
209        return Err(format!(
210            "pad id mismatch: runtime={} oracle={expected_pad}",
211            hash.pad_id
212        )
213        .into());
214    }
215    let expected_multipliers: Vec<[u64; 4]> =
216        serde_json::from_value(reference["multipliers"].clone())?;
217    let expected_primes: Vec<Vec<Vec<u64>>> = serde_json::from_value(reference["primes"].clone())?;
218    let expected_offsets: Vec<Vec<u64>> = serde_json::from_value(reference["offsets"].clone())?;
219    if hash.multipliers != expected_multipliers {
220        return Err(format!(
221            "multiplier mismatch: runtime={:?} oracle={expected_multipliers:?}",
222            hash.multipliers
223        )
224        .into());
225    }
226    if hash.primes != expected_primes {
227        return Err("prime layout mismatch".into());
228    }
229    let actual_offsets: Vec<Vec<u64>> = hash
230        .offsets
231        .iter()
232        .zip(&hash.primes)
233        .map(|(starts, per_ngram)| {
234            per_ngram
235                .iter()
236                .zip(starts)
237                .flat_map(|(primes, &start)| {
238                    let mut offset = start;
239                    primes.iter().map(move |&prime| {
240                        let current = offset;
241                        offset += prime;
242                        current
243                    })
244                })
245                .collect()
246        })
247        .collect();
248    if actual_offsets != expected_offsets {
249        return Err("offset layout mismatch".into());
250    }
251    println!(
252        "hash layout layers={} primes={} offsets={} multipliers=exact",
253        hash.layer_ids.len(),
254        hash.primes
255            .iter()
256            .map(|x| x.iter().map(Vec::len).sum::<usize>())
257            .sum::<usize>(),
258        hash.offsets.iter().map(Vec::len).sum::<usize>()
259    );
260
261    let cases = reference["cases"]
262        .as_array()
263        .ok_or("reference cases missing")?;
264    let mut all_hashes = Vec::with_capacity(cases.len());
265    for (case_no, case) in cases.iter().enumerate() {
266        let ids: Vec<u32> = serde_json::from_value(case["input_ids"].clone())?;
267        let mask: Option<Vec<bool>> = if case["token_mask"].is_null() {
268            None
269        } else {
270            Some(serde_json::from_value(case["token_mask"].clone())?)
271        };
272        let expected_hashes: Vec<Vec<Vec<usize>>> = serde_json::from_value(case["hashes"].clone())?;
273        let mut state = hash.clone();
274        state.reset();
275        let mut got_hashes = Vec::with_capacity(ids.len());
276        for (pos, &id) in ids.iter().enumerate() {
277            got_hashes.push(state.push(id, mask.as_ref().map(|m| m[pos]).unwrap_or(true)));
278        }
279        if got_hashes != expected_hashes {
280            let first = got_hashes
281                .iter()
282                .zip(&expected_hashes)
283                .enumerate()
284                .find(|(_, (a, b))| a != b)
285                .map(|(i, (a, b))| (i, a, b));
286            return Err(format!("hash case {case_no} mismatch: {first:?}").into());
287        }
288        let cuts: Vec<usize> = serde_json::from_value(case["chunk_ends"].clone())?;
289        let mut chunk_state = hash.clone();
290        chunk_state.reset();
291        let mut chunk_hashes = Vec::with_capacity(ids.len());
292        let mut start = 0;
293        for end in cuts {
294            for pos in start..end {
295                chunk_hashes.push(
296                    chunk_state.push(ids[pos], mask.as_ref().map(|m| m[pos]).unwrap_or(true)),
297                );
298            }
299            start = end;
300        }
301        if start != ids.len() || chunk_hashes != expected_hashes {
302            return Err(format!("hash case {case_no} chunked sequence mismatch").into());
303        }
304        println!(
305            "hash case={} name={} seq={} exact_indices=true chunked=true",
306            case_no,
307            case["name"].as_str().unwrap_or("?"),
308            ids.len()
309        );
310        all_hashes.push(expected_hashes);
311    }
312    Ok((reference, all_hashes))
313}

Trait Implementations§

impl Clone for EngramHash

fn clone(&self) -> EngramHash

Returns a duplicate of the value. Read more

1.0.0 (const: unstable) · Source§

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source. Read more

Auto Trait Implementations§

impl Freeze for EngramHash

impl RefUnwindSafe for EngramHash

impl Send for EngramHash

impl Sync for EngramHash

impl Unpin for EngramHash

impl UnsafeUnpin for EngramHash

impl UnwindSafe for EngramHash

Blanket Implementations§

impl<T> Any for T
where T: 'static + ?Sized,

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more

impl<T> Borrow<T> for T
where T: ?Sized,

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more

impl<T> BorrowMut<T> for T
where T: ?Sized,

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more

impl<T> CloneToUninit for T
where T: Clone,

unsafe fn clone_to_uninit(&self, dest: *mut u8)

🔬This is a nightly-only experimental API. (clone_to_uninit)

Performs copy-assignment from self to dest. Read more

impl<T> From<T> for T

fn from(t: T) -> T

Returns the argument unchanged.

impl<T> Instrument for T

fn instrument(self, span: Span) -> Instrumented<Self> ⓘ

Instruments this type with the provided Span, returning an Instrumented wrapper. Read more

fn in_current_span(self) -> Instrumented<Self> ⓘ

Instruments this type with the current Span, returning an Instrumented wrapper. Read more

impl<T, U> Into<U> for T
where U: From<T>,

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

impl<T> ToOwned for T
where T: Clone,

type Owned = T

The resulting type after obtaining ownership.

fn to_owned(&self) -> T

Creates owned data from borrowed data, usually by cloning. Read more

fn clone_into(&self, target: &mut T)

Uses borrowed data to replace owned data, usually by cloning. Read more

impl<T, U> TryFrom<U> for T
where U: Into<T>,

type Error = !

The type returned in the event of a conversion error.

fn try_from(value: U) -> Result<T, !>

Performs the conversion.

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.

impl<T> WithSubscriber for T

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self> ⓘ
where S: Into<Dispatch>,

Attaches the provided Subscriber to this type, returning a WithDispatch wrapper. Read more

fn with_current_subscriber(self) -> WithDispatch<Self> ⓘ

Attaches the current default Subscriber to this type, returning a WithDispatch wrapper. Read more