sva_engine/cache/
codec.rs

1// Concern: one stored node value, tagged with the store's format, and one staged run, as bytes and back | Non-concern: where the bytes live or when | IO: (Stored or Buffer) <-> bytes, a bad one None
2
3use sva_formula::Codomain;
4use sva_samples::{
5    Buffer, Cost, Detail, Dropped, Extent, Grid, Label, PSYCHOACOUSTIC_V1, Rule, Source,
6};
7
8use sva_formula::Hash;
9
10use super::Stored;
11use super::stored::{Header, Laid, Samples};
12
13/// Bumped by, and only by, a change to a stored value's bytes.
14pub const STORE_FORMAT: u32 = 12;
15
16/// Every entry opens with its format, so one another format wrote is never read as a value,
17/// even where a wipe left it.
18fn entry_tag() -> Vec<u8> {
19    [&b"SVAh"[..], &STORE_FORMAT.to_le_bytes()].concat()
20}
21
22const STAGED_RUN: &[u8; 4] = b"SVAc";
23
24/// Samples per chunk of a run: what one checksum covers, and the least a read loads. Samples
25/// stay f64, as the render computes them: a stored value is read by others, and f32 would
26/// change the bits of every value that reads it.
27pub(crate) const CHUNK: usize = 4096;
28
29/// A header's length, then the header, then each run's samples chunk by chunk, every plane of
30/// a chunk together, so a stretch of a run is one read.
31pub(crate) fn entry(head: &Header, runs: &[Buffer]) -> Vec<u8> {
32    let mut laid = Vec::new();
33    let mut body = Vec::new();
34    for run in runs {
35        let mut sums = Vec::new();
36        let at = body.len() as u64;
37        for from in (0..run.len()).step_by(CHUNK) {
38            let to = (from + CHUNK).min(run.len());
39            let chunk = body.len();
40            for plane in &run.planes {
41                for sample in &plane[from..to] {
42                    word(&mut body, sample.to_bits());
43                }
44            }
45            sums.push(checksum(&body[chunk..]));
46        }
47        laid.push(Laid {
48            rate: run.rate,
49            start: run.start,
50            width: run.width,
51            len: run.len(),
52            at,
53            sums,
54        });
55    }
56    let head = sealed(header(head, &laid));
57    let mut out = Vec::with_capacity(8 + head.len() + body.len());
58    word(&mut out, head.len() as u64);
59    out.extend_from_slice(&head);
60    out.extend_from_slice(&body);
61    out
62}
63
64fn header(head: &Header, laid: &[Laid]) -> Vec<u8> {
65    let stored = head.stored();
66    let mut out = entry_tag();
67    word(&mut out, stored.key.0);
68    word(&mut out, stored.key.1);
69    labelled(&mut out, &stored.label);
70    out.push(stored.width);
71    out.push(match stored.codomain {
72        Codomain::Real => 0,
73        Codomain::Complex => 1,
74    });
75    maybe(&mut out, stored.rate.map(u64::from));
76    word(&mut out, u64::from(stored.grid.rate));
77    out.extend_from_slice(&stored.grid.a.to_le_bytes());
78    out.extend_from_slice(&stored.grid.d.to_le_bytes());
79    word(&mut out, stored.support.start as u64);
80    word(&mut out, stored.support.end as u64);
81    out.extend_from_slice(&stored.priced.to_le_bytes());
82    word(&mut out, stored.moved.to_bits());
83    out.push(u8::from(stored.readable));
84    match head.samples() {
85        &Samples::Of { key, by } => {
86            out.push(1);
87            word(&mut out, key.0);
88            word(&mut out, key.1);
89            word(&mut out, by as u64);
90        }
91        _ => out.push(0),
92    }
93    word(&mut out, laid.len() as u64);
94    for run in laid {
95        word(&mut out, u64::from(run.rate));
96        word(&mut out, run.start as u64);
97        word(&mut out, run.width as u64);
98        word(&mut out, run.len as u64);
99        for sum in &run.sums {
100            word(&mut out, *sum);
101        }
102    }
103    out
104}
105
106/// How many of an entry's first bytes its header takes, off its first eight.
107pub(crate) fn head_len(first: &[u8]) -> Option<usize> {
108    let len = u64::from_le_bytes(first.get(..8)?.try_into().ok()?);
109    8usize.checked_add(usize::try_from(len).ok()?)
110}
111
112/// The meta an entry's header states, its samples laid in `file`, and the entry's length; a
113/// truncated, corrupt or foreign header is `None`, never a partial value.
114pub(crate) fn read_head(bytes: &[u8], file: Hash) -> Option<(Header, u64)> {
115    let span = head_len(bytes)?;
116    let mut r = Reader(opened(bytes.get(8..span)?, &entry_tag())?);
117    let key = Hash(r.word()?, r.word()?);
118    let label = r.label()?;
119    let width = r.byte()?;
120    let codomain = match r.byte()? {
121        0 => Codomain::Real,
122        1 => Codomain::Complex,
123        _ => return None,
124    };
125    let rate = match r.maybe()? {
126        Some(rate) => Some(u32::try_from(rate).ok()?),
127        None => None,
128    };
129    let grid = Grid {
130        rate: u32::try_from(r.word()?).ok()?,
131        a: r.wide()? as i128,
132        d: r.wide()? as i128,
133    };
134    let (start, end) = (r.word()? as i64, r.word()? as i64);
135    let support = (start <= end).then(|| Extent::new(start, end))?;
136    let priced = r.wide()?;
137    let moved = f64::from_bits(r.word()?);
138    let readable = match r.byte()? {
139        0 => false,
140        1 => true,
141        _ => return None,
142    };
143    let of = match r.byte()? {
144        0 => None,
145        1 => Some(Samples::Of {
146            key: Hash(r.word()?, r.word()?),
147            by: r.word()? as i64,
148        }),
149        _ => return None,
150    };
151    let count = r.word()? as usize;
152    let mut runs = Vec::new();
153    let mut at = span as u64;
154    for _ in 0..count.min(r.0.len()) {
155        let rate = u32::try_from(r.word()?).ok()?;
156        let start = r.word()? as i64;
157        let (width, len) = (r.word()? as usize, r.word()? as usize);
158        let sums = (0..len.div_ceil(CHUNK).min(r.0.len()))
159            .map(|_| r.word())
160            .collect::<Option<Vec<u64>>>()?;
161        if sums.len() != len.div_ceil(CHUNK) {
162            return None;
163        }
164        runs.push(Laid {
165            rate,
166            start,
167            width,
168            len,
169            at,
170            sums,
171        });
172        at = at.checked_add((width.checked_mul(len)?.checked_mul(8)?) as u64)?;
173    }
174    let samples = match (of, runs.is_empty()) {
175        (Some(of), true) => of,
176        (Some(_), false) => return None,
177        (None, true) => Samples::None,
178        (None, false) => Samples::Entry {
179            file,
180            runs,
181            shift: 0,
182        },
183    };
184    let stored = Stored {
185        key,
186        label,
187        width,
188        codomain,
189        rate,
190        grid,
191        support,
192        priced,
193        moved,
194        readable,
195        held: Vec::new(),
196    };
197    let whole = runs_whole(count, &samples) && r.0.is_empty();
198    whole.then(|| (Header::new(stored, samples), at))
199}
200
201fn runs_whole(count: usize, samples: &Samples) -> bool {
202    match samples {
203        Samples::Entry { runs, .. } => runs.len() == count,
204        _ => count == 0,
205    }
206}
207
208/// Where in its file a run's chunks `from..to` lie, and how many bytes.
209pub(crate) fn span_of(run: &Laid, from: usize, to: usize) -> (u64, u64) {
210    let bytes = |chunks: usize| ((chunks * CHUNK).min(run.len) * run.width * 8) as u64;
211    (run.at + bytes(from), bytes(to) - bytes(from))
212}
213
214/// A run's chunks `from..to` off the bytes `span_of` names; `None` where a checksum fails.
215pub(crate) fn read_chunks(bytes: &[u8], run: &Laid, from: usize, to: usize) -> Option<Buffer> {
216    let first = from * CHUNK;
217    let len = (to * CHUNK).min(run.len).checked_sub(first)?;
218    if bytes.len() != len * run.width * 8 {
219        return None;
220    }
221    let mut planes = vec![Vec::with_capacity(len); run.width];
222    let mut at = 0;
223    for (k, sum) in run.sums[from..to].iter().enumerate() {
224        let n = (CHUNK).min(run.len - first - k * CHUNK);
225        let chunk = &bytes[at..at + n * run.width * 8];
226        if checksum(chunk) != *sum {
227            return None;
228        }
229        for (c, plane) in planes.iter_mut().enumerate() {
230            let own = &chunk[c * n * 8..(c + 1) * n * 8];
231            plane.extend(
232                own.chunks_exact(8).map(|b| {
233                    f64::from_bits(u64::from_le_bytes(b.try_into().expect("eight bytes")))
234                }),
235            );
236        }
237        at += chunk.len();
238    }
239    let mut out = Buffer::of_planes(run.rate, planes);
240    out.start = run.start + first as i64;
241    Some(out)
242}
243
244/// Every run an entry under `key` lays out; `None` where any is corrupt.
245pub(crate) fn read_runs(bytes: &[u8], key: Hash) -> Option<Vec<Buffer>> {
246    let (head, _) = read_head(bytes, key)?;
247    let Samples::Entry { runs, .. } = head.samples() else {
248        return Some(Vec::new());
249    };
250    let read = |run: &Laid| {
251        let n = run.len.div_ceil(CHUNK);
252        let (at, len) = span_of(run, 0, n);
253        let span = bytes.get(at as usize..(at + len) as usize)?;
254        read_chunks(span, run, 0, n)
255    };
256    runs.iter().map(read).collect()
257}
258
259pub(crate) fn chunk(samples: &Buffer) -> Vec<u8> {
260    let mut out = STAGED_RUN.to_vec();
261    segments(&mut out, std::slice::from_ref(samples));
262    sealed(out)
263}
264
265pub(crate) fn read_chunk(bytes: &[u8]) -> Option<Buffer> {
266    let mut r = Reader(opened(bytes, STAGED_RUN)?);
267    let mut parts = r.segments()?;
268    (r.0.is_empty() && parts.len() == 1).then(|| parts.remove(0))
269}
270
271fn sealed(mut out: Vec<u8>) -> Vec<u8> {
272    let sum = checksum(&out);
273    word(&mut out, sum);
274    out
275}
276
277/// The body past `magic`, where the checksum holds.
278fn opened<'b>(bytes: &'b [u8], magic: &[u8]) -> Option<&'b [u8]> {
279    let body = bytes.len().checked_sub(8)?;
280    let (body, sum) = bytes.split_at(body);
281    (checksum(body).to_le_bytes() == sum && body.starts_with(magic)).then(|| &body[magic.len()..])
282}
283
284fn segments(out: &mut Vec<u8>, parts: &[Buffer]) {
285    word(out, parts.len() as u64);
286    for part in parts {
287        word(out, u64::from(part.rate));
288        word(out, part.start as u64);
289        word(out, part.width as u64);
290        word(out, part.len() as u64);
291        for plane in &part.planes {
292            for sample in plane {
293                word(out, sample.to_bits());
294            }
295        }
296    }
297}
298
299fn checksum(bytes: &[u8]) -> u64 {
300    bytes.iter().fold(0xcbf2_9ce4_8422_2325, |h, b| {
301        (h ^ u64::from(*b)).wrapping_mul(0x0100_0000_01b3)
302    })
303}
304
305fn word(out: &mut Vec<u8>, w: u64) {
306    out.extend_from_slice(&w.to_le_bytes());
307}
308
309fn float(out: &mut Vec<u8>, v: Option<f64>) {
310    maybe(out, v.map(f64::to_bits));
311}
312
313fn maybe(out: &mut Vec<u8>, w: Option<u64>) {
314    match w {
315        None => out.push(0),
316        Some(w) => {
317            out.push(1);
318            word(out, w);
319        }
320    }
321}
322
323fn text(out: &mut Vec<u8>, s: &str) {
324    word(out, s.len() as u64);
325    out.extend_from_slice(s.as_bytes());
326}
327
328fn labelled(out: &mut Vec<u8>, label: &Label) {
329    out.push(match label.source {
330        Source::Exact => 0,
331        Source::Measured => 1,
332    });
333    text(out, label.profile);
334    word(out, u64::from(label.rate));
335    detailed(out, &label.detail);
336    match label.cost {
337        None => out.push(0),
338        Some(Cost { flops, budget }) => {
339            out.push(1);
340            out.extend_from_slice(&flops.to_le_bytes());
341            out.extend_from_slice(&budget.to_le_bytes());
342        }
343    }
344    float(out, label.moved);
345    match &label.pruned {
346        None => out.push(0),
347        Some(pruned) => {
348            out.push(1);
349            word(out, pruned.db.to_bits());
350            word(out, pruned.cuts.len() as u64);
351            for (node, at) in &pruned.cuts {
352                text(out, node);
353                word(out, *at as u64);
354            }
355        }
356    }
357}
358
359fn detailed(out: &mut Vec<u8>, detail: &Detail) {
360    let rule = |out: &mut Vec<u8>, rule: &Rule| text(out, rule.as_str());
361    match detail {
362        Detail::Lines {
363            rule: r,
364            placed,
365            summed,
366            dropped,
367            dropped_more,
368            terms,
369            tail_db,
370        } => {
371            out.push(0);
372            rule(out, r);
373            word(out, *placed as u64);
374            word(out, *summed as u64);
375            word(out, dropped.len() as u64);
376            for d in dropped {
377                word(out, d.hz.to_bits());
378                word(out, d.db.to_bits());
379            }
380            word(out, *dropped_more as u64);
381            maybe(out, terms.map(|t| t as u64));
382            float(out, *tail_db);
383        }
384        Detail::Continuous { rule: r } => {
385            out.push(1);
386            rule(out, r);
387        }
388        Detail::Cropped { rule: r, tail_db } => {
389            out.push(2);
390            rule(out, r);
391            float(out, *tail_db);
392        }
393        Detail::Point { rule: r, alias_db } => {
394            out.push(3);
395            rule(out, r);
396            float(out, *alias_db);
397        }
398        Detail::Spectrum { rule: r, wrap_db } => {
399            out.push(4);
400            rule(out, r);
401            word(out, wrap_db.to_bits());
402        }
403        Detail::Roundtrip { rule: r, edited } => {
404            out.push(5);
405            rule(out, r);
406            out.push(u8::from(*edited));
407        }
408        Detail::Reading { rule: r } => {
409            out.push(6);
410            rule(out, r);
411        }
412        Detail::Added { parts } => {
413            out.push(7);
414            word(out, parts.len() as u64);
415            for part in parts {
416                detailed(out, part);
417            }
418        }
419    }
420}
421
422struct Reader<'b>(&'b [u8]);
423
424impl Reader<'_> {
425    fn take(&mut self, n: usize) -> Option<&[u8]> {
426        if self.0.len() < n {
427            return None;
428        }
429        let (head, rest) = self.0.split_at(n);
430        self.0 = rest;
431        Some(head)
432    }
433
434    fn byte(&mut self) -> Option<u8> {
435        self.take(1).map(|b| b[0])
436    }
437
438    fn word(&mut self) -> Option<u64> {
439        Some(u64::from_le_bytes(self.take(8)?.try_into().ok()?))
440    }
441
442    fn segments(&mut self) -> Option<Vec<Buffer>> {
443        let count = self.word()? as usize;
444        let mut parts = Vec::new();
445        for _ in 0..count.min(self.0.len()) {
446            let rate = u32::try_from(self.word()?).ok()?;
447            let start = self.word()? as i64;
448            let (width, len) = (self.word()? as usize, self.word()? as usize);
449            if width.saturating_mul(len).saturating_mul(8) > self.0.len() {
450                return None;
451            }
452            let planes = (0..width)
453                .map(|_| (0..len).map(|_| self.word().map(f64::from_bits)).collect())
454                .collect::<Option<Vec<Vec<f64>>>>()?;
455            let mut part = Buffer::of_planes(rate, planes);
456            part.start = start;
457            parts.push(part);
458        }
459        (parts.len() == count).then_some(parts)
460    }
461
462    fn wide(&mut self) -> Option<u128> {
463        Some(u128::from_le_bytes(self.take(16)?.try_into().ok()?))
464    }
465
466    fn maybe(&mut self) -> Option<Option<u64>> {
467        match self.byte()? {
468            0 => Some(None),
469            1 => Some(Some(self.word()?)),
470            _ => None,
471        }
472    }
473
474    fn float(&mut self) -> Option<Option<f64>> {
475        Some(self.maybe()?.map(f64::from_bits))
476    }
477
478    fn text(&mut self) -> Option<&str> {
479        let len = self.word()? as usize;
480        std::str::from_utf8(self.take(len)?).ok()
481    }
482
483    fn rule(&mut self) -> Option<Rule> {
484        Rule::named(self.text()?)
485    }
486
487    fn label(&mut self) -> Option<Label> {
488        let source = match self.byte()? {
489            0 => Source::Exact,
490            1 => Source::Measured,
491            _ => return None,
492        };
493        let profile = match self.text()? {
494            name if name == PSYCHOACOUSTIC_V1.name => PSYCHOACOUSTIC_V1.name,
495            _ => return None,
496        };
497        let rate = u32::try_from(self.word()?).ok()?;
498        let detail = self.detail()?;
499        let cost = match self.byte()? {
500            0 => None,
501            1 => Some(Cost {
502                flops: self.wide()?,
503                budget: self.wide()?,
504            }),
505            _ => return None,
506        };
507        let moved = self.float()?;
508        let pruned = match self.byte()? {
509            0 => None,
510            1 => {
511                let db = f64::from_bits(self.word()?);
512                let count = usize::try_from(self.word()?).ok()?;
513                let cuts = (0..count)
514                    .map(|_| Some((self.text()?.to_string(), self.word()? as i64)))
515                    .collect::<Option<Vec<_>>>()?;
516                Some(sva_samples::Pruned { db, cuts })
517            }
518            _ => return None,
519        };
520        Some(Label {
521            source,
522            profile,
523            rate,
524            detail,
525            cost,
526            moved,
527            pruned,
528        })
529    }
530
531    fn detail(&mut self) -> Option<Detail> {
532        Some(match self.byte()? {
533            0 => {
534                let rule = self.rule()?;
535                let (placed, summed) = (self.word()? as usize, self.word()? as usize);
536                let count = self.word()? as usize;
537                let dropped = (0..count.min(self.0.len()))
538                    .map(|_| {
539                        Some(Dropped {
540                            hz: f64::from_bits(self.word()?),
541                            db: f64::from_bits(self.word()?),
542                        })
543                    })
544                    .collect::<Option<Vec<_>>>()?;
545                if dropped.len() != count {
546                    return None;
547                }
548                Detail::Lines {
549                    rule,
550                    placed,
551                    summed,
552                    dropped,
553                    dropped_more: self.word()? as usize,
554                    terms: self.maybe()?.map(|t| t as usize),
555                    tail_db: self.float()?,
556                }
557            }
558            1 => Detail::Continuous { rule: self.rule()? },
559            2 => Detail::Cropped {
560                rule: self.rule()?,
561                tail_db: self.float()?,
562            },
563            3 => Detail::Point {
564                rule: self.rule()?,
565                alias_db: self.float()?,
566            },
567            4 => Detail::Spectrum {
568                rule: self.rule()?,
569                wrap_db: f64::from_bits(self.word()?),
570            },
571            5 => Detail::Roundtrip {
572                rule: self.rule()?,
573                edited: match self.byte()? {
574                    0 => false,
575                    1 => true,
576                    _ => return None,
577                },
578            },
579            6 => Detail::Reading { rule: self.rule()? },
580            7 => {
581                let count = self.word()? as usize;
582                let parts = (0..count.min(self.0.len()))
583                    .map(|_| self.detail())
584                    .collect::<Option<Vec<_>>>()?;
585                if parts.len() != count {
586                    return None;
587                }
588                Detail::Added { parts }
589            }
590            _ => return None,
591        })
592    }
593}
sva_engine/cache/codec.rs

sva_engine/cache/
codec.rs
