sva_engine/cache/
codec.rs

1// Concern: one stored node value, tagged with the store's format, and one staged run, as bytes and back | Non-concern: where the bytes live or when | IO: (Stored or Buffer) <-> bytes, a bad one None
2
3use sva_formula::Codomain;
4use sva_samples::{
5    BLOCK, Buffer, Detail, Dropped, Extent, Grid, Label, PSYCHOACOUSTIC_V1, Rule, Source, block_end,
6};
7
8use sva_formula::{FOURIER_DUAL_RULES_VERSION, Hash};
9
10use super::Stored;
11use super::stored::{Header, Laid, Samples};
12
13/// Bumped by, and only by, a change to a stored value's bytes or to the key it is stored under.
14pub const STORE_FORMAT: u32 = 44;
15
16/// Every entry opens with its format and Fourier dual rules version, so one another wrote is
17/// never read as a value.
18fn entry_tag() -> Vec<u8> {
19    [
20        &b"SVAh"[..],
21        &STORE_FORMAT.to_le_bytes(),
22        &FOURIER_DUAL_RULES_VERSION.to_le_bytes(),
23    ]
24    .concat()
25}
26
27const STAGED_RUN: &[u8; 4] = b"SVAc";
28
29/// A run's chunks are its samples within each block of the grid: what one checksum covers,
30/// and the least a read loads. Samples stay f64, as the render computes them: a stored value
31/// is read by others, and f32 would change the bits of every value that reads it.
32fn chunks(start: i64, len: usize) -> Option<usize> {
33    if len == 0 {
34        return Some(0);
35    }
36    let last = start.checked_add(i64::try_from(len).ok()? - 1)?;
37    usize::try_from(last.div_euclid(BLOCK) - start.div_euclid(BLOCK) + 1).ok()
38}
39
40/// How many of a run's samples come before its chunk `k`.
41fn before(run: &Laid, k: usize) -> usize {
42    let edge = match k {
43        0 => run.start,
44        k => (run.start.div_euclid(BLOCK) + k as i64).saturating_mul(BLOCK),
45    };
46    (edge.min(run.start + run.len as i64) - run.start) as usize
47}
48
49/// The chunks of `run` that hold `over`, met with the run.
50pub(crate) fn chunks_over(run: &Laid, over: Extent) -> (usize, usize) {
51    let first = run.start.div_euclid(BLOCK);
52    let at = |n: i64| (n.div_euclid(BLOCK) - first) as usize;
53    (at(over.start), at(over.end - 1) + 1)
54}
55
56/// A header's length, then the header, then each run's samples chunk by chunk, every plane of
57/// a chunk together, so a stretch of a run is one read.
58pub(crate) fn entry(head: &Header, runs: &[Buffer]) -> Vec<u8> {
59    let mut laid = Vec::new();
60    let mut body = Vec::new();
61    for run in runs {
62        let mut sums = Vec::new();
63        let at = body.len() as u64;
64        let mut from = 0;
65        while from < run.len() {
66            let to = (block_end(run.start + from as i64) - run.start).min(run.len() as i64);
67            let to = to as usize;
68            let chunk = body.len();
69            for plane in &run.planes {
70                for sample in &plane[from..to] {
71                    word(&mut body, sample.to_bits());
72                }
73            }
74            sums.push(checksum(&body[chunk..]));
75            from = to;
76        }
77        laid.push(Laid {
78            rate: run.rate,
79            start: run.start,
80            width: run.width(),
81            len: run.len(),
82            at,
83            sums,
84        });
85    }
86    let head = sealed(header(head, &laid));
87    let mut out = Vec::with_capacity(8 + head.len() + body.len());
88    word(&mut out, head.len() as u64);
89    out.extend_from_slice(&head);
90    out.extend_from_slice(&body);
91    out
92}
93
94fn header(head: &Header, laid: &[Laid]) -> Vec<u8> {
95    let stored = head.stored();
96    let mut out = entry_tag();
97    word(&mut out, stored.key.0);
98    word(&mut out, stored.key.1);
99    word(&mut out, stored.identity.0);
100    word(&mut out, stored.identity.1);
101    labelled(&mut out, &stored.label);
102    out.push(stored.width);
103    out.push(match stored.codomain {
104        Codomain::Real => 0,
105        Codomain::Complex => 1,
106    });
107    maybe(&mut out, stored.rate.map(u64::from));
108    word(&mut out, u64::from(stored.grid.rate));
109    out.extend_from_slice(&stored.grid.a.to_le_bytes());
110    out.extend_from_slice(&stored.grid.d.to_le_bytes());
111    word(&mut out, stored.support.start as u64);
112    word(&mut out, stored.support.end as u64);
113    word(&mut out, stored.moved.to_bits());
114    out.push(u8::from(stored.readable));
115    out.push(u8::from(stored.sampled));
116    match head.samples() {
117        &Samples::Of { key, by } => {
118            out.push(1);
119            word(&mut out, key.0);
120            word(&mut out, key.1);
121            word(&mut out, by as u64);
122        }
123        _ => out.push(0),
124    }
125    word(&mut out, laid.len() as u64);
126    for run in laid {
127        word(&mut out, u64::from(run.rate));
128        word(&mut out, run.start as u64);
129        word(&mut out, run.width as u64);
130        word(&mut out, run.len as u64);
131        for sum in &run.sums {
132            word(&mut out, *sum);
133        }
134    }
135    out
136}
137
138/// How many of an entry's first bytes its header takes, off its first eight.
139pub(crate) fn head_len(first: &[u8]) -> Option<usize> {
140    let len = u64::from_le_bytes(first.get(..8)?.try_into().ok()?);
141    8usize.checked_add(usize::try_from(len).ok()?)
142}
143
144/// The meta an entry's header states, its samples laid in `file`, and the entry's length; a
145/// truncated, corrupt or foreign header is `None`, never a partial value.
146pub(crate) fn read_head(bytes: &[u8], file: Hash) -> Option<(Header, u64)> {
147    let span = head_len(bytes)?;
148    let mut r = Reader(opened(bytes.get(8..span)?, &entry_tag())?);
149    let key = Hash(r.word()?, r.word()?);
150    let identity = Hash(r.word()?, r.word()?);
151    let label = r.label()?;
152    let width = r.byte()?;
153    let codomain = match r.byte()? {
154        0 => Codomain::Real,
155        1 => Codomain::Complex,
156        _ => return None,
157    };
158    let rate = match r.maybe()? {
159        Some(rate) => Some(u32::try_from(rate).ok()?),
160        None => None,
161    };
162    let grid = Grid {
163        rate: u32::try_from(r.word()?).ok()?,
164        a: r.wide()? as i128,
165        d: r.wide()? as i128,
166    };
167    let (start, end) = (r.word()? as i64, r.word()? as i64);
168    let support = (start <= end).then(|| Extent::new(start, end))?;
169    let moved = f64::from_bits(r.word()?);
170    let readable = r.flag()?;
171    let sampled = r.flag()?;
172    let of = match r.byte()? {
173        0 => None,
174        1 => Some(Samples::Of {
175            key: Hash(r.word()?, r.word()?),
176            by: r.word()? as i64,
177        }),
178        _ => return None,
179    };
180    let count = r.word()? as usize;
181    let mut runs = Vec::new();
182    let mut at = span as u64;
183    for _ in 0..count.min(r.0.len()) {
184        let rate = u32::try_from(r.word()?).ok()?;
185        let start = r.word()? as i64;
186        let (width, len) = (r.word()? as usize, r.word()? as usize);
187        let count = chunks(start, len)?;
188        let sums = (0..count.min(r.0.len()))
189            .map(|_| r.word())
190            .collect::<Option<Vec<u64>>>()?;
191        if sums.len() != count {
192            return None;
193        }
194        runs.push(Laid {
195            rate,
196            start,
197            width,
198            len,
199            at,
200            sums,
201        });
202        at = at.checked_add((width.checked_mul(len)?.checked_mul(8)?) as u64)?;
203    }
204    let samples = match (of, runs.is_empty()) {
205        (Some(of), true) => of,
206        (Some(_), false) => return None,
207        (None, true) => Samples::None,
208        (None, false) => Samples::Entry {
209            file,
210            runs,
211            shift: 0,
212        },
213    };
214    let stored = Stored {
215        key,
216        identity,
217        label,
218        width,
219        codomain,
220        rate,
221        grid,
222        support,
223        moved,
224        readable,
225        sampled,
226        held: Vec::new(),
227    };
228    let whole = runs_whole(count, &samples) && r.0.is_empty();
229    whole.then(|| (Header::new(stored, samples), at))
230}
231
232fn runs_whole(count: usize, samples: &Samples) -> bool {
233    match samples {
234        Samples::Entry { runs, .. } => runs.len() == count,
235        _ => count == 0,
236    }
237}
238
239/// Where in its file a run's chunks `from..to` lie, and how many bytes.
240pub(crate) fn span_of(run: &Laid, from: usize, to: usize) -> (u64, u64) {
241    let bytes = |k: usize| (before(run, k) * run.width * 8) as u64;
242    (run.at + bytes(from), bytes(to) - bytes(from))
243}
244
245/// A run's chunks `from..to` off the bytes `span_of` names; `None` where a checksum fails.
246pub(crate) fn read_chunks(bytes: &[u8], run: &Laid, from: usize, to: usize) -> Option<Buffer> {
247    let first = before(run, from);
248    let len = before(run, to).checked_sub(first)?;
249    if bytes.len() != len * run.width * 8 {
250        return None;
251    }
252    let mut planes = vec![Vec::with_capacity(len); run.width];
253    let mut at = 0;
254    for (k, sum) in run.sums[from..to].iter().enumerate() {
255        let n = before(run, from + k + 1) - before(run, from + k);
256        let chunk = &bytes[at..at + n * run.width * 8];
257        if checksum(chunk) != *sum {
258            return None;
259        }
260        for (c, plane) in planes.iter_mut().enumerate() {
261            let own = &chunk[c * n * 8..(c + 1) * n * 8];
262            plane.extend(
263                own.chunks_exact(8).map(|b| {
264                    f64::from_bits(u64::from_le_bytes(b.try_into().expect("eight bytes")))
265                }),
266            );
267        }
268        at += chunk.len();
269    }
270    let mut out = Buffer::of_planes(run.rate, planes);
271    out.start = run.start + first as i64;
272    Some(out)
273}
274
275/// Every run an entry under `key` lays out; `None` where any is corrupt.
276pub(crate) fn read_runs(bytes: &[u8], key: Hash) -> Option<Vec<Buffer>> {
277    let (head, _) = read_head(bytes, key)?;
278    let Samples::Entry { runs, .. } = head.samples() else {
279        return Some(Vec::new());
280    };
281    let read = |run: &Laid| {
282        let n = chunks(run.start, run.len)?;
283        let (at, len) = span_of(run, 0, n);
284        let span = bytes.get(at as usize..(at + len) as usize)?;
285        read_chunks(span, run, 0, n)
286    };
287    runs.iter().map(read).collect()
288}
289
290pub(crate) fn chunk(samples: &Buffer) -> Vec<u8> {
291    let mut out = STAGED_RUN.to_vec();
292    segments(&mut out, std::slice::from_ref(samples));
293    sealed(out)
294}
295
296pub(crate) fn read_chunk(bytes: &[u8]) -> Option<Buffer> {
297    let mut r = Reader(opened(bytes, STAGED_RUN)?);
298    let mut parts = r.segments()?;
299    (r.0.is_empty() && parts.len() == 1).then(|| parts.remove(0))
300}
301
302fn sealed(mut out: Vec<u8>) -> Vec<u8> {
303    let sum = checksum(&out);
304    word(&mut out, sum);
305    out
306}
307
308/// The body past `magic`, where the checksum holds.
309fn opened<'b>(bytes: &'b [u8], magic: &[u8]) -> Option<&'b [u8]> {
310    let body = bytes.len().checked_sub(8)?;
311    let (body, sum) = bytes.split_at(body);
312    (checksum(body).to_le_bytes() == sum && body.starts_with(magic)).then(|| &body[magic.len()..])
313}
314
315fn segments(out: &mut Vec<u8>, parts: &[Buffer]) {
316    word(out, parts.len() as u64);
317    for part in parts {
318        word(out, u64::from(part.rate));
319        word(out, part.start as u64);
320        word(out, part.width() as u64);
321        word(out, part.len() as u64);
322        for plane in &part.planes {
323            for sample in plane {
324                word(out, sample.to_bits());
325            }
326        }
327    }
328}
329
330fn checksum(bytes: &[u8]) -> u64 {
331    bytes.iter().fold(0xcbf2_9ce4_8422_2325, |h, b| {
332        (h ^ u64::from(*b)).wrapping_mul(0x0100_0000_01b3)
333    })
334}
335
336fn word(out: &mut Vec<u8>, w: u64) {
337    out.extend_from_slice(&w.to_le_bytes());
338}
339
340fn float(out: &mut Vec<u8>, v: Option<f64>) {
341    maybe(out, v.map(f64::to_bits));
342}
343
344fn maybe(out: &mut Vec<u8>, w: Option<u64>) {
345    match w {
346        None => out.push(0),
347        Some(w) => {
348            out.push(1);
349            word(out, w);
350        }
351    }
352}
353
354fn text(out: &mut Vec<u8>, s: &str) {
355    word(out, s.len() as u64);
356    out.extend_from_slice(s.as_bytes());
357}
358
359fn labelled(out: &mut Vec<u8>, label: &Label) {
360    out.push(match label.source {
361        Source::Exact => 0,
362        Source::Measured => 1,
363    });
364    text(out, label.profile);
365    word(out, u64::from(label.rate));
366    detailed(out, &label.detail);
367    float(out, label.moved);
368    match &label.cutting_below_silence_threshold {
369        None => out.push(0),
370        Some(cutting) => {
371            out.push(1);
372            word(out, cutting.silence_threshold_dbfs.to_bits());
373            word(out, cutting.treated_as_silent_from_sample.len() as u64);
374            for (node, at) in &cutting.treated_as_silent_from_sample {
375                text(out, node);
376                word(out, *at as u64);
377            }
378        }
379    }
380}
381
382fn detailed(out: &mut Vec<u8>, detail: &Detail) {
383    let rule = |out: &mut Vec<u8>, rule: &Rule| text(out, rule.as_str());
384    match detail {
385        Detail::Lines {
386            rule: r,
387            summed,
388            dropped,
389            dropped_more,
390            tail_db,
391        } => {
392            out.push(0);
393            rule(out, r);
394            word(out, *summed as u64);
395            word(out, dropped.len() as u64);
396            for d in dropped {
397                word(out, d.hz.to_bits());
398                word(out, d.db.to_bits());
399            }
400            word(out, *dropped_more as u64);
401            float(out, *tail_db);
402        }
403        Detail::Continuous { rule: r } => {
404            out.push(1);
405            rule(out, r);
406        }
407        Detail::Cropped { rule: r, tail_db } => {
408            out.push(2);
409            rule(out, r);
410            float(out, *tail_db);
411        }
412        Detail::Point {
413            rule: r,
414            alias_db,
415            tail_db,
416        } => {
417            out.push(3);
418            rule(out, r);
419            float(out, *alias_db);
420            float(out, *tail_db);
421        }
422        Detail::Roundtrip { rule: r, edited } => {
423            out.push(5);
424            rule(out, r);
425            out.push(u8::from(*edited));
426        }
427        Detail::Reading { rule: r } => {
428            out.push(6);
429            rule(out, r);
430        }
431        Detail::Added { parts } => {
432            out.push(7);
433            word(out, parts.len() as u64);
434            for part in parts {
435                detailed(out, part);
436            }
437        }
438    }
439}
440
441struct Reader<'b>(&'b [u8]);
442
443impl Reader<'_> {
444    fn take(&mut self, n: usize) -> Option<&[u8]> {
445        if self.0.len() < n {
446            return None;
447        }
448        let (head, rest) = self.0.split_at(n);
449        self.0 = rest;
450        Some(head)
451    }
452
453    fn byte(&mut self) -> Option<u8> {
454        self.take(1).map(|b| b[0])
455    }
456
457    fn flag(&mut self) -> Option<bool> {
458        match self.byte()? {
459            0 => Some(false),
460            1 => Some(true),
461            _ => None,
462        }
463    }
464
465    fn word(&mut self) -> Option<u64> {
466        Some(u64::from_le_bytes(self.take(8)?.try_into().ok()?))
467    }
468
469    fn segments(&mut self) -> Option<Vec<Buffer>> {
470        let count = self.word()? as usize;
471        let mut parts = Vec::new();
472        for _ in 0..count.min(self.0.len()) {
473            let rate = u32::try_from(self.word()?).ok()?;
474            let start = self.word()? as i64;
475            let (width, len) = (self.word()? as usize, self.word()? as usize);
476            if width.saturating_mul(len).saturating_mul(8) > self.0.len() {
477                return None;
478            }
479            let planes = (0..width)
480                .map(|_| (0..len).map(|_| self.word().map(f64::from_bits)).collect())
481                .collect::<Option<Vec<Vec<f64>>>>()?;
482            let mut part = Buffer::of_planes(rate, planes);
483            part.start = start;
484            parts.push(part);
485        }
486        (parts.len() == count).then_some(parts)
487    }
488
489    fn wide(&mut self) -> Option<u128> {
490        Some(u128::from_le_bytes(self.take(16)?.try_into().ok()?))
491    }
492
493    fn maybe(&mut self) -> Option<Option<u64>> {
494        match self.byte()? {
495            0 => Some(None),
496            1 => Some(Some(self.word()?)),
497            _ => None,
498        }
499    }
500
501    fn float(&mut self) -> Option<Option<f64>> {
502        Some(self.maybe()?.map(f64::from_bits))
503    }
504
505    fn text(&mut self) -> Option<&str> {
506        let len = self.word()? as usize;
507        std::str::from_utf8(self.take(len)?).ok()
508    }
509
510    fn rule(&mut self) -> Option<Rule> {
511        Rule::named(self.text()?)
512    }
513
514    fn label(&mut self) -> Option<Label> {
515        let source = match self.byte()? {
516            0 => Source::Exact,
517            1 => Source::Measured,
518            _ => return None,
519        };
520        let profile = match self.text()? {
521            name if name == PSYCHOACOUSTIC_V1.name => PSYCHOACOUSTIC_V1.name,
522            _ => return None,
523        };
524        let rate = u32::try_from(self.word()?).ok()?;
525        let detail = self.detail()?;
526        let moved = self.float()?;
527        let cutting_below_silence_threshold = match self.byte()? {
528            0 => None,
529            1 => {
530                let silence_threshold_dbfs = f64::from_bits(self.word()?);
531                let count = usize::try_from(self.word()?).ok()?;
532                let treated_as_silent_from_sample = (0..count)
533                    .map(|_| Some((self.text()?.to_string(), self.word()? as i64)))
534                    .collect::<Option<Vec<_>>>()?;
535                Some(sva_samples::CuttingBelowSilenceThreshold {
536                    silence_threshold_dbfs,
537                    treated_as_silent_from_sample,
538                })
539            }
540            _ => return None,
541        };
542        Some(Label {
543            source,
544            profile,
545            rate,
546            detail,
547            moved,
548            cutting_below_silence_threshold,
549        })
550    }
551
552    fn detail(&mut self) -> Option<Detail> {
553        Some(match self.byte()? {
554            0 => {
555                let rule = self.rule()?;
556                let summed = self.word()? as usize;
557                let count = self.word()? as usize;
558                let dropped = (0..count.min(self.0.len()))
559                    .map(|_| {
560                        Some(Dropped {
561                            hz: f64::from_bits(self.word()?),
562                            db: f64::from_bits(self.word()?),
563                        })
564                    })
565                    .collect::<Option<Vec<_>>>()?;
566                if dropped.len() != count {
567                    return None;
568                }
569                Detail::Lines {
570                    rule,
571                    summed,
572                    dropped,
573                    dropped_more: self.word()? as usize,
574                    tail_db: self.float()?,
575                }
576            }
577            1 => Detail::Continuous { rule: self.rule()? },
578            2 => Detail::Cropped {
579                rule: self.rule()?,
580                tail_db: self.float()?,
581            },
582            3 => Detail::Point {
583                rule: self.rule()?,
584                alias_db: self.float()?,
585                tail_db: self.float()?,
586            },
587            5 => Detail::Roundtrip {
588                rule: self.rule()?,
589                edited: match self.byte()? {
590                    0 => false,
591                    1 => true,
592                    _ => return None,
593                },
594            },
595            6 => Detail::Reading { rule: self.rule()? },
596            7 => {
597                let count = self.word()? as usize;
598                let parts = (0..count.min(self.0.len()))
599                    .map(|_| self.detail())
600                    .collect::<Option<Vec<_>>>()?;
601                if parts.len() != count {
602                    return None;
603                }
604                Detail::Added { parts }
605            }
606            _ => return None,
607        })
608    }
609}
sva_engine/cache/codec.rs

sva_engine/cache/
codec.rs
